srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) Console() string {
|
||||
func (h *Handler) Console() (string, error) {
|
||||
viper.SetDefault("env", "devel")
|
||||
cmd := viper.GetString("create")
|
||||
cmdList := viper.Sub("console.cmd")
|
||||
@@ -16,11 +16,11 @@ func (h *Handler) Console() string {
|
||||
|
||||
if !slices.Contains(allowCreate, cmd) {
|
||||
fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd))
|
||||
return ""
|
||||
return "", fmt.Errorf("command %q is not allowed", cmd)
|
||||
}
|
||||
|
||||
fmt.Printf("%v\n", slices.Contains(allowCreate, cmd))
|
||||
|
||||
return viper.GetString("env")
|
||||
return viper.GetString("env"), nil
|
||||
|
||||
}
|
||||
|
||||
+18
-10
@@ -1,7 +1,7 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"log" // Додано імпорт log
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
)
|
||||
|
||||
// consoleCommandHandler визначає тип для функцій-обробників консольних команд.
|
||||
type consoleCommandHandler func() string
|
||||
// The error it returns decides the process exit code, so cron can tell a failed
|
||||
// run from a successful one.
|
||||
type consoleCommandHandler func() (string, error)
|
||||
|
||||
type Handler struct {
|
||||
services *service.Service
|
||||
@@ -29,24 +31,30 @@ func New(services *service.Service) *Handler {
|
||||
"prescene": h.Prescene,
|
||||
"rutracker": h.Rutracker,
|
||||
"stb": h.STB,
|
||||
"srrdb": h.Srrdb,
|
||||
//"web": h.Web,
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *Handler) InitConsole() string {
|
||||
// InitConsole runs the command selected by --scope-enable. The text is returned
|
||||
// even together with an error, so the caller can print what happened before it
|
||||
// reports the failure.
|
||||
func (h *Handler) InitConsole() (string, error) {
|
||||
commandName := viper.GetString(constant.FlagScopeEnable)
|
||||
|
||||
if cmdFunc, ok := h.consoleCommands[commandName]; ok {
|
||||
return commandName + " launched, " + cmdFunc() + "\n"
|
||||
} else {
|
||||
log.Printf("Error: unknown console command '%s'", commandName)
|
||||
return "Error: unknown console command"
|
||||
cmdFunc, ok := h.consoleCommands[commandName]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("unknown console command %q", commandName)
|
||||
}
|
||||
|
||||
result, err := cmdFunc()
|
||||
|
||||
return commandName + " launched, " + result + "\n", err
|
||||
}
|
||||
|
||||
func (h *Handler) InitRoutes() string {
|
||||
func (h *Handler) InitRoutes() (string, error) {
|
||||
//TODO:
|
||||
return "i am the web initiator"
|
||||
return "i am the web initiator", nil
|
||||
}
|
||||
|
||||
+3
-2
@@ -12,13 +12,14 @@ import (
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) Info() string {
|
||||
func (h *Handler) Info() (string, error) {
|
||||
md5()
|
||||
|
||||
fmt.Printf("%s: %s; %s: %s\n",
|
||||
aurora.Cyan("ENV"),
|
||||
viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable))
|
||||
return "info"
|
||||
|
||||
return "info", nil
|
||||
}
|
||||
|
||||
func md5() {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
package handler
|
||||
|
||||
func (h *Handler) MetalArchives() string {
|
||||
return "MetalArchives................."
|
||||
func (h *Handler) MetalArchives() (string, error) {
|
||||
return "MetalArchives.................", nil
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -12,7 +11,7 @@ import (
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) Prescene() string {
|
||||
func (h *Handler) Prescene() (string, error) {
|
||||
currMinute := time.Now().Minute()
|
||||
levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
|
||||
initPage := strconv.Itoa(currMinute/(60/levels) + 1)
|
||||
@@ -30,17 +29,15 @@ func (h *Handler) Prescene() string {
|
||||
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
||||
}
|
||||
|
||||
_, err := h.services.Prescene.GetSrrDb()
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||
if _, err := h.services.Prescene.GetSrrDb(); err != nil {
|
||||
return "ps", fmt.Errorf("getting srr db: %w", err)
|
||||
}
|
||||
|
||||
_, err = h.services.Prescene.GetPage(pagesToScrape)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||
if _, err := h.services.Prescene.GetPage(pagesToScrape); err != nil {
|
||||
return "ps", fmt.Errorf("getting page: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("%+v\n", pagesToScrape)
|
||||
|
||||
return "ps"
|
||||
return "ps", nil
|
||||
}
|
||||
|
||||
@@ -2,20 +2,18 @@ package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) Rutracker() string {
|
||||
func (h *Handler) Rutracker() (string, error) {
|
||||
key := fmt.Sprintf("topic.%v", time.Now().Hour())
|
||||
topics := viper.GetStringSlice(key)
|
||||
|
||||
_, err := h.services.Rutracker.GetTopic(topics)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
||||
if _, err := h.services.Rutracker.GetTopic(topics); err != nil {
|
||||
return "rt", fmt.Errorf("getting topic: %w", err)
|
||||
}
|
||||
|
||||
return "rt"
|
||||
return "rt", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
// Srrdb runs one srrDB search pass over the raw path given by --single-uri.
|
||||
func (h *Handler) Srrdb() (string, error) {
|
||||
uri := viper.GetString(constant.FlagSingleUri)
|
||||
if uri == "" {
|
||||
return "srrdb", errors.New("no search path, pass --single-uri")
|
||||
}
|
||||
|
||||
stored, err := h.services.Srrdb.Search(uri)
|
||||
|
||||
return fmt.Sprintf("%d records\n", stored), err
|
||||
}
|
||||
+3
-5
@@ -3,20 +3,18 @@ package handler
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) STB() string {
|
||||
func (h *Handler) STB() (string, error) {
|
||||
endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint)
|
||||
endpoint = viper.GetString(endpoint)
|
||||
|
||||
if endpoint == "" {
|
||||
slog.Error("getting endpoint from config", "err", errors.New("no endpoint provided"))
|
||||
return "stb"
|
||||
return "stb", errors.New("no endpoint provided")
|
||||
}
|
||||
|
||||
es, ms := h.services.ShareTheBrutality.GetMail(endpoint)
|
||||
@@ -32,5 +30,5 @@ func (h *Handler) STB() string {
|
||||
slog.Error("error occurred while getting topic: ", "err", err)
|
||||
}*/
|
||||
|
||||
return fmt.Sprintf("Added %d records\n", len(es))
|
||||
return fmt.Sprintf("Added %d records\n", len(es)), nil
|
||||
}
|
||||
|
||||
@@ -10,12 +10,16 @@ type Repository struct {
|
||||
_interface.Rutracker
|
||||
_interface.Prescene
|
||||
_interface.ShareTheBrutality
|
||||
_interface.Srrdb
|
||||
}
|
||||
|
||||
func New(db *sqlx.DB) *Repository {
|
||||
// New wires the collectors. sqlite serves the srrdb scope only and may be nil
|
||||
// for every other source; it is connected lazily in Bootstrap.
|
||||
func New(db, sqlite *sqlx.DB) *Repository {
|
||||
return &Repository{
|
||||
Rutracker: NewRutrackerRepository(db),
|
||||
Prescene: NewPresceneRepository(db),
|
||||
ShareTheBrutality: NewShareTheBrutalityRepository(db),
|
||||
Srrdb: NewSrrdbRepository(sqlite),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,140 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
// srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the
|
||||
// API itself uses in its date field, because the column is TEXT.
|
||||
const srrdbTimestamp = "2006-01-02 15:04:05"
|
||||
|
||||
// Srrdb harvests the srrDB search API into the separate SQLite database.
|
||||
type Srrdb struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
// NewSrrdbRepository returns a collector writing into the SQLite handle. The
|
||||
// handle is nil for every scope except srrdb, because Bootstrap connects the
|
||||
// SQLite database lazily, so the collector refuses to run without it instead of
|
||||
// failing later on a nil handle.
|
||||
func NewSrrdbRepository(db *sqlx.DB) *Srrdb {
|
||||
return &Srrdb{db: db}
|
||||
}
|
||||
|
||||
// Search harvests one search path. Nothing is written to the database until the
|
||||
// whole set is collected and its size matches what the API reported, so the
|
||||
// dump and the success log can only ever describe a complete day.
|
||||
func (s *Srrdb) Search(rawURI string) (int, error) {
|
||||
if s.db == nil {
|
||||
err := errors.New("srrdb collector has no database handle")
|
||||
srrdbLogError(rawURI, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !strings.Contains(rawURI, "category:") {
|
||||
err := fmt.Errorf("search path %q has no category: segment", rawURI)
|
||||
srrdbLogError(rawURI, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
auto := strings.Contains(rawURI, srrdbAutoSegment)
|
||||
|
||||
now := time.Now()
|
||||
|
||||
uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, errSrrdbFloor):
|
||||
slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI)
|
||||
|
||||
return 0, nil
|
||||
case err != nil:
|
||||
srrdbLogError(rawURI, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// A slash typed on either end of the command line would build a double
|
||||
// separator in the URL and an odd dump file name.
|
||||
uri = strings.Trim(uri, "/")
|
||||
|
||||
// The guard applies to the automatic walk only. A day named on the command
|
||||
// line is fetched again and its dump rewritten every time: a day can still
|
||||
// gain releases later, either from another time zone or from a late upload,
|
||||
// and cron refreshes today and yesterday on purpose.
|
||||
dump := srrdbDumpPath(uri)
|
||||
|
||||
if auto {
|
||||
exists, err := srrdbExists(dump)
|
||||
if err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if exists {
|
||||
slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump)
|
||||
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
collected, err := srrdbCollectAll(uri)
|
||||
if err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// An empty day is a normal outcome: zero results and no warnings. The dump
|
||||
// records it as harvested, which keeps the walk moving. Warnings mean the
|
||||
// path was not understood, and that must never be remembered as done.
|
||||
switch {
|
||||
case len(collected.Warnings) > 0:
|
||||
srrdbLogWarning(uri, collected.Warnings)
|
||||
|
||||
return 0, fmt.Errorf("srrdb %q reported warnings: %s",
|
||||
uri, strings.Join(collected.Warnings, "; "))
|
||||
case len(collected.Results) != collected.ResultsCount:
|
||||
err := fmt.Errorf("srrdb %q pulled %d of %d reported releases",
|
||||
uri, len(collected.Results), collected.ResultsCount)
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
run := time.Now().Format(srrdbTimestamp)
|
||||
category := srrdbCategory(uri)
|
||||
rows := make([]model.SrrdbRelease, 0, len(collected.Results))
|
||||
|
||||
for _, result := range collected.Results {
|
||||
rows = append(rows, srrdbRow(result, run, category))
|
||||
}
|
||||
|
||||
if err := table.UpsertAllSrrdb(s.db, rows); err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := srrdbWriteDump(dump, collected); err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return len(rows), err
|
||||
}
|
||||
|
||||
srrdbLogStatus(uri, len(rows), collected.ResultsCount)
|
||||
|
||||
return len(rows), nil
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
||||
)
|
||||
|
||||
// srrdbPageSegment locates the page:N segment so a run can continue from N into
|
||||
// the following pages.
|
||||
var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`)
|
||||
|
||||
// srrdbCollectAll walks every page before anything is written, so the caller
|
||||
// only ever receives a complete set or an error.
|
||||
func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) {
|
||||
endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/")
|
||||
|
||||
// The flag binds to its own key and wins over the config value; an unused
|
||||
// flag yields zero.
|
||||
sleep := viper.GetDuration(constant.FlagSleep)
|
||||
if sleep <= 0 {
|
||||
sleep = viper.GetDuration("srrdb.sleep")
|
||||
}
|
||||
if sleep <= 0 {
|
||||
sleep = 2 * time.Second
|
||||
}
|
||||
|
||||
slog.Info("srrdb pacing", "uri", uri, "sleep", sleep)
|
||||
|
||||
page, paginated := srrdbStartPage(uri)
|
||||
collected := &resource.SrrdbSearchResponse{}
|
||||
|
||||
for {
|
||||
url := endpoint + "/" + srrdbWithPage(uri, page, paginated)
|
||||
|
||||
response, err := srrdbFetch(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
srrdbCollect(collected, response)
|
||||
|
||||
slog.Info("srrdb page fetched",
|
||||
"url", url,
|
||||
"page", page,
|
||||
"fetched", len(response.Results),
|
||||
"reported", response.ResultsCount)
|
||||
|
||||
if !paginated || len(response.Results) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Everything the API reported is already collected: one more page would
|
||||
// cost a request and a pause to receive an empty answer.
|
||||
if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount {
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
// srrdbCollect merges one page into the answer kept for the dump: results are
|
||||
// appended, while resultsCount and query come from the first page.
|
||||
func srrdbCollect(all, page *resource.SrrdbSearchResponse) {
|
||||
if all.ResultsCount == 0 {
|
||||
all.ResultsCount = page.ResultsCount
|
||||
}
|
||||
|
||||
if len(all.Query) == 0 {
|
||||
all.Query = page.Query
|
||||
}
|
||||
|
||||
all.Results = append(all.Results, page.Results...)
|
||||
all.Warnings = append(all.Warnings, page.Warnings...)
|
||||
}
|
||||
|
||||
// srrdbStartPage reads the page:N segment. Without it the path is a single
|
||||
// request and no further pages are attempted.
|
||||
func srrdbStartPage(uri string) (int, bool) {
|
||||
match := srrdbPageSegment.FindStringSubmatch(uri)
|
||||
if match == nil {
|
||||
return 1, false
|
||||
}
|
||||
|
||||
page, err := strconv.Atoi(match[1])
|
||||
if err != nil || page < 1 {
|
||||
return 1, true
|
||||
}
|
||||
|
||||
return page, true
|
||||
}
|
||||
|
||||
// srrdbWithPage swaps the page:N segment for the page requested now.
|
||||
func srrdbWithPage(uri string, page int, paginated bool) string {
|
||||
if !paginated {
|
||||
return uri
|
||||
}
|
||||
|
||||
return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page))
|
||||
}
|
||||
|
||||
// srrdbFetch performs one search request and decodes the JSON answer.
|
||||
func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
request, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build srrdb request %q: %w", url, err)
|
||||
}
|
||||
|
||||
request.Header.Set("Accept", "application/json")
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request srrdb %q: %w", url, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := response.Body.Close(); cerr != nil {
|
||||
slog.Error("closing srrdb response body", "url", url, "err", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read srrdb body %q: %w", url, err)
|
||||
}
|
||||
|
||||
decoded := &resource.SrrdbSearchResponse{}
|
||||
if err := json.Unmarshal(body, decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode srrdb body %q: %w", url, err)
|
||||
}
|
||||
|
||||
return decoded, nil
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// srrdbAutoSegment is the placeholder replaced by the date the run has to
|
||||
// harvest, so cron can repeat the same command every day.
|
||||
const srrdbAutoSegment = "date:auto"
|
||||
|
||||
// srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for
|
||||
// the neighbouring days, so cron can name a day without hard-coding a date.
|
||||
const (
|
||||
srrdbYesterdaySegment = "date:yesterday"
|
||||
srrdbTodaySegment = "date:today"
|
||||
srrdbTomorrowSegment = "date:tomorrow"
|
||||
)
|
||||
|
||||
// srrdbDayLayout is the layout of a date segment in a search path.
|
||||
const srrdbDayLayout = "2006-01-02"
|
||||
|
||||
// errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is
|
||||
// the archive being fully covered rather than a failure.
|
||||
var errSrrdbFloor = errors.New("srrdb.floor reached")
|
||||
|
||||
// srrdbResolveAutoDate replaces date:auto with the newest day in
|
||||
// [srrdb.floor..yesterday] that has not been harvested yet.
|
||||
//
|
||||
// Scanning from the top-down means a new day is picked up the moment it appears,
|
||||
// and a hole left in the middle is closed once the walk reaches it. The previous
|
||||
// rule (the day before the earliest dump) treated the harvested days as one
|
||||
// uninterrupted run, so a single dump below the floor made it declare the whole
|
||||
// archive covered while everything above stayed missing.
|
||||
//
|
||||
// The current moment is passed in rather than read here, so the walk can be tested
|
||||
// without depending on the wall clock.
|
||||
func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) {
|
||||
if !strings.Contains(rawURI, srrdbAutoSegment) {
|
||||
return rawURI, nil
|
||||
}
|
||||
|
||||
floor, err := srrdbFloorDate()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
harvested, err := srrdbDumpedDays(rawURI)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
done := make(map[string]bool, len(harvested))
|
||||
for _, day := range harvested {
|
||||
done[day.Format(srrdbDayLayout)] = true
|
||||
}
|
||||
|
||||
// The current day is never taken: it is still being filled, so it would have
|
||||
// no dump and would be chosen by every single run, freezing the walk. cron
|
||||
// refreshes today explicitly instead.
|
||||
for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) {
|
||||
if !done[day.Format(srrdbDayLayout)] {
|
||||
return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", errSrrdbFloor
|
||||
}
|
||||
|
||||
// srrdbDumpedDays lists the days already harvested for the same query, read back
|
||||
// from the dump file names inside srrdb.dump_dir.
|
||||
//
|
||||
// A missing directory is an error rather than "nothing harvested yet": the dumps
|
||||
// are the cursor of the walk, so a renamed, unmounted or mistyped directory would
|
||||
// otherwise send the run silently back to the beginning. The directory is created
|
||||
// once by hand, exactly like the database file.
|
||||
func srrdbDumpedDays(uri string) ([]time.Time, error) {
|
||||
pattern, err := srrdbDumpPattern(uri)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dir := viper.GetString("srrdb.dump_dir")
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err)
|
||||
}
|
||||
|
||||
days := make([]time.Time, 0, len(entries))
|
||||
|
||||
for _, entry := range entries {
|
||||
match := pattern.FindStringSubmatch(entry.Name())
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
day, err := time.Parse(srrdbDayLayout, match[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
days = append(days, day)
|
||||
}
|
||||
|
||||
return days, nil
|
||||
}
|
||||
|
||||
// srrdbDumpPattern builds the regular expression matching the dump names of the
|
||||
// same query, capturing the day segment so it can be read back. The literal part
|
||||
// is quoted first, therefore the capture group is the only pattern syntax left in
|
||||
// the result.
|
||||
func srrdbDumpPattern(uri string) (*regexp.Regexp, error) {
|
||||
const normalizedAuto = "date:auto"
|
||||
|
||||
escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri))
|
||||
|
||||
head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto))
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri)
|
||||
}
|
||||
|
||||
return regexp.Compile("^" + head + `date:(\d{4}-\d{2}-\d{2})` + tail + "$")
|
||||
}
|
||||
|
||||
// srrdbFloorDate is the earliest day the automatic walk may ask for, kept in
|
||||
// configuration because it is a property of the archive, not of the code.
|
||||
//
|
||||
// The value is read through viper.Get rather than viper.GetString: YAML decodes
|
||||
// an unquoted date as a timestamp, and asking for its string form yields
|
||||
// "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the
|
||||
// quoted and the unquoted spelling are accepted here.
|
||||
func srrdbFloorDate() (time.Time, error) {
|
||||
switch raw := viper.Get("srrdb.floor").(type) {
|
||||
case nil:
|
||||
return time.Time{}, errors.New("srrdb.floor is not configured")
|
||||
case time.Time:
|
||||
return raw, nil
|
||||
case string:
|
||||
floor, err := time.Parse(srrdbDayLayout, raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err)
|
||||
}
|
||||
|
||||
return floor, nil
|
||||
default:
|
||||
return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// srrdbReplaceDate swaps a date placeholder for a concrete day.
|
||||
func srrdbReplaceDate(uri, segment string, day time.Time) string {
|
||||
return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1)
|
||||
}
|
||||
|
||||
// srrdbRelativeDays holds the date placeholders that stand for a day counted
|
||||
// from now. date:auto is deliberately absent: it means "the next day the walk
|
||||
// has to cover", which only srrdbResolveAutoDate can decide.
|
||||
var srrdbRelativeDays = []struct {
|
||||
segment string
|
||||
offset int
|
||||
}{
|
||||
{segment: srrdbYesterdaySegment, offset: -1},
|
||||
{segment: srrdbTodaySegment, offset: 0},
|
||||
{segment: srrdbTomorrowSegment, offset: 1},
|
||||
}
|
||||
|
||||
// srrdbResolveRelativeDays expands every relative date placeholder present in the
|
||||
// path. The current moment is passed in for the same reason as in
|
||||
// srrdbResolveAutoDate: a test has to be able to pin it.
|
||||
func srrdbResolveRelativeDays(rawURI string, now time.Time) string {
|
||||
uri := rawURI
|
||||
|
||||
for _, relative := range srrdbRelativeDays {
|
||||
if !strings.Contains(uri, relative.segment) {
|
||||
continue
|
||||
}
|
||||
|
||||
uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset))
|
||||
}
|
||||
|
||||
return uri
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// srrdbTestNow pins the moment the date logic is tested against, so no assertion
|
||||
// depends on the wall clock.
|
||||
var srrdbTestNow = time.Date(2026, 9, 30, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// srrdbTestDir gives every test a clean viper and an empty dump directory, because
|
||||
// the package reads srrdb.dump_dir from the global configuration instance.
|
||||
func srrdbTestDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
viper.Reset()
|
||||
|
||||
dir := t.TempDir()
|
||||
viper.Set("srrdb.dump_dir", dir)
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
// srrdbTestFloor points srrdb.floor at the given day.
|
||||
func srrdbTestFloor(t *testing.T, day time.Time) {
|
||||
t.Helper()
|
||||
|
||||
viper.Set("srrdb.floor", day.Format(srrdbDayLayout))
|
||||
}
|
||||
|
||||
// srrdbTestDump writes a dump file for a query, which is how the walk learns that
|
||||
// a day is done.
|
||||
func srrdbTestDump(t *testing.T, uri string) {
|
||||
t.Helper()
|
||||
|
||||
path := srrdbDumpPath(uri)
|
||||
if err := os.WriteFile(path, []byte("{}"), 0o644); err != nil {
|
||||
t.Fatalf("write dump %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbFloorDateFromString(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
viper.Set("srrdb.floor", "2013-03-16")
|
||||
|
||||
floor, err := srrdbFloorDate()
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbFloorDate: %v", err)
|
||||
}
|
||||
|
||||
want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
|
||||
if !floor.Equal(want) {
|
||||
t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The regression this test exists for: YAML decodes an unquoted date as a
|
||||
// timestamp, so the value reaches the code as a time.Time and not as a string. It
|
||||
// used to fail with "extra text: 00:00:00 +0000 UTC".
|
||||
func TestSrrdbFloorDateFromTimestamp(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
|
||||
viper.Set("srrdb.floor", want)
|
||||
|
||||
floor, err := srrdbFloorDate()
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbFloorDate: %v", err)
|
||||
}
|
||||
|
||||
if !floor.Equal(want) {
|
||||
t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbFloorDateNotConfigured(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
if _, err := srrdbFloorDate(); err == nil {
|
||||
t.Error("srrdbFloorDate without srrdb.floor returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbDumpPattern(t *testing.T) {
|
||||
pattern, err := srrdbDumpPattern("category:flac/date:auto/page:1")
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbDumpPattern: %v", err)
|
||||
}
|
||||
|
||||
match := pattern.FindStringSubmatch(srrdbNormalizedPath("category:flac/date:2026-09-28/page:1"))
|
||||
if match == nil {
|
||||
t.Fatal("the pattern did not match the dump name of the same query")
|
||||
}
|
||||
|
||||
if match[1] != "2026-09-28" {
|
||||
t.Errorf("captured day = %q, want 2026-09-28", match[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbDumpPatternNeedsAutoSegment(t *testing.T) {
|
||||
if _, err := srrdbDumpPattern("category:flac/date:2026-09-28/page:1"); err == nil {
|
||||
t.Error("srrdbDumpPattern without date:auto returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbDumpedDaysIgnoresForeignQueries(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
|
||||
srrdbTestDump(t, "category:mp3/date:2026-09-26/page:1")
|
||||
|
||||
days, err := srrdbDumpedDays("category:flac/date:auto/page:1")
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbDumpedDays: %v", err)
|
||||
}
|
||||
|
||||
if len(days) != 2 {
|
||||
t.Fatalf("len(days) = %d, want 2: only dumps of the same query may count", len(days))
|
||||
}
|
||||
}
|
||||
|
||||
// An absent directory must not read as "nothing harvested yet": the dumps are the
|
||||
// cursor of the walk, so a renamed or unmounted directory would otherwise send the
|
||||
// run silently back to the beginning.
|
||||
func TestSrrdbDumpedDaysMissingDirectory(t *testing.T) {
|
||||
viper.Reset()
|
||||
viper.Set("srrdb.dump_dir", filepath.Join(t.TempDir(), "absent"))
|
||||
|
||||
if _, err := srrdbDumpedDays("category:flac/date:auto/page:1"); err == nil {
|
||||
t.Error("srrdbDumpedDays on an absent directory returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDateStartsFromYesterday(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
|
||||
|
||||
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
want := "category:flac/date:2026-09-29/page:1"
|
||||
if got != want {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q: the current day is still being filled", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDatePicksTheNewestMissingDay(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
|
||||
|
||||
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
want := "category:flac/date:2026-09-27/page:1"
|
||||
if got != want {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A hole in the middle must be closed once the walk reaches it, not declared
|
||||
// covered: this is the defect that made the previous cursor stop at the floor.
|
||||
func TestSrrdbResolveAutoDateClosesHoles(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
|
||||
|
||||
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
want := "category:flac/date:2026-09-28/page:1"
|
||||
if got != want {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q: the gap above the floor must be filled first", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDateStopsAtFloor(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -1))
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
|
||||
|
||||
_, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if !errors.Is(err, errSrrdbFloor) {
|
||||
t.Errorf("srrdbResolveAutoDate = %v, want errSrrdbFloor", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDateIgnoresConcreteDates(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
uri := "category:flac/date:2026-09-28/page:1"
|
||||
|
||||
got, err := srrdbResolveAutoDate(uri, srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
if got != uri {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q left untouched", got, uri)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveRelativeDays(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uri string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "today",
|
||||
uri: "category:flac/date:today/page:1",
|
||||
want: "category:flac/date:2026-09-30/page:1",
|
||||
},
|
||||
{
|
||||
name: "yesterday",
|
||||
uri: "category:flac/date:yesterday/page:1",
|
||||
want: "category:flac/date:2026-09-29/page:1",
|
||||
},
|
||||
{
|
||||
name: "tomorrow",
|
||||
uri: "category:flac/date:tomorrow/page:1",
|
||||
want: "category:flac/date:2026-10-01/page:1",
|
||||
},
|
||||
{
|
||||
name: "auto belongs to the walk, not here",
|
||||
uri: "category:flac/date:auto/page:1",
|
||||
want: "category:flac/date:auto/page:1",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := srrdbResolveRelativeDays(test.uri, srrdbTestNow); got != test.want {
|
||||
t.Errorf("srrdbResolveRelativeDays(%q) = %q, want %q", test.uri, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
||||
)
|
||||
|
||||
// srrdbNormalizedPath turns a search path into a file name. The two characters
|
||||
// that carry structure in a path are swapped for look-alikes, so the name still
|
||||
// reads as the query it came from and can be reversed character by character.
|
||||
func srrdbNormalizedPath(uri string) string {
|
||||
return strings.NewReplacer(":", ":", "/", "∕").Replace(uri) + ".json"
|
||||
}
|
||||
|
||||
// srrdbDumpPath is the guard file of a search path inside srrdb.dump_dir.
|
||||
func srrdbDumpPath(uri string) string {
|
||||
return filepath.Join(viper.GetString("srrdb.dump_dir"), srrdbNormalizedPath(uri))
|
||||
}
|
||||
|
||||
// srrdbExists reports whether a dump file is present. Any stat failure other than
|
||||
// "not found" is returned as an error: treating it as absent would let the run
|
||||
// overwrite a dump it has no right to read.
|
||||
func srrdbExists(path string) (bool, error) {
|
||||
_, err := os.Stat(path)
|
||||
|
||||
switch {
|
||||
case err == nil:
|
||||
return true, nil
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("stat srrdb dump %q: %w", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// srrdbWriteDump stores the merged answer as indented JSON. It runs only after
|
||||
// the database is updated, therefore the presence of the file means the whole
|
||||
// run completed.
|
||||
//
|
||||
// The directory is never created here: it holds the cursor of the walk and is made
|
||||
// once by hand, like the database file.
|
||||
func srrdbWriteDump(path string, all *resource.SrrdbSearchResponse) error {
|
||||
data, err := json.MarshalIndent(all, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Errorf("encode srrdb dump %q: %w", path, err)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
return fmt.Errorf("write srrdb dump %q: %w", path, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSrrdbDumpPath(t *testing.T) {
|
||||
dir := srrdbTestDir(t)
|
||||
|
||||
got := srrdbDumpPath("category:flac/date:2026-09-28/page:1")
|
||||
want := filepath.Join(dir, "category:flac∕date:2026-09-28∕page:1.json")
|
||||
|
||||
if got != want {
|
||||
t.Errorf("srrdbDumpPath = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbExists(t *testing.T) {
|
||||
dir := srrdbTestDir(t)
|
||||
|
||||
present := filepath.Join(dir, "present.json")
|
||||
if err := os.WriteFile(present, []byte("{}"), 0o644); err != nil {
|
||||
t.Fatalf("write %q: %v", present, err)
|
||||
}
|
||||
|
||||
exists, err := srrdbExists(present)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbExists(%q) returned an error: %v", present, err)
|
||||
}
|
||||
|
||||
if !exists {
|
||||
t.Errorf("srrdbExists(%q) = false, want true", present)
|
||||
}
|
||||
|
||||
exists, err = srrdbExists(filepath.Join(dir, "absent.json"))
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbExists on a missing file returned an error: %v", err)
|
||||
}
|
||||
|
||||
if exists {
|
||||
t.Error("srrdbExists on a missing file = true, want false")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// srrdbAppendJSONL adds one object as a single line to a .jsonl log. A missing
|
||||
// or failing log never changes the outcome of the run itself.
|
||||
func srrdbAppendJSONL(key string, record map[string]any) {
|
||||
path := viper.GetString(key)
|
||||
if path == "" {
|
||||
return
|
||||
}
|
||||
|
||||
data, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
slog.Error("encode srrdb log record", "key", key, "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
slog.Error("open srrdb log", "path", path, "err", err)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := file.Close(); cerr != nil {
|
||||
slog.Error("close srrdb log", "path", path, "err", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err := file.Write(append(data, '\n')); err != nil {
|
||||
slog.Error("write srrdb log", "path", path, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// srrdbLogStatus appends the outcome of one finished run to srrdb.log.
|
||||
func srrdbLogStatus(uri string, stored, reported int) {
|
||||
srrdbAppendJSONL("srrdb.log", map[string]any{
|
||||
"time": time.Now().Format(srrdbTimestamp),
|
||||
"uri": uri,
|
||||
"stored": stored,
|
||||
"reported": reported,
|
||||
})
|
||||
}
|
||||
|
||||
// srrdbLogWarning records warnings reported by the API. They share the failure
|
||||
// file but carry their own kind, because the endpoint answers an unrecognized
|
||||
// keyword with HTTP 200 and an empty result set: the text is the only trace of a
|
||||
// bad path, and it must stay distinguishable from a real breakdown.
|
||||
func srrdbLogWarning(uri string, warnings []string) {
|
||||
srrdbAppendJSONL("srrdb.error_log", map[string]any{
|
||||
"kind": "warning",
|
||||
"time": time.Now().Format(srrdbTimestamp),
|
||||
"uri": uri,
|
||||
"warnings": strings.Join(warnings, "; "),
|
||||
})
|
||||
}
|
||||
|
||||
// srrdbLogError appends one failure to srrdb.error_log.
|
||||
func srrdbLogError(uri string, err error) {
|
||||
srrdbAppendJSONL("srrdb.error_log", map[string]any{
|
||||
"kind": "error",
|
||||
"time": time.Now().Format(srrdbTimestamp),
|
||||
"uri": uri,
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
||||
)
|
||||
|
||||
// srrdbGroupSegment cuts the group off the tail of a release line, which always
|
||||
// sits last. Compiled once at package level instead of per record.
|
||||
//
|
||||
// The pattern takes the trailing word, so a group whose own name contains a
|
||||
// hyphen or a dot would lose everything before that character. Nothing in the
|
||||
// line tells such a name apart from the tokens in front of it, so widening the
|
||||
// character class without a list of real group names would only start eating the
|
||||
// title.
|
||||
var srrdbGroupSegment = regexp.MustCompile(`-(\w+)$`)
|
||||
|
||||
// srrdbCategory reads the value of the category: segment of a search path, which
|
||||
// is guaranteed to be present by the time this is called.
|
||||
func srrdbCategory(uri string) string {
|
||||
_, rest, ok := strings.Cut(uri, "category:")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
|
||||
value, _, _ := strings.Cut(rest, "/")
|
||||
|
||||
return value
|
||||
}
|
||||
|
||||
// srrdbRow maps one API record onto a table row. status, nfo, content and
|
||||
// synced_at stay nil and are stored as NULL.
|
||||
func srrdbRow(result resource.SrrdbResult, run, category string) model.SrrdbRelease {
|
||||
return model.SrrdbRelease{
|
||||
Release: result.Release,
|
||||
Date: result.Date,
|
||||
HasNfo: result.HasNFO,
|
||||
HasSrs: result.HasSRS,
|
||||
IsForeign: result.IsForeign,
|
||||
ImdbID: result.ImdbID,
|
||||
Size: result.Size,
|
||||
GroupName: srrdbGroupName(result.Release),
|
||||
Category: category,
|
||||
CreatedAt: run,
|
||||
}
|
||||
}
|
||||
|
||||
// srrdbGroupName returns the trailing group of a release line, or an empty
|
||||
// string when the tail is not a plain word. The nil check is deliberate:
|
||||
// FindStringSubmatch returns nil on no match, and indexing it directly panics.
|
||||
func srrdbGroupName(release string) string {
|
||||
match := srrdbGroupSegment.FindStringSubmatch(release)
|
||||
if match == nil {
|
||||
return ""
|
||||
}
|
||||
|
||||
return match[1]
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
||||
)
|
||||
|
||||
func TestSrrdbNormalizedPath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uri string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "full search path",
|
||||
uri: "category:flac/date:2026-09-28/page:1",
|
||||
want: "category:flac∕date:2026-09-28∕page:1.json",
|
||||
},
|
||||
{
|
||||
name: "single segment",
|
||||
uri: "category:flac",
|
||||
want: "category:flac.json",
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
uri: "",
|
||||
want: ".json",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := srrdbNormalizedPath(test.uri); got != test.want {
|
||||
t.Errorf("srrdbNormalizedPath(%q) = %q, want %q", test.uri, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbGroupName(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
release string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "music release",
|
||||
release: "A-Reis-Aetherial_Flow-16BIT-WEB-FLAC-2025-NeonDreams",
|
||||
want: "NeonDreams",
|
||||
},
|
||||
{
|
||||
name: "group containing digits",
|
||||
release: "Old_Man-Grim_Tales-16BIT-WEB-FLAC-2020-86D",
|
||||
want: "86D",
|
||||
},
|
||||
{
|
||||
name: "two tokens only",
|
||||
release: "Artist-Title",
|
||||
want: "Title",
|
||||
},
|
||||
{
|
||||
// The pattern takes the trailing word, so a group that has a hyphen
|
||||
// in its own name loses everything before it. Pinned here on purpose:
|
||||
// it is the known limitation, and this test will fail the moment the
|
||||
// pattern is improved.
|
||||
name: "hyphen inside the group",
|
||||
release: "Artist-Title-16BIT-WEB-FLAC-2024-MOON-BLOOD",
|
||||
want: "BLOOD",
|
||||
},
|
||||
{
|
||||
name: "trailing hyphen",
|
||||
release: "Artist-Title-FLAC-2024-",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "empty line",
|
||||
release: "",
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := srrdbGroupName(test.release); got != test.want {
|
||||
t.Errorf("srrdbGroupName(%q) = %q, want %q", test.release, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbCategory(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uri string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "flac with further segments",
|
||||
uri: "category:flac/date:2026-09-28/page:1",
|
||||
want: "flac",
|
||||
},
|
||||
{
|
||||
name: "category alone",
|
||||
uri: "category:mp3",
|
||||
want: "mp3",
|
||||
},
|
||||
{
|
||||
name: "no category segment",
|
||||
uri: "date:2026-09-28/page:1",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "empty path",
|
||||
uri: "",
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := srrdbCategory(test.uri); got != test.want {
|
||||
t.Errorf("srrdbCategory(%q) = %q, want %q", test.uri, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbStartPage(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uri string
|
||||
wantPage int
|
||||
wantPaginated bool
|
||||
}{
|
||||
{
|
||||
name: "first page",
|
||||
uri: "category:flac/date:2026-09-28/page:1",
|
||||
wantPage: 1,
|
||||
wantPaginated: true,
|
||||
},
|
||||
{
|
||||
name: "later page",
|
||||
uri: "category:flac/date:2026-09-28/page:7",
|
||||
wantPage: 7,
|
||||
wantPaginated: true,
|
||||
},
|
||||
{
|
||||
name: "no page segment",
|
||||
uri: "category:flac/date:2026-09-28",
|
||||
wantPage: 1,
|
||||
wantPaginated: false,
|
||||
},
|
||||
{
|
||||
name: "zero page falls back to one",
|
||||
uri: "category:flac/date:2026-09-28/page:0",
|
||||
wantPage: 1,
|
||||
wantPaginated: true,
|
||||
},
|
||||
{
|
||||
name: "non numeric page is not a page segment",
|
||||
uri: "category:flac/date:2026-09-28/page:abc",
|
||||
wantPage: 1,
|
||||
wantPaginated: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
page, paginated := srrdbStartPage(test.uri)
|
||||
if page != test.wantPage || paginated != test.wantPaginated {
|
||||
t.Errorf("srrdbStartPage(%q) = %d, %t, want %d, %t",
|
||||
test.uri, page, paginated, test.wantPage, test.wantPaginated)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbWithPage(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uri string
|
||||
page int
|
||||
paginated bool
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "swaps the page segment",
|
||||
uri: "category:flac/date:2026-09-28/page:3",
|
||||
page: 4,
|
||||
paginated: true,
|
||||
want: "category:flac/date:2026-09-28/page:4",
|
||||
},
|
||||
{
|
||||
name: "single request keeps the path untouched",
|
||||
uri: "category:flac/date:2026-09-28/page:3",
|
||||
page: 4,
|
||||
paginated: false,
|
||||
want: "category:flac/date:2026-09-28/page:3",
|
||||
},
|
||||
{
|
||||
name: "no page segment to swap",
|
||||
uri: "category:flac/date:2026-09-28",
|
||||
page: 2,
|
||||
paginated: true,
|
||||
want: "category:flac/date:2026-09-28",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := srrdbWithPage(test.uri, test.page, test.paginated); got != test.want {
|
||||
t.Errorf("srrdbWithPage(%q, %d, %t) = %q, want %q",
|
||||
test.uri, test.page, test.paginated, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbCollectKeepsFirstPageEnvelope(t *testing.T) {
|
||||
collected := &resource.SrrdbSearchResponse{}
|
||||
|
||||
srrdbCollect(collected, &resource.SrrdbSearchResponse{
|
||||
ResultsCount: 155,
|
||||
Query: json.RawMessage(`[21]`),
|
||||
Results: []resource.SrrdbResult{
|
||||
{Release: "First-Release-16BIT-WEB-FLAC-2026-GROUP"},
|
||||
{Release: "Second-Release-16BIT-WEB-FLAC-2026-GROUP"},
|
||||
},
|
||||
Warnings: []string{"first warning"},
|
||||
})
|
||||
|
||||
srrdbCollect(collected, &resource.SrrdbSearchResponse{
|
||||
ResultsCount: 999,
|
||||
Query: json.RawMessage(`[22]`),
|
||||
Results: []resource.SrrdbResult{
|
||||
{Release: "Third-Release-16BIT-WEB-FLAC-2026-GROUP"},
|
||||
},
|
||||
Warnings: []string{"second warning"},
|
||||
})
|
||||
|
||||
if collected.ResultsCount != 155 {
|
||||
t.Errorf("ResultsCount = %d, want 155: only the first page may set it", collected.ResultsCount)
|
||||
}
|
||||
|
||||
if got := string(collected.Query); got != "[21]" {
|
||||
t.Errorf("Query = %s, want [21]: only the first page may set it", got)
|
||||
}
|
||||
|
||||
if len(collected.Results) != 3 {
|
||||
t.Errorf("len(Results) = %d, want 3", len(collected.Results))
|
||||
}
|
||||
|
||||
if len(collected.Warnings) != 2 {
|
||||
t.Errorf("len(Warnings) = %d, want 2", len(collected.Warnings))
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbCollectZeroCountOnFirstPageIsNotSticky(t *testing.T) {
|
||||
collected := &resource.SrrdbSearchResponse{}
|
||||
|
||||
srrdbCollect(collected, &resource.SrrdbSearchResponse{})
|
||||
srrdbCollect(collected, &resource.SrrdbSearchResponse{ResultsCount: 42})
|
||||
|
||||
if collected.ResultsCount != 42 {
|
||||
t.Errorf("ResultsCount = %d, want 42: a zero count must not freeze the envelope", collected.ResultsCount)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package table
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
// srrdbUpsert keys on the "release" primary key: a repeated search refreshes the
|
||||
// existing row instead of inserting a copy. synced_at is stamped with the same
|
||||
// run timestamp as created_at, so the two differ only on a release that was
|
||||
// already stored by an earlier run.
|
||||
const srrdbUpsert = `
|
||||
INSERT INTO srrdb_releases
|
||||
("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at)
|
||||
VALUES
|
||||
(:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at)
|
||||
ON CONFLICT("release") DO UPDATE SET
|
||||
date = excluded.date,
|
||||
has_nfo = excluded.has_nfo,
|
||||
has_srs = excluded.has_srs,
|
||||
is_foreign = excluded.is_foreign,
|
||||
imdb_id = excluded.imdb_id,
|
||||
size = excluded.size,
|
||||
group_name = excluded.group_name,
|
||||
category = excluded.category,
|
||||
synced_at = excluded.created_at`
|
||||
|
||||
// UpsertAllSrrdb writes a whole set in one transaction: either every release
|
||||
// lands or none does, so a failure halfway through cannot leave a day half
|
||||
// harvested.
|
||||
func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error {
|
||||
if len(rows) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
tx, err := db.Beginx()
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin srrdb transaction: %w", err)
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
if _, err := tx.NamedExec(srrdbUpsert, &row); err != nil {
|
||||
// Rollback failure is not reported: the insert error is the one that
|
||||
// matters to the caller, and the transaction is gone either way.
|
||||
_ = tx.Rollback()
|
||||
|
||||
return fmt.Errorf("upsert srrdb release %q: %w", row.Release, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := tx.Commit(); err != nil {
|
||||
return fmt.Errorf("commit srrdb transaction: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -10,6 +10,7 @@ type Service struct {
|
||||
_interface.Prescene
|
||||
_interface.Info
|
||||
_interface.ShareTheBrutality
|
||||
_interface.Srrdb
|
||||
}
|
||||
|
||||
func New(repos *repository.Repository) *Service {
|
||||
@@ -18,5 +19,6 @@ func New(repos *repository.Repository) *Service {
|
||||
Prescene: NewPresceneService(repos.Prescene),
|
||||
Info: NewInfoService(),
|
||||
ShareTheBrutality: NewShareTheBrutalityService(repos.ShareTheBrutality),
|
||||
Srrdb: NewSrrdbService(repos.Srrdb),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
|
||||
)
|
||||
|
||||
type SrrdbService struct {
|
||||
repo _interface.Srrdb
|
||||
}
|
||||
|
||||
func NewSrrdbService(repo _interface.Srrdb) *SrrdbService {
|
||||
return &SrrdbService{repo: repo}
|
||||
}
|
||||
|
||||
func (s *SrrdbService) Search(uri string) (int, error) {
|
||||
return s.repo.Search(uri)
|
||||
}
|
||||
Reference in New Issue
Block a user