srrdb: harvest the srrDB search API into a separate SQLite database

New scope reading the API instead of scraping HTML: one search path per run,
passed raw through --single-uri, and the answer stored as-is into its own
SQLite file created from the embedded schema.

Command line: --single-uri carries the path after /v1/search/
(category:flac/date:today/page:1) and must name a category; --sleep sets the
pause between pages, with the config value as fallback. A day named on the
command line is always fetched again and its dump rewritten, so cron can
refresh today hourly and yesterday daily.

Date placeholders: date:auto walks backwards to the newest day in
[srrdb.floor..yesterday] that has no dump yet, closing holes instead of
declaring the archive covered. date:today, date:yesterday and date:tomorrow
stand for the neighbouring days. Today is never taken by the walk, because a
day still being filled would be frozen by its own dump after the first run.

Dumps are the cursor as well as the record, so a missing dump directory is an
error instead of "nothing harvested yet": a renamed, unmounted or mistyped
directory must not send the run silently back to the beginning. An empty day is
a normal result and is remembered as harvested; API warnings never are, and
they reach the exit code.

Writes go through one transaction, so a day lands completely or not at all.
group and category come from the release line and the search path; status, nfo,
content and synced_at stay NULL. srrdb.log.jsonl records one line per finished
run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind
field.

Also: the console command contract returns (string, error), so a failing scope
reaches the process exit code and cron can tell a failed run from a successful
one. config/scope/srrdb.yaml is untracked because its paths differ per host.
modernc.org/sqlite replaces the archived libsql client, which only ever talked
to a server. pkg/repository/srrdb.go is split by responsibility and covered by
table-driven tests.

pkg/repository/repository.go

@@ -10,12 +10,16 @@ type Repository struct {
 	_interface.Rutracker
 	_interface.Prescene
 	_interface.ShareTheBrutality
+	_interface.Srrdb
 }
 
-func New(db *sqlx.DB) *Repository {
+// New wires the collectors. sqlite serves the srrdb scope only and may be nil
+// for every other source; it is connected lazily in Bootstrap.
+func New(db, sqlite *sqlx.DB) *Repository {
 	return &Repository{
 		Rutracker:         NewRutrackerRepository(db),
 		Prescene:          NewPresceneRepository(db),
 		ShareTheBrutality: NewShareTheBrutalityRepository(db),
+		Srrdb:             NewSrrdbRepository(sqlite),
 	}
 }

pkg/repository/srrdb.go

@@ -0,0 +1,140 @@
+package repository
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"strings"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+// srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the
+// API itself uses in its date field, because the column is TEXT.
+const srrdbTimestamp = "2006-01-02 15:04:05"
+
+// Srrdb harvests the srrDB search API into the separate SQLite database.
+type Srrdb struct {
+	db *sqlx.DB
+}
+
+// NewSrrdbRepository returns a collector writing into the SQLite handle. The
+// handle is nil for every scope except srrdb, because Bootstrap connects the
+// SQLite database lazily, so the collector refuses to run without it instead of
+// failing later on a nil handle.
+func NewSrrdbRepository(db *sqlx.DB) *Srrdb {
+	return &Srrdb{db: db}
+}
+
+// Search harvests one search path. Nothing is written to the database until the
+// whole set is collected and its size matches what the API reported, so the
+// dump and the success log can only ever describe a complete day.
+func (s *Srrdb) Search(rawURI string) (int, error) {
+	if s.db == nil {
+		err := errors.New("srrdb collector has no database handle")
+		srrdbLogError(rawURI, err)
+
+		return 0, err
+	}
+
+	if !strings.Contains(rawURI, "category:") {
+		err := fmt.Errorf("search path %q has no category: segment", rawURI)
+		srrdbLogError(rawURI, err)
+
+		return 0, err
+	}
+
+	auto := strings.Contains(rawURI, srrdbAutoSegment)
+
+	now := time.Now()
+
+	uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now)
+
+	switch {
+	case errors.Is(err, errSrrdbFloor):
+		slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI)
+
+		return 0, nil
+	case err != nil:
+		srrdbLogError(rawURI, err)
+
+		return 0, err
+	}
+
+	// A slash typed on either end of the command line would build a double
+	// separator in the URL and an odd dump file name.
+	uri = strings.Trim(uri, "/")
+
+	// The guard applies to the automatic walk only. A day named on the command
+	// line is fetched again and its dump rewritten every time: a day can still
+	// gain releases later, either from another time zone or from a late upload,
+	// and cron refreshes today and yesterday on purpose.
+	dump := srrdbDumpPath(uri)
+
+	if auto {
+		exists, err := srrdbExists(dump)
+		if err != nil {
+			srrdbLogError(uri, err)
+
+			return 0, err
+		}
+
+		if exists {
+			slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump)
+
+			return 0, nil
+		}
+	}
+
+	collected, err := srrdbCollectAll(uri)
+	if err != nil {
+		srrdbLogError(uri, err)
+
+		return 0, err
+	}
+
+	// An empty day is a normal outcome: zero results and no warnings. The dump
+	// records it as harvested, which keeps the walk moving. Warnings mean the
+	// path was not understood, and that must never be remembered as done.
+	switch {
+	case len(collected.Warnings) > 0:
+		srrdbLogWarning(uri, collected.Warnings)
+
+		return 0, fmt.Errorf("srrdb %q reported warnings: %s",
+			uri, strings.Join(collected.Warnings, "; "))
+	case len(collected.Results) != collected.ResultsCount:
+		err := fmt.Errorf("srrdb %q pulled %d of %d reported releases",
+			uri, len(collected.Results), collected.ResultsCount)
+		srrdbLogError(uri, err)
+
+		return 0, err
+	}
+
+	run := time.Now().Format(srrdbTimestamp)
+	category := srrdbCategory(uri)
+	rows := make([]model.SrrdbRelease, 0, len(collected.Results))
+
+	for _, result := range collected.Results {
+		rows = append(rows, srrdbRow(result, run, category))
+	}
+
+	if err := table.UpsertAllSrrdb(s.db, rows); err != nil {
+		srrdbLogError(uri, err)
+
+		return 0, err
+	}
+
+	if err := srrdbWriteDump(dump, collected); err != nil {
+		srrdbLogError(uri, err)
+
+		return len(rows), err
+	}
+
+	srrdbLogStatus(uri, len(rows), collected.ResultsCount)
+
+	return len(rows), nil
+}

pkg/repository/srrdb_api.go

@@ -0,0 +1,155 @@
+package repository
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"regexp"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
+)
+
+// srrdbPageSegment locates the page:N segment so a run can continue from N into
+// the following pages.
+var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`)
+
+// srrdbCollectAll walks every page before anything is written, so the caller
+// only ever receives a complete set or an error.
+func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) {
+	endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/")
+
+	// The flag binds to its own key and wins over the config value; an unused
+	// flag yields zero.
+	sleep := viper.GetDuration(constant.FlagSleep)
+	if sleep <= 0 {
+		sleep = viper.GetDuration("srrdb.sleep")
+	}
+	if sleep <= 0 {
+		sleep = 2 * time.Second
+	}
+
+	slog.Info("srrdb pacing", "uri", uri, "sleep", sleep)
+
+	page, paginated := srrdbStartPage(uri)
+	collected := &resource.SrrdbSearchResponse{}
+
+	for {
+		url := endpoint + "/" + srrdbWithPage(uri, page, paginated)
+
+		response, err := srrdbFetch(url)
+		if err != nil {
+			return nil, err
+		}
+
+		srrdbCollect(collected, response)
+
+		slog.Info("srrdb page fetched",
+			"url", url,
+			"page", page,
+			"fetched", len(response.Results),
+			"reported", response.ResultsCount)
+
+		if !paginated || len(response.Results) == 0 {
+			break
+		}
+
+		// Everything the API reported is already collected: one more page would
+		// cost a request and a pause to receive an empty answer.
+		if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount {
+			break
+		}
+
+		page++
+
+		time.Sleep(sleep)
+	}
+
+	return collected, nil
+}
+
+// srrdbCollect merges one page into the answer kept for the dump: results are
+// appended, while resultsCount and query come from the first page.
+func srrdbCollect(all, page *resource.SrrdbSearchResponse) {
+	if all.ResultsCount == 0 {
+		all.ResultsCount = page.ResultsCount
+	}
+
+	if len(all.Query) == 0 {
+		all.Query = page.Query
+	}
+
+	all.Results = append(all.Results, page.Results...)
+	all.Warnings = append(all.Warnings, page.Warnings...)
+}
+
+// srrdbStartPage reads the page:N segment. Without it the path is a single
+// request and no further pages are attempted.
+func srrdbStartPage(uri string) (int, bool) {
+	match := srrdbPageSegment.FindStringSubmatch(uri)
+	if match == nil {
+		return 1, false
+	}
+
+	page, err := strconv.Atoi(match[1])
+	if err != nil || page < 1 {
+		return 1, true
+	}
+
+	return page, true
+}
+
+// srrdbWithPage swaps the page:N segment for the page requested now.
+func srrdbWithPage(uri string, page int, paginated bool) string {
+	if !paginated {
+		return uri
+	}
+
+	return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page))
+}
+
+// srrdbFetch performs one search request and decodes the JSON answer.
+func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) {
+	client := &http.Client{Timeout: 30 * time.Second}
+
+	request, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("build srrdb request %q: %w", url, err)
+	}
+
+	request.Header.Set("Accept", "application/json")
+
+	response, err := client.Do(request)
+	if err != nil {
+		return nil, fmt.Errorf("request srrdb %q: %w", url, err)
+	}
+
+	defer func() {
+		if cerr := response.Body.Close(); cerr != nil {
+			slog.Error("closing srrdb response body", "url", url, "err", cerr)
+		}
+	}()
+
+	if response.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status)
+	}
+
+	body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
+	if err != nil {
+		return nil, fmt.Errorf("read srrdb body %q: %w", url, err)
+	}
+
+	decoded := &resource.SrrdbSearchResponse{}
+	if err := json.Unmarshal(body, decoded); err != nil {
+		return nil, fmt.Errorf("decode srrdb body %q: %w", url, err)
+	}
+
+	return decoded, nil
+}

pkg/repository/srrdb_date.go

@@ -0,0 +1,189 @@
+package repository
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+// srrdbAutoSegment is the placeholder replaced by the date the run has to
+// harvest, so cron can repeat the same command every day.
+const srrdbAutoSegment = "date:auto"
+
+// srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for
+// the neighbouring days, so cron can name a day without hard-coding a date.
+const (
+	srrdbYesterdaySegment = "date:yesterday"
+	srrdbTodaySegment     = "date:today"
+	srrdbTomorrowSegment  = "date:tomorrow"
+)
+
+// srrdbDayLayout is the layout of a date segment in a search path.
+const srrdbDayLayout = "2006-01-02"
+
+// errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is
+// the archive being fully covered rather than a failure.
+var errSrrdbFloor = errors.New("srrdb.floor reached")
+
+// srrdbResolveAutoDate replaces date:auto with the newest day in
+// [srrdb.floor..yesterday] that has not been harvested yet.
+//
+// Scanning from the top-down means a new day is picked up the moment it appears,
+// and a hole left in the middle is closed once the walk reaches it. The previous
+// rule (the day before the earliest dump) treated the harvested days as one
+// uninterrupted run, so a single dump below the floor made it declare the whole
+// archive covered while everything above stayed missing.
+//
+// The current moment is passed in rather than read here, so the walk can be tested
+// without depending on the wall clock.
+func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) {
+	if !strings.Contains(rawURI, srrdbAutoSegment) {
+		return rawURI, nil
+	}
+
+	floor, err := srrdbFloorDate()
+	if err != nil {
+		return "", err
+	}
+
+	harvested, err := srrdbDumpedDays(rawURI)
+	if err != nil {
+		return "", err
+	}
+
+	done := make(map[string]bool, len(harvested))
+	for _, day := range harvested {
+		done[day.Format(srrdbDayLayout)] = true
+	}
+
+	// The current day is never taken: it is still being filled, so it would have
+	// no dump and would be chosen by every single run, freezing the walk. cron
+	// refreshes today explicitly instead.
+	for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) {
+		if !done[day.Format(srrdbDayLayout)] {
+			return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil
+		}
+	}
+
+	return "", errSrrdbFloor
+}
+
+// srrdbDumpedDays lists the days already harvested for the same query, read back
+// from the dump file names inside srrdb.dump_dir.
+//
+// A missing directory is an error rather than "nothing harvested yet": the dumps
+// are the cursor of the walk, so a renamed, unmounted or mistyped directory would
+// otherwise send the run silently back to the beginning. The directory is created
+// once by hand, exactly like the database file.
+func srrdbDumpedDays(uri string) ([]time.Time, error) {
+	pattern, err := srrdbDumpPattern(uri)
+	if err != nil {
+		return nil, err
+	}
+
+	dir := viper.GetString("srrdb.dump_dir")
+
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err)
+	}
+
+	days := make([]time.Time, 0, len(entries))
+
+	for _, entry := range entries {
+		match := pattern.FindStringSubmatch(entry.Name())
+		if match == nil {
+			continue
+		}
+
+		day, err := time.Parse(srrdbDayLayout, match[1])
+		if err != nil {
+			continue
+		}
+
+		days = append(days, day)
+	}
+
+	return days, nil
+}
+
+// srrdbDumpPattern builds the regular expression matching the dump names of the
+// same query, capturing the day segment so it can be read back. The literal part
+// is quoted first, therefore the capture group is the only pattern syntax left in
+// the result.
+func srrdbDumpPattern(uri string) (*regexp.Regexp, error) {
+	const normalizedAuto = "date：auto"
+
+	escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri))
+
+	head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto))
+	if !ok {
+		return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri)
+	}
+
+	return regexp.Compile("^" + head + `date：(\d{4}-\d{2}-\d{2})` + tail + "$")
+}
+
+// srrdbFloorDate is the earliest day the automatic walk may ask for, kept in
+// configuration because it is a property of the archive, not of the code.
+//
+// The value is read through viper.Get rather than viper.GetString: YAML decodes
+// an unquoted date as a timestamp, and asking for its string form yields
+// "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the
+// quoted and the unquoted spelling are accepted here.
+func srrdbFloorDate() (time.Time, error) {
+	switch raw := viper.Get("srrdb.floor").(type) {
+	case nil:
+		return time.Time{}, errors.New("srrdb.floor is not configured")
+	case time.Time:
+		return raw, nil
+	case string:
+		floor, err := time.Parse(srrdbDayLayout, raw)
+		if err != nil {
+			return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err)
+		}
+
+		return floor, nil
+	default:
+		return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw)
+	}
+}
+
+// srrdbReplaceDate swaps a date placeholder for a concrete day.
+func srrdbReplaceDate(uri, segment string, day time.Time) string {
+	return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1)
+}
+
+// srrdbRelativeDays holds the date placeholders that stand for a day counted
+// from now. date:auto is deliberately absent: it means "the next day the walk
+// has to cover", which only srrdbResolveAutoDate can decide.
+var srrdbRelativeDays = []struct {
+	segment string
+	offset  int
+}{
+	{segment: srrdbYesterdaySegment, offset: -1},
+	{segment: srrdbTodaySegment, offset: 0},
+	{segment: srrdbTomorrowSegment, offset: 1},
+}
+
+// srrdbResolveRelativeDays expands every relative date placeholder present in the
+// path. The current moment is passed in for the same reason as in
+// srrdbResolveAutoDate: a test has to be able to pin it.
+func srrdbResolveRelativeDays(rawURI string, now time.Time) string {
+	uri := rawURI
+
+	for _, relative := range srrdbRelativeDays {
+		if !strings.Contains(uri, relative.segment) {
+			continue
+		}
+
+		uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset))
+	}
+
+	return uri
+}

pkg/repository/srrdb_date_test.go

@@ -0,0 +1,256 @@
+package repository
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+// srrdbTestNow pins the moment the date logic is tested against, so no assertion
+// depends on the wall clock.
+var srrdbTestNow = time.Date(2026, 9, 30, 12, 0, 0, 0, time.UTC)
+
+// srrdbTestDir gives every test a clean viper and an empty dump directory, because
+// the package reads srrdb.dump_dir from the global configuration instance.
+func srrdbTestDir(t *testing.T) string {
+	t.Helper()
+
+	viper.Reset()
+
+	dir := t.TempDir()
+	viper.Set("srrdb.dump_dir", dir)
+
+	return dir
+}
+
+// srrdbTestFloor points srrdb.floor at the given day.
+func srrdbTestFloor(t *testing.T, day time.Time) {
+	t.Helper()
+
+	viper.Set("srrdb.floor", day.Format(srrdbDayLayout))
+}
+
+// srrdbTestDump writes a dump file for a query, which is how the walk learns that
+// a day is done.
+func srrdbTestDump(t *testing.T, uri string) {
+	t.Helper()
+
+	path := srrdbDumpPath(uri)
+	if err := os.WriteFile(path, []byte("{}"), 0o644); err != nil {
+		t.Fatalf("write dump %q: %v", path, err)
+	}
+}
+
+func TestSrrdbFloorDateFromString(t *testing.T) {
+	srrdbTestDir(t)
+	viper.Set("srrdb.floor", "2013-03-16")
+
+	floor, err := srrdbFloorDate()
+	if err != nil {
+		t.Fatalf("srrdbFloorDate: %v", err)
+	}
+
+	want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
+	if !floor.Equal(want) {
+		t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
+	}
+}
+
+// The regression this test exists for: YAML decodes an unquoted date as a
+// timestamp, so the value reaches the code as a time.Time and not as a string. It
+// used to fail with "extra text: 00:00:00 +0000 UTC".
+func TestSrrdbFloorDateFromTimestamp(t *testing.T) {
+	srrdbTestDir(t)
+
+	want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
+	viper.Set("srrdb.floor", want)
+
+	floor, err := srrdbFloorDate()
+	if err != nil {
+		t.Fatalf("srrdbFloorDate: %v", err)
+	}
+
+	if !floor.Equal(want) {
+		t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
+	}
+}
+
+func TestSrrdbFloorDateNotConfigured(t *testing.T) {
+	srrdbTestDir(t)
+
+	if _, err := srrdbFloorDate(); err == nil {
+		t.Error("srrdbFloorDate without srrdb.floor returned no error")
+	}
+}
+
+func TestSrrdbDumpPattern(t *testing.T) {
+	pattern, err := srrdbDumpPattern("category:flac/date:auto/page:1")
+	if err != nil {
+		t.Fatalf("srrdbDumpPattern: %v", err)
+	}
+
+	match := pattern.FindStringSubmatch(srrdbNormalizedPath("category:flac/date:2026-09-28/page:1"))
+	if match == nil {
+		t.Fatal("the pattern did not match the dump name of the same query")
+	}
+
+	if match[1] != "2026-09-28" {
+		t.Errorf("captured day = %q, want 2026-09-28", match[1])
+	}
+}
+
+func TestSrrdbDumpPatternNeedsAutoSegment(t *testing.T) {
+	if _, err := srrdbDumpPattern("category:flac/date:2026-09-28/page:1"); err == nil {
+		t.Error("srrdbDumpPattern without date:auto returned no error")
+	}
+}
+
+func TestSrrdbDumpedDaysIgnoresForeignQueries(t *testing.T) {
+	srrdbTestDir(t)
+
+	srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
+	srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
+	srrdbTestDump(t, "category:mp3/date:2026-09-26/page:1")
+
+	days, err := srrdbDumpedDays("category:flac/date:auto/page:1")
+	if err != nil {
+		t.Fatalf("srrdbDumpedDays: %v", err)
+	}
+
+	if len(days) != 2 {
+		t.Fatalf("len(days) = %d, want 2: only dumps of the same query may count", len(days))
+	}
+}
+
+// An absent directory must not read as "nothing harvested yet": the dumps are the
+// cursor of the walk, so a renamed or unmounted directory would otherwise send the
+// run silently back to the beginning.
+func TestSrrdbDumpedDaysMissingDirectory(t *testing.T) {
+	viper.Reset()
+	viper.Set("srrdb.dump_dir", filepath.Join(t.TempDir(), "absent"))
+
+	if _, err := srrdbDumpedDays("category:flac/date:auto/page:1"); err == nil {
+		t.Error("srrdbDumpedDays on an absent directory returned no error")
+	}
+}
+
+func TestSrrdbResolveAutoDateStartsFromYesterday(t *testing.T) {
+	srrdbTestDir(t)
+	srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
+
+	got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
+	if err != nil {
+		t.Fatalf("srrdbResolveAutoDate: %v", err)
+	}
+
+	want := "category:flac/date:2026-09-29/page:1"
+	if got != want {
+		t.Errorf("srrdbResolveAutoDate = %q, want %q: the current day is still being filled", got, want)
+	}
+}
+
+func TestSrrdbResolveAutoDatePicksTheNewestMissingDay(t *testing.T) {
+	srrdbTestDir(t)
+	srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
+
+	srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
+	srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
+
+	got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
+	if err != nil {
+		t.Fatalf("srrdbResolveAutoDate: %v", err)
+	}
+
+	want := "category:flac/date:2026-09-27/page:1"
+	if got != want {
+		t.Errorf("srrdbResolveAutoDate = %q, want %q", got, want)
+	}
+}
+
+// A hole in the middle must be closed once the walk reaches it, not declared
+// covered: this is the defect that made the previous cursor stop at the floor.
+func TestSrrdbResolveAutoDateClosesHoles(t *testing.T) {
+	srrdbTestDir(t)
+	srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
+
+	srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
+	srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
+
+	got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
+	if err != nil {
+		t.Fatalf("srrdbResolveAutoDate: %v", err)
+	}
+
+	want := "category:flac/date:2026-09-28/page:1"
+	if got != want {
+		t.Errorf("srrdbResolveAutoDate = %q, want %q: the gap above the floor must be filled first", got, want)
+	}
+}
+
+func TestSrrdbResolveAutoDateStopsAtFloor(t *testing.T) {
+	srrdbTestDir(t)
+	srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -1))
+
+	srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
+
+	_, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
+	if !errors.Is(err, errSrrdbFloor) {
+		t.Errorf("srrdbResolveAutoDate = %v, want errSrrdbFloor", err)
+	}
+}
+
+func TestSrrdbResolveAutoDateIgnoresConcreteDates(t *testing.T) {
+	srrdbTestDir(t)
+
+	uri := "category:flac/date:2026-09-28/page:1"
+
+	got, err := srrdbResolveAutoDate(uri, srrdbTestNow)
+	if err != nil {
+		t.Fatalf("srrdbResolveAutoDate: %v", err)
+	}
+
+	if got != uri {
+		t.Errorf("srrdbResolveAutoDate = %q, want %q left untouched", got, uri)
+	}
+}
+
+func TestSrrdbResolveRelativeDays(t *testing.T) {
+	tests := []struct {
+		name string
+		uri  string
+		want string
+	}{
+		{
+			name: "today",
+			uri:  "category:flac/date:today/page:1",
+			want: "category:flac/date:2026-09-30/page:1",
+		},
+		{
+			name: "yesterday",
+			uri:  "category:flac/date:yesterday/page:1",
+			want: "category:flac/date:2026-09-29/page:1",
+		},
+		{
+			name: "tomorrow",
+			uri:  "category:flac/date:tomorrow/page:1",
+			want: "category:flac/date:2026-10-01/page:1",
+		},
+		{
+			name: "auto belongs to the walk, not here",
+			uri:  "category:flac/date:auto/page:1",
+			want: "category:flac/date:auto/page:1",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := srrdbResolveRelativeDays(test.uri, srrdbTestNow); got != test.want {
+				t.Errorf("srrdbResolveRelativeDays(%q) = %q, want %q", test.uri, got, test.want)
+			}
+		})
+	}
+}

pkg/repository/srrdb_dump.go

@@ -0,0 +1,62 @@
+package repository
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
+)
+
+// srrdbNormalizedPath turns a search path into a file name. The two characters
+// that carry structure in a path are swapped for look-alikes, so the name still
+// reads as the query it came from and can be reversed character by character.
+func srrdbNormalizedPath(uri string) string {
+	return strings.NewReplacer(":", "：", "/", "∕").Replace(uri) + ".json"
+}
+
+// srrdbDumpPath is the guard file of a search path inside srrdb.dump_dir.
+func srrdbDumpPath(uri string) string {
+	return filepath.Join(viper.GetString("srrdb.dump_dir"), srrdbNormalizedPath(uri))
+}
+
+// srrdbExists reports whether a dump file is present. Any stat failure other than
+// "not found" is returned as an error: treating it as absent would let the run
+// overwrite a dump it has no right to read.
+func srrdbExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+
+	switch {
+	case err == nil:
+		return true, nil
+	case errors.Is(err, fs.ErrNotExist):
+		return false, nil
+	default:
+		return false, fmt.Errorf("stat srrdb dump %q: %w", path, err)
+	}
+}
+
+// srrdbWriteDump stores the merged answer as indented JSON. It runs only after
+// the database is updated, therefore the presence of the file means the whole
+// run completed.
+//
+// The directory is never created here: it holds the cursor of the walk and is made
+// once by hand, like the database file.
+func srrdbWriteDump(path string, all *resource.SrrdbSearchResponse) error {
+	data, err := json.MarshalIndent(all, "", "  ")
+	if err != nil {
+		return fmt.Errorf("encode srrdb dump %q: %w", path, err)
+	}
+
+	if err := os.WriteFile(path, data, 0o644); err != nil {
+		return fmt.Errorf("write srrdb dump %q: %w", path, err)
+	}
+
+	return nil
+}

pkg/repository/srrdb_dump_test.go

@@ -0,0 +1,45 @@
+package repository
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestSrrdbDumpPath(t *testing.T) {
+	dir := srrdbTestDir(t)
+
+	got := srrdbDumpPath("category:flac/date:2026-09-28/page:1")
+	want := filepath.Join(dir, "category：flac∕date：2026-09-28∕page：1.json")
+
+	if got != want {
+		t.Errorf("srrdbDumpPath = %q, want %q", got, want)
+	}
+}
+
+func TestSrrdbExists(t *testing.T) {
+	dir := srrdbTestDir(t)
+
+	present := filepath.Join(dir, "present.json")
+	if err := os.WriteFile(present, []byte("{}"), 0o644); err != nil {
+		t.Fatalf("write %q: %v", present, err)
+	}
+
+	exists, err := srrdbExists(present)
+	if err != nil {
+		t.Fatalf("srrdbExists(%q) returned an error: %v", present, err)
+	}
+
+	if !exists {
+		t.Errorf("srrdbExists(%q) = false, want true", present)
+	}
+
+	exists, err = srrdbExists(filepath.Join(dir, "absent.json"))
+	if err != nil {
+		t.Fatalf("srrdbExists on a missing file returned an error: %v", err)
+	}
+
+	if exists {
+		t.Error("srrdbExists on a missing file = true, want false")
+	}
+}

pkg/repository/srrdb_log.go

@@ -0,0 +1,77 @@
+package repository
+
+import (
+	"encoding/json"
+	"log/slog"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+// srrdbAppendJSONL adds one object as a single line to a .jsonl log. A missing
+// or failing log never changes the outcome of the run itself.
+func srrdbAppendJSONL(key string, record map[string]any) {
+	path := viper.GetString(key)
+	if path == "" {
+		return
+	}
+
+	data, err := json.Marshal(record)
+	if err != nil {
+		slog.Error("encode srrdb log record", "key", key, "err", err)
+
+		return
+	}
+
+	file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
+	if err != nil {
+		slog.Error("open srrdb log", "path", path, "err", err)
+
+		return
+	}
+
+	defer func() {
+		if cerr := file.Close(); cerr != nil {
+			slog.Error("close srrdb log", "path", path, "err", cerr)
+		}
+	}()
+
+	if _, err := file.Write(append(data, '\n')); err != nil {
+		slog.Error("write srrdb log", "path", path, "err", err)
+	}
+}
+
+// srrdbLogStatus appends the outcome of one finished run to srrdb.log.
+func srrdbLogStatus(uri string, stored, reported int) {
+	srrdbAppendJSONL("srrdb.log", map[string]any{
+		"time":     time.Now().Format(srrdbTimestamp),
+		"uri":      uri,
+		"stored":   stored,
+		"reported": reported,
+	})
+}
+
+// srrdbLogWarning records warnings reported by the API. They share the failure
+// file but carry their own kind, because the endpoint answers an unrecognized
+// keyword with HTTP 200 and an empty result set: the text is the only trace of a
+// bad path, and it must stay distinguishable from a real breakdown.
+func srrdbLogWarning(uri string, warnings []string) {
+	srrdbAppendJSONL("srrdb.error_log", map[string]any{
+		"kind":     "warning",
+		"time":     time.Now().Format(srrdbTimestamp),
+		"uri":      uri,
+		"warnings": strings.Join(warnings, "; "),
+	})
+}
+
+// srrdbLogError appends one failure to srrdb.error_log.
+func srrdbLogError(uri string, err error) {
+	srrdbAppendJSONL("srrdb.error_log", map[string]any{
+		"kind":  "error",
+		"time":  time.Now().Format(srrdbTimestamp),
+		"uri":   uri,
+		"error": err.Error(),
+	})
+}

pkg/repository/srrdb_row.go

@@ -0,0 +1,61 @@
+package repository
+
+import (
+	"regexp"
+	"strings"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
+)
+
+// srrdbGroupSegment cuts the group off the tail of a release line, which always
+// sits last. Compiled once at package level instead of per record.
+//
+// The pattern takes the trailing word, so a group whose own name contains a
+// hyphen or a dot would lose everything before that character. Nothing in the
+// line tells such a name apart from the tokens in front of it, so widening the
+// character class without a list of real group names would only start eating the
+// title.
+var srrdbGroupSegment = regexp.MustCompile(`-(\w+)$`)
+
+// srrdbCategory reads the value of the category: segment of a search path, which
+// is guaranteed to be present by the time this is called.
+func srrdbCategory(uri string) string {
+	_, rest, ok := strings.Cut(uri, "category:")
+	if !ok {
+		return ""
+	}
+
+	value, _, _ := strings.Cut(rest, "/")
+
+	return value
+}
+
+// srrdbRow maps one API record onto a table row. status, nfo, content and
+// synced_at stay nil and are stored as NULL.
+func srrdbRow(result resource.SrrdbResult, run, category string) model.SrrdbRelease {
+	return model.SrrdbRelease{
+		Release:   result.Release,
+		Date:      result.Date,
+		HasNfo:    result.HasNFO,
+		HasSrs:    result.HasSRS,
+		IsForeign: result.IsForeign,
+		ImdbID:    result.ImdbID,
+		Size:      result.Size,
+		GroupName: srrdbGroupName(result.Release),
+		Category:  category,
+		CreatedAt: run,
+	}
+}
+
+// srrdbGroupName returns the trailing group of a release line, or an empty
+// string when the tail is not a plain word. The nil check is deliberate:
+// FindStringSubmatch returns nil on no match, and indexing it directly panics.
+func srrdbGroupName(release string) string {
+	match := srrdbGroupSegment.FindStringSubmatch(release)
+	if match == nil {
+		return ""
+	}
+
+	return match[1]
+}

pkg/repository/srrdb_test.go

@@ -0,0 +1,269 @@
+package repository
+
+import (
+	"encoding/json"
+	"testing"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
+)
+
+func TestSrrdbNormalizedPath(t *testing.T) {
+	tests := []struct {
+		name string
+		uri  string
+		want string
+	}{
+		{
+			name: "full search path",
+			uri:  "category:flac/date:2026-09-28/page:1",
+			want: "category：flac∕date：2026-09-28∕page：1.json",
+		},
+		{
+			name: "single segment",
+			uri:  "category:flac",
+			want: "category：flac.json",
+		},
+		{
+			name: "empty path",
+			uri:  "",
+			want: ".json",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := srrdbNormalizedPath(test.uri); got != test.want {
+				t.Errorf("srrdbNormalizedPath(%q) = %q, want %q", test.uri, got, test.want)
+			}
+		})
+	}
+}
+
+func TestSrrdbGroupName(t *testing.T) {
+	tests := []struct {
+		name    string
+		release string
+		want    string
+	}{
+		{
+			name:    "music release",
+			release: "A-Reis-Aetherial_Flow-16BIT-WEB-FLAC-2025-NeonDreams",
+			want:    "NeonDreams",
+		},
+		{
+			name:    "group containing digits",
+			release: "Old_Man-Grim_Tales-16BIT-WEB-FLAC-2020-86D",
+			want:    "86D",
+		},
+		{
+			name:    "two tokens only",
+			release: "Artist-Title",
+			want:    "Title",
+		},
+		{
+			// The pattern takes the trailing word, so a group that has a hyphen
+			// in its own name loses everything before it. Pinned here on purpose:
+			// it is the known limitation, and this test will fail the moment the
+			// pattern is improved.
+			name:    "hyphen inside the group",
+			release: "Artist-Title-16BIT-WEB-FLAC-2024-MOON-BLOOD",
+			want:    "BLOOD",
+		},
+		{
+			name:    "trailing hyphen",
+			release: "Artist-Title-FLAC-2024-",
+			want:    "",
+		},
+		{
+			name:    "empty line",
+			release: "",
+			want:    "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := srrdbGroupName(test.release); got != test.want {
+				t.Errorf("srrdbGroupName(%q) = %q, want %q", test.release, got, test.want)
+			}
+		})
+	}
+}
+
+func TestSrrdbCategory(t *testing.T) {
+	tests := []struct {
+		name string
+		uri  string
+		want string
+	}{
+		{
+			name: "flac with further segments",
+			uri:  "category:flac/date:2026-09-28/page:1",
+			want: "flac",
+		},
+		{
+			name: "category alone",
+			uri:  "category:mp3",
+			want: "mp3",
+		},
+		{
+			name: "no category segment",
+			uri:  "date:2026-09-28/page:1",
+			want: "",
+		},
+		{
+			name: "empty path",
+			uri:  "",
+			want: "",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := srrdbCategory(test.uri); got != test.want {
+				t.Errorf("srrdbCategory(%q) = %q, want %q", test.uri, got, test.want)
+			}
+		})
+	}
+}
+
+func TestSrrdbStartPage(t *testing.T) {
+	tests := []struct {
+		name          string
+		uri           string
+		wantPage      int
+		wantPaginated bool
+	}{
+		{
+			name:          "first page",
+			uri:           "category:flac/date:2026-09-28/page:1",
+			wantPage:      1,
+			wantPaginated: true,
+		},
+		{
+			name:          "later page",
+			uri:           "category:flac/date:2026-09-28/page:7",
+			wantPage:      7,
+			wantPaginated: true,
+		},
+		{
+			name:          "no page segment",
+			uri:           "category:flac/date:2026-09-28",
+			wantPage:      1,
+			wantPaginated: false,
+		},
+		{
+			name:          "zero page falls back to one",
+			uri:           "category:flac/date:2026-09-28/page:0",
+			wantPage:      1,
+			wantPaginated: true,
+		},
+		{
+			name:          "non numeric page is not a page segment",
+			uri:           "category:flac/date:2026-09-28/page:abc",
+			wantPage:      1,
+			wantPaginated: false,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			page, paginated := srrdbStartPage(test.uri)
+			if page != test.wantPage || paginated != test.wantPaginated {
+				t.Errorf("srrdbStartPage(%q) = %d, %t, want %d, %t",
+					test.uri, page, paginated, test.wantPage, test.wantPaginated)
+			}
+		})
+	}
+}
+
+func TestSrrdbWithPage(t *testing.T) {
+	tests := []struct {
+		name      string
+		uri       string
+		page      int
+		paginated bool
+		want      string
+	}{
+		{
+			name:      "swaps the page segment",
+			uri:       "category:flac/date:2026-09-28/page:3",
+			page:      4,
+			paginated: true,
+			want:      "category:flac/date:2026-09-28/page:4",
+		},
+		{
+			name:      "single request keeps the path untouched",
+			uri:       "category:flac/date:2026-09-28/page:3",
+			page:      4,
+			paginated: false,
+			want:      "category:flac/date:2026-09-28/page:3",
+		},
+		{
+			name:      "no page segment to swap",
+			uri:       "category:flac/date:2026-09-28",
+			page:      2,
+			paginated: true,
+			want:      "category:flac/date:2026-09-28",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if got := srrdbWithPage(test.uri, test.page, test.paginated); got != test.want {
+				t.Errorf("srrdbWithPage(%q, %d, %t) = %q, want %q",
+					test.uri, test.page, test.paginated, got, test.want)
+			}
+		})
+	}
+}
+
+func TestSrrdbCollectKeepsFirstPageEnvelope(t *testing.T) {
+	collected := &resource.SrrdbSearchResponse{}
+
+	srrdbCollect(collected, &resource.SrrdbSearchResponse{
+		ResultsCount: 155,
+		Query:        json.RawMessage(`[21]`),
+		Results: []resource.SrrdbResult{
+			{Release: "First-Release-16BIT-WEB-FLAC-2026-GROUP"},
+			{Release: "Second-Release-16BIT-WEB-FLAC-2026-GROUP"},
+		},
+		Warnings: []string{"first warning"},
+	})
+
+	srrdbCollect(collected, &resource.SrrdbSearchResponse{
+		ResultsCount: 999,
+		Query:        json.RawMessage(`[22]`),
+		Results: []resource.SrrdbResult{
+			{Release: "Third-Release-16BIT-WEB-FLAC-2026-GROUP"},
+		},
+		Warnings: []string{"second warning"},
+	})
+
+	if collected.ResultsCount != 155 {
+		t.Errorf("ResultsCount = %d, want 155: only the first page may set it", collected.ResultsCount)
+	}
+
+	if got := string(collected.Query); got != "[21]" {
+		t.Errorf("Query = %s, want [21]: only the first page may set it", got)
+	}
+
+	if len(collected.Results) != 3 {
+		t.Errorf("len(Results) = %d, want 3", len(collected.Results))
+	}
+
+	if len(collected.Warnings) != 2 {
+		t.Errorf("len(Warnings) = %d, want 2", len(collected.Warnings))
+	}
+}
+
+func TestSrrdbCollectZeroCountOnFirstPageIsNotSticky(t *testing.T) {
+	collected := &resource.SrrdbSearchResponse{}
+
+	srrdbCollect(collected, &resource.SrrdbSearchResponse{})
+	srrdbCollect(collected, &resource.SrrdbSearchResponse{ResultsCount: 42})
+
+	if collected.ResultsCount != 42 {
+		t.Errorf("ResultsCount = %d, want 42: a zero count must not freeze the envelope", collected.ResultsCount)
+	}
+}

pkg/repository/table/srrdb.go

@@ -0,0 +1,59 @@
+package table
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+// srrdbUpsert keys on the "release" primary key: a repeated search refreshes the
+// existing row instead of inserting a copy. synced_at is stamped with the same
+// run timestamp as created_at, so the two differ only on a release that was
+// already stored by an earlier run.
+const srrdbUpsert = `
+	INSERT INTO srrdb_releases
+		("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at)
+	VALUES
+		(:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at)
+	ON CONFLICT("release") DO UPDATE SET
+		date       = excluded.date,
+		has_nfo    = excluded.has_nfo,
+		has_srs    = excluded.has_srs,
+		is_foreign = excluded.is_foreign,
+		imdb_id    = excluded.imdb_id,
+		size       = excluded.size,
+		group_name = excluded.group_name,
+	    category   = excluded.category,
+		synced_at  = excluded.created_at`
+
+// UpsertAllSrrdb writes a whole set in one transaction: either every release
+// lands or none does, so a failure halfway through cannot leave a day half
+// harvested.
+func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error {
+	if len(rows) == 0 {
+		return nil
+	}
+
+	tx, err := db.Beginx()
+	if err != nil {
+		return fmt.Errorf("begin srrdb transaction: %w", err)
+	}
+
+	for _, row := range rows {
+		if _, err := tx.NamedExec(srrdbUpsert, &row); err != nil {
+			// Rollback failure is not reported: the insert error is the one that
+			// matters to the caller, and the transaction is gone either way.
+			_ = tx.Rollback()
+
+			return fmt.Errorf("upsert srrdb release %q: %w", row.Release, err)
+		}
+	}
+
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("commit srrdb transaction: %w", err)
+	}
+
+	return nil
+}