From a2bfa8ea71e257151548daf583aad4f70906b014 Mon Sep 17 00:00:00 2001 From: Yevhen Odynets <6lo5jp@proton.me> Date: Wed, 30 Sep 2026 12:44:40 +0300 Subject: [PATCH] srrdb: persist per-day harvest ledger alongside releases Add srrdb_log table with composite primary key (day, category) and counters found, stored, ignored plus run_at. Replace the release-only persistence path with UpsertAllSrrdbWithDay so releases and their corresponding srrdb_log row are written in one SQLite transaction. Preserve existing error handling: rollback reports the release/day that failed, and commit errors return without writing a dump. Handle fully ignored days as closed. When every fetched release is skipped by ignore rules, still write the ledger row with zero stored releases. This prevents the automatic day walk from endlessly retrying a day whose releases are filtered out. Derive and validate the ledger day from the resolved request URI. Missing or invalid date: segments now fail before database writes. Keep run_at independent from created_at semantics and continue populating created_at from the run time. Preserve column and upsert semantics. category continues to use NOT NULL DEFAULT empty string; status keeps COALESCE behavior on conflict; nfo, content, and synced_at remain unchanged. --- internal/db/srrdb.sql | 13 +++- pkg/repository/srrdb.go | 49 ++++++++++++- pkg/repository/srrdb_group.go | 100 ++++++++++++++++++++++++++ pkg/repository/srrdb_group_test.go | 109 +++++++++++++++++++++++++++++ pkg/repository/table/srrdb.go | 30 +++++++- types/model/srrdb.go | 12 ++++ 6 files changed, 307 insertions(+), 6 deletions(-) create mode 100644 pkg/repository/srrdb_group.go create mode 100644 pkg/repository/srrdb_group_test.go diff --git a/internal/db/srrdb.sql b/internal/db/srrdb.sql index b265544..bd9eb54 100644 --- a/internal/db/srrdb.sql +++ b/internal/db/srrdb.sql @@ -36,4 +36,15 @@ CREATE INDEX IF NOT EXISTS idx_srrdb_group ON srrdb_releases (group_name); CREATE INDEX IF NOT EXISTS idx_srrdb_status - ON srrdb_releases (status); \ No newline at end of file + ON srrdb_releases (status); + +CREATE TABLE IF NOT EXISTS srrdb_log +( + day TEXT NOT NULL, + category TEXT NOT NULL, + found INTEGER NOT NULL, + stored INTEGER NOT NULL, + ignored INTEGER NOT NULL, + run_at TEXT NOT NULL, + PRIMARY KEY (day, category) +); \ No newline at end of file diff --git a/pkg/repository/srrdb.go b/pkg/repository/srrdb.go index 554a10d..0cb55c9 100644 --- a/pkg/repository/srrdb.go +++ b/pkg/repository/srrdb.go @@ -48,6 +48,13 @@ func (s *Srrdb) Search(rawURI string) (int, error) { return 0, err } + rules, err := srrdbLoadGroupRules() + if err != nil { + srrdbLogError(rawURI, err) + + return 0, err + } + auto := strings.Contains(rawURI, srrdbAutoSegment) now := time.Now() @@ -117,12 +124,50 @@ func (s *Srrdb) Search(rawURI string) (int, error) { run := time.Now().Format(srrdbTimestamp) category := srrdbCategory(uri) rows := make([]model.SrrdbRelease, 0, len(collected.Results)) + day := &model.SrrdbLog{ + Category: category, + RunAt: run, + } + day.Found = len(collected.Results) for _, result := range collected.Results { - rows = append(rows, srrdbRow(result, run, category)) + row := srrdbRow(result, run, category) + + if rules.ignore[row.GroupName] { + slog.Info("srrdb release ignored by group rule", "release", row.Release, "group", row.GroupName) + day.Ignored++ + + continue + } + + if status, exists := rules.status[row.GroupName]; exists { + row.Status = &status + } + + rows = append(rows, row) } - if err := table.UpsertAllSrrdb(s.db, rows); err != nil { + day.Stored = len(rows) + + _, dateRest, ok := strings.Cut(uri, "date:") + if !ok { + err := fmt.Errorf("search path %q has no date: segment", uri) + srrdbLogError(uri, err) + + return 0, err + } + + dayName, _, _ := strings.Cut(dateRest, "/") + if _, err := time.Parse(srrdbDayLayout, dayName); err != nil { + err := fmt.Errorf("search path %q has an invalid date %q", uri, dayName) + srrdbLogError(uri, err) + + return 0, err + } + + day.Day = dayName + + if err := table.UpsertAllSrrdbWithDay(s.db, rows, day); err != nil { srrdbLogError(uri, err) return 0, err diff --git a/pkg/repository/srrdb_group.go b/pkg/repository/srrdb_group.go new file mode 100644 index 0000000..facd207 --- /dev/null +++ b/pkg/repository/srrdb_group.go @@ -0,0 +1,100 @@ +package repository + +import ( + "fmt" + + "github.com/spf13/viper" +) + +const ( + // srrdbGroupsKey is the configuration section holding the group rules: + // groups..: [group, ...]. + srrdbGroupsKey = "groups" + + // srrdbStatusColumn is the only column the rules may target. Rows are written + // through a fixed statement, so a second column would need a different way of + // building the insert. + srrdbStatusColumn = "status" + + // srrdbIgnoreValue marks groups that must not reach the database at all. It is + // deliberately not a status of its own: the table rejects it with a CHECK, and + // an ignored release has no reason to be stored. + srrdbIgnoreValue = "ignore" +) + +// srrdbGroupRules is the resolved rule set: the status every configured group +// receives, and the groups skipped before the insert. +type srrdbGroupRules struct { + status map[string]string + ignore map[string]bool +} + +// srrdbLoadGroupRules reads groups.status.: [group, ...]. +// +// A group listed under two values is a configuration error rather than a question +// of precedence: the values arrive in a map, Go iterates it in an arbitrary order, +// and the status a release receives would then change from run to run. +func srrdbLoadGroupRules() (*srrdbGroupRules, error) { + rules := &srrdbGroupRules{ + status: make(map[string]string), + ignore: make(map[string]bool), + } + + for column, rawValues := range viper.GetStringMap(srrdbGroupsKey) { + if column != srrdbStatusColumn { + return nil, fmt.Errorf("%s.%s is not a column of srrdb_releases", srrdbGroupsKey, column) + } + + values, ok := rawValues.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s.%s must map a value to a list of groups", srrdbGroupsKey, column) + } + + for value, rawList := range values { + groups, err := srrdbGroupList(value, rawList) + if err != nil { + return nil, err + } + + for _, group := range groups { + if assigned, exists := rules.status[group]; exists { + return nil, fmt.Errorf("group %q is listed under both %q and %q", group, assigned, value) + } + + rules.status[group] = value + + if value == srrdbIgnoreValue { + rules.ignore[group] = true + } + } + } + } + + return rules, nil +} + +// srrdbGroupList converts one value's group list. An element that is not a string +// is rejected rather than dropped: YAML reads an unquoted 401 as a number, and +// losing that group silently would look like a rule that simply does not work. +func srrdbGroupList(value string, raw any) ([]string, error) { + switch list := raw.(type) { + case []string: + return list, nil + case []any: + groups := make([]string, 0, len(list)) + + for _, item := range list { + group, ok := item.(string) + if !ok { + return nil, fmt.Errorf("%s.%s.%s holds %v, which is not a group name: quote it in the configuration", + srrdbGroupsKey, srrdbStatusColumn, value, item) + } + + groups = append(groups, group) + } + + return groups, nil + default: + return nil, fmt.Errorf("%s.%s.%s must be a list of groups", srrdbGroupsKey, srrdbStatusColumn, value) + } +} diff --git a/pkg/repository/srrdb_group_test.go b/pkg/repository/srrdb_group_test.go new file mode 100644 index 0000000..357d3cd --- /dev/null +++ b/pkg/repository/srrdb_group_test.go @@ -0,0 +1,109 @@ +package repository + +import ( + "strings" + "testing" + + "github.com/spf13/viper" +) + +func TestSrrdbLoadGroupRules(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "neformat": []string{"SHGZ"}, + "remove": []any{"BIGLOVE", "CT"}, + "ignore": []string{"BABAS"}, + }, + }) + + rules, err := srrdbLoadGroupRules() + if err != nil { + t.Fatalf("srrdbLoadGroupRules: %v", err) + } + + if got := rules.status["SHGZ"]; got != "neformat" { + t.Errorf("status[SHGZ] = %q, want neformat", got) + } + + if got := rules.status["CT"]; got != "remove" { + t.Errorf("status[CT] = %q, want remove", got) + } + + if !rules.ignore["BABAS"] { + t.Error("BABAS must be ignored") + } + + if rules.ignore["CT"] { + t.Error("CT must not be ignored") + } +} + +// The regression this guards: viper lowercases keys, and if the group names were +// read as keys, SHGZ would arrive as shgz and would never match a release line. +func TestSrrdbLoadGroupRulesKeepsGroupCase(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "neformat": []string{"SHGZ", "ENViED"}, + }, + }) + + rules, err := srrdbLoadGroupRules() + if err != nil { + t.Fatalf("srrdbLoadGroupRules: %v", err) + } + + for _, group := range []string{"SHGZ", "ENViED"} { + if _, exists := rules.status[group]; !exists { + t.Errorf("group %q was not registered with its original case", group) + } + } +} + +func TestSrrdbLoadGroupRulesRejectsGroupInTwoLists(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "neformat": []string{"SHGZ"}, + "remove": []string{"SHGZ"}, + }, + }) + + _, err := srrdbLoadGroupRules() + if err == nil { + t.Fatal("a group listed under two values must be rejected") + } + + for _, want := range []string{"SHGZ", "neformat", "remove"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %q", err, want) + } + } +} + +func TestSrrdbLoadGroupRulesRejectsNumericGroup(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "ignore": []any{401}, + }, + }) + + if _, err := srrdbLoadGroupRules(); err == nil { + t.Error("an unquoted numeric group must be rejected instead of dropped silently") + } +} + +func TestSrrdbLoadGroupRulesRejectsUnknownColumn(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "nfo": map[string]any{ + "x": []string{"Y"}, + }, + }) + + if _, err := srrdbLoadGroupRules(); err == nil { + t.Error("a rule targeting a column other than status must be rejected") + } +} diff --git a/pkg/repository/table/srrdb.go b/pkg/repository/table/srrdb.go index 8a4d8ee..ff37cc9 100644 --- a/pkg/repository/table/srrdb.go +++ b/pkg/repository/table/srrdb.go @@ -14,9 +14,9 @@ import ( // already stored by an earlier run. const srrdbUpsert = ` INSERT INTO srrdb_releases - ("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at) + ("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, status, created_at) VALUES - (:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at) + (:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :status, :created_at) ON CONFLICT("release") DO UPDATE SET date = excluded.date, has_nfo = excluded.has_nfo, @@ -26,13 +26,29 @@ const srrdbUpsert = ` size = excluded.size, group_name = excluded.group_name, category = excluded.category, + status = COALESCE(excluded.status, status), synced_at = excluded.created_at` +const srrdbUpsertDay = ` + INSERT INTO srrdb_log + ("day", "category", "found", "stored", "ignored", "run_at") + VALUES + (:day, :category, :found, :stored, :ignored, :run_at) + ON CONFLICT("day", "category") DO UPDATE SET + found = excluded.found, + stored = excluded.stored, + ignored = excluded.ignored, + run_at = excluded.run_at` + // UpsertAllSrrdb writes a whole set in one transaction: either every release // lands or none does, so a failure halfway through cannot leave a day half // harvested. func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error { - if len(rows) == 0 { + return UpsertAllSrrdbWithDay(db, rows, nil) +} + +func UpsertAllSrrdbWithDay(db *sqlx.DB, rows []model.SrrdbRelease, day *model.SrrdbLog) error { + if len(rows) == 0 && day == nil { return nil } @@ -51,6 +67,14 @@ func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error { } } + if day != nil { + if _, err := tx.NamedExec(srrdbUpsertDay, day); err != nil { + _ = tx.Rollback() + + return fmt.Errorf("upsert srrdb day %q/%q: %w", day.Day, day.Category, err) + } + } + if err := tx.Commit(); err != nil { return fmt.Errorf("commit srrdb transaction: %w", err) } diff --git a/types/model/srrdb.go b/types/model/srrdb.go index 6bd8e44..26e0b3c 100644 --- a/types/model/srrdb.go +++ b/types/model/srrdb.go @@ -20,3 +20,15 @@ type SrrdbRelease struct { CreatedAt string `db:"created_at"` SyncedAt *string `db:"synced_at"` } + +// SrrdbLog is one row of the srrdb_log table: per-day, per-category +// accounting of a harvest run. Written in the same transaction as the +// releases, so a day marked there always matches what landed in srrdb_releases. +type SrrdbLog struct { + Day string `db:"day"` + Category string `db:"category"` + Found int `db:"found"` + Stored int `db:"stored"` + Ignored int `db:"ignored"` + RunAt string `db:"run_at"` +}