diff --git a/internal/db/srrdb.sql b/internal/db/srrdb.sql index b265544..bd9eb54 100644 --- a/internal/db/srrdb.sql +++ b/internal/db/srrdb.sql @@ -36,4 +36,15 @@ CREATE INDEX IF NOT EXISTS idx_srrdb_group ON srrdb_releases (group_name); CREATE INDEX IF NOT EXISTS idx_srrdb_status - ON srrdb_releases (status); \ No newline at end of file + ON srrdb_releases (status); + +CREATE TABLE IF NOT EXISTS srrdb_log +( + day TEXT NOT NULL, + category TEXT NOT NULL, + found INTEGER NOT NULL, + stored INTEGER NOT NULL, + ignored INTEGER NOT NULL, + run_at TEXT NOT NULL, + PRIMARY KEY (day, category) +); \ No newline at end of file diff --git a/pkg/repository/srrdb.go b/pkg/repository/srrdb.go index 554a10d..0cb55c9 100644 --- a/pkg/repository/srrdb.go +++ b/pkg/repository/srrdb.go @@ -48,6 +48,13 @@ func (s *Srrdb) Search(rawURI string) (int, error) { return 0, err } + rules, err := srrdbLoadGroupRules() + if err != nil { + srrdbLogError(rawURI, err) + + return 0, err + } + auto := strings.Contains(rawURI, srrdbAutoSegment) now := time.Now() @@ -117,12 +124,50 @@ func (s *Srrdb) Search(rawURI string) (int, error) { run := time.Now().Format(srrdbTimestamp) category := srrdbCategory(uri) rows := make([]model.SrrdbRelease, 0, len(collected.Results)) + day := &model.SrrdbLog{ + Category: category, + RunAt: run, + } + day.Found = len(collected.Results) for _, result := range collected.Results { - rows = append(rows, srrdbRow(result, run, category)) + row := srrdbRow(result, run, category) + + if rules.ignore[row.GroupName] { + slog.Info("srrdb release ignored by group rule", "release", row.Release, "group", row.GroupName) + day.Ignored++ + + continue + } + + if status, exists := rules.status[row.GroupName]; exists { + row.Status = &status + } + + rows = append(rows, row) } - if err := table.UpsertAllSrrdb(s.db, rows); err != nil { + day.Stored = len(rows) + + _, dateRest, ok := strings.Cut(uri, "date:") + if !ok { + err := fmt.Errorf("search path %q has no date: segment", uri) + srrdbLogError(uri, err) + + return 0, err + } + + dayName, _, _ := strings.Cut(dateRest, "/") + if _, err := time.Parse(srrdbDayLayout, dayName); err != nil { + err := fmt.Errorf("search path %q has an invalid date %q", uri, dayName) + srrdbLogError(uri, err) + + return 0, err + } + + day.Day = dayName + + if err := table.UpsertAllSrrdbWithDay(s.db, rows, day); err != nil { srrdbLogError(uri, err) return 0, err diff --git a/pkg/repository/srrdb_group.go b/pkg/repository/srrdb_group.go new file mode 100644 index 0000000..facd207 --- /dev/null +++ b/pkg/repository/srrdb_group.go @@ -0,0 +1,100 @@ +package repository + +import ( + "fmt" + + "github.com/spf13/viper" +) + +const ( + // srrdbGroupsKey is the configuration section holding the group rules: + // groups..: [group, ...]. + srrdbGroupsKey = "groups" + + // srrdbStatusColumn is the only column the rules may target. Rows are written + // through a fixed statement, so a second column would need a different way of + // building the insert. + srrdbStatusColumn = "status" + + // srrdbIgnoreValue marks groups that must not reach the database at all. It is + // deliberately not a status of its own: the table rejects it with a CHECK, and + // an ignored release has no reason to be stored. + srrdbIgnoreValue = "ignore" +) + +// srrdbGroupRules is the resolved rule set: the status every configured group +// receives, and the groups skipped before the insert. +type srrdbGroupRules struct { + status map[string]string + ignore map[string]bool +} + +// srrdbLoadGroupRules reads groups.status.: [group, ...]. +// +// A group listed under two values is a configuration error rather than a question +// of precedence: the values arrive in a map, Go iterates it in an arbitrary order, +// and the status a release receives would then change from run to run. +func srrdbLoadGroupRules() (*srrdbGroupRules, error) { + rules := &srrdbGroupRules{ + status: make(map[string]string), + ignore: make(map[string]bool), + } + + for column, rawValues := range viper.GetStringMap(srrdbGroupsKey) { + if column != srrdbStatusColumn { + return nil, fmt.Errorf("%s.%s is not a column of srrdb_releases", srrdbGroupsKey, column) + } + + values, ok := rawValues.(map[string]any) + if !ok { + return nil, fmt.Errorf("%s.%s must map a value to a list of groups", srrdbGroupsKey, column) + } + + for value, rawList := range values { + groups, err := srrdbGroupList(value, rawList) + if err != nil { + return nil, err + } + + for _, group := range groups { + if assigned, exists := rules.status[group]; exists { + return nil, fmt.Errorf("group %q is listed under both %q and %q", group, assigned, value) + } + + rules.status[group] = value + + if value == srrdbIgnoreValue { + rules.ignore[group] = true + } + } + } + } + + return rules, nil +} + +// srrdbGroupList converts one value's group list. An element that is not a string +// is rejected rather than dropped: YAML reads an unquoted 401 as a number, and +// losing that group silently would look like a rule that simply does not work. +func srrdbGroupList(value string, raw any) ([]string, error) { + switch list := raw.(type) { + case []string: + return list, nil + case []any: + groups := make([]string, 0, len(list)) + + for _, item := range list { + group, ok := item.(string) + if !ok { + return nil, fmt.Errorf("%s.%s.%s holds %v, which is not a group name: quote it in the configuration", + srrdbGroupsKey, srrdbStatusColumn, value, item) + } + + groups = append(groups, group) + } + + return groups, nil + default: + return nil, fmt.Errorf("%s.%s.%s must be a list of groups", srrdbGroupsKey, srrdbStatusColumn, value) + } +} diff --git a/pkg/repository/srrdb_group_test.go b/pkg/repository/srrdb_group_test.go new file mode 100644 index 0000000..357d3cd --- /dev/null +++ b/pkg/repository/srrdb_group_test.go @@ -0,0 +1,109 @@ +package repository + +import ( + "strings" + "testing" + + "github.com/spf13/viper" +) + +func TestSrrdbLoadGroupRules(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "neformat": []string{"SHGZ"}, + "remove": []any{"BIGLOVE", "CT"}, + "ignore": []string{"BABAS"}, + }, + }) + + rules, err := srrdbLoadGroupRules() + if err != nil { + t.Fatalf("srrdbLoadGroupRules: %v", err) + } + + if got := rules.status["SHGZ"]; got != "neformat" { + t.Errorf("status[SHGZ] = %q, want neformat", got) + } + + if got := rules.status["CT"]; got != "remove" { + t.Errorf("status[CT] = %q, want remove", got) + } + + if !rules.ignore["BABAS"] { + t.Error("BABAS must be ignored") + } + + if rules.ignore["CT"] { + t.Error("CT must not be ignored") + } +} + +// The regression this guards: viper lowercases keys, and if the group names were +// read as keys, SHGZ would arrive as shgz and would never match a release line. +func TestSrrdbLoadGroupRulesKeepsGroupCase(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "neformat": []string{"SHGZ", "ENViED"}, + }, + }) + + rules, err := srrdbLoadGroupRules() + if err != nil { + t.Fatalf("srrdbLoadGroupRules: %v", err) + } + + for _, group := range []string{"SHGZ", "ENViED"} { + if _, exists := rules.status[group]; !exists { + t.Errorf("group %q was not registered with its original case", group) + } + } +} + +func TestSrrdbLoadGroupRulesRejectsGroupInTwoLists(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "neformat": []string{"SHGZ"}, + "remove": []string{"SHGZ"}, + }, + }) + + _, err := srrdbLoadGroupRules() + if err == nil { + t.Fatal("a group listed under two values must be rejected") + } + + for _, want := range []string{"SHGZ", "neformat", "remove"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not name %q", err, want) + } + } +} + +func TestSrrdbLoadGroupRulesRejectsNumericGroup(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "status": map[string]any{ + "ignore": []any{401}, + }, + }) + + if _, err := srrdbLoadGroupRules(); err == nil { + t.Error("an unquoted numeric group must be rejected instead of dropped silently") + } +} + +func TestSrrdbLoadGroupRulesRejectsUnknownColumn(t *testing.T) { + viper.Reset() + viper.Set("groups", map[string]any{ + "nfo": map[string]any{ + "x": []string{"Y"}, + }, + }) + + if _, err := srrdbLoadGroupRules(); err == nil { + t.Error("a rule targeting a column other than status must be rejected") + } +} diff --git a/pkg/repository/table/srrdb.go b/pkg/repository/table/srrdb.go index 8a4d8ee..ff37cc9 100644 --- a/pkg/repository/table/srrdb.go +++ b/pkg/repository/table/srrdb.go @@ -14,9 +14,9 @@ import ( // already stored by an earlier run. const srrdbUpsert = ` INSERT INTO srrdb_releases - ("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at) + ("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, status, created_at) VALUES - (:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at) + (:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :status, :created_at) ON CONFLICT("release") DO UPDATE SET date = excluded.date, has_nfo = excluded.has_nfo, @@ -26,13 +26,29 @@ const srrdbUpsert = ` size = excluded.size, group_name = excluded.group_name, category = excluded.category, + status = COALESCE(excluded.status, status), synced_at = excluded.created_at` +const srrdbUpsertDay = ` + INSERT INTO srrdb_log + ("day", "category", "found", "stored", "ignored", "run_at") + VALUES + (:day, :category, :found, :stored, :ignored, :run_at) + ON CONFLICT("day", "category") DO UPDATE SET + found = excluded.found, + stored = excluded.stored, + ignored = excluded.ignored, + run_at = excluded.run_at` + // UpsertAllSrrdb writes a whole set in one transaction: either every release // lands or none does, so a failure halfway through cannot leave a day half // harvested. func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error { - if len(rows) == 0 { + return UpsertAllSrrdbWithDay(db, rows, nil) +} + +func UpsertAllSrrdbWithDay(db *sqlx.DB, rows []model.SrrdbRelease, day *model.SrrdbLog) error { + if len(rows) == 0 && day == nil { return nil } @@ -51,6 +67,14 @@ func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error { } } + if day != nil { + if _, err := tx.NamedExec(srrdbUpsertDay, day); err != nil { + _ = tx.Rollback() + + return fmt.Errorf("upsert srrdb day %q/%q: %w", day.Day, day.Category, err) + } + } + if err := tx.Commit(); err != nil { return fmt.Errorf("commit srrdb transaction: %w", err) } diff --git a/types/model/srrdb.go b/types/model/srrdb.go index 6bd8e44..26e0b3c 100644 --- a/types/model/srrdb.go +++ b/types/model/srrdb.go @@ -20,3 +20,15 @@ type SrrdbRelease struct { CreatedAt string `db:"created_at"` SyncedAt *string `db:"synced_at"` } + +// SrrdbLog is one row of the srrdb_log table: per-day, per-category +// accounting of a harvest run. Written in the same transaction as the +// releases, so a day marked there always matches what landed in srrdb_releases. +type SrrdbLog struct { + Day string `db:"day"` + Category string `db:"category"` + Found int `db:"found"` + Stored int `db:"stored"` + Ignored int `db:"ignored"` + RunAt string `db:"run_at"` +}