yevhen/resource-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

srrdb: persist per-day harvest ledger alongside releases

Browse Source 
Add srrdb_log table with composite primary key (day, category) and counters
found, stored, ignored plus run_at.

Replace the release-only persistence path with UpsertAllSrrdbWithDay so releases
and their corresponding srrdb_log row are written in one SQLite transaction.
Preserve existing error handling: rollback reports the release/day that failed,
and commit errors return without writing a dump.

Handle fully ignored days as closed. When every fetched release is skipped by
ignore rules, still write the ledger row with zero stored releases. This prevents
the automatic day walk from endlessly retrying a day whose releases are filtered
out.

Derive and validate the ledger day from the resolved request URI. Missing or
invalid date: segments now fail before database writes. Keep run_at independent
from created_at semantics and continue populating created_at from the run time.

Preserve column and upsert semantics. category continues to use NOT NULL DEFAULT
empty string; status keeps COALESCE behavior on conflict; nfo, content, and
synced_at remain unchanged.
This commit is contained in:
Yevhen Odynets
2026-09-30 12:44:40 +03:00
parent 8e986ad2f1
commit a2bfa8ea71
6 changed files with 307 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/db/srrdb.sql
+11
View File
@@ -37,3 +37,14 @@ CREATE INDEX IF NOT EXISTS idx_srrdb_group
CREATE INDEX IF NOT EXISTS idx_srrdb_status
ON srrdb_releases (status);
CREATE TABLE IF NOT EXISTS srrdb_log
(
day TEXT NOT NULL,
category TEXT NOT NULL,
found INTEGER NOT NULL,
stored INTEGER NOT NULL,
ignored INTEGER NOT NULL,
run_at TEXT NOT NULL,
PRIMARY KEY (day, category)
);
pkg/repository/srrdb.go
+47 -2
View File
@@ -48,6 +48,13 @@ func (s *Srrdb) Search(rawURI string) (int, error) {
return 0, err
}
rules, err := srrdbLoadGroupRules()
if err != nil {
srrdbLogError(rawURI, err)
return 0, err
}
auto := strings.Contains(rawURI, srrdbAutoSegment)
now := time.Now()
@@ -117,12 +124,50 @@ func (s *Srrdb) Search(rawURI string) (int, error) {
run := time.Now().Format(srrdbTimestamp)
category := srrdbCategory(uri)
rows := make([]model.SrrdbRelease, 0, len(collected.Results))
day := &model.SrrdbLog{
Category: category,
RunAt: run,
}
day.Found = len(collected.Results)
for _, result := range collected.Results {
rows = append(rows, srrdbRow(result, run, category))
row := srrdbRow(result, run, category)
if rules.ignore[row.GroupName] {
slog.Info("srrdb release ignored by group rule", "release", row.Release, "group", row.GroupName)
day.Ignored++
continue
}
if status, exists := rules.status[row.GroupName]; exists {
row.Status = &status
}
rows = append(rows, row)
}
if err := table.UpsertAllSrrdb(s.db, rows); err != nil {
day.Stored = len(rows)
_, dateRest, ok := strings.Cut(uri, "date:")
if !ok {
err := fmt.Errorf("search path %q has no date: segment", uri)
srrdbLogError(uri, err)
return 0, err
}
dayName, _, _ := strings.Cut(dateRest, "/")
if _, err := time.Parse(srrdbDayLayout, dayName); err != nil {
err := fmt.Errorf("search path %q has an invalid date %q", uri, dayName)
srrdbLogError(uri, err)
return 0, err
}
day.Day = dayName
if err := table.UpsertAllSrrdbWithDay(s.db, rows, day); err != nil {
srrdbLogError(uri, err)
return 0, err
pkg/repository/srrdb_group.go
+100
View File
@@ -0,0 +1,100 @@
package repository
import (
"fmt"
"github.com/spf13/viper"
)
const (
// srrdbGroupsKey is the configuration section holding the group rules:
// groups.<column>.<value>: [group, ...].
srrdbGroupsKey = "groups"
// srrdbStatusColumn is the only column the rules may target. Rows are written
// through a fixed statement, so a second column would need a different way of
// building the insert.
srrdbStatusColumn = "status"
// srrdbIgnoreValue marks groups that must not reach the database at all. It is
// deliberately not a status of its own: the table rejects it with a CHECK, and
// an ignored release has no reason to be stored.
srrdbIgnoreValue = "ignore"
)
// srrdbGroupRules is the resolved rule set: the status every configured group
// receives, and the groups skipped before the insert.
type srrdbGroupRules struct {
status map[string]string
ignore map[string]bool
}
// srrdbLoadGroupRules reads groups.status.<value>: [group, ...].
//
// A group listed under two values is a configuration error rather than a question
// of precedence: the values arrive in a map, Go iterates it in an arbitrary order,
// and the status a release receives would then change from run to run.
func srrdbLoadGroupRules() (*srrdbGroupRules, error) {
rules := &srrdbGroupRules{
status: make(map[string]string),
ignore: make(map[string]bool),
}
for column, rawValues := range viper.GetStringMap(srrdbGroupsKey) {
if column != srrdbStatusColumn {
return nil, fmt.Errorf("%s.%s is not a column of srrdb_releases", srrdbGroupsKey, column)
}
values, ok := rawValues.(map[string]any)
if !ok {
return nil, fmt.Errorf("%s.%s must map a value to a list of groups", srrdbGroupsKey, column)
}
for value, rawList := range values {
groups, err := srrdbGroupList(value, rawList)
if err != nil {
return nil, err
}
for _, group := range groups {
if assigned, exists := rules.status[group]; exists {
return nil, fmt.Errorf("group %q is listed under both %q and %q", group, assigned, value)
}
rules.status[group] = value
if value == srrdbIgnoreValue {
rules.ignore[group] = true
}
}
}
}
return rules, nil
}
// srrdbGroupList converts one value's group list. An element that is not a string
// is rejected rather than dropped: YAML reads an unquoted 401 as a number, and
// losing that group silently would look like a rule that simply does not work.
func srrdbGroupList(value string, raw any) ([]string, error) {
switch list := raw.(type) {
case []string:
return list, nil
case []any:
groups := make([]string, 0, len(list))
for _, item := range list {
group, ok := item.(string)
if !ok {
return nil, fmt.Errorf("%s.%s.%s holds %v, which is not a group name: quote it in the configuration",
srrdbGroupsKey, srrdbStatusColumn, value, item)
}
groups = append(groups, group)
}
return groups, nil
default:
return nil, fmt.Errorf("%s.%s.%s must be a list of groups", srrdbGroupsKey, srrdbStatusColumn, value)
}
}
pkg/repository/srrdb_group_test.go
+109
View File
@@ -0,0 +1,109 @@
package repository
import (
"strings"
"testing"
"github.com/spf13/viper"
)
func TestSrrdbLoadGroupRules(t *testing.T) {
viper.Reset()
viper.Set("groups", map[string]any{
"status": map[string]any{
"neformat": []string{"SHGZ"},
"remove": []any{"BIGLOVE", "CT"},
"ignore": []string{"BABAS"},
},
})
rules, err := srrdbLoadGroupRules()
if err != nil {
t.Fatalf("srrdbLoadGroupRules: %v", err)
}
if got := rules.status["SHGZ"]; got != "neformat" {
t.Errorf("status[SHGZ] = %q, want neformat", got)
}
if got := rules.status["CT"]; got != "remove" {
t.Errorf("status[CT] = %q, want remove", got)
}
if !rules.ignore["BABAS"] {
t.Error("BABAS must be ignored")
}
if rules.ignore["CT"] {
t.Error("CT must not be ignored")
}
}
// The regression this guards: viper lowercases keys, and if the group names were
// read as keys, SHGZ would arrive as shgz and would never match a release line.
func TestSrrdbLoadGroupRulesKeepsGroupCase(t *testing.T) {
viper.Reset()
viper.Set("groups", map[string]any{
"status": map[string]any{
"neformat": []string{"SHGZ", "ENViED"},
},
})
rules, err := srrdbLoadGroupRules()
if err != nil {
t.Fatalf("srrdbLoadGroupRules: %v", err)
}
for _, group := range []string{"SHGZ", "ENViED"} {
if _, exists := rules.status[group]; !exists {
t.Errorf("group %q was not registered with its original case", group)
}
}
}
func TestSrrdbLoadGroupRulesRejectsGroupInTwoLists(t *testing.T) {
viper.Reset()
viper.Set("groups", map[string]any{
"status": map[string]any{
"neformat": []string{"SHGZ"},
"remove": []string{"SHGZ"},
},
})
_, err := srrdbLoadGroupRules()
if err == nil {
t.Fatal("a group listed under two values must be rejected")
}
for _, want := range []string{"SHGZ", "neformat", "remove"} {
if !strings.Contains(err.Error(), want) {
t.Errorf("error %q does not name %q", err, want)
}
}
}
func TestSrrdbLoadGroupRulesRejectsNumericGroup(t *testing.T) {
viper.Reset()
viper.Set("groups", map[string]any{
"status": map[string]any{
"ignore": []any{401},
},
})
if _, err := srrdbLoadGroupRules(); err == nil {
t.Error("an unquoted numeric group must be rejected instead of dropped silently")
}
}
func TestSrrdbLoadGroupRulesRejectsUnknownColumn(t *testing.T) {
viper.Reset()
viper.Set("groups", map[string]any{
"nfo": map[string]any{
"x": []string{"Y"},
},
})
if _, err := srrdbLoadGroupRules(); err == nil {
t.Error("a rule targeting a column other than status must be rejected")
}
}
pkg/repository/table/srrdb.go
+27 -3
View File
@@ -14,9 +14,9 @@ import (
// already stored by an earlier run.
const srrdbUpsert = `
INSERT INTO srrdb_releases
("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at)
("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, status, created_at)
VALUES
(:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at)
(:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :status, :created_at)
ON CONFLICT("release") DO UPDATE SET
date = excluded.date,
has_nfo = excluded.has_nfo,
@@ -26,13 +26,29 @@ const srrdbUpsert = `
size = excluded.size,
group_name = excluded.group_name,
category = excluded.category,
status = COALESCE(excluded.status, status),
synced_at = excluded.created_at`
const srrdbUpsertDay = `
INSERT INTO srrdb_log
("day", "category", "found", "stored", "ignored", "run_at")
VALUES
(:day, :category, :found, :stored, :ignored, :run_at)
ON CONFLICT("day", "category") DO UPDATE SET
found = excluded.found,
stored = excluded.stored,
ignored = excluded.ignored,
run_at = excluded.run_at`
// UpsertAllSrrdb writes a whole set in one transaction: either every release
// lands or none does, so a failure halfway through cannot leave a day half
// harvested.
func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error {
if len(rows) == 0 {
return UpsertAllSrrdbWithDay(db, rows, nil)
}
func UpsertAllSrrdbWithDay(db *sqlx.DB, rows []model.SrrdbRelease, day *model.SrrdbLog) error {
if len(rows) == 0 && day == nil {
return nil
}
@@ -51,6 +67,14 @@ func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error {
}
}
if day != nil {
if _, err := tx.NamedExec(srrdbUpsertDay, day); err != nil {
_ = tx.Rollback()
return fmt.Errorf("upsert srrdb day %q/%q: %w", day.Day, day.Category, err)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit srrdb transaction: %w", err)
}
types/model/srrdb.go
+12
View File
@@ -20,3 +20,15 @@ type SrrdbRelease struct {
CreatedAt string `db:"created_at"`
SyncedAt *string `db:"synced_at"`
}
// SrrdbLog is one row of the srrdb_log table: per-day, per-category
// accounting of a harvest run. Written in the same transaction as the
// releases, so a day marked there always matches what landed in srrdb_releases.
type SrrdbLog struct {
Day string `db:"day"`
Category string `db:"category"`
Found int `db:"found"`
Stored int `db:"stored"`
Ignored int `db:"ignored"`
RunAt string `db:"run_at"`
}