srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
Yevhen Odynets
@@ -0,0 +1,256 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// srrdbTestNow pins the moment the date logic is tested against, so no assertion
|
||||
// depends on the wall clock.
|
||||
var srrdbTestNow = time.Date(2026, 9, 30, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
// srrdbTestDir gives every test a clean viper and an empty dump directory, because
|
||||
// the package reads srrdb.dump_dir from the global configuration instance.
|
||||
func srrdbTestDir(t *testing.T) string {
|
||||
t.Helper()
|
||||
|
||||
viper.Reset()
|
||||
|
||||
dir := t.TempDir()
|
||||
viper.Set("srrdb.dump_dir", dir)
|
||||
|
||||
return dir
|
||||
}
|
||||
|
||||
// srrdbTestFloor points srrdb.floor at the given day.
|
||||
func srrdbTestFloor(t *testing.T, day time.Time) {
|
||||
t.Helper()
|
||||
|
||||
viper.Set("srrdb.floor", day.Format(srrdbDayLayout))
|
||||
}
|
||||
|
||||
// srrdbTestDump writes a dump file for a query, which is how the walk learns that
|
||||
// a day is done.
|
||||
func srrdbTestDump(t *testing.T, uri string) {
|
||||
t.Helper()
|
||||
|
||||
path := srrdbDumpPath(uri)
|
||||
if err := os.WriteFile(path, []byte("{}"), 0o644); err != nil {
|
||||
t.Fatalf("write dump %q: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbFloorDateFromString(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
viper.Set("srrdb.floor", "2013-03-16")
|
||||
|
||||
floor, err := srrdbFloorDate()
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbFloorDate: %v", err)
|
||||
}
|
||||
|
||||
want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
|
||||
if !floor.Equal(want) {
|
||||
t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
|
||||
}
|
||||
}
|
||||
|
||||
// The regression this test exists for: YAML decodes an unquoted date as a
|
||||
// timestamp, so the value reaches the code as a time.Time and not as a string. It
|
||||
// used to fail with "extra text: 00:00:00 +0000 UTC".
|
||||
func TestSrrdbFloorDateFromTimestamp(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
|
||||
viper.Set("srrdb.floor", want)
|
||||
|
||||
floor, err := srrdbFloorDate()
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbFloorDate: %v", err)
|
||||
}
|
||||
|
||||
if !floor.Equal(want) {
|
||||
t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbFloorDateNotConfigured(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
if _, err := srrdbFloorDate(); err == nil {
|
||||
t.Error("srrdbFloorDate without srrdb.floor returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbDumpPattern(t *testing.T) {
|
||||
pattern, err := srrdbDumpPattern("category:flac/date:auto/page:1")
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbDumpPattern: %v", err)
|
||||
}
|
||||
|
||||
match := pattern.FindStringSubmatch(srrdbNormalizedPath("category:flac/date:2026-09-28/page:1"))
|
||||
if match == nil {
|
||||
t.Fatal("the pattern did not match the dump name of the same query")
|
||||
}
|
||||
|
||||
if match[1] != "2026-09-28" {
|
||||
t.Errorf("captured day = %q, want 2026-09-28", match[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbDumpPatternNeedsAutoSegment(t *testing.T) {
|
||||
if _, err := srrdbDumpPattern("category:flac/date:2026-09-28/page:1"); err == nil {
|
||||
t.Error("srrdbDumpPattern without date:auto returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbDumpedDaysIgnoresForeignQueries(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
|
||||
srrdbTestDump(t, "category:mp3/date:2026-09-26/page:1")
|
||||
|
||||
days, err := srrdbDumpedDays("category:flac/date:auto/page:1")
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbDumpedDays: %v", err)
|
||||
}
|
||||
|
||||
if len(days) != 2 {
|
||||
t.Fatalf("len(days) = %d, want 2: only dumps of the same query may count", len(days))
|
||||
}
|
||||
}
|
||||
|
||||
// An absent directory must not read as "nothing harvested yet": the dumps are the
|
||||
// cursor of the walk, so a renamed or unmounted directory would otherwise send the
|
||||
// run silently back to the beginning.
|
||||
func TestSrrdbDumpedDaysMissingDirectory(t *testing.T) {
|
||||
viper.Reset()
|
||||
viper.Set("srrdb.dump_dir", filepath.Join(t.TempDir(), "absent"))
|
||||
|
||||
if _, err := srrdbDumpedDays("category:flac/date:auto/page:1"); err == nil {
|
||||
t.Error("srrdbDumpedDays on an absent directory returned no error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDateStartsFromYesterday(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
|
||||
|
||||
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
want := "category:flac/date:2026-09-29/page:1"
|
||||
if got != want {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q: the current day is still being filled", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDatePicksTheNewestMissingDay(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
|
||||
|
||||
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
want := "category:flac/date:2026-09-27/page:1"
|
||||
if got != want {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
// A hole in the middle must be closed once the walk reaches it, not declared
|
||||
// covered: this is the defect that made the previous cursor stop at the floor.
|
||||
func TestSrrdbResolveAutoDateClosesHoles(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
|
||||
|
||||
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
want := "category:flac/date:2026-09-28/page:1"
|
||||
if got != want {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q: the gap above the floor must be filled first", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDateStopsAtFloor(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -1))
|
||||
|
||||
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
|
||||
|
||||
_, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
|
||||
if !errors.Is(err, errSrrdbFloor) {
|
||||
t.Errorf("srrdbResolveAutoDate = %v, want errSrrdbFloor", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveAutoDateIgnoresConcreteDates(t *testing.T) {
|
||||
srrdbTestDir(t)
|
||||
|
||||
uri := "category:flac/date:2026-09-28/page:1"
|
||||
|
||||
got, err := srrdbResolveAutoDate(uri, srrdbTestNow)
|
||||
if err != nil {
|
||||
t.Fatalf("srrdbResolveAutoDate: %v", err)
|
||||
}
|
||||
|
||||
if got != uri {
|
||||
t.Errorf("srrdbResolveAutoDate = %q, want %q left untouched", got, uri)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSrrdbResolveRelativeDays(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
uri string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "today",
|
||||
uri: "category:flac/date:today/page:1",
|
||||
want: "category:flac/date:2026-09-30/page:1",
|
||||
},
|
||||
{
|
||||
name: "yesterday",
|
||||
uri: "category:flac/date:yesterday/page:1",
|
||||
want: "category:flac/date:2026-09-29/page:1",
|
||||
},
|
||||
{
|
||||
name: "tomorrow",
|
||||
uri: "category:flac/date:tomorrow/page:1",
|
||||
want: "category:flac/date:2026-10-01/page:1",
|
||||
},
|
||||
{
|
||||
name: "auto belongs to the walk, not here",
|
||||
uri: "category:flac/date:auto/page:1",
|
||||
want: "category:flac/date:auto/page:1",
|
||||
},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
t.Run(test.name, func(t *testing.T) {
|
||||
if got := srrdbResolveRelativeDays(test.uri, srrdbTestNow); got != test.want {
|
||||
t.Errorf("srrdbResolveRelativeDays(%q) = %q, want %q", test.uri, got, test.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user