Files
Yevhen Odynets 346369ca11 srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run,
passed raw through --single-uri, and the answer stored as-is into its own
SQLite file created from the embedded schema.

Command line: --single-uri carries the path after /v1/search/
(category:flac/date:today/page:1) and must name a category; --sleep sets the
pause between pages, with the config value as fallback. A day named on the
command line is always fetched again and its dump rewritten, so cron can
refresh today hourly and yesterday daily.

Date placeholders: date:auto walks backwards to the newest day in
[srrdb.floor..yesterday] that has no dump yet, closing holes instead of
declaring the archive covered. date:today, date:yesterday and date:tomorrow
stand for the neighbouring days. Today is never taken by the walk, because a
day still being filled would be frozen by its own dump after the first run.

Dumps are the cursor as well as the record, so a missing dump directory is an
error instead of "nothing harvested yet": a renamed, unmounted or mistyped
directory must not send the run silently back to the beginning. An empty day is
a normal result and is remembered as harvested; API warnings never are, and
they reach the exit code.

Writes go through one transaction, so a day lands completely or not at all.
group and category come from the release line and the search path; status, nfo,
content and synced_at stay NULL. srrdb.log.jsonl records one line per finished
run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind
field.

Also: the console command contract returns (string, error), so a failing scope
reaches the process exit code and cron can tell a failed run from a successful
one. config/scope/srrdb.yaml is untracked because its paths differ per host.
modernc.org/sqlite replaces the archived libsql client, which only ever talked
to a server. pkg/repository/srrdb.go is split by responsibility and covered by
table-driven tests.
2026-09-30 09:27:33 +03:00

270 lines
6.3 KiB
Go
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package repository
import (
"encoding/json"
"testing"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
func TestSrrdbNormalizedPath(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "full search path",
uri: "category:flac/date:2026-09-28/page:1",
want: "category:flac∕date:2026-09-28∕page:1.json",
},
{
name: "single segment",
uri: "category:flac",
want: "category:flac.json",
},
{
name: "empty path",
uri: "",
want: ".json",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbNormalizedPath(test.uri); got != test.want {
t.Errorf("srrdbNormalizedPath(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
func TestSrrdbGroupName(t *testing.T) {
tests := []struct {
name string
release string
want string
}{
{
name: "music release",
release: "A-Reis-Aetherial_Flow-16BIT-WEB-FLAC-2025-NeonDreams",
want: "NeonDreams",
},
{
name: "group containing digits",
release: "Old_Man-Grim_Tales-16BIT-WEB-FLAC-2020-86D",
want: "86D",
},
{
name: "two tokens only",
release: "Artist-Title",
want: "Title",
},
{
// The pattern takes the trailing word, so a group that has a hyphen
// in its own name loses everything before it. Pinned here on purpose:
// it is the known limitation, and this test will fail the moment the
// pattern is improved.
name: "hyphen inside the group",
release: "Artist-Title-16BIT-WEB-FLAC-2024-MOON-BLOOD",
want: "BLOOD",
},
{
name: "trailing hyphen",
release: "Artist-Title-FLAC-2024-",
want: "",
},
{
name: "empty line",
release: "",
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbGroupName(test.release); got != test.want {
t.Errorf("srrdbGroupName(%q) = %q, want %q", test.release, got, test.want)
}
})
}
}
func TestSrrdbCategory(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "flac with further segments",
uri: "category:flac/date:2026-09-28/page:1",
want: "flac",
},
{
name: "category alone",
uri: "category:mp3",
want: "mp3",
},
{
name: "no category segment",
uri: "date:2026-09-28/page:1",
want: "",
},
{
name: "empty path",
uri: "",
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbCategory(test.uri); got != test.want {
t.Errorf("srrdbCategory(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
func TestSrrdbStartPage(t *testing.T) {
tests := []struct {
name string
uri string
wantPage int
wantPaginated bool
}{
{
name: "first page",
uri: "category:flac/date:2026-09-28/page:1",
wantPage: 1,
wantPaginated: true,
},
{
name: "later page",
uri: "category:flac/date:2026-09-28/page:7",
wantPage: 7,
wantPaginated: true,
},
{
name: "no page segment",
uri: "category:flac/date:2026-09-28",
wantPage: 1,
wantPaginated: false,
},
{
name: "zero page falls back to one",
uri: "category:flac/date:2026-09-28/page:0",
wantPage: 1,
wantPaginated: true,
},
{
name: "non numeric page is not a page segment",
uri: "category:flac/date:2026-09-28/page:abc",
wantPage: 1,
wantPaginated: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
page, paginated := srrdbStartPage(test.uri)
if page != test.wantPage || paginated != test.wantPaginated {
t.Errorf("srrdbStartPage(%q) = %d, %t, want %d, %t",
test.uri, page, paginated, test.wantPage, test.wantPaginated)
}
})
}
}
func TestSrrdbWithPage(t *testing.T) {
tests := []struct {
name string
uri string
page int
paginated bool
want string
}{
{
name: "swaps the page segment",
uri: "category:flac/date:2026-09-28/page:3",
page: 4,
paginated: true,
want: "category:flac/date:2026-09-28/page:4",
},
{
name: "single request keeps the path untouched",
uri: "category:flac/date:2026-09-28/page:3",
page: 4,
paginated: false,
want: "category:flac/date:2026-09-28/page:3",
},
{
name: "no page segment to swap",
uri: "category:flac/date:2026-09-28",
page: 2,
paginated: true,
want: "category:flac/date:2026-09-28",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbWithPage(test.uri, test.page, test.paginated); got != test.want {
t.Errorf("srrdbWithPage(%q, %d, %t) = %q, want %q",
test.uri, test.page, test.paginated, got, test.want)
}
})
}
}
func TestSrrdbCollectKeepsFirstPageEnvelope(t *testing.T) {
collected := &resource.SrrdbSearchResponse{}
srrdbCollect(collected, &resource.SrrdbSearchResponse{
ResultsCount: 155,
Query: json.RawMessage(`[21]`),
Results: []resource.SrrdbResult{
{Release: "First-Release-16BIT-WEB-FLAC-2026-GROUP"},
{Release: "Second-Release-16BIT-WEB-FLAC-2026-GROUP"},
},
Warnings: []string{"first warning"},
})
srrdbCollect(collected, &resource.SrrdbSearchResponse{
ResultsCount: 999,
Query: json.RawMessage(`[22]`),
Results: []resource.SrrdbResult{
{Release: "Third-Release-16BIT-WEB-FLAC-2026-GROUP"},
},
Warnings: []string{"second warning"},
})
if collected.ResultsCount != 155 {
t.Errorf("ResultsCount = %d, want 155: only the first page may set it", collected.ResultsCount)
}
if got := string(collected.Query); got != "[21]" {
t.Errorf("Query = %s, want [21]: only the first page may set it", got)
}
if len(collected.Results) != 3 {
t.Errorf("len(Results) = %d, want 3", len(collected.Results))
}
if len(collected.Warnings) != 2 {
t.Errorf("len(Warnings) = %d, want 2", len(collected.Warnings))
}
}
func TestSrrdbCollectZeroCountOnFirstPageIsNotSticky(t *testing.T) {
collected := &resource.SrrdbSearchResponse{}
srrdbCollect(collected, &resource.SrrdbSearchResponse{})
srrdbCollect(collected, &resource.SrrdbSearchResponse{ResultsCount: 42})
if collected.ResultsCount != 42 {
t.Errorf("ResultsCount = %d, want 42: a zero count must not freeze the envelope", collected.ResultsCount)
}
}