yevhen/resource-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

srrdb: harvest the srrDB search API into a separate SQLite database

Browse Source 
New scope reading the API instead of scraping HTML: one search path per run,
passed raw through --single-uri, and the answer stored as-is into its own
SQLite file created from the embedded schema.

Command line: --single-uri carries the path after /v1/search/
(category:flac/date:today/page:1) and must name a category; --sleep sets the
pause between pages, with the config value as fallback. A day named on the
command line is always fetched again and its dump rewritten, so cron can
refresh today hourly and yesterday daily.

Date placeholders: date:auto walks backwards to the newest day in
[srrdb.floor..yesterday] that has no dump yet, closing holes instead of
declaring the archive covered. date:today, date:yesterday and date:tomorrow
stand for the neighbouring days. Today is never taken by the walk, because a
day still being filled would be frozen by its own dump after the first run.

Dumps are the cursor as well as the record, so a missing dump directory is an
error instead of "nothing harvested yet": a renamed, unmounted or mistyped
directory must not send the run silently back to the beginning. An empty day is
a normal result and is remembered as harvested; API warnings never are, and
they reach the exit code.

Writes go through one transaction, so a day lands completely or not at all.
group and category come from the release line and the search path; status, nfo,
content and synced_at stay NULL. srrdb.log.jsonl records one line per finished
run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind
field.

Also: the console command contract returns (string, error), so a failing scope
reaches the process exit code and cron can tell a failed run from a successful
one. config/scope/srrdb.yaml is untracked because its paths differ per host.
modernc.org/sqlite replaces the archived libsql client, which only ever talked
to a server. pkg/repository/srrdb.go is split by responsibility and covered by
table-driven tests.
This commit is contained in:
Yevhen Odynets
2026-09-30 09:27:33 +03:00
parent 332d143f1f
commit 346369ca11
35 changed files with 1653 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/repository/srrdb_test.go
+269
View File
@@ -0,0 +1,269 @@
package repository
import (
"encoding/json"
"testing"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
func TestSrrdbNormalizedPath(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "full search path",
uri: "category:flac/date:2026-09-28/page:1",
want: "category：flac∕date：2026-09-28∕page：1.json",
},
{
name: "single segment",
uri: "category:flac",
want: "category：flac.json",
},
{
name: "empty path",
uri: "",
want: ".json",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbNormalizedPath(test.uri); got != test.want {
t.Errorf("srrdbNormalizedPath(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
func TestSrrdbGroupName(t *testing.T) {
tests := []struct {
name string
release string
want string
}{
{
name: "music release",
release: "A-Reis-Aetherial_Flow-16BIT-WEB-FLAC-2025-NeonDreams",
want: "NeonDreams",
},
{
name: "group containing digits",
release: "Old_Man-Grim_Tales-16BIT-WEB-FLAC-2020-86D",
want: "86D",
},
{
name: "two tokens only",
release: "Artist-Title",
want: "Title",
},
{
// The pattern takes the trailing word, so a group that has a hyphen
// in its own name loses everything before it. Pinned here on purpose:
// it is the known limitation, and this test will fail the moment the
// pattern is improved.
name: "hyphen inside the group",
release: "Artist-Title-16BIT-WEB-FLAC-2024-MOON-BLOOD",
want: "BLOOD",
},
{
name: "trailing hyphen",
release: "Artist-Title-FLAC-2024-",
want: "",
},
{
name: "empty line",
release: "",
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbGroupName(test.release); got != test.want {
t.Errorf("srrdbGroupName(%q) = %q, want %q", test.release, got, test.want)
}
})
}
}
func TestSrrdbCategory(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "flac with further segments",
uri: "category:flac/date:2026-09-28/page:1",
want: "flac",
},
{
name: "category alone",
uri: "category:mp3",
want: "mp3",
},
{
name: "no category segment",
uri: "date:2026-09-28/page:1",
want: "",
},
{
name: "empty path",
uri: "",
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbCategory(test.uri); got != test.want {
t.Errorf("srrdbCategory(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
func TestSrrdbStartPage(t *testing.T) {
tests := []struct {
name string
uri string
wantPage int
wantPaginated bool
}{
{
name: "first page",
uri: "category:flac/date:2026-09-28/page:1",
wantPage: 1,
wantPaginated: true,
},
{
name: "later page",
uri: "category:flac/date:2026-09-28/page:7",
wantPage: 7,
wantPaginated: true,
},
{
name: "no page segment",
uri: "category:flac/date:2026-09-28",
wantPage: 1,
wantPaginated: false,
},
{
name: "zero page falls back to one",
uri: "category:flac/date:2026-09-28/page:0",
wantPage: 1,
wantPaginated: true,
},
{
name: "non numeric page is not a page segment",
uri: "category:flac/date:2026-09-28/page:abc",
wantPage: 1,
wantPaginated: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
page, paginated := srrdbStartPage(test.uri)
if page != test.wantPage || paginated != test.wantPaginated {
t.Errorf("srrdbStartPage(%q) = %d, %t, want %d, %t",
test.uri, page, paginated, test.wantPage, test.wantPaginated)
}
})
}
}
func TestSrrdbWithPage(t *testing.T) {
tests := []struct {
name string
uri string
page int
paginated bool
want string
}{
{
name: "swaps the page segment",
uri: "category:flac/date:2026-09-28/page:3",
page: 4,
paginated: true,
want: "category:flac/date:2026-09-28/page:4",
},
{
name: "single request keeps the path untouched",
uri: "category:flac/date:2026-09-28/page:3",
page: 4,
paginated: false,
want: "category:flac/date:2026-09-28/page:3",
},
{
name: "no page segment to swap",
uri: "category:flac/date:2026-09-28",
page: 2,
paginated: true,
want: "category:flac/date:2026-09-28",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbWithPage(test.uri, test.page, test.paginated); got != test.want {
t.Errorf("srrdbWithPage(%q, %d, %t) = %q, want %q",
test.uri, test.page, test.paginated, got, test.want)
}
})
}
}
func TestSrrdbCollectKeepsFirstPageEnvelope(t *testing.T) {
collected := &resource.SrrdbSearchResponse{}
srrdbCollect(collected, &resource.SrrdbSearchResponse{
ResultsCount: 155,
Query: json.RawMessage(`[21]`),
Results: []resource.SrrdbResult{
{Release: "First-Release-16BIT-WEB-FLAC-2026-GROUP"},
{Release: "Second-Release-16BIT-WEB-FLAC-2026-GROUP"},
},
Warnings: []string{"first warning"},
})
srrdbCollect(collected, &resource.SrrdbSearchResponse{
ResultsCount: 999,
Query: json.RawMessage(`[22]`),
Results: []resource.SrrdbResult{
{Release: "Third-Release-16BIT-WEB-FLAC-2026-GROUP"},
},
Warnings: []string{"second warning"},
})
if collected.ResultsCount != 155 {
t.Errorf("ResultsCount = %d, want 155: only the first page may set it", collected.ResultsCount)
}
if got := string(collected.Query); got != "[21]" {
t.Errorf("Query = %s, want [21]: only the first page may set it", got)
}
if len(collected.Results) != 3 {
t.Errorf("len(Results) = %d, want 3", len(collected.Results))
}
if len(collected.Warnings) != 2 {
t.Errorf("len(Warnings) = %d, want 2", len(collected.Warnings))
}
}
func TestSrrdbCollectZeroCountOnFirstPageIsNotSticky(t *testing.T) {
collected := &resource.SrrdbSearchResponse{}
srrdbCollect(collected, &resource.SrrdbSearchResponse{})
srrdbCollect(collected, &resource.SrrdbSearchResponse{ResultsCount: 42})
if collected.ResultsCount != 42 {
t.Errorf("ResultsCount = %d, want 42: a zero count must not freeze the envelope", collected.ResultsCount)
}
}