Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
62 lines
1.9 KiB
Go
62 lines
1.9 KiB
Go
package repository
|
|
|
|
import (
|
|
"regexp"
|
|
"strings"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
|
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
|
)
|
|
|
|
// srrdbGroupSegment cuts the group off the tail of a release line, which always
|
|
// sits last. Compiled once at package level instead of per record.
|
|
//
|
|
// The pattern takes the trailing word, so a group whose own name contains a
|
|
// hyphen or a dot would lose everything before that character. Nothing in the
|
|
// line tells such a name apart from the tokens in front of it, so widening the
|
|
// character class without a list of real group names would only start eating the
|
|
// title.
|
|
var srrdbGroupSegment = regexp.MustCompile(`-(\w+)$`)
|
|
|
|
// srrdbCategory reads the value of the category: segment of a search path, which
|
|
// is guaranteed to be present by the time this is called.
|
|
func srrdbCategory(uri string) string {
|
|
_, rest, ok := strings.Cut(uri, "category:")
|
|
if !ok {
|
|
return ""
|
|
}
|
|
|
|
value, _, _ := strings.Cut(rest, "/")
|
|
|
|
return value
|
|
}
|
|
|
|
// srrdbRow maps one API record onto a table row. status, nfo, content and
|
|
// synced_at stay nil and are stored as NULL.
|
|
func srrdbRow(result resource.SrrdbResult, run, category string) model.SrrdbRelease {
|
|
return model.SrrdbRelease{
|
|
Release: result.Release,
|
|
Date: result.Date,
|
|
HasNfo: result.HasNFO,
|
|
HasSrs: result.HasSRS,
|
|
IsForeign: result.IsForeign,
|
|
ImdbID: result.ImdbID,
|
|
Size: result.Size,
|
|
GroupName: srrdbGroupName(result.Release),
|
|
Category: category,
|
|
CreatedAt: run,
|
|
}
|
|
}
|
|
|
|
// srrdbGroupName returns the trailing group of a release line, or an empty
|
|
// string when the tail is not a plain word. The nil check is deliberate:
|
|
// FindStringSubmatch returns nil on no match, and indexing it directly panics.
|
|
func srrdbGroupName(release string) string {
|
|
match := srrdbGroupSegment.FindStringSubmatch(release)
|
|
if match == nil {
|
|
return ""
|
|
}
|
|
|
|
return match[1]
|
|
}
|