srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
Yevhen Odynets
@@ -0,0 +1,189 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// srrdbAutoSegment is the placeholder replaced by the date the run has to
|
||||
// harvest, so cron can repeat the same command every day.
|
||||
const srrdbAutoSegment = "date:auto"
|
||||
|
||||
// srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for
|
||||
// the neighbouring days, so cron can name a day without hard-coding a date.
|
||||
const (
|
||||
srrdbYesterdaySegment = "date:yesterday"
|
||||
srrdbTodaySegment = "date:today"
|
||||
srrdbTomorrowSegment = "date:tomorrow"
|
||||
)
|
||||
|
||||
// srrdbDayLayout is the layout of a date segment in a search path.
|
||||
const srrdbDayLayout = "2006-01-02"
|
||||
|
||||
// errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is
|
||||
// the archive being fully covered rather than a failure.
|
||||
var errSrrdbFloor = errors.New("srrdb.floor reached")
|
||||
|
||||
// srrdbResolveAutoDate replaces date:auto with the newest day in
|
||||
// [srrdb.floor..yesterday] that has not been harvested yet.
|
||||
//
|
||||
// Scanning from the top-down means a new day is picked up the moment it appears,
|
||||
// and a hole left in the middle is closed once the walk reaches it. The previous
|
||||
// rule (the day before the earliest dump) treated the harvested days as one
|
||||
// uninterrupted run, so a single dump below the floor made it declare the whole
|
||||
// archive covered while everything above stayed missing.
|
||||
//
|
||||
// The current moment is passed in rather than read here, so the walk can be tested
|
||||
// without depending on the wall clock.
|
||||
func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) {
|
||||
if !strings.Contains(rawURI, srrdbAutoSegment) {
|
||||
return rawURI, nil
|
||||
}
|
||||
|
||||
floor, err := srrdbFloorDate()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
harvested, err := srrdbDumpedDays(rawURI)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
done := make(map[string]bool, len(harvested))
|
||||
for _, day := range harvested {
|
||||
done[day.Format(srrdbDayLayout)] = true
|
||||
}
|
||||
|
||||
// The current day is never taken: it is still being filled, so it would have
|
||||
// no dump and would be chosen by every single run, freezing the walk. cron
|
||||
// refreshes today explicitly instead.
|
||||
for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) {
|
||||
if !done[day.Format(srrdbDayLayout)] {
|
||||
return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil
|
||||
}
|
||||
}
|
||||
|
||||
return "", errSrrdbFloor
|
||||
}
|
||||
|
||||
// srrdbDumpedDays lists the days already harvested for the same query, read back
|
||||
// from the dump file names inside srrdb.dump_dir.
|
||||
//
|
||||
// A missing directory is an error rather than "nothing harvested yet": the dumps
|
||||
// are the cursor of the walk, so a renamed, unmounted or mistyped directory would
|
||||
// otherwise send the run silently back to the beginning. The directory is created
|
||||
// once by hand, exactly like the database file.
|
||||
func srrdbDumpedDays(uri string) ([]time.Time, error) {
|
||||
pattern, err := srrdbDumpPattern(uri)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dir := viper.GetString("srrdb.dump_dir")
|
||||
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err)
|
||||
}
|
||||
|
||||
days := make([]time.Time, 0, len(entries))
|
||||
|
||||
for _, entry := range entries {
|
||||
match := pattern.FindStringSubmatch(entry.Name())
|
||||
if match == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
day, err := time.Parse(srrdbDayLayout, match[1])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
days = append(days, day)
|
||||
}
|
||||
|
||||
return days, nil
|
||||
}
|
||||
|
||||
// srrdbDumpPattern builds the regular expression matching the dump names of the
|
||||
// same query, capturing the day segment so it can be read back. The literal part
|
||||
// is quoted first, therefore the capture group is the only pattern syntax left in
|
||||
// the result.
|
||||
func srrdbDumpPattern(uri string) (*regexp.Regexp, error) {
|
||||
const normalizedAuto = "date：auto"
|
||||
|
||||
escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri))
|
||||
|
||||
head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto))
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri)
|
||||
}
|
||||
|
||||
return regexp.Compile("^" + head + `date：(\d{4}-\d{2}-\d{2})` + tail + "$")
|
||||
}
|
||||
|
||||
// srrdbFloorDate is the earliest day the automatic walk may ask for, kept in
|
||||
// configuration because it is a property of the archive, not of the code.
|
||||
//
|
||||
// The value is read through viper.Get rather than viper.GetString: YAML decodes
|
||||
// an unquoted date as a timestamp, and asking for its string form yields
|
||||
// "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the
|
||||
// quoted and the unquoted spelling are accepted here.
|
||||
func srrdbFloorDate() (time.Time, error) {
|
||||
switch raw := viper.Get("srrdb.floor").(type) {
|
||||
case nil:
|
||||
return time.Time{}, errors.New("srrdb.floor is not configured")
|
||||
case time.Time:
|
||||
return raw, nil
|
||||
case string:
|
||||
floor, err := time.Parse(srrdbDayLayout, raw)
|
||||
if err != nil {
|
||||
return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err)
|
||||
}
|
||||
|
||||
return floor, nil
|
||||
default:
|
||||
return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw)
|
||||
}
|
||||
}
|
||||
|
||||
// srrdbReplaceDate swaps a date placeholder for a concrete day.
|
||||
func srrdbReplaceDate(uri, segment string, day time.Time) string {
|
||||
return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1)
|
||||
}
|
||||
|
||||
// srrdbRelativeDays holds the date placeholders that stand for a day counted
|
||||
// from now. date:auto is deliberately absent: it means "the next day the walk
|
||||
// has to cover", which only srrdbResolveAutoDate can decide.
|
||||
var srrdbRelativeDays = []struct {
|
||||
segment string
|
||||
offset int
|
||||
}{
|
||||
{segment: srrdbYesterdaySegment, offset: -1},
|
||||
{segment: srrdbTodaySegment, offset: 0},
|
||||
{segment: srrdbTomorrowSegment, offset: 1},
|
||||
}
|
||||
|
||||
// srrdbResolveRelativeDays expands every relative date placeholder present in the
|
||||
// path. The current moment is passed in for the same reason as in
|
||||
// srrdbResolveAutoDate: a test has to be able to pin it.
|
||||
func srrdbResolveRelativeDays(rawURI string, now time.Time) string {
|
||||
uri := rawURI
|
||||
|
||||
for _, relative := range srrdbRelativeDays {
|
||||
if !strings.Contains(uri, relative.segment) {
|
||||
continue
|
||||
}
|
||||
|
||||
uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset))
|
||||
}
|
||||
|
||||
return uri
|
||||
}
|
||||
Reference in New Issue
Block a user