srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
@@ -0,0 +1,140 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
||||
)
|
||||
|
||||
// srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the
|
||||
// API itself uses in its date field, because the column is TEXT.
|
||||
const srrdbTimestamp = "2006-01-02 15:04:05"
|
||||
|
||||
// Srrdb harvests the srrDB search API into the separate SQLite database.
|
||||
type Srrdb struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
// NewSrrdbRepository returns a collector writing into the SQLite handle. The
|
||||
// handle is nil for every scope except srrdb, because Bootstrap connects the
|
||||
// SQLite database lazily, so the collector refuses to run without it instead of
|
||||
// failing later on a nil handle.
|
||||
func NewSrrdbRepository(db *sqlx.DB) *Srrdb {
|
||||
return &Srrdb{db: db}
|
||||
}
|
||||
|
||||
// Search harvests one search path. Nothing is written to the database until the
|
||||
// whole set is collected and its size matches what the API reported, so the
|
||||
// dump and the success log can only ever describe a complete day.
|
||||
func (s *Srrdb) Search(rawURI string) (int, error) {
|
||||
if s.db == nil {
|
||||
err := errors.New("srrdb collector has no database handle")
|
||||
srrdbLogError(rawURI, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !strings.Contains(rawURI, "category:") {
|
||||
err := fmt.Errorf("search path %q has no category: segment", rawURI)
|
||||
srrdbLogError(rawURI, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
auto := strings.Contains(rawURI, srrdbAutoSegment)
|
||||
|
||||
now := time.Now()
|
||||
|
||||
uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now)
|
||||
|
||||
switch {
|
||||
case errors.Is(err, errSrrdbFloor):
|
||||
slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI)
|
||||
|
||||
return 0, nil
|
||||
case err != nil:
|
||||
srrdbLogError(rawURI, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// A slash typed on either end of the command line would build a double
|
||||
// separator in the URL and an odd dump file name.
|
||||
uri = strings.Trim(uri, "/")
|
||||
|
||||
// The guard applies to the automatic walk only. A day named on the command
|
||||
// line is fetched again and its dump rewritten every time: a day can still
|
||||
// gain releases later, either from another time zone or from a late upload,
|
||||
// and cron refreshes today and yesterday on purpose.
|
||||
dump := srrdbDumpPath(uri)
|
||||
|
||||
if auto {
|
||||
exists, err := srrdbExists(dump)
|
||||
if err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if exists {
|
||||
slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump)
|
||||
|
||||
return 0, nil
|
||||
}
|
||||
}
|
||||
|
||||
collected, err := srrdbCollectAll(uri)
|
||||
if err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// An empty day is a normal outcome: zero results and no warnings. The dump
|
||||
// records it as harvested, which keeps the walk moving. Warnings mean the
|
||||
// path was not understood, and that must never be remembered as done.
|
||||
switch {
|
||||
case len(collected.Warnings) > 0:
|
||||
srrdbLogWarning(uri, collected.Warnings)
|
||||
|
||||
return 0, fmt.Errorf("srrdb %q reported warnings: %s",
|
||||
uri, strings.Join(collected.Warnings, "; "))
|
||||
case len(collected.Results) != collected.ResultsCount:
|
||||
err := fmt.Errorf("srrdb %q pulled %d of %d reported releases",
|
||||
uri, len(collected.Results), collected.ResultsCount)
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
run := time.Now().Format(srrdbTimestamp)
|
||||
category := srrdbCategory(uri)
|
||||
rows := make([]model.SrrdbRelease, 0, len(collected.Results))
|
||||
|
||||
for _, result := range collected.Results {
|
||||
rows = append(rows, srrdbRow(result, run, category))
|
||||
}
|
||||
|
||||
if err := table.UpsertAllSrrdb(s.db, rows); err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if err := srrdbWriteDump(dump, collected); err != nil {
|
||||
srrdbLogError(uri, err)
|
||||
|
||||
return len(rows), err
|
||||
}
|
||||
|
||||
srrdbLogStatus(uri, len(rows), collected.ResultsCount)
|
||||
|
||||
return len(rows), nil
|
||||
}
|
||||
Reference in New Issue
Block a user