Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
60 lines
1.8 KiB
Go
60 lines
1.8 KiB
Go
package table
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/jmoiron/sqlx"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
|
)
|
|
|
|
// srrdbUpsert keys on the "release" primary key: a repeated search refreshes the
|
|
// existing row instead of inserting a copy. synced_at is stamped with the same
|
|
// run timestamp as created_at, so the two differ only on a release that was
|
|
// already stored by an earlier run.
|
|
const srrdbUpsert = `
|
|
INSERT INTO srrdb_releases
|
|
("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at)
|
|
VALUES
|
|
(:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at)
|
|
ON CONFLICT("release") DO UPDATE SET
|
|
date = excluded.date,
|
|
has_nfo = excluded.has_nfo,
|
|
has_srs = excluded.has_srs,
|
|
is_foreign = excluded.is_foreign,
|
|
imdb_id = excluded.imdb_id,
|
|
size = excluded.size,
|
|
group_name = excluded.group_name,
|
|
category = excluded.category,
|
|
synced_at = excluded.created_at`
|
|
|
|
// UpsertAllSrrdb writes a whole set in one transaction: either every release
|
|
// lands or none does, so a failure halfway through cannot leave a day half
|
|
// harvested.
|
|
func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error {
|
|
if len(rows) == 0 {
|
|
return nil
|
|
}
|
|
|
|
tx, err := db.Beginx()
|
|
if err != nil {
|
|
return fmt.Errorf("begin srrdb transaction: %w", err)
|
|
}
|
|
|
|
for _, row := range rows {
|
|
if _, err := tx.NamedExec(srrdbUpsert, &row); err != nil {
|
|
// Rollback failure is not reported: the insert error is the one that
|
|
// matters to the caller, and the transaction is gone either way.
|
|
_ = tx.Rollback()
|
|
|
|
return fmt.Errorf("upsert srrdb release %q: %w", row.Release, err)
|
|
}
|
|
}
|
|
|
|
if err := tx.Commit(); err != nil {
|
|
return fmt.Errorf("commit srrdb transaction: %w", err)
|
|
}
|
|
|
|
return nil
|
|
}
|