Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
78 lines
2.0 KiB
Go
78 lines
2.0 KiB
Go
package repository
|
|
|
|
import (
|
|
"encoding/json"
|
|
"log/slog"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/spf13/viper"
|
|
)
|
|
|
|
// srrdbAppendJSONL adds one object as a single line to a .jsonl log. A missing
|
|
// or failing log never changes the outcome of the run itself.
|
|
func srrdbAppendJSONL(key string, record map[string]any) {
|
|
path := viper.GetString(key)
|
|
if path == "" {
|
|
return
|
|
}
|
|
|
|
data, err := json.Marshal(record)
|
|
if err != nil {
|
|
slog.Error("encode srrdb log record", "key", key, "err", err)
|
|
|
|
return
|
|
}
|
|
|
|
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
|
if err != nil {
|
|
slog.Error("open srrdb log", "path", path, "err", err)
|
|
|
|
return
|
|
}
|
|
|
|
defer func() {
|
|
if cerr := file.Close(); cerr != nil {
|
|
slog.Error("close srrdb log", "path", path, "err", cerr)
|
|
}
|
|
}()
|
|
|
|
if _, err := file.Write(append(data, '\n')); err != nil {
|
|
slog.Error("write srrdb log", "path", path, "err", err)
|
|
}
|
|
}
|
|
|
|
// srrdbLogStatus appends the outcome of one finished run to srrdb.log.
|
|
func srrdbLogStatus(uri string, stored, reported int) {
|
|
srrdbAppendJSONL("srrdb.log", map[string]any{
|
|
"time": time.Now().Format(srrdbTimestamp),
|
|
"uri": uri,
|
|
"stored": stored,
|
|
"reported": reported,
|
|
})
|
|
}
|
|
|
|
// srrdbLogWarning records warnings reported by the API. They share the failure
|
|
// file but carry their own kind, because the endpoint answers an unrecognized
|
|
// keyword with HTTP 200 and an empty result set: the text is the only trace of a
|
|
// bad path, and it must stay distinguishable from a real breakdown.
|
|
func srrdbLogWarning(uri string, warnings []string) {
|
|
srrdbAppendJSONL("srrdb.error_log", map[string]any{
|
|
"kind": "warning",
|
|
"time": time.Now().Format(srrdbTimestamp),
|
|
"uri": uri,
|
|
"warnings": strings.Join(warnings, "; "),
|
|
})
|
|
}
|
|
|
|
// srrdbLogError appends one failure to srrdb.error_log.
|
|
func srrdbLogError(uri string, err error) {
|
|
srrdbAppendJSONL("srrdb.error_log", map[string]any{
|
|
"kind": "error",
|
|
"time": time.Now().Format(srrdbTimestamp),
|
|
"uri": uri,
|
|
"error": err.Error(),
|
|
})
|
|
}
|