srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
Yevhen Odynets
+34
-5
@@ -2,8 +2,10 @@ package scraper
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/iancoleman/strcase"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/logrusorgru/aurora/v4"
|
||||
"github.com/spf13/viper"
|
||||
"golang.org/x/exp/slices"
|
||||
@@ -26,22 +28,49 @@ func init() {
|
||||
strcase.ConfigureAcronym("stb", "STB")
|
||||
}
|
||||
|
||||
func Bootstrap() {
|
||||
// Bootstrap runs the selected scope and returns the process exit code: zero on
|
||||
// success, non-zero as soon as the scope reports a failure.
|
||||
func Bootstrap() int {
|
||||
if !isAllowScope() {
|
||||
fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml"))
|
||||
return
|
||||
|
||||
return 2
|
||||
}
|
||||
|
||||
dbase := db.New()
|
||||
repos := repository.New(dbase)
|
||||
|
||||
// The SQLite handle is opened only for the srrdb scope, so that a missing
|
||||
// or unreadable srrdb.sqlite does not break the other sources.
|
||||
var srrdb *sqlx.DB
|
||||
if viper.GetString(constant.FlagScopeEnable) == constant.ScopeSrrdb {
|
||||
srrdb = db.NewSrrdb()
|
||||
}
|
||||
|
||||
repos := repository.New(dbase, srrdb)
|
||||
services := service.New(repos)
|
||||
handlers := handler.New(services)
|
||||
|
||||
switch viper.GetString("role") {
|
||||
case constant.RoleConsole:
|
||||
fmt.Printf("init console console: %s\n", handlers.InitConsole())
|
||||
result, err := handlers.InitConsole()
|
||||
fmt.Printf("init console console: %s\n", result)
|
||||
|
||||
if err != nil {
|
||||
slog.Error("console command failed", "scope", viper.GetString(constant.FlagScopeEnable), "err", err)
|
||||
|
||||
return 1
|
||||
}
|
||||
case constant.RoleWeb:
|
||||
fmt.Printf("who: %s\n", handlers.InitRoutes())
|
||||
result, err := handlers.InitRoutes()
|
||||
fmt.Printf("who: %s\n", result)
|
||||
///http.Run()
|
||||
|
||||
if err != nil {
|
||||
slog.Error("web init failed", "err", err)
|
||||
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// srrdbSchema is the DDL of the srrdb database. Keeping it in a file lets the
|
||||
// schema be read and edited without touching Go code; go:embed pulls it into the
|
||||
// binary at build time.
|
||||
//
|
||||
//go:embed srrdb.sql
|
||||
var srrdbSchema string
|
||||
|
||||
// NewSrrdb opens the separate SQLite database that holds srrdb_releases.
|
||||
// The driver name is read from db.ext.driver (sqlite, modernc.org/sqlite), so
|
||||
// the driver can be replaced in configuration without touching this file.
|
||||
func NewSrrdb() *sqlx.DB {
|
||||
driver := viper.GetString("db.ext.driver")
|
||||
path := viper.GetString("srrdb.database")
|
||||
|
||||
db, err := sqlx.Connect(driver, path)
|
||||
if err != nil {
|
||||
panic("Failed to connect to the srrdb database: " + err.Error())
|
||||
}
|
||||
|
||||
if err = db.Ping(); err != nil {
|
||||
panic("Failed to ping the srrdb database: " + err.Error())
|
||||
}
|
||||
|
||||
if err = srrdbCreate(db); err != nil {
|
||||
panic("Failed to create the srrdb schema: " + err.Error())
|
||||
}
|
||||
|
||||
return db
|
||||
}
|
||||
|
||||
// srrdbCreate applies the embedded DDL statement by statement. Every statement is
|
||||
// written to be repeatable, so an existing database is left alone: this creates
|
||||
// missing objects, it does not alter existing ones.
|
||||
//
|
||||
// Splitting on ";" is safe for this schema because no statement holds a semicolon
|
||||
// inside a literal.
|
||||
func srrdbCreate(db *sqlx.DB) error {
|
||||
for _, statement := range strings.Split(srrdbSchema, ";") {
|
||||
if statement = strings.TrimSpace(statement); statement == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
if _, err := db.Exec(statement); err != nil {
|
||||
return fmt.Errorf("apply srrdb schema statement %q: %w", statement, err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
CREATE TABLE IF NOT EXISTS srrdb_releases
|
||||
(
|
||||
"release" TEXT PRIMARY KEY NOT NULL,
|
||||
date TEXT,
|
||||
created_at TEXT NOT NULL,
|
||||
synced_at TEXT,
|
||||
category TEXT NOT NULL DEFAULT '',
|
||||
status TEXT,
|
||||
group_name TEXT,
|
||||
has_nfo TEXT,
|
||||
has_srs TEXT,
|
||||
is_foreign TEXT,
|
||||
imdb_id TEXT,
|
||||
size INTEGER,
|
||||
comment TEXT,
|
||||
nfo BLOB,
|
||||
content TEXT,
|
||||
source TEXT,
|
||||
CHECK (
|
||||
status IN (
|
||||
'on', 'hdd', 'hdd_e', 'catalogued',
|
||||
'deferred', 'ext', 'cloud', 'burnt', 'updated',
|
||||
'onhold', 'watch', 'hidden', 'duplicate', 'trash',
|
||||
'closed', 'request', '2cat', '2compare', 'neformat',
|
||||
'sf88', 'BATA', 'away', 'ready2burn', 'accurip',
|
||||
'replace', '2burn', 'remove', 'paid', 'free', 'ftp',
|
||||
'stream', 'ai'
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_srrdb_date
|
||||
ON srrdb_releases (date);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_srrdb_group
|
||||
ON srrdb_releases (group_name);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_srrdb_status
|
||||
ON srrdb_releases (status);
|
||||
+2
-1
@@ -42,7 +42,8 @@ func ParseFlags() {
|
||||
flag.String(constant.FlagScopeEnable, "", "")
|
||||
flag.String(constant.FlagSingleUri, "", "")
|
||||
flag.String(constant.FlagEnv, defaultConfigEnv, "")
|
||||
flag.String("create", defaultConfigEnv, "used to create e.g. scope etc.")
|
||||
flag.String(constant.FlagCreate, defaultConfigEnv, "used to create e.g. scope etc.")
|
||||
flag.Duration(constant.FlagSleep, 0, "pause between requests, e.g. 2s")
|
||||
|
||||
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
||||
pflag.Parse()
|
||||
|
||||
Reference in New Issue
Block a user