Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
77 lines
1.9 KiB
Go
77 lines
1.9 KiB
Go
package scraper
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
|
|
"github.com/iancoleman/strcase"
|
|
"github.com/jmoiron/sqlx"
|
|
"github.com/logrusorgru/aurora/v4"
|
|
"github.com/spf13/viper"
|
|
"golang.org/x/exp/slices"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/db"
|
|
"git.kplus.net.ua/yevhen/resource-scraper/pkg/handler"
|
|
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository"
|
|
"git.kplus.net.ua/yevhen/resource-scraper/pkg/service"
|
|
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
|
)
|
|
|
|
func isAllowScope() bool {
|
|
scopesAllow := viper.GetStringSlice("scope.allow")
|
|
scopeEnabled := viper.GetString(constant.FlagScopeEnable)
|
|
|
|
return slices.Contains(scopesAllow, scopeEnabled)
|
|
}
|
|
|
|
func init() {
|
|
strcase.ConfigureAcronym("stb", "STB")
|
|
}
|
|
|
|
// Bootstrap runs the selected scope and returns the process exit code: zero on
|
|
// success, non-zero as soon as the scope reports a failure.
|
|
func Bootstrap() int {
|
|
if !isAllowScope() {
|
|
fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml"))
|
|
|
|
return 2
|
|
}
|
|
|
|
dbase := db.New()
|
|
|
|
// The SQLite handle is opened only for the srrdb scope, so that a missing
|
|
// or unreadable srrdb.sqlite does not break the other sources.
|
|
var srrdb *sqlx.DB
|
|
if viper.GetString(constant.FlagScopeEnable) == constant.ScopeSrrdb {
|
|
srrdb = db.NewSrrdb()
|
|
}
|
|
|
|
repos := repository.New(dbase, srrdb)
|
|
services := service.New(repos)
|
|
handlers := handler.New(services)
|
|
|
|
switch viper.GetString("role") {
|
|
case constant.RoleConsole:
|
|
result, err := handlers.InitConsole()
|
|
fmt.Printf("init console console: %s\n", result)
|
|
|
|
if err != nil {
|
|
slog.Error("console command failed", "scope", viper.GetString(constant.FlagScopeEnable), "err", err)
|
|
|
|
return 1
|
|
}
|
|
case constant.RoleWeb:
|
|
result, err := handlers.InitRoutes()
|
|
fmt.Printf("who: %s\n", result)
|
|
///http.Run()
|
|
|
|
if err != nil {
|
|
slog.Error("web init failed", "err", err)
|
|
|
|
return 1
|
|
}
|
|
}
|
|
|
|
return 0
|
|
}
|