Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
62 lines
1.8 KiB
Go
62 lines
1.8 KiB
Go
package scraper
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"log"
|
|
"runtime"
|
|
|
|
"github.com/spf13/pflag"
|
|
"github.com/spf13/viper"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
|
)
|
|
|
|
const usage = `Music Database (MDB) server/cli craftware'
|
|
|
|
Usage:
|
|
mdb [command] -c [config-file-path]
|
|
|
|
Commands:
|
|
-help, -h Print this message
|
|
-version -v Print version.
|
|
-config -c Set configuration file without extension. It is %s by default.
|
|
-section -s Select section e.g.: rutracker, bandcamp etc.
|
|
|
|
The Go runtime version: %s
|
|
Report bugs to https://git.kplus.net.ua/yevhen/resource-scraper/issues`
|
|
|
|
const (
|
|
version = "0.1"
|
|
defaultConfigPath = constant.DefaultConfigPath
|
|
defaultConfigEnv = constant.DefaultEnvProd
|
|
)
|
|
|
|
func ParseFlags() {
|
|
flag.Bool(constant.FlagHelpShort, false, "")
|
|
flag.Bool(constant.FlagHelp, false, "")
|
|
flag.Bool(constant.FlagVersionShort, false, "")
|
|
flag.Bool(constant.FlagVersion, false, "")
|
|
flag.Bool(constant.FlagDebug, false, "")
|
|
flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location")
|
|
flag.String(constant.FlagScopeEnable, "", "")
|
|
flag.String(constant.FlagSingleUri, "", "")
|
|
flag.String(constant.FlagEnv, defaultConfigEnv, "")
|
|
flag.String(constant.FlagCreate, defaultConfigEnv, "used to create e.g. scope etc.")
|
|
flag.Duration(constant.FlagSleep, 0, "pause between requests, e.g. 2s")
|
|
|
|
pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
|
|
pflag.Parse()
|
|
err := viper.BindPFlags(pflag.CommandLine)
|
|
if err != nil {
|
|
log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
|
|
}
|
|
|
|
if viper.GetBool(constant.FlagHelpShort) || viper.GetBool(constant.FlagHelp) {
|
|
msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
|
|
fmt.Println(msg)
|
|
} else if viper.GetBool(constant.FlagVersionShort) || viper.GetBool(constant.FlagVersion) {
|
|
fmt.Println("MDB version", version)
|
|
}
|
|
}
|