Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
61 lines
1.7 KiB
Go
61 lines
1.7 KiB
Go
package handler
|
|
|
|
import (
|
|
"fmt"
|
|
|
|
"github.com/spf13/viper"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/pkg/service"
|
|
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
|
)
|
|
|
|
// consoleCommandHandler визначає тип для функцій-обробників консольних команд.
|
|
// The error it returns decides the process exit code, so cron can tell a failed
|
|
// run from a successful one.
|
|
type consoleCommandHandler func() (string, error)
|
|
|
|
type Handler struct {
|
|
services *service.Service
|
|
// consoleCommands зберігає мапу назв команд до відповідних функцій-обробників.
|
|
consoleCommands map[string]consoleCommandHandler
|
|
}
|
|
|
|
func New(services *service.Service) *Handler {
|
|
h := &Handler{services: services}
|
|
|
|
// Ініціалізація мапи команд.
|
|
h.consoleCommands = map[string]consoleCommandHandler{
|
|
"console": h.Console,
|
|
"info": h.Info,
|
|
"metal-archives": h.MetalArchives,
|
|
"prescene": h.Prescene,
|
|
"rutracker": h.Rutracker,
|
|
"stb": h.STB,
|
|
"srrdb": h.Srrdb,
|
|
//"web": h.Web,
|
|
}
|
|
|
|
return h
|
|
}
|
|
|
|
// InitConsole runs the command selected by --scope-enable. The text is returned
|
|
// even together with an error, so the caller can print what happened before it
|
|
// reports the failure.
|
|
func (h *Handler) InitConsole() (string, error) {
|
|
commandName := viper.GetString(constant.FlagScopeEnable)
|
|
|
|
cmdFunc, ok := h.consoleCommands[commandName]
|
|
if !ok {
|
|
return "", fmt.Errorf("unknown console command %q", commandName)
|
|
}
|
|
|
|
result, err := cmdFunc()
|
|
|
|
return commandName + " launched, " + result + "\n", err
|
|
}
|
|
|
|
func (h *Handler) InitRoutes() (string, error) {
|
|
//TODO:
|
|
return "i am the web initiator", nil
|
|
}
|