srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
@@ -8,7 +8,7 @@ import (
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) Console() string {
|
||||
func (h *Handler) Console() (string, error) {
|
||||
viper.SetDefault("env", "devel")
|
||||
cmd := viper.GetString("create")
|
||||
cmdList := viper.Sub("console.cmd")
|
||||
@@ -16,11 +16,11 @@ func (h *Handler) Console() string {
|
||||
|
||||
if !slices.Contains(allowCreate, cmd) {
|
||||
fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd))
|
||||
return ""
|
||||
return "", fmt.Errorf("command %q is not allowed", cmd)
|
||||
}
|
||||
|
||||
fmt.Printf("%v\n", slices.Contains(allowCreate, cmd))
|
||||
|
||||
return viper.GetString("env")
|
||||
return viper.GetString("env"), nil
|
||||
|
||||
}
|
||||
|
||||
+18
-10
@@ -1,7 +1,7 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"log" // Додано імпорт log
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
@@ -10,7 +10,9 @@ import (
|
||||
)
|
||||
|
||||
// consoleCommandHandler визначає тип для функцій-обробників консольних команд.
|
||||
type consoleCommandHandler func() string
|
||||
// The error it returns decides the process exit code, so cron can tell a failed
|
||||
// run from a successful one.
|
||||
type consoleCommandHandler func() (string, error)
|
||||
|
||||
type Handler struct {
|
||||
services *service.Service
|
||||
@@ -29,24 +31,30 @@ func New(services *service.Service) *Handler {
|
||||
"prescene": h.Prescene,
|
||||
"rutracker": h.Rutracker,
|
||||
"stb": h.STB,
|
||||
"srrdb": h.Srrdb,
|
||||
//"web": h.Web,
|
||||
}
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
func (h *Handler) InitConsole() string {
|
||||
// InitConsole runs the command selected by --scope-enable. The text is returned
|
||||
// even together with an error, so the caller can print what happened before it
|
||||
// reports the failure.
|
||||
func (h *Handler) InitConsole() (string, error) {
|
||||
commandName := viper.GetString(constant.FlagScopeEnable)
|
||||
|
||||
if cmdFunc, ok := h.consoleCommands[commandName]; ok {
|
||||
return commandName + " launched, " + cmdFunc() + "\n"
|
||||
} else {
|
||||
log.Printf("Error: unknown console command '%s'", commandName)
|
||||
return "Error: unknown console command"
|
||||
cmdFunc, ok := h.consoleCommands[commandName]
|
||||
if !ok {
|
||||
return "", fmt.Errorf("unknown console command %q", commandName)
|
||||
}
|
||||
|
||||
result, err := cmdFunc()
|
||||
|
||||
return commandName + " launched, " + result + "\n", err
|
||||
}
|
||||
|
||||
func (h *Handler) InitRoutes() string {
|
||||
func (h *Handler) InitRoutes() (string, error) {
|
||||
//TODO:
|
||||
return "i am the web initiator"
|
||||
return "i am the web initiator", nil
|
||||
}
|
||||
|
||||
+3
-2
@@ -12,13 +12,14 @@ import (
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) Info() string {
|
||||
func (h *Handler) Info() (string, error) {
|
||||
md5()
|
||||
|
||||
fmt.Printf("%s: %s; %s: %s\n",
|
||||
aurora.Cyan("ENV"),
|
||||
viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable))
|
||||
return "info"
|
||||
|
||||
return "info", nil
|
||||
}
|
||||
|
||||
func md5() {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
package handler
|
||||
|
||||
func (h *Handler) MetalArchives() string {
|
||||
return "MetalArchives................."
|
||||
func (h *Handler) MetalArchives() (string, error) {
|
||||
return "MetalArchives.................", nil
|
||||
}
|
||||
|
||||
@@ -2,7 +2,6 @@ package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"math"
|
||||
"strconv"
|
||||
"time"
|
||||
@@ -12,7 +11,7 @@ import (
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) Prescene() string {
|
||||
func (h *Handler) Prescene() (string, error) {
|
||||
currMinute := time.Now().Minute()
|
||||
levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
|
||||
initPage := strconv.Itoa(currMinute/(60/levels) + 1)
|
||||
@@ -30,17 +29,15 @@ func (h *Handler) Prescene() string {
|
||||
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
||||
}
|
||||
|
||||
_, err := h.services.Prescene.GetSrrDb()
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||
if _, err := h.services.Prescene.GetSrrDb(); err != nil {
|
||||
return "ps", fmt.Errorf("getting srr db: %w", err)
|
||||
}
|
||||
|
||||
_, err = h.services.Prescene.GetPage(pagesToScrape)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||
if _, err := h.services.Prescene.GetPage(pagesToScrape); err != nil {
|
||||
return "ps", fmt.Errorf("getting page: %w", err)
|
||||
}
|
||||
|
||||
fmt.Printf("%+v\n", pagesToScrape)
|
||||
|
||||
return "ps"
|
||||
return "ps", nil
|
||||
}
|
||||
|
||||
@@ -2,20 +2,18 @@ package handler
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
func (h *Handler) Rutracker() string {
|
||||
func (h *Handler) Rutracker() (string, error) {
|
||||
key := fmt.Sprintf("topic.%v", time.Now().Hour())
|
||||
topics := viper.GetStringSlice(key)
|
||||
|
||||
_, err := h.services.Rutracker.GetTopic(topics)
|
||||
if err != nil {
|
||||
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
||||
if _, err := h.services.Rutracker.GetTopic(topics); err != nil {
|
||||
return "rt", fmt.Errorf("getting topic: %w", err)
|
||||
}
|
||||
|
||||
return "rt"
|
||||
return "rt", nil
|
||||
}
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
// Srrdb runs one srrDB search pass over the raw path given by --single-uri.
|
||||
func (h *Handler) Srrdb() (string, error) {
|
||||
uri := viper.GetString(constant.FlagSingleUri)
|
||||
if uri == "" {
|
||||
return "srrdb", errors.New("no search path, pass --single-uri")
|
||||
}
|
||||
|
||||
stored, err := h.services.Srrdb.Search(uri)
|
||||
|
||||
return fmt.Sprintf("%d records\n", stored), err
|
||||
}
|
||||
+3
-5
@@ -3,20 +3,18 @@ package handler
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
)
|
||||
|
||||
func (h *Handler) STB() string {
|
||||
func (h *Handler) STB() (string, error) {
|
||||
endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint)
|
||||
endpoint = viper.GetString(endpoint)
|
||||
|
||||
if endpoint == "" {
|
||||
slog.Error("getting endpoint from config", "err", errors.New("no endpoint provided"))
|
||||
return "stb"
|
||||
return "stb", errors.New("no endpoint provided")
|
||||
}
|
||||
|
||||
es, ms := h.services.ShareTheBrutality.GetMail(endpoint)
|
||||
@@ -32,5 +30,5 @@ func (h *Handler) STB() string {
|
||||
slog.Error("error occurred while getting topic: ", "err", err)
|
||||
}*/
|
||||
|
||||
return fmt.Sprintf("Added %d records\n", len(es))
|
||||
return fmt.Sprintf("Added %d records\n", len(es)), nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user