Yevhen Odynets
346369ca11
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
63 lines
1.8 KiB
Go
63 lines
1.8 KiB
Go
package repository
|
||
|
||
import (
|
||
"encoding/json"
|
||
"errors"
|
||
"fmt"
|
||
"io/fs"
|
||
"os"
|
||
"path/filepath"
|
||
"strings"
|
||
|
||
"github.com/spf13/viper"
|
||
|
||
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
||
)
|
||
|
||
// srrdbNormalizedPath turns a search path into a file name. The two characters
|
||
// that carry structure in a path are swapped for look-alikes, so the name still
|
||
// reads as the query it came from and can be reversed character by character.
|
||
func srrdbNormalizedPath(uri string) string {
|
||
return strings.NewReplacer(":", "：", "/", "∕").Replace(uri) + ".json"
|
||
}
|
||
|
||
// srrdbDumpPath is the guard file of a search path inside srrdb.dump_dir.
|
||
func srrdbDumpPath(uri string) string {
|
||
return filepath.Join(viper.GetString("srrdb.dump_dir"), srrdbNormalizedPath(uri))
|
||
}
|
||
|
||
// srrdbExists reports whether a dump file is present. Any stat failure other than
|
||
// "not found" is returned as an error: treating it as absent would let the run
|
||
// overwrite a dump it has no right to read.
|
||
func srrdbExists(path string) (bool, error) {
|
||
_, err := os.Stat(path)
|
||
|
||
switch {
|
||
case err == nil:
|
||
return true, nil
|
||
case errors.Is(err, fs.ErrNotExist):
|
||
return false, nil
|
||
default:
|
||
return false, fmt.Errorf("stat srrdb dump %q: %w", path, err)
|
||
}
|
||
}
|
||
|
||
// srrdbWriteDump stores the merged answer as indented JSON. It runs only after
|
||
// the database is updated, therefore the presence of the file means the whole
|
||
// run completed.
|
||
//
|
||
// The directory is never created here: it holds the cursor of the walk and is made
|
||
// once by hand, like the database file.
|
||
func srrdbWriteDump(path string, all *resource.SrrdbSearchResponse) error {
|
||
data, err := json.MarshalIndent(all, "", " ")
|
||
if err != nil {
|
||
return fmt.Errorf("encode srrdb dump %q: %w", path, err)
|
||
}
|
||
|
||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||
return fmt.Errorf("write srrdb dump %q: %w", path, err)
|
||
}
|
||
|
||
return nil
|
||
}
|