srrdb: harvest the srrDB search API into a separate SQLite database
New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests.
This commit is contained in:
@@ -0,0 +1,155 @@
|
||||
package repository
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/viper"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
|
||||
)
|
||||
|
||||
// srrdbPageSegment locates the page:N segment so a run can continue from N into
|
||||
// the following pages.
|
||||
var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`)
|
||||
|
||||
// srrdbCollectAll walks every page before anything is written, so the caller
|
||||
// only ever receives a complete set or an error.
|
||||
func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) {
|
||||
endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/")
|
||||
|
||||
// The flag binds to its own key and wins over the config value; an unused
|
||||
// flag yields zero.
|
||||
sleep := viper.GetDuration(constant.FlagSleep)
|
||||
if sleep <= 0 {
|
||||
sleep = viper.GetDuration("srrdb.sleep")
|
||||
}
|
||||
if sleep <= 0 {
|
||||
sleep = 2 * time.Second
|
||||
}
|
||||
|
||||
slog.Info("srrdb pacing", "uri", uri, "sleep", sleep)
|
||||
|
||||
page, paginated := srrdbStartPage(uri)
|
||||
collected := &resource.SrrdbSearchResponse{}
|
||||
|
||||
for {
|
||||
url := endpoint + "/" + srrdbWithPage(uri, page, paginated)
|
||||
|
||||
response, err := srrdbFetch(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
srrdbCollect(collected, response)
|
||||
|
||||
slog.Info("srrdb page fetched",
|
||||
"url", url,
|
||||
"page", page,
|
||||
"fetched", len(response.Results),
|
||||
"reported", response.ResultsCount)
|
||||
|
||||
if !paginated || len(response.Results) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
// Everything the API reported is already collected: one more page would
|
||||
// cost a request and a pause to receive an empty answer.
|
||||
if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount {
|
||||
break
|
||||
}
|
||||
|
||||
page++
|
||||
|
||||
time.Sleep(sleep)
|
||||
}
|
||||
|
||||
return collected, nil
|
||||
}
|
||||
|
||||
// srrdbCollect merges one page into the answer kept for the dump: results are
|
||||
// appended, while resultsCount and query come from the first page.
|
||||
func srrdbCollect(all, page *resource.SrrdbSearchResponse) {
|
||||
if all.ResultsCount == 0 {
|
||||
all.ResultsCount = page.ResultsCount
|
||||
}
|
||||
|
||||
if len(all.Query) == 0 {
|
||||
all.Query = page.Query
|
||||
}
|
||||
|
||||
all.Results = append(all.Results, page.Results...)
|
||||
all.Warnings = append(all.Warnings, page.Warnings...)
|
||||
}
|
||||
|
||||
// srrdbStartPage reads the page:N segment. Without it the path is a single
|
||||
// request and no further pages are attempted.
|
||||
func srrdbStartPage(uri string) (int, bool) {
|
||||
match := srrdbPageSegment.FindStringSubmatch(uri)
|
||||
if match == nil {
|
||||
return 1, false
|
||||
}
|
||||
|
||||
page, err := strconv.Atoi(match[1])
|
||||
if err != nil || page < 1 {
|
||||
return 1, true
|
||||
}
|
||||
|
||||
return page, true
|
||||
}
|
||||
|
||||
// srrdbWithPage swaps the page:N segment for the page requested now.
|
||||
func srrdbWithPage(uri string, page int, paginated bool) string {
|
||||
if !paginated {
|
||||
return uri
|
||||
}
|
||||
|
||||
return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page))
|
||||
}
|
||||
|
||||
// srrdbFetch performs one search request and decodes the JSON answer.
|
||||
func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) {
|
||||
client := &http.Client{Timeout: 30 * time.Second}
|
||||
|
||||
request, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("build srrdb request %q: %w", url, err)
|
||||
}
|
||||
|
||||
request.Header.Set("Accept", "application/json")
|
||||
|
||||
response, err := client.Do(request)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("request srrdb %q: %w", url, err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if cerr := response.Body.Close(); cerr != nil {
|
||||
slog.Error("closing srrdb response body", "url", url, "err", cerr)
|
||||
}
|
||||
}()
|
||||
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status)
|
||||
}
|
||||
|
||||
body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("read srrdb body %q: %w", url, err)
|
||||
}
|
||||
|
||||
decoded := &resource.SrrdbSearchResponse{}
|
||||
if err := json.Unmarshal(body, decoded); err != nil {
|
||||
return nil, fmt.Errorf("decode srrdb body %q: %w", url, err)
|
||||
}
|
||||
|
||||
return decoded, nil
|
||||
}
|
||||
Reference in New Issue
Block a user