srrdb: harvest the srrDB search API into a separate SQLite database

New scope reading the API instead of scraping HTML: one search path per run,
passed raw through --single-uri, and the answer stored as-is into its own
SQLite file created from the embedded schema.

Command line: --single-uri carries the path after /v1/search/
(category:flac/date:today/page:1) and must name a category; --sleep sets the
pause between pages, with the config value as fallback. A day named on the
command line is always fetched again and its dump rewritten, so cron can
refresh today hourly and yesterday daily.

Date placeholders: date:auto walks backwards to the newest day in
[srrdb.floor..yesterday] that has no dump yet, closing holes instead of
declaring the archive covered. date:today, date:yesterday and date:tomorrow
stand for the neighbouring days. Today is never taken by the walk, because a
day still being filled would be frozen by its own dump after the first run.

Dumps are the cursor as well as the record, so a missing dump directory is an
error instead of "nothing harvested yet": a renamed, unmounted or mistyped
directory must not send the run silently back to the beginning. An empty day is
a normal result and is remembered as harvested; API warnings never are, and
they reach the exit code.

Writes go through one transaction, so a day lands completely or not at all.
group and category come from the release line and the search path; status, nfo,
content and synced_at stay NULL. srrdb.log.jsonl records one line per finished
run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind
field.

Also: the console command contract returns (string, error), so a failing scope
reaches the process exit code and cron can tell a failed run from a successful
one. config/scope/srrdb.yaml is untracked because its paths differ per host.
modernc.org/sqlite replaces the archived libsql client, which only ever talked
to a server. pkg/repository/srrdb.go is split by responsibility and covered by
table-driven tests.
This commit is contained in:
Yevhen Odynets
2026-09-30 09:27:33 +03:00
parent 332d143f1f
commit 346369ca11
35 changed files with 1653 additions and 53 deletions
+1
View File
@@ -12,6 +12,7 @@ fabric.properties
tmp/ tmp/
vendor/ vendor/
config/secret.yaml config/secret.yaml
config/scope/srrdb.yaml
*.md *.md
mdb mdb
internal/releaseparser/groups/resources/* internal/releaseparser/groups/resources/*
+6 -8
View File
@@ -2,9 +2,11 @@ package main
import ( import (
"fmt" "fmt"
"os"
"time" "time"
_ "github.com/go-sql-driver/mysql" _ "github.com/go-sql-driver/mysql"
_ "modernc.org/sqlite"
scraper "git.kplus.net.ua/yevhen/resource-scraper/internal" scraper "git.kplus.net.ua/yevhen/resource-scraper/internal"
"git.kplus.net.ua/yevhen/resource-scraper/internal/config" "git.kplus.net.ua/yevhen/resource-scraper/internal/config"
@@ -33,14 +35,10 @@ func init() {
} }
func main() { func main() {
defer func() { code := scraper.Bootstrap()
fmt.Printf("\nExecution took %v", time.Since(st))
}()
//a, b := runtime.GOOS fmt.Printf("\nExecution took %v\n", time.Since(st))
//fmt.Printf("%v - - - \n", runtime.GOOS)
scraper.Bootstrap() // os.Exit skips deferred calls, so the timing above is printed before it.
//fmt.Printf("%v\n", viper.Get("23").([]interface{})) os.Exit(code)
// Кінґстонська стіна
} }
+1 -1
View File
@@ -1,5 +1,5 @@
scope: scope:
allow: [info, web, rutracker, prescene, stb, metal-archives, console] allow: [info, web, rutracker, prescene, stb, metal-archives, srrdb, console]
default: console default: console
mail: mail:
kplus.net.ua: kplus.net.ua:
+9
View File
@@ -18,20 +18,26 @@ require (
github.com/spf13/viper v1.21.0 github.com/spf13/viper v1.21.0
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
golang.org/x/net v0.59.0 golang.org/x/net v0.59.0
modernc.org/sqlite v1.60.0
) )
require ( require (
filippo.io/edwards25519 v1.2.0 // indirect filippo.io/edwards25519 v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.5 // indirect github.com/andybalholm/cascadia v1.3.5 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/icza/bitio v1.1.0 // indirect github.com/icza/bitio v1.1.0 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 // indirect github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 // indirect
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect github.com/spf13/cast v1.10.0 // indirect
@@ -39,4 +45,7 @@ require (
go.yaml.in/yaml/v3 v3.0.5 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/sys v0.48.0 // indirect golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect golang.org/x/text v0.42.0 // indirect
modernc.org/libc v1.77.1 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
) )
+48
View File
@@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dromara/carbon/v2 v2.6.17 h1:QL1LVcp5A/NQtIPbc1vrL9qIXxPNWnaDOBaFlmP1iDg= github.com/dromara/carbon/v2 v2.6.17 h1:QL1LVcp5A/NQtIPbc1vrL9qIXxPNWnaDOBaFlmP1iDg=
github.com/dromara/carbon/v2 v2.6.17/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo= github.com/dromara/carbon/v2 v2.6.17/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug= github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48= github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -32,6 +34,12 @@ github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7w
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0=
@@ -48,6 +56,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mewkiz/flac v1.0.14 h1:hyRGAM8NCKznoPmIi9zz2jyO+nfmxY2ErqBnHZ+gxh4= github.com/mewkiz/flac v1.0.14 h1:hyRGAM8NCKznoPmIi9zz2jyO+nfmxY2ErqBnHZ+gxh4=
@@ -56,10 +66,14 @@ github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 h1:fpWNLIg7nX2f+D+6ZveQ
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87/go.mod h1:wohiqwTwXA5twjDES74xIJLGjAxo3/0pHSbTXLbSDW8= github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87/go.mod h1:wohiqwTwXA5twjDES74xIJLGjAxo3/0pHSbTXLbSDW8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8= github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4= github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
@@ -85,6 +99,8 @@ golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
@@ -94,6 +110,8 @@ golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -116,6 +134,36 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.60.0 h1:7AZh8lREDo8x3j7aSdF7KGpAKUkJExJ1p67tcRnmttM=
modernc.org/sqlite v1.60.0/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+34 -5
View File
@@ -2,8 +2,10 @@ package scraper
import ( import (
"fmt" "fmt"
"log/slog"
"github.com/iancoleman/strcase" "github.com/iancoleman/strcase"
"github.com/jmoiron/sqlx"
"github.com/logrusorgru/aurora/v4" "github.com/logrusorgru/aurora/v4"
"github.com/spf13/viper" "github.com/spf13/viper"
"golang.org/x/exp/slices" "golang.org/x/exp/slices"
@@ -26,22 +28,49 @@ func init() {
strcase.ConfigureAcronym("stb", "STB") strcase.ConfigureAcronym("stb", "STB")
} }
func Bootstrap() { // Bootstrap runs the selected scope and returns the process exit code: zero on
// success, non-zero as soon as the scope reports a failure.
func Bootstrap() int {
if !isAllowScope() { if !isAllowScope() {
fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml")) fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml"))
return
return 2
} }
dbase := db.New() dbase := db.New()
repos := repository.New(dbase)
// The SQLite handle is opened only for the srrdb scope, so that a missing
// or unreadable srrdb.sqlite does not break the other sources.
var srrdb *sqlx.DB
if viper.GetString(constant.FlagScopeEnable) == constant.ScopeSrrdb {
srrdb = db.NewSrrdb()
}
repos := repository.New(dbase, srrdb)
services := service.New(repos) services := service.New(repos)
handlers := handler.New(services) handlers := handler.New(services)
switch viper.GetString("role") { switch viper.GetString("role") {
case constant.RoleConsole: case constant.RoleConsole:
fmt.Printf("init console console: %s\n", handlers.InitConsole()) result, err := handlers.InitConsole()
fmt.Printf("init console console: %s\n", result)
if err != nil {
slog.Error("console command failed", "scope", viper.GetString(constant.FlagScopeEnable), "err", err)
return 1
}
case constant.RoleWeb: case constant.RoleWeb:
fmt.Printf("who: %s\n", handlers.InitRoutes()) result, err := handlers.InitRoutes()
fmt.Printf("who: %s\n", result)
///http.Run() ///http.Run()
if err != nil {
slog.Error("web init failed", "err", err)
return 1
} }
} }
return 0
}
+60
View File
@@ -0,0 +1,60 @@
package db
import (
_ "embed"
"fmt"
"strings"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
)
// srrdbSchema is the DDL of the srrdb database. Keeping it in a file lets the
// schema be read and edited without touching Go code; go:embed pulls it into the
// binary at build time.
//
//go:embed srrdb.sql
var srrdbSchema string
// NewSrrdb opens the separate SQLite database that holds srrdb_releases.
// The driver name is read from db.ext.driver (sqlite, modernc.org/sqlite), so
// the driver can be replaced in configuration without touching this file.
func NewSrrdb() *sqlx.DB {
driver := viper.GetString("db.ext.driver")
path := viper.GetString("srrdb.database")
db, err := sqlx.Connect(driver, path)
if err != nil {
panic("Failed to connect to the srrdb database: " + err.Error())
}
if err = db.Ping(); err != nil {
panic("Failed to ping the srrdb database: " + err.Error())
}
if err = srrdbCreate(db); err != nil {
panic("Failed to create the srrdb schema: " + err.Error())
}
return db
}
// srrdbCreate applies the embedded DDL statement by statement. Every statement is
// written to be repeatable, so an existing database is left alone: this creates
// missing objects, it does not alter existing ones.
//
// Splitting on ";" is safe for this schema because no statement holds a semicolon
// inside a literal.
func srrdbCreate(db *sqlx.DB) error {
for _, statement := range strings.Split(srrdbSchema, ";") {
if statement = strings.TrimSpace(statement); statement == "" {
continue
}
if _, err := db.Exec(statement); err != nil {
return fmt.Errorf("apply srrdb schema statement %q: %w", statement, err)
}
}
return nil
}
+39
View File
@@ -0,0 +1,39 @@
CREATE TABLE IF NOT EXISTS srrdb_releases
(
"release" TEXT PRIMARY KEY NOT NULL,
date TEXT,
created_at TEXT NOT NULL,
synced_at TEXT,
category TEXT NOT NULL DEFAULT '',
status TEXT,
group_name TEXT,
has_nfo TEXT,
has_srs TEXT,
is_foreign TEXT,
imdb_id TEXT,
size INTEGER,
comment TEXT,
nfo BLOB,
content TEXT,
source TEXT,
CHECK (
status IN (
'on', 'hdd', 'hdd_e', 'catalogued',
'deferred', 'ext', 'cloud', 'burnt', 'updated',
'onhold', 'watch', 'hidden', 'duplicate', 'trash',
'closed', 'request', '2cat', '2compare', 'neformat',
'sf88', 'BATA', 'away', 'ready2burn', 'accurip',
'replace', '2burn', 'remove', 'paid', 'free', 'ftp',
'stream', 'ai'
)
)
);
CREATE INDEX IF NOT EXISTS idx_srrdb_date
ON srrdb_releases (date);
CREATE INDEX IF NOT EXISTS idx_srrdb_group
ON srrdb_releases (group_name);
CREATE INDEX IF NOT EXISTS idx_srrdb_status
ON srrdb_releases (status);
+2 -1
View File
@@ -42,7 +42,8 @@ func ParseFlags() {
flag.String(constant.FlagScopeEnable, "", "") flag.String(constant.FlagScopeEnable, "", "")
flag.String(constant.FlagSingleUri, "", "") flag.String(constant.FlagSingleUri, "", "")
flag.String(constant.FlagEnv, defaultConfigEnv, "") flag.String(constant.FlagEnv, defaultConfigEnv, "")
flag.String("create", defaultConfigEnv, "used to create e.g. scope etc.") flag.String(constant.FlagCreate, defaultConfigEnv, "used to create e.g. scope etc.")
flag.Duration(constant.FlagSleep, 0, "pause between requests, e.g. 2s")
pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
pflag.Parse() pflag.Parse()
+3 -3
View File
@@ -8,7 +8,7 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
) )
func (h *Handler) Console() string { func (h *Handler) Console() (string, error) {
viper.SetDefault("env", "devel") viper.SetDefault("env", "devel")
cmd := viper.GetString("create") cmd := viper.GetString("create")
cmdList := viper.Sub("console.cmd") cmdList := viper.Sub("console.cmd")
@@ -16,11 +16,11 @@ func (h *Handler) Console() string {
if !slices.Contains(allowCreate, cmd) { if !slices.Contains(allowCreate, cmd) {
fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd)) fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd))
return "" return "", fmt.Errorf("command %q is not allowed", cmd)
} }
fmt.Printf("%v\n", slices.Contains(allowCreate, cmd)) fmt.Printf("%v\n", slices.Contains(allowCreate, cmd))
return viper.GetString("env") return viper.GetString("env"), nil
} }
+20 -12
View File
@@ -1,7 +1,7 @@
package handler package handler
import ( import (
"log" // Додано імпорт log "fmt"
"github.com/spf13/viper" "github.com/spf13/viper"
@@ -10,7 +10,9 @@ import (
) )
// consoleCommandHandler визначає тип для функцій-обробників консольних команд. // consoleCommandHandler визначає тип для функцій-обробників консольних команд.
type consoleCommandHandler func() string // The error it returns decides the process exit code, so cron can tell a failed
// run from a successful one.
type consoleCommandHandler func() (string, error)
type Handler struct { type Handler struct {
services *service.Service services *service.Service
@@ -29,24 +31,30 @@ func New(services *service.Service) *Handler {
"prescene": h.Prescene, "prescene": h.Prescene,
"rutracker": h.Rutracker, "rutracker": h.Rutracker,
"stb": h.STB, "stb": h.STB,
"srrdb": h.Srrdb,
//"web": h.Web, //"web": h.Web,
} }
return h return h
} }
func (h *Handler) InitConsole() string { // InitConsole runs the command selected by --scope-enable. The text is returned
// even together with an error, so the caller can print what happened before it
// reports the failure.
func (h *Handler) InitConsole() (string, error) {
commandName := viper.GetString(constant.FlagScopeEnable) commandName := viper.GetString(constant.FlagScopeEnable)
if cmdFunc, ok := h.consoleCommands[commandName]; ok { cmdFunc, ok := h.consoleCommands[commandName]
return commandName + " launched, " + cmdFunc() + "\n" if !ok {
} else { return "", fmt.Errorf("unknown console command %q", commandName)
log.Printf("Error: unknown console command '%s'", commandName)
return "Error: unknown console command"
}
} }
func (h *Handler) InitRoutes() string { result, err := cmdFunc()
//TODO:
return "i am the web initiator" return commandName + " launched, " + result + "\n", err
}
func (h *Handler) InitRoutes() (string, error) {
//TODO:
return "i am the web initiator", nil
} }
+3 -2
View File
@@ -12,13 +12,14 @@ import (
"git.kplus.net.ua/yevhen/resource-scraper/types/constant" "git.kplus.net.ua/yevhen/resource-scraper/types/constant"
) )
func (h *Handler) Info() string { func (h *Handler) Info() (string, error) {
md5() md5()
fmt.Printf("%s: %s; %s: %s\n", fmt.Printf("%s: %s; %s: %s\n",
aurora.Cyan("ENV"), aurora.Cyan("ENV"),
viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable)) viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable))
return "info"
return "info", nil
} }
func md5() { func md5() {
+2 -2
View File
@@ -1,5 +1,5 @@
package handler package handler
func (h *Handler) MetalArchives() string { func (h *Handler) MetalArchives() (string, error) {
return "MetalArchives................." return "MetalArchives.................", nil
} }
+6 -9
View File
@@ -2,7 +2,6 @@ package handler
import ( import (
"fmt" "fmt"
"log/slog"
"math" "math"
"strconv" "strconv"
"time" "time"
@@ -12,7 +11,7 @@ import (
"git.kplus.net.ua/yevhen/resource-scraper/types/constant" "git.kplus.net.ua/yevhen/resource-scraper/types/constant"
) )
func (h *Handler) Prescene() string { func (h *Handler) Prescene() (string, error) {
currMinute := time.Now().Minute() currMinute := time.Now().Minute()
levels := viper.GetInt(constant.CfgKeyLevelsToScrape) levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
initPage := strconv.Itoa(currMinute/(60/levels) + 1) initPage := strconv.Itoa(currMinute/(60/levels) + 1)
@@ -30,17 +29,15 @@ func (h *Handler) Prescene() string {
pagesToScrape = append(pagesToScrape, strconv.Itoa(page)) pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
} }
_, err := h.services.Prescene.GetSrrDb() if _, err := h.services.Prescene.GetSrrDb(); err != nil {
if err != nil { return "ps", fmt.Errorf("getting srr db: %w", err)
slog.Error("error occurred while getting page: ", "err", err.Error())
} }
_, err = h.services.Prescene.GetPage(pagesToScrape) if _, err := h.services.Prescene.GetPage(pagesToScrape); err != nil {
if err != nil { return "ps", fmt.Errorf("getting page: %w", err)
slog.Error("error occurred while getting page: ", "err", err.Error())
} }
fmt.Printf("%+v\n", pagesToScrape) fmt.Printf("%+v\n", pagesToScrape)
return "ps" return "ps", nil
} }
+4 -6
View File
@@ -2,20 +2,18 @@ package handler
import ( import (
"fmt" "fmt"
"log/slog"
"time" "time"
"github.com/spf13/viper" "github.com/spf13/viper"
) )
func (h *Handler) Rutracker() string { func (h *Handler) Rutracker() (string, error) {
key := fmt.Sprintf("topic.%v", time.Now().Hour()) key := fmt.Sprintf("topic.%v", time.Now().Hour())
topics := viper.GetStringSlice(key) topics := viper.GetStringSlice(key)
_, err := h.services.Rutracker.GetTopic(topics) if _, err := h.services.Rutracker.GetTopic(topics); err != nil {
if err != nil { return "rt", fmt.Errorf("getting topic: %w", err)
slog.Error("error occurred while getting topic: ", "err", err.Error())
} }
return "rt" return "rt", nil
} }
+22
View File
@@ -0,0 +1,22 @@
package handler
import (
"errors"
"fmt"
"github.com/spf13/viper"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
// Srrdb runs one srrDB search pass over the raw path given by --single-uri.
func (h *Handler) Srrdb() (string, error) {
uri := viper.GetString(constant.FlagSingleUri)
if uri == "" {
return "srrdb", errors.New("no search path, pass --single-uri")
}
stored, err := h.services.Srrdb.Search(uri)
return fmt.Sprintf("%d records\n", stored), err
}
+3 -5
View File
@@ -3,20 +3,18 @@ package handler
import ( import (
"errors" "errors"
"fmt" "fmt"
"log/slog"
"github.com/spf13/viper" "github.com/spf13/viper"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant" "git.kplus.net.ua/yevhen/resource-scraper/types/constant"
) )
func (h *Handler) STB() string { func (h *Handler) STB() (string, error) {
endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint) endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint)
endpoint = viper.GetString(endpoint) endpoint = viper.GetString(endpoint)
if endpoint == "" { if endpoint == "" {
slog.Error("getting endpoint from config", "err", errors.New("no endpoint provided")) return "stb", errors.New("no endpoint provided")
return "stb"
} }
es, ms := h.services.ShareTheBrutality.GetMail(endpoint) es, ms := h.services.ShareTheBrutality.GetMail(endpoint)
@@ -32,5 +30,5 @@ func (h *Handler) STB() string {
slog.Error("error occurred while getting topic: ", "err", err) slog.Error("error occurred while getting topic: ", "err", err)
}*/ }*/
return fmt.Sprintf("Added %d records\n", len(es)) return fmt.Sprintf("Added %d records\n", len(es)), nil
} }
+5 -1
View File
@@ -10,12 +10,16 @@ type Repository struct {
_interface.Rutracker _interface.Rutracker
_interface.Prescene _interface.Prescene
_interface.ShareTheBrutality _interface.ShareTheBrutality
_interface.Srrdb
} }
func New(db *sqlx.DB) *Repository { // New wires the collectors. sqlite serves the srrdb scope only and may be nil
// for every other source; it is connected lazily in Bootstrap.
func New(db, sqlite *sqlx.DB) *Repository {
return &Repository{ return &Repository{
Rutracker: NewRutrackerRepository(db), Rutracker: NewRutrackerRepository(db),
Prescene: NewPresceneRepository(db), Prescene: NewPresceneRepository(db),
ShareTheBrutality: NewShareTheBrutalityRepository(db), ShareTheBrutality: NewShareTheBrutalityRepository(db),
Srrdb: NewSrrdbRepository(sqlite),
} }
} }
+140
View File
@@ -0,0 +1,140 @@
package repository
import (
"errors"
"fmt"
"log/slog"
"strings"
"time"
"github.com/jmoiron/sqlx"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
// srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the
// API itself uses in its date field, because the column is TEXT.
const srrdbTimestamp = "2006-01-02 15:04:05"
// Srrdb harvests the srrDB search API into the separate SQLite database.
type Srrdb struct {
db *sqlx.DB
}
// NewSrrdbRepository returns a collector writing into the SQLite handle. The
// handle is nil for every scope except srrdb, because Bootstrap connects the
// SQLite database lazily, so the collector refuses to run without it instead of
// failing later on a nil handle.
func NewSrrdbRepository(db *sqlx.DB) *Srrdb {
return &Srrdb{db: db}
}
// Search harvests one search path. Nothing is written to the database until the
// whole set is collected and its size matches what the API reported, so the
// dump and the success log can only ever describe a complete day.
func (s *Srrdb) Search(rawURI string) (int, error) {
if s.db == nil {
err := errors.New("srrdb collector has no database handle")
srrdbLogError(rawURI, err)
return 0, err
}
if !strings.Contains(rawURI, "category:") {
err := fmt.Errorf("search path %q has no category: segment", rawURI)
srrdbLogError(rawURI, err)
return 0, err
}
auto := strings.Contains(rawURI, srrdbAutoSegment)
now := time.Now()
uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now)
switch {
case errors.Is(err, errSrrdbFloor):
slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI)
return 0, nil
case err != nil:
srrdbLogError(rawURI, err)
return 0, err
}
// A slash typed on either end of the command line would build a double
// separator in the URL and an odd dump file name.
uri = strings.Trim(uri, "/")
// The guard applies to the automatic walk only. A day named on the command
// line is fetched again and its dump rewritten every time: a day can still
// gain releases later, either from another time zone or from a late upload,
// and cron refreshes today and yesterday on purpose.
dump := srrdbDumpPath(uri)
if auto {
exists, err := srrdbExists(dump)
if err != nil {
srrdbLogError(uri, err)
return 0, err
}
if exists {
slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump)
return 0, nil
}
}
collected, err := srrdbCollectAll(uri)
if err != nil {
srrdbLogError(uri, err)
return 0, err
}
// An empty day is a normal outcome: zero results and no warnings. The dump
// records it as harvested, which keeps the walk moving. Warnings mean the
// path was not understood, and that must never be remembered as done.
switch {
case len(collected.Warnings) > 0:
srrdbLogWarning(uri, collected.Warnings)
return 0, fmt.Errorf("srrdb %q reported warnings: %s",
uri, strings.Join(collected.Warnings, "; "))
case len(collected.Results) != collected.ResultsCount:
err := fmt.Errorf("srrdb %q pulled %d of %d reported releases",
uri, len(collected.Results), collected.ResultsCount)
srrdbLogError(uri, err)
return 0, err
}
run := time.Now().Format(srrdbTimestamp)
category := srrdbCategory(uri)
rows := make([]model.SrrdbRelease, 0, len(collected.Results))
for _, result := range collected.Results {
rows = append(rows, srrdbRow(result, run, category))
}
if err := table.UpsertAllSrrdb(s.db, rows); err != nil {
srrdbLogError(uri, err)
return 0, err
}
if err := srrdbWriteDump(dump, collected); err != nil {
srrdbLogError(uri, err)
return len(rows), err
}
srrdbLogStatus(uri, len(rows), collected.ResultsCount)
return len(rows), nil
}
+155
View File
@@ -0,0 +1,155 @@
package repository
import (
"encoding/json"
"fmt"
"io"
"log/slog"
"net/http"
"regexp"
"strconv"
"strings"
"time"
"github.com/spf13/viper"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
// srrdbPageSegment locates the page:N segment so a run can continue from N into
// the following pages.
var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`)
// srrdbCollectAll walks every page before anything is written, so the caller
// only ever receives a complete set or an error.
func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) {
endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/")
// The flag binds to its own key and wins over the config value; an unused
// flag yields zero.
sleep := viper.GetDuration(constant.FlagSleep)
if sleep <= 0 {
sleep = viper.GetDuration("srrdb.sleep")
}
if sleep <= 0 {
sleep = 2 * time.Second
}
slog.Info("srrdb pacing", "uri", uri, "sleep", sleep)
page, paginated := srrdbStartPage(uri)
collected := &resource.SrrdbSearchResponse{}
for {
url := endpoint + "/" + srrdbWithPage(uri, page, paginated)
response, err := srrdbFetch(url)
if err != nil {
return nil, err
}
srrdbCollect(collected, response)
slog.Info("srrdb page fetched",
"url", url,
"page", page,
"fetched", len(response.Results),
"reported", response.ResultsCount)
if !paginated || len(response.Results) == 0 {
break
}
// Everything the API reported is already collected: one more page would
// cost a request and a pause to receive an empty answer.
if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount {
break
}
page++
time.Sleep(sleep)
}
return collected, nil
}
// srrdbCollect merges one page into the answer kept for the dump: results are
// appended, while resultsCount and query come from the first page.
func srrdbCollect(all, page *resource.SrrdbSearchResponse) {
if all.ResultsCount == 0 {
all.ResultsCount = page.ResultsCount
}
if len(all.Query) == 0 {
all.Query = page.Query
}
all.Results = append(all.Results, page.Results...)
all.Warnings = append(all.Warnings, page.Warnings...)
}
// srrdbStartPage reads the page:N segment. Without it the path is a single
// request and no further pages are attempted.
func srrdbStartPage(uri string) (int, bool) {
match := srrdbPageSegment.FindStringSubmatch(uri)
if match == nil {
return 1, false
}
page, err := strconv.Atoi(match[1])
if err != nil || page < 1 {
return 1, true
}
return page, true
}
// srrdbWithPage swaps the page:N segment for the page requested now.
func srrdbWithPage(uri string, page int, paginated bool) string {
if !paginated {
return uri
}
return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page))
}
// srrdbFetch performs one search request and decodes the JSON answer.
func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) {
client := &http.Client{Timeout: 30 * time.Second}
request, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("build srrdb request %q: %w", url, err)
}
request.Header.Set("Accept", "application/json")
response, err := client.Do(request)
if err != nil {
return nil, fmt.Errorf("request srrdb %q: %w", url, err)
}
defer func() {
if cerr := response.Body.Close(); cerr != nil {
slog.Error("closing srrdb response body", "url", url, "err", cerr)
}
}()
if response.StatusCode != http.StatusOK {
return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status)
}
body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
if err != nil {
return nil, fmt.Errorf("read srrdb body %q: %w", url, err)
}
decoded := &resource.SrrdbSearchResponse{}
if err := json.Unmarshal(body, decoded); err != nil {
return nil, fmt.Errorf("decode srrdb body %q: %w", url, err)
}
return decoded, nil
}
+189
View File
@@ -0,0 +1,189 @@
package repository
import (
"errors"
"fmt"
"os"
"regexp"
"strings"
"time"
"github.com/spf13/viper"
)
// srrdbAutoSegment is the placeholder replaced by the date the run has to
// harvest, so cron can repeat the same command every day.
const srrdbAutoSegment = "date:auto"
// srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for
// the neighbouring days, so cron can name a day without hard-coding a date.
const (
srrdbYesterdaySegment = "date:yesterday"
srrdbTodaySegment = "date:today"
srrdbTomorrowSegment = "date:tomorrow"
)
// srrdbDayLayout is the layout of a date segment in a search path.
const srrdbDayLayout = "2006-01-02"
// errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is
// the archive being fully covered rather than a failure.
var errSrrdbFloor = errors.New("srrdb.floor reached")
// srrdbResolveAutoDate replaces date:auto with the newest day in
// [srrdb.floor..yesterday] that has not been harvested yet.
//
// Scanning from the top-down means a new day is picked up the moment it appears,
// and a hole left in the middle is closed once the walk reaches it. The previous
// rule (the day before the earliest dump) treated the harvested days as one
// uninterrupted run, so a single dump below the floor made it declare the whole
// archive covered while everything above stayed missing.
//
// The current moment is passed in rather than read here, so the walk can be tested
// without depending on the wall clock.
func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) {
if !strings.Contains(rawURI, srrdbAutoSegment) {
return rawURI, nil
}
floor, err := srrdbFloorDate()
if err != nil {
return "", err
}
harvested, err := srrdbDumpedDays(rawURI)
if err != nil {
return "", err
}
done := make(map[string]bool, len(harvested))
for _, day := range harvested {
done[day.Format(srrdbDayLayout)] = true
}
// The current day is never taken: it is still being filled, so it would have
// no dump and would be chosen by every single run, freezing the walk. cron
// refreshes today explicitly instead.
for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) {
if !done[day.Format(srrdbDayLayout)] {
return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil
}
}
return "", errSrrdbFloor
}
// srrdbDumpedDays lists the days already harvested for the same query, read back
// from the dump file names inside srrdb.dump_dir.
//
// A missing directory is an error rather than "nothing harvested yet": the dumps
// are the cursor of the walk, so a renamed, unmounted or mistyped directory would
// otherwise send the run silently back to the beginning. The directory is created
// once by hand, exactly like the database file.
func srrdbDumpedDays(uri string) ([]time.Time, error) {
pattern, err := srrdbDumpPattern(uri)
if err != nil {
return nil, err
}
dir := viper.GetString("srrdb.dump_dir")
entries, err := os.ReadDir(dir)
if err != nil {
return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err)
}
days := make([]time.Time, 0, len(entries))
for _, entry := range entries {
match := pattern.FindStringSubmatch(entry.Name())
if match == nil {
continue
}
day, err := time.Parse(srrdbDayLayout, match[1])
if err != nil {
continue
}
days = append(days, day)
}
return days, nil
}
// srrdbDumpPattern builds the regular expression matching the dump names of the
// same query, capturing the day segment so it can be read back. The literal part
// is quoted first, therefore the capture group is the only pattern syntax left in
// the result.
func srrdbDumpPattern(uri string) (*regexp.Regexp, error) {
const normalizedAuto = "date:auto"
escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri))
head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto))
if !ok {
return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri)
}
return regexp.Compile("^" + head + `date:(\d{4}-\d{2}-\d{2})` + tail + "$")
}
// srrdbFloorDate is the earliest day the automatic walk may ask for, kept in
// configuration because it is a property of the archive, not of the code.
//
// The value is read through viper.Get rather than viper.GetString: YAML decodes
// an unquoted date as a timestamp, and asking for its string form yields
// "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the
// quoted and the unquoted spelling are accepted here.
func srrdbFloorDate() (time.Time, error) {
switch raw := viper.Get("srrdb.floor").(type) {
case nil:
return time.Time{}, errors.New("srrdb.floor is not configured")
case time.Time:
return raw, nil
case string:
floor, err := time.Parse(srrdbDayLayout, raw)
if err != nil {
return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err)
}
return floor, nil
default:
return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw)
}
}
// srrdbReplaceDate swaps a date placeholder for a concrete day.
func srrdbReplaceDate(uri, segment string, day time.Time) string {
return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1)
}
// srrdbRelativeDays holds the date placeholders that stand for a day counted
// from now. date:auto is deliberately absent: it means "the next day the walk
// has to cover", which only srrdbResolveAutoDate can decide.
var srrdbRelativeDays = []struct {
segment string
offset int
}{
{segment: srrdbYesterdaySegment, offset: -1},
{segment: srrdbTodaySegment, offset: 0},
{segment: srrdbTomorrowSegment, offset: 1},
}
// srrdbResolveRelativeDays expands every relative date placeholder present in the
// path. The current moment is passed in for the same reason as in
// srrdbResolveAutoDate: a test has to be able to pin it.
func srrdbResolveRelativeDays(rawURI string, now time.Time) string {
uri := rawURI
for _, relative := range srrdbRelativeDays {
if !strings.Contains(uri, relative.segment) {
continue
}
uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset))
}
return uri
}
+256
View File
@@ -0,0 +1,256 @@
package repository
import (
"errors"
"os"
"path/filepath"
"testing"
"time"
"github.com/spf13/viper"
)
// srrdbTestNow pins the moment the date logic is tested against, so no assertion
// depends on the wall clock.
var srrdbTestNow = time.Date(2026, 9, 30, 12, 0, 0, 0, time.UTC)
// srrdbTestDir gives every test a clean viper and an empty dump directory, because
// the package reads srrdb.dump_dir from the global configuration instance.
func srrdbTestDir(t *testing.T) string {
t.Helper()
viper.Reset()
dir := t.TempDir()
viper.Set("srrdb.dump_dir", dir)
return dir
}
// srrdbTestFloor points srrdb.floor at the given day.
func srrdbTestFloor(t *testing.T, day time.Time) {
t.Helper()
viper.Set("srrdb.floor", day.Format(srrdbDayLayout))
}
// srrdbTestDump writes a dump file for a query, which is how the walk learns that
// a day is done.
func srrdbTestDump(t *testing.T, uri string) {
t.Helper()
path := srrdbDumpPath(uri)
if err := os.WriteFile(path, []byte("{}"), 0o644); err != nil {
t.Fatalf("write dump %q: %v", path, err)
}
}
func TestSrrdbFloorDateFromString(t *testing.T) {
srrdbTestDir(t)
viper.Set("srrdb.floor", "2013-03-16")
floor, err := srrdbFloorDate()
if err != nil {
t.Fatalf("srrdbFloorDate: %v", err)
}
want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
if !floor.Equal(want) {
t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
}
}
// The regression this test exists for: YAML decodes an unquoted date as a
// timestamp, so the value reaches the code as a time.Time and not as a string. It
// used to fail with "extra text: 00:00:00 +0000 UTC".
func TestSrrdbFloorDateFromTimestamp(t *testing.T) {
srrdbTestDir(t)
want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC)
viper.Set("srrdb.floor", want)
floor, err := srrdbFloorDate()
if err != nil {
t.Fatalf("srrdbFloorDate: %v", err)
}
if !floor.Equal(want) {
t.Errorf("srrdbFloorDate = %s, want %s", floor, want)
}
}
func TestSrrdbFloorDateNotConfigured(t *testing.T) {
srrdbTestDir(t)
if _, err := srrdbFloorDate(); err == nil {
t.Error("srrdbFloorDate without srrdb.floor returned no error")
}
}
func TestSrrdbDumpPattern(t *testing.T) {
pattern, err := srrdbDumpPattern("category:flac/date:auto/page:1")
if err != nil {
t.Fatalf("srrdbDumpPattern: %v", err)
}
match := pattern.FindStringSubmatch(srrdbNormalizedPath("category:flac/date:2026-09-28/page:1"))
if match == nil {
t.Fatal("the pattern did not match the dump name of the same query")
}
if match[1] != "2026-09-28" {
t.Errorf("captured day = %q, want 2026-09-28", match[1])
}
}
func TestSrrdbDumpPatternNeedsAutoSegment(t *testing.T) {
if _, err := srrdbDumpPattern("category:flac/date:2026-09-28/page:1"); err == nil {
t.Error("srrdbDumpPattern without date:auto returned no error")
}
}
func TestSrrdbDumpedDaysIgnoresForeignQueries(t *testing.T) {
srrdbTestDir(t)
srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
srrdbTestDump(t, "category:mp3/date:2026-09-26/page:1")
days, err := srrdbDumpedDays("category:flac/date:auto/page:1")
if err != nil {
t.Fatalf("srrdbDumpedDays: %v", err)
}
if len(days) != 2 {
t.Fatalf("len(days) = %d, want 2: only dumps of the same query may count", len(days))
}
}
// An absent directory must not read as "nothing harvested yet": the dumps are the
// cursor of the walk, so a renamed or unmounted directory would otherwise send the
// run silently back to the beginning.
func TestSrrdbDumpedDaysMissingDirectory(t *testing.T) {
viper.Reset()
viper.Set("srrdb.dump_dir", filepath.Join(t.TempDir(), "absent"))
if _, err := srrdbDumpedDays("category:flac/date:auto/page:1"); err == nil {
t.Error("srrdbDumpedDays on an absent directory returned no error")
}
}
func TestSrrdbResolveAutoDateStartsFromYesterday(t *testing.T) {
srrdbTestDir(t)
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
if err != nil {
t.Fatalf("srrdbResolveAutoDate: %v", err)
}
want := "category:flac/date:2026-09-29/page:1"
if got != want {
t.Errorf("srrdbResolveAutoDate = %q, want %q: the current day is still being filled", got, want)
}
}
func TestSrrdbResolveAutoDatePicksTheNewestMissingDay(t *testing.T) {
srrdbTestDir(t)
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
srrdbTestDump(t, "category:flac/date:2026-09-28/page:1")
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
if err != nil {
t.Fatalf("srrdbResolveAutoDate: %v", err)
}
want := "category:flac/date:2026-09-27/page:1"
if got != want {
t.Errorf("srrdbResolveAutoDate = %q, want %q", got, want)
}
}
// A hole in the middle must be closed once the walk reaches it, not declared
// covered: this is the defect that made the previous cursor stop at the floor.
func TestSrrdbResolveAutoDateClosesHoles(t *testing.T) {
srrdbTestDir(t)
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10))
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
srrdbTestDump(t, "category:flac/date:2026-09-27/page:1")
got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
if err != nil {
t.Fatalf("srrdbResolveAutoDate: %v", err)
}
want := "category:flac/date:2026-09-28/page:1"
if got != want {
t.Errorf("srrdbResolveAutoDate = %q, want %q: the gap above the floor must be filled first", got, want)
}
}
func TestSrrdbResolveAutoDateStopsAtFloor(t *testing.T) {
srrdbTestDir(t)
srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -1))
srrdbTestDump(t, "category:flac/date:2026-09-29/page:1")
_, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow)
if !errors.Is(err, errSrrdbFloor) {
t.Errorf("srrdbResolveAutoDate = %v, want errSrrdbFloor", err)
}
}
func TestSrrdbResolveAutoDateIgnoresConcreteDates(t *testing.T) {
srrdbTestDir(t)
uri := "category:flac/date:2026-09-28/page:1"
got, err := srrdbResolveAutoDate(uri, srrdbTestNow)
if err != nil {
t.Fatalf("srrdbResolveAutoDate: %v", err)
}
if got != uri {
t.Errorf("srrdbResolveAutoDate = %q, want %q left untouched", got, uri)
}
}
func TestSrrdbResolveRelativeDays(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "today",
uri: "category:flac/date:today/page:1",
want: "category:flac/date:2026-09-30/page:1",
},
{
name: "yesterday",
uri: "category:flac/date:yesterday/page:1",
want: "category:flac/date:2026-09-29/page:1",
},
{
name: "tomorrow",
uri: "category:flac/date:tomorrow/page:1",
want: "category:flac/date:2026-10-01/page:1",
},
{
name: "auto belongs to the walk, not here",
uri: "category:flac/date:auto/page:1",
want: "category:flac/date:auto/page:1",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbResolveRelativeDays(test.uri, srrdbTestNow); got != test.want {
t.Errorf("srrdbResolveRelativeDays(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
+62
View File
@@ -0,0 +1,62 @@
package repository
import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"strings"
"github.com/spf13/viper"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
// srrdbNormalizedPath turns a search path into a file name. The two characters
// that carry structure in a path are swapped for look-alikes, so the name still
// reads as the query it came from and can be reversed character by character.
func srrdbNormalizedPath(uri string) string {
return strings.NewReplacer(":", ":", "/", "∕").Replace(uri) + ".json"
}
// srrdbDumpPath is the guard file of a search path inside srrdb.dump_dir.
func srrdbDumpPath(uri string) string {
return filepath.Join(viper.GetString("srrdb.dump_dir"), srrdbNormalizedPath(uri))
}
// srrdbExists reports whether a dump file is present. Any stat failure other than
// "not found" is returned as an error: treating it as absent would let the run
// overwrite a dump it has no right to read.
func srrdbExists(path string) (bool, error) {
_, err := os.Stat(path)
switch {
case err == nil:
return true, nil
case errors.Is(err, fs.ErrNotExist):
return false, nil
default:
return false, fmt.Errorf("stat srrdb dump %q: %w", path, err)
}
}
// srrdbWriteDump stores the merged answer as indented JSON. It runs only after
// the database is updated, therefore the presence of the file means the whole
// run completed.
//
// The directory is never created here: it holds the cursor of the walk and is made
// once by hand, like the database file.
func srrdbWriteDump(path string, all *resource.SrrdbSearchResponse) error {
data, err := json.MarshalIndent(all, "", " ")
if err != nil {
return fmt.Errorf("encode srrdb dump %q: %w", path, err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
return fmt.Errorf("write srrdb dump %q: %w", path, err)
}
return nil
}
+45
View File
@@ -0,0 +1,45 @@
package repository
import (
"os"
"path/filepath"
"testing"
)
func TestSrrdbDumpPath(t *testing.T) {
dir := srrdbTestDir(t)
got := srrdbDumpPath("category:flac/date:2026-09-28/page:1")
want := filepath.Join(dir, "category:flac∕date:2026-09-28∕page:1.json")
if got != want {
t.Errorf("srrdbDumpPath = %q, want %q", got, want)
}
}
func TestSrrdbExists(t *testing.T) {
dir := srrdbTestDir(t)
present := filepath.Join(dir, "present.json")
if err := os.WriteFile(present, []byte("{}"), 0o644); err != nil {
t.Fatalf("write %q: %v", present, err)
}
exists, err := srrdbExists(present)
if err != nil {
t.Fatalf("srrdbExists(%q) returned an error: %v", present, err)
}
if !exists {
t.Errorf("srrdbExists(%q) = false, want true", present)
}
exists, err = srrdbExists(filepath.Join(dir, "absent.json"))
if err != nil {
t.Fatalf("srrdbExists on a missing file returned an error: %v", err)
}
if exists {
t.Error("srrdbExists on a missing file = true, want false")
}
}
+77
View File
@@ -0,0 +1,77 @@
package repository
import (
"encoding/json"
"log/slog"
"os"
"strings"
"time"
"github.com/spf13/viper"
)
// srrdbAppendJSONL adds one object as a single line to a .jsonl log. A missing
// or failing log never changes the outcome of the run itself.
func srrdbAppendJSONL(key string, record map[string]any) {
path := viper.GetString(key)
if path == "" {
return
}
data, err := json.Marshal(record)
if err != nil {
slog.Error("encode srrdb log record", "key", key, "err", err)
return
}
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
if err != nil {
slog.Error("open srrdb log", "path", path, "err", err)
return
}
defer func() {
if cerr := file.Close(); cerr != nil {
slog.Error("close srrdb log", "path", path, "err", cerr)
}
}()
if _, err := file.Write(append(data, '\n')); err != nil {
slog.Error("write srrdb log", "path", path, "err", err)
}
}
// srrdbLogStatus appends the outcome of one finished run to srrdb.log.
func srrdbLogStatus(uri string, stored, reported int) {
srrdbAppendJSONL("srrdb.log", map[string]any{
"time": time.Now().Format(srrdbTimestamp),
"uri": uri,
"stored": stored,
"reported": reported,
})
}
// srrdbLogWarning records warnings reported by the API. They share the failure
// file but carry their own kind, because the endpoint answers an unrecognized
// keyword with HTTP 200 and an empty result set: the text is the only trace of a
// bad path, and it must stay distinguishable from a real breakdown.
func srrdbLogWarning(uri string, warnings []string) {
srrdbAppendJSONL("srrdb.error_log", map[string]any{
"kind": "warning",
"time": time.Now().Format(srrdbTimestamp),
"uri": uri,
"warnings": strings.Join(warnings, "; "),
})
}
// srrdbLogError appends one failure to srrdb.error_log.
func srrdbLogError(uri string, err error) {
srrdbAppendJSONL("srrdb.error_log", map[string]any{
"kind": "error",
"time": time.Now().Format(srrdbTimestamp),
"uri": uri,
"error": err.Error(),
})
}
+61
View File
@@ -0,0 +1,61 @@
package repository
import (
"regexp"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
// srrdbGroupSegment cuts the group off the tail of a release line, which always
// sits last. Compiled once at package level instead of per record.
//
// The pattern takes the trailing word, so a group whose own name contains a
// hyphen or a dot would lose everything before that character. Nothing in the
// line tells such a name apart from the tokens in front of it, so widening the
// character class without a list of real group names would only start eating the
// title.
var srrdbGroupSegment = regexp.MustCompile(`-(\w+)$`)
// srrdbCategory reads the value of the category: segment of a search path, which
// is guaranteed to be present by the time this is called.
func srrdbCategory(uri string) string {
_, rest, ok := strings.Cut(uri, "category:")
if !ok {
return ""
}
value, _, _ := strings.Cut(rest, "/")
return value
}
// srrdbRow maps one API record onto a table row. status, nfo, content and
// synced_at stay nil and are stored as NULL.
func srrdbRow(result resource.SrrdbResult, run, category string) model.SrrdbRelease {
return model.SrrdbRelease{
Release: result.Release,
Date: result.Date,
HasNfo: result.HasNFO,
HasSrs: result.HasSRS,
IsForeign: result.IsForeign,
ImdbID: result.ImdbID,
Size: result.Size,
GroupName: srrdbGroupName(result.Release),
Category: category,
CreatedAt: run,
}
}
// srrdbGroupName returns the trailing group of a release line, or an empty
// string when the tail is not a plain word. The nil check is deliberate:
// FindStringSubmatch returns nil on no match, and indexing it directly panics.
func srrdbGroupName(release string) string {
match := srrdbGroupSegment.FindStringSubmatch(release)
if match == nil {
return ""
}
return match[1]
}
+269
View File
@@ -0,0 +1,269 @@
package repository
import (
"encoding/json"
"testing"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
func TestSrrdbNormalizedPath(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "full search path",
uri: "category:flac/date:2026-09-28/page:1",
want: "category:flac∕date:2026-09-28∕page:1.json",
},
{
name: "single segment",
uri: "category:flac",
want: "category:flac.json",
},
{
name: "empty path",
uri: "",
want: ".json",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbNormalizedPath(test.uri); got != test.want {
t.Errorf("srrdbNormalizedPath(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
func TestSrrdbGroupName(t *testing.T) {
tests := []struct {
name string
release string
want string
}{
{
name: "music release",
release: "A-Reis-Aetherial_Flow-16BIT-WEB-FLAC-2025-NeonDreams",
want: "NeonDreams",
},
{
name: "group containing digits",
release: "Old_Man-Grim_Tales-16BIT-WEB-FLAC-2020-86D",
want: "86D",
},
{
name: "two tokens only",
release: "Artist-Title",
want: "Title",
},
{
// The pattern takes the trailing word, so a group that has a hyphen
// in its own name loses everything before it. Pinned here on purpose:
// it is the known limitation, and this test will fail the moment the
// pattern is improved.
name: "hyphen inside the group",
release: "Artist-Title-16BIT-WEB-FLAC-2024-MOON-BLOOD",
want: "BLOOD",
},
{
name: "trailing hyphen",
release: "Artist-Title-FLAC-2024-",
want: "",
},
{
name: "empty line",
release: "",
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbGroupName(test.release); got != test.want {
t.Errorf("srrdbGroupName(%q) = %q, want %q", test.release, got, test.want)
}
})
}
}
func TestSrrdbCategory(t *testing.T) {
tests := []struct {
name string
uri string
want string
}{
{
name: "flac with further segments",
uri: "category:flac/date:2026-09-28/page:1",
want: "flac",
},
{
name: "category alone",
uri: "category:mp3",
want: "mp3",
},
{
name: "no category segment",
uri: "date:2026-09-28/page:1",
want: "",
},
{
name: "empty path",
uri: "",
want: "",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbCategory(test.uri); got != test.want {
t.Errorf("srrdbCategory(%q) = %q, want %q", test.uri, got, test.want)
}
})
}
}
func TestSrrdbStartPage(t *testing.T) {
tests := []struct {
name string
uri string
wantPage int
wantPaginated bool
}{
{
name: "first page",
uri: "category:flac/date:2026-09-28/page:1",
wantPage: 1,
wantPaginated: true,
},
{
name: "later page",
uri: "category:flac/date:2026-09-28/page:7",
wantPage: 7,
wantPaginated: true,
},
{
name: "no page segment",
uri: "category:flac/date:2026-09-28",
wantPage: 1,
wantPaginated: false,
},
{
name: "zero page falls back to one",
uri: "category:flac/date:2026-09-28/page:0",
wantPage: 1,
wantPaginated: true,
},
{
name: "non numeric page is not a page segment",
uri: "category:flac/date:2026-09-28/page:abc",
wantPage: 1,
wantPaginated: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
page, paginated := srrdbStartPage(test.uri)
if page != test.wantPage || paginated != test.wantPaginated {
t.Errorf("srrdbStartPage(%q) = %d, %t, want %d, %t",
test.uri, page, paginated, test.wantPage, test.wantPaginated)
}
})
}
}
func TestSrrdbWithPage(t *testing.T) {
tests := []struct {
name string
uri string
page int
paginated bool
want string
}{
{
name: "swaps the page segment",
uri: "category:flac/date:2026-09-28/page:3",
page: 4,
paginated: true,
want: "category:flac/date:2026-09-28/page:4",
},
{
name: "single request keeps the path untouched",
uri: "category:flac/date:2026-09-28/page:3",
page: 4,
paginated: false,
want: "category:flac/date:2026-09-28/page:3",
},
{
name: "no page segment to swap",
uri: "category:flac/date:2026-09-28",
page: 2,
paginated: true,
want: "category:flac/date:2026-09-28",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if got := srrdbWithPage(test.uri, test.page, test.paginated); got != test.want {
t.Errorf("srrdbWithPage(%q, %d, %t) = %q, want %q",
test.uri, test.page, test.paginated, got, test.want)
}
})
}
}
func TestSrrdbCollectKeepsFirstPageEnvelope(t *testing.T) {
collected := &resource.SrrdbSearchResponse{}
srrdbCollect(collected, &resource.SrrdbSearchResponse{
ResultsCount: 155,
Query: json.RawMessage(`[21]`),
Results: []resource.SrrdbResult{
{Release: "First-Release-16BIT-WEB-FLAC-2026-GROUP"},
{Release: "Second-Release-16BIT-WEB-FLAC-2026-GROUP"},
},
Warnings: []string{"first warning"},
})
srrdbCollect(collected, &resource.SrrdbSearchResponse{
ResultsCount: 999,
Query: json.RawMessage(`[22]`),
Results: []resource.SrrdbResult{
{Release: "Third-Release-16BIT-WEB-FLAC-2026-GROUP"},
},
Warnings: []string{"second warning"},
})
if collected.ResultsCount != 155 {
t.Errorf("ResultsCount = %d, want 155: only the first page may set it", collected.ResultsCount)
}
if got := string(collected.Query); got != "[21]" {
t.Errorf("Query = %s, want [21]: only the first page may set it", got)
}
if len(collected.Results) != 3 {
t.Errorf("len(Results) = %d, want 3", len(collected.Results))
}
if len(collected.Warnings) != 2 {
t.Errorf("len(Warnings) = %d, want 2", len(collected.Warnings))
}
}
func TestSrrdbCollectZeroCountOnFirstPageIsNotSticky(t *testing.T) {
collected := &resource.SrrdbSearchResponse{}
srrdbCollect(collected, &resource.SrrdbSearchResponse{})
srrdbCollect(collected, &resource.SrrdbSearchResponse{ResultsCount: 42})
if collected.ResultsCount != 42 {
t.Errorf("ResultsCount = %d, want 42: a zero count must not freeze the envelope", collected.ResultsCount)
}
}
+59
View File
@@ -0,0 +1,59 @@
package table
import (
"fmt"
"github.com/jmoiron/sqlx"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
// srrdbUpsert keys on the "release" primary key: a repeated search refreshes the
// existing row instead of inserting a copy. synced_at is stamped with the same
// run timestamp as created_at, so the two differ only on a release that was
// already stored by an earlier run.
const srrdbUpsert = `
INSERT INTO srrdb_releases
("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at)
VALUES
(:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at)
ON CONFLICT("release") DO UPDATE SET
date = excluded.date,
has_nfo = excluded.has_nfo,
has_srs = excluded.has_srs,
is_foreign = excluded.is_foreign,
imdb_id = excluded.imdb_id,
size = excluded.size,
group_name = excluded.group_name,
category = excluded.category,
synced_at = excluded.created_at`
// UpsertAllSrrdb writes a whole set in one transaction: either every release
// lands or none does, so a failure halfway through cannot leave a day half
// harvested.
func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error {
if len(rows) == 0 {
return nil
}
tx, err := db.Beginx()
if err != nil {
return fmt.Errorf("begin srrdb transaction: %w", err)
}
for _, row := range rows {
if _, err := tx.NamedExec(srrdbUpsert, &row); err != nil {
// Rollback failure is not reported: the insert error is the one that
// matters to the caller, and the transaction is gone either way.
_ = tx.Rollback()
return fmt.Errorf("upsert srrdb release %q: %w", row.Release, err)
}
}
if err := tx.Commit(); err != nil {
return fmt.Errorf("commit srrdb transaction: %w", err)
}
return nil
}
+2
View File
@@ -10,6 +10,7 @@ type Service struct {
_interface.Prescene _interface.Prescene
_interface.Info _interface.Info
_interface.ShareTheBrutality _interface.ShareTheBrutality
_interface.Srrdb
} }
func New(repos *repository.Repository) *Service { func New(repos *repository.Repository) *Service {
@@ -18,5 +19,6 @@ func New(repos *repository.Repository) *Service {
Prescene: NewPresceneService(repos.Prescene), Prescene: NewPresceneService(repos.Prescene),
Info: NewInfoService(), Info: NewInfoService(),
ShareTheBrutality: NewShareTheBrutalityService(repos.ShareTheBrutality), ShareTheBrutality: NewShareTheBrutalityService(repos.ShareTheBrutality),
Srrdb: NewSrrdbService(repos.Srrdb),
} }
} }
+17
View File
@@ -0,0 +1,17 @@
package service
import (
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
)
type SrrdbService struct {
repo _interface.Srrdb
}
func NewSrrdbService(repo _interface.Srrdb) *SrrdbService {
return &SrrdbService{repo: repo}
}
func (s *SrrdbService) Search(uri string) (int, error) {
return s.repo.Search(uri)
}
+1
View File
@@ -6,6 +6,7 @@ const (
ScopeWeb string = "web" ScopeWeb string = "web"
ScopeInfo string = "info" ScopeInfo string = "info"
ScopeShareTheBrutality string = "stb" ScopeShareTheBrutality string = "stb"
ScopeSrrdb string = "srrdb"
) )
const ( const (
+2
View File
@@ -10,6 +10,8 @@ const (
FlagScopeEnable = "scope-enable" FlagScopeEnable = "scope-enable"
FlagSingleUri = "single-uri" FlagSingleUri = "single-uri"
FlagEnv = "env" FlagEnv = "env"
FlagCreate = "create"
FlagSleep = "sleep"
DefaultEnvProd = "prod" DefaultEnvProd = "prod"
DefaultEnvDev = "devel" DefaultEnvDev = "devel"
+4
View File
@@ -22,4 +22,8 @@ type MetalArchiveInterface interface {
GetSearchResults(searchResults []model.ExternalSources) ([]model.ExternalSources, error) GetSearchResults(searchResults []model.ExternalSources) ([]model.ExternalSources, error)
} }
type Srrdb interface {
Search(uri string) (int, error)
}
type Info interface{} type Info interface{}
+22
View File
@@ -0,0 +1,22 @@
package model
// SrrdbRelease is one row of the srrdb_releases table described by
// internal/db/srrdb.sql. Release is the primary key, therefore a repeated
// search rewrites the previous row of the same release instead of inserting
// a copy. Columns left nil are stored as NULL.
type SrrdbRelease struct {
Release string `db:"release"`
Date string `db:"date"`
HasNfo string `db:"has_nfo"`
HasSrs string `db:"has_srs"`
IsForeign string `db:"is_foreign"`
ImdbID *string `db:"imdb_id"`
Size int64 `db:"size"`
GroupName string `db:"group_name"`
Category string `db:"category"`
Status *string `db:"status"`
Nfo []byte `db:"nfo"`
Content *string `db:"content"`
CreatedAt string `db:"created_at"`
SyncedAt *string `db:"synced_at"`
}
+26
View File
@@ -0,0 +1,26 @@
package resource
import "encoding/json"
// SrrdbSearchResponse is the payload of GET https://api.srrdb.com/v1/search/{path}.
// Query stays raw because the API returns a heterogeneous array there: category
// ids as numbers, search terms as strings.
type SrrdbSearchResponse struct {
Results []SrrdbResult `json:"results"`
ResultsCount int `json:"resultsCount"`
Warnings []string `json:"warnings"`
Query json.RawMessage `json:"query"`
}
// SrrdbResult is a single release as reported by the search endpoint.
// HasNFO, HasSRS and IsForeign arrive as the literal strings "yes" and "no";
// ImdbID is null for music releases.
type SrrdbResult struct {
Release string `json:"release"`
Date string `json:"date"`
HasNFO string `json:"hasNFO"`
HasSRS string `json:"hasSRS"`
IsForeign string `json:"isForeign"`
ImdbID *string `json:"imdbId"`
Size int64 `json:"size"`
}