From 346369ca11592219322d9350efc96bcfc5d10b54 Mon Sep 17 00:00:00 2001 From: Yevhen Odynets <6lo5jp@proton.me> Date: Wed, 30 Sep 2026 09:27:33 +0300 Subject: [PATCH] srrdb: harvest the srrDB search API into a separate SQLite database New scope reading the API instead of scraping HTML: one search path per run, passed raw through --single-uri, and the answer stored as-is into its own SQLite file created from the embedded schema. Command line: --single-uri carries the path after /v1/search/ (category:flac/date:today/page:1) and must name a category; --sleep sets the pause between pages, with the config value as fallback. A day named on the command line is always fetched again and its dump rewritten, so cron can refresh today hourly and yesterday daily. Date placeholders: date:auto walks backwards to the newest day in [srrdb.floor..yesterday] that has no dump yet, closing holes instead of declaring the archive covered. date:today, date:yesterday and date:tomorrow stand for the neighbouring days. Today is never taken by the walk, because a day still being filled would be frozen by its own dump after the first run. Dumps are the cursor as well as the record, so a missing dump directory is an error instead of "nothing harvested yet": a renamed, unmounted or mistyped directory must not send the run silently back to the beginning. An empty day is a normal result and is remembered as harvested; API warnings never are, and they reach the exit code. Writes go through one transaction, so a day lands completely or not at all. group and category come from the release line and the search path; status, nfo, content and synced_at stay NULL. srrdb.log.jsonl records one line per finished run, srrdb.error_log.jsonl keeps failures and API warnings apart by a kind field. Also: the console command contract returns (string, error), so a failing scope reaches the process exit code and cron can tell a failed run from a successful one. config/scope/srrdb.yaml is untracked because its paths differ per host. modernc.org/sqlite replaces the archived libsql client, which only ever talked to a server. pkg/repository/srrdb.go is split by responsibility and covered by table-driven tests. --- .gitignore | 1 + cmd/main.go | 14 +- config/default.yaml | 2 +- go.mod | 9 + go.sum | 48 ++++++ internal/bootstrap.go | 39 ++++- internal/db/srrdb.go | 60 +++++++ internal/db/srrdb.sql | 39 +++++ internal/flag.go | 3 +- pkg/handler/console.go | 6 +- pkg/handler/handler.go | 28 ++-- pkg/handler/info.go | 5 +- pkg/handler/metal-archives.go | 4 +- pkg/handler/prescene.go | 15 +- pkg/handler/rutracker.go | 10 +- pkg/handler/srrdb.go | 22 +++ pkg/handler/stb.go | 8 +- pkg/repository/repository.go | 6 +- pkg/repository/srrdb.go | 140 ++++++++++++++++ pkg/repository/srrdb_api.go | 155 +++++++++++++++++ pkg/repository/srrdb_date.go | 189 +++++++++++++++++++++ pkg/repository/srrdb_date_test.go | 256 ++++++++++++++++++++++++++++ pkg/repository/srrdb_dump.go | 62 +++++++ pkg/repository/srrdb_dump_test.go | 45 +++++ pkg/repository/srrdb_log.go | 77 +++++++++ pkg/repository/srrdb_row.go | 61 +++++++ pkg/repository/srrdb_test.go | 269 ++++++++++++++++++++++++++++++ pkg/repository/table/srrdb.go | 59 +++++++ pkg/service/service.go | 2 + pkg/service/srrdb.go | 17 ++ types/constant/constants.go | 1 + types/constant/flag.go | 2 + types/interface/interfaces.go | 4 + types/model/srrdb.go | 22 +++ types/resource/srrdb.go | 26 +++ 35 files changed, 1653 insertions(+), 53 deletions(-) create mode 100644 internal/db/srrdb.go create mode 100644 internal/db/srrdb.sql create mode 100644 pkg/handler/srrdb.go create mode 100644 pkg/repository/srrdb.go create mode 100644 pkg/repository/srrdb_api.go create mode 100644 pkg/repository/srrdb_date.go create mode 100644 pkg/repository/srrdb_date_test.go create mode 100644 pkg/repository/srrdb_dump.go create mode 100644 pkg/repository/srrdb_dump_test.go create mode 100644 pkg/repository/srrdb_log.go create mode 100644 pkg/repository/srrdb_row.go create mode 100644 pkg/repository/srrdb_test.go create mode 100644 pkg/repository/table/srrdb.go create mode 100644 pkg/service/srrdb.go create mode 100644 types/model/srrdb.go create mode 100644 types/resource/srrdb.go diff --git a/.gitignore b/.gitignore index cc96208..4aafb34 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ fabric.properties tmp/ vendor/ config/secret.yaml +config/scope/srrdb.yaml *.md mdb internal/releaseparser/groups/resources/* diff --git a/cmd/main.go b/cmd/main.go index 0484673..c0d6284 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -2,9 +2,11 @@ package main import ( "fmt" + "os" "time" _ "github.com/go-sql-driver/mysql" + _ "modernc.org/sqlite" scraper "git.kplus.net.ua/yevhen/resource-scraper/internal" "git.kplus.net.ua/yevhen/resource-scraper/internal/config" @@ -33,14 +35,10 @@ func init() { } func main() { - defer func() { - fmt.Printf("\nExecution took %v", time.Since(st)) - }() + code := scraper.Bootstrap() - //a, b := runtime.GOOS - //fmt.Printf("%v - - - \n", runtime.GOOS) + fmt.Printf("\nExecution took %v\n", time.Since(st)) - scraper.Bootstrap() - //fmt.Printf("%v\n", viper.Get("23").([]interface{})) - // Кінґстонська стіна + // os.Exit skips deferred calls, so the timing above is printed before it. + os.Exit(code) } diff --git a/config/default.yaml b/config/default.yaml index 946c6e2..accfa8f 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -1,5 +1,5 @@ scope: - allow: [info, web, rutracker, prescene, stb, metal-archives, console] + allow: [info, web, rutracker, prescene, stb, metal-archives, srrdb, console] default: console mail: kplus.net.ua: diff --git a/go.mod b/go.mod index 3684682..d165bcd 100644 --- a/go.mod +++ b/go.mod @@ -18,20 +18,26 @@ require ( github.com/spf13/viper v1.21.0 golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba golang.org/x/net v0.59.0 + modernc.org/sqlite v1.60.0 ) require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/andybalholm/cascadia v1.3.5 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emersion/go-message v0.18.2 // indirect github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect github.com/fsnotify/fsnotify v1.10.1 // indirect github.com/go-viper/mapstructure/v2 v2.5.0 // indirect github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect + github.com/google/uuid v1.6.0 // indirect github.com/icza/bitio v1.1.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 // indirect github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect github.com/pelletier/go-toml/v2 v2.4.3 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sagikazarmark/locafero v0.12.0 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/cast v1.10.0 // indirect @@ -39,4 +45,7 @@ require ( go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.42.0 // indirect + modernc.org/libc v1.77.1 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect ) diff --git a/go.sum b/go.sum index cc4afb5..b2420f4 100644 --- a/go.sum +++ b/go.sum @@ -7,6 +7,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dromara/carbon/v2 v2.6.17 h1:QL1LVcp5A/NQtIPbc1vrL9qIXxPNWnaDOBaFlmP1iDg= github.com/dromara/carbon/v2 v2.6.17/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug= github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48= github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg= @@ -32,6 +34,12 @@ github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7w github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= +github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI= github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0= @@ -48,6 +56,8 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA= github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/mewkiz/flac v1.0.14 h1:hyRGAM8NCKznoPmIi9zz2jyO+nfmxY2ErqBnHZ+gxh4= @@ -56,10 +66,14 @@ github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 h1:fpWNLIg7nX2f+D+6ZveQ github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87/go.mod h1:wohiqwTwXA5twjDES74xIJLGjAxo3/0pHSbTXLbSDW8= github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8= github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4= @@ -85,6 +99,8 @@ golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+ golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c= +golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= @@ -94,6 +110,8 @@ golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -116,6 +134,36 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU= +golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg= +modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE= +modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc= +modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs= +modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.60.0 h1:7AZh8lREDo8x3j7aSdF7KGpAKUkJExJ1p67tcRnmttM= +modernc.org/sqlite v1.60.0/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/internal/bootstrap.go b/internal/bootstrap.go index 6d258d7..52b1c84 100644 --- a/internal/bootstrap.go +++ b/internal/bootstrap.go @@ -2,8 +2,10 @@ package scraper import ( "fmt" + "log/slog" "github.com/iancoleman/strcase" + "github.com/jmoiron/sqlx" "github.com/logrusorgru/aurora/v4" "github.com/spf13/viper" "golang.org/x/exp/slices" @@ -26,22 +28,49 @@ func init() { strcase.ConfigureAcronym("stb", "STB") } -func Bootstrap() { +// Bootstrap runs the selected scope and returns the process exit code: zero on +// success, non-zero as soon as the scope reports a failure. +func Bootstrap() int { if !isAllowScope() { fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml")) - return + + return 2 } dbase := db.New() - repos := repository.New(dbase) + + // The SQLite handle is opened only for the srrdb scope, so that a missing + // or unreadable srrdb.sqlite does not break the other sources. + var srrdb *sqlx.DB + if viper.GetString(constant.FlagScopeEnable) == constant.ScopeSrrdb { + srrdb = db.NewSrrdb() + } + + repos := repository.New(dbase, srrdb) services := service.New(repos) handlers := handler.New(services) switch viper.GetString("role") { case constant.RoleConsole: - fmt.Printf("init console console: %s\n", handlers.InitConsole()) + result, err := handlers.InitConsole() + fmt.Printf("init console console: %s\n", result) + + if err != nil { + slog.Error("console command failed", "scope", viper.GetString(constant.FlagScopeEnable), "err", err) + + return 1 + } case constant.RoleWeb: - fmt.Printf("who: %s\n", handlers.InitRoutes()) + result, err := handlers.InitRoutes() + fmt.Printf("who: %s\n", result) ///http.Run() + + if err != nil { + slog.Error("web init failed", "err", err) + + return 1 + } } + + return 0 } diff --git a/internal/db/srrdb.go b/internal/db/srrdb.go new file mode 100644 index 0000000..0026b98 --- /dev/null +++ b/internal/db/srrdb.go @@ -0,0 +1,60 @@ +package db + +import ( + _ "embed" + "fmt" + "strings" + + "github.com/jmoiron/sqlx" + "github.com/spf13/viper" +) + +// srrdbSchema is the DDL of the srrdb database. Keeping it in a file lets the +// schema be read and edited without touching Go code; go:embed pulls it into the +// binary at build time. +// +//go:embed srrdb.sql +var srrdbSchema string + +// NewSrrdb opens the separate SQLite database that holds srrdb_releases. +// The driver name is read from db.ext.driver (sqlite, modernc.org/sqlite), so +// the driver can be replaced in configuration without touching this file. +func NewSrrdb() *sqlx.DB { + driver := viper.GetString("db.ext.driver") + path := viper.GetString("srrdb.database") + + db, err := sqlx.Connect(driver, path) + if err != nil { + panic("Failed to connect to the srrdb database: " + err.Error()) + } + + if err = db.Ping(); err != nil { + panic("Failed to ping the srrdb database: " + err.Error()) + } + + if err = srrdbCreate(db); err != nil { + panic("Failed to create the srrdb schema: " + err.Error()) + } + + return db +} + +// srrdbCreate applies the embedded DDL statement by statement. Every statement is +// written to be repeatable, so an existing database is left alone: this creates +// missing objects, it does not alter existing ones. +// +// Splitting on ";" is safe for this schema because no statement holds a semicolon +// inside a literal. +func srrdbCreate(db *sqlx.DB) error { + for _, statement := range strings.Split(srrdbSchema, ";") { + if statement = strings.TrimSpace(statement); statement == "" { + continue + } + + if _, err := db.Exec(statement); err != nil { + return fmt.Errorf("apply srrdb schema statement %q: %w", statement, err) + } + } + + return nil +} diff --git a/internal/db/srrdb.sql b/internal/db/srrdb.sql new file mode 100644 index 0000000..b265544 --- /dev/null +++ b/internal/db/srrdb.sql @@ -0,0 +1,39 @@ +CREATE TABLE IF NOT EXISTS srrdb_releases +( + "release" TEXT PRIMARY KEY NOT NULL, + date TEXT, + created_at TEXT NOT NULL, + synced_at TEXT, + category TEXT NOT NULL DEFAULT '', + status TEXT, + group_name TEXT, + has_nfo TEXT, + has_srs TEXT, + is_foreign TEXT, + imdb_id TEXT, + size INTEGER, + comment TEXT, + nfo BLOB, + content TEXT, + source TEXT, + CHECK ( + status IN ( + 'on', 'hdd', 'hdd_e', 'catalogued', + 'deferred', 'ext', 'cloud', 'burnt', 'updated', + 'onhold', 'watch', 'hidden', 'duplicate', 'trash', + 'closed', 'request', '2cat', '2compare', 'neformat', + 'sf88', 'BATA', 'away', 'ready2burn', 'accurip', + 'replace', '2burn', 'remove', 'paid', 'free', 'ftp', + 'stream', 'ai' + ) + ) +); + +CREATE INDEX IF NOT EXISTS idx_srrdb_date + ON srrdb_releases (date); + +CREATE INDEX IF NOT EXISTS idx_srrdb_group + ON srrdb_releases (group_name); + +CREATE INDEX IF NOT EXISTS idx_srrdb_status + ON srrdb_releases (status); \ No newline at end of file diff --git a/internal/flag.go b/internal/flag.go index f51f889..7748425 100644 --- a/internal/flag.go +++ b/internal/flag.go @@ -42,7 +42,8 @@ func ParseFlags() { flag.String(constant.FlagScopeEnable, "", "") flag.String(constant.FlagSingleUri, "", "") flag.String(constant.FlagEnv, defaultConfigEnv, "") - flag.String("create", defaultConfigEnv, "used to create e.g. scope etc.") + flag.String(constant.FlagCreate, defaultConfigEnv, "used to create e.g. scope etc.") + flag.Duration(constant.FlagSleep, 0, "pause between requests, e.g. 2s") pflag.CommandLine.AddGoFlagSet(flag.CommandLine) pflag.Parse() diff --git a/pkg/handler/console.go b/pkg/handler/console.go index 5d40c3c..bdde231 100644 --- a/pkg/handler/console.go +++ b/pkg/handler/console.go @@ -8,7 +8,7 @@ import ( "github.com/spf13/viper" ) -func (h *Handler) Console() string { +func (h *Handler) Console() (string, error) { viper.SetDefault("env", "devel") cmd := viper.GetString("create") cmdList := viper.Sub("console.cmd") @@ -16,11 +16,11 @@ func (h *Handler) Console() string { if !slices.Contains(allowCreate, cmd) { fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd)) - return "" + return "", fmt.Errorf("command %q is not allowed", cmd) } fmt.Printf("%v\n", slices.Contains(allowCreate, cmd)) - return viper.GetString("env") + return viper.GetString("env"), nil } diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go index 6d33264..c8c951a 100644 --- a/pkg/handler/handler.go +++ b/pkg/handler/handler.go @@ -1,7 +1,7 @@ package handler import ( - "log" // Додано імпорт log + "fmt" "github.com/spf13/viper" @@ -10,7 +10,9 @@ import ( ) // consoleCommandHandler визначає тип для функцій-обробників консольних команд. -type consoleCommandHandler func() string +// The error it returns decides the process exit code, so cron can tell a failed +// run from a successful one. +type consoleCommandHandler func() (string, error) type Handler struct { services *service.Service @@ -29,24 +31,30 @@ func New(services *service.Service) *Handler { "prescene": h.Prescene, "rutracker": h.Rutracker, "stb": h.STB, + "srrdb": h.Srrdb, //"web": h.Web, } return h } -func (h *Handler) InitConsole() string { +// InitConsole runs the command selected by --scope-enable. The text is returned +// even together with an error, so the caller can print what happened before it +// reports the failure. +func (h *Handler) InitConsole() (string, error) { commandName := viper.GetString(constant.FlagScopeEnable) - if cmdFunc, ok := h.consoleCommands[commandName]; ok { - return commandName + " launched, " + cmdFunc() + "\n" - } else { - log.Printf("Error: unknown console command '%s'", commandName) - return "Error: unknown console command" + cmdFunc, ok := h.consoleCommands[commandName] + if !ok { + return "", fmt.Errorf("unknown console command %q", commandName) } + + result, err := cmdFunc() + + return commandName + " launched, " + result + "\n", err } -func (h *Handler) InitRoutes() string { +func (h *Handler) InitRoutes() (string, error) { //TODO: - return "i am the web initiator" + return "i am the web initiator", nil } diff --git a/pkg/handler/info.go b/pkg/handler/info.go index 25747ec..b329693 100644 --- a/pkg/handler/info.go +++ b/pkg/handler/info.go @@ -12,13 +12,14 @@ import ( "git.kplus.net.ua/yevhen/resource-scraper/types/constant" ) -func (h *Handler) Info() string { +func (h *Handler) Info() (string, error) { md5() fmt.Printf("%s: %s; %s: %s\n", aurora.Cyan("ENV"), viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable)) - return "info" + + return "info", nil } func md5() { diff --git a/pkg/handler/metal-archives.go b/pkg/handler/metal-archives.go index 3f49316..5b79eaa 100644 --- a/pkg/handler/metal-archives.go +++ b/pkg/handler/metal-archives.go @@ -1,5 +1,5 @@ package handler -func (h *Handler) MetalArchives() string { - return "MetalArchives................." +func (h *Handler) MetalArchives() (string, error) { + return "MetalArchives.................", nil } diff --git a/pkg/handler/prescene.go b/pkg/handler/prescene.go index 63ad139..b544375 100644 --- a/pkg/handler/prescene.go +++ b/pkg/handler/prescene.go @@ -2,7 +2,6 @@ package handler import ( "fmt" - "log/slog" "math" "strconv" "time" @@ -12,7 +11,7 @@ import ( "git.kplus.net.ua/yevhen/resource-scraper/types/constant" ) -func (h *Handler) Prescene() string { +func (h *Handler) Prescene() (string, error) { currMinute := time.Now().Minute() levels := viper.GetInt(constant.CfgKeyLevelsToScrape) initPage := strconv.Itoa(currMinute/(60/levels) + 1) @@ -30,17 +29,15 @@ func (h *Handler) Prescene() string { pagesToScrape = append(pagesToScrape, strconv.Itoa(page)) } - _, err := h.services.Prescene.GetSrrDb() - if err != nil { - slog.Error("error occurred while getting page: ", "err", err.Error()) + if _, err := h.services.Prescene.GetSrrDb(); err != nil { + return "ps", fmt.Errorf("getting srr db: %w", err) } - _, err = h.services.Prescene.GetPage(pagesToScrape) - if err != nil { - slog.Error("error occurred while getting page: ", "err", err.Error()) + if _, err := h.services.Prescene.GetPage(pagesToScrape); err != nil { + return "ps", fmt.Errorf("getting page: %w", err) } fmt.Printf("%+v\n", pagesToScrape) - return "ps" + return "ps", nil } diff --git a/pkg/handler/rutracker.go b/pkg/handler/rutracker.go index c7d746f..7a69315 100644 --- a/pkg/handler/rutracker.go +++ b/pkg/handler/rutracker.go @@ -2,20 +2,18 @@ package handler import ( "fmt" - "log/slog" "time" "github.com/spf13/viper" ) -func (h *Handler) Rutracker() string { +func (h *Handler) Rutracker() (string, error) { key := fmt.Sprintf("topic.%v", time.Now().Hour()) topics := viper.GetStringSlice(key) - _, err := h.services.Rutracker.GetTopic(topics) - if err != nil { - slog.Error("error occurred while getting topic: ", "err", err.Error()) + if _, err := h.services.Rutracker.GetTopic(topics); err != nil { + return "rt", fmt.Errorf("getting topic: %w", err) } - return "rt" + return "rt", nil } diff --git a/pkg/handler/srrdb.go b/pkg/handler/srrdb.go new file mode 100644 index 0000000..dcf9508 --- /dev/null +++ b/pkg/handler/srrdb.go @@ -0,0 +1,22 @@ +package handler + +import ( + "errors" + "fmt" + + "github.com/spf13/viper" + + "git.kplus.net.ua/yevhen/resource-scraper/types/constant" +) + +// Srrdb runs one srrDB search pass over the raw path given by --single-uri. +func (h *Handler) Srrdb() (string, error) { + uri := viper.GetString(constant.FlagSingleUri) + if uri == "" { + return "srrdb", errors.New("no search path, pass --single-uri") + } + + stored, err := h.services.Srrdb.Search(uri) + + return fmt.Sprintf("%d records\n", stored), err +} diff --git a/pkg/handler/stb.go b/pkg/handler/stb.go index 9eff239..b834be6 100644 --- a/pkg/handler/stb.go +++ b/pkg/handler/stb.go @@ -3,20 +3,18 @@ package handler import ( "errors" "fmt" - "log/slog" "github.com/spf13/viper" "git.kplus.net.ua/yevhen/resource-scraper/types/constant" ) -func (h *Handler) STB() string { +func (h *Handler) STB() (string, error) { endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint) endpoint = viper.GetString(endpoint) if endpoint == "" { - slog.Error("getting endpoint from config", "err", errors.New("no endpoint provided")) - return "stb" + return "stb", errors.New("no endpoint provided") } es, ms := h.services.ShareTheBrutality.GetMail(endpoint) @@ -32,5 +30,5 @@ func (h *Handler) STB() string { slog.Error("error occurred while getting topic: ", "err", err) }*/ - return fmt.Sprintf("Added %d records\n", len(es)) + return fmt.Sprintf("Added %d records\n", len(es)), nil } diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index 6cf81e9..39bc886 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -10,12 +10,16 @@ type Repository struct { _interface.Rutracker _interface.Prescene _interface.ShareTheBrutality + _interface.Srrdb } -func New(db *sqlx.DB) *Repository { +// New wires the collectors. sqlite serves the srrdb scope only and may be nil +// for every other source; it is connected lazily in Bootstrap. +func New(db, sqlite *sqlx.DB) *Repository { return &Repository{ Rutracker: NewRutrackerRepository(db), Prescene: NewPresceneRepository(db), ShareTheBrutality: NewShareTheBrutalityRepository(db), + Srrdb: NewSrrdbRepository(sqlite), } } diff --git a/pkg/repository/srrdb.go b/pkg/repository/srrdb.go new file mode 100644 index 0000000..554a10d --- /dev/null +++ b/pkg/repository/srrdb.go @@ -0,0 +1,140 @@ +package repository + +import ( + "errors" + "fmt" + "log/slog" + "strings" + "time" + + "github.com/jmoiron/sqlx" + + "git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table" + "git.kplus.net.ua/yevhen/resource-scraper/types/model" +) + +// srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the +// API itself uses in its date field, because the column is TEXT. +const srrdbTimestamp = "2006-01-02 15:04:05" + +// Srrdb harvests the srrDB search API into the separate SQLite database. +type Srrdb struct { + db *sqlx.DB +} + +// NewSrrdbRepository returns a collector writing into the SQLite handle. The +// handle is nil for every scope except srrdb, because Bootstrap connects the +// SQLite database lazily, so the collector refuses to run without it instead of +// failing later on a nil handle. +func NewSrrdbRepository(db *sqlx.DB) *Srrdb { + return &Srrdb{db: db} +} + +// Search harvests one search path. Nothing is written to the database until the +// whole set is collected and its size matches what the API reported, so the +// dump and the success log can only ever describe a complete day. +func (s *Srrdb) Search(rawURI string) (int, error) { + if s.db == nil { + err := errors.New("srrdb collector has no database handle") + srrdbLogError(rawURI, err) + + return 0, err + } + + if !strings.Contains(rawURI, "category:") { + err := fmt.Errorf("search path %q has no category: segment", rawURI) + srrdbLogError(rawURI, err) + + return 0, err + } + + auto := strings.Contains(rawURI, srrdbAutoSegment) + + now := time.Now() + + uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now) + + switch { + case errors.Is(err, errSrrdbFloor): + slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI) + + return 0, nil + case err != nil: + srrdbLogError(rawURI, err) + + return 0, err + } + + // A slash typed on either end of the command line would build a double + // separator in the URL and an odd dump file name. + uri = strings.Trim(uri, "/") + + // The guard applies to the automatic walk only. A day named on the command + // line is fetched again and its dump rewritten every time: a day can still + // gain releases later, either from another time zone or from a late upload, + // and cron refreshes today and yesterday on purpose. + dump := srrdbDumpPath(uri) + + if auto { + exists, err := srrdbExists(dump) + if err != nil { + srrdbLogError(uri, err) + + return 0, err + } + + if exists { + slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump) + + return 0, nil + } + } + + collected, err := srrdbCollectAll(uri) + if err != nil { + srrdbLogError(uri, err) + + return 0, err + } + + // An empty day is a normal outcome: zero results and no warnings. The dump + // records it as harvested, which keeps the walk moving. Warnings mean the + // path was not understood, and that must never be remembered as done. + switch { + case len(collected.Warnings) > 0: + srrdbLogWarning(uri, collected.Warnings) + + return 0, fmt.Errorf("srrdb %q reported warnings: %s", + uri, strings.Join(collected.Warnings, "; ")) + case len(collected.Results) != collected.ResultsCount: + err := fmt.Errorf("srrdb %q pulled %d of %d reported releases", + uri, len(collected.Results), collected.ResultsCount) + srrdbLogError(uri, err) + + return 0, err + } + + run := time.Now().Format(srrdbTimestamp) + category := srrdbCategory(uri) + rows := make([]model.SrrdbRelease, 0, len(collected.Results)) + + for _, result := range collected.Results { + rows = append(rows, srrdbRow(result, run, category)) + } + + if err := table.UpsertAllSrrdb(s.db, rows); err != nil { + srrdbLogError(uri, err) + + return 0, err + } + + if err := srrdbWriteDump(dump, collected); err != nil { + srrdbLogError(uri, err) + + return len(rows), err + } + + srrdbLogStatus(uri, len(rows), collected.ResultsCount) + + return len(rows), nil +} diff --git a/pkg/repository/srrdb_api.go b/pkg/repository/srrdb_api.go new file mode 100644 index 0000000..e51ce83 --- /dev/null +++ b/pkg/repository/srrdb_api.go @@ -0,0 +1,155 @@ +package repository + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/spf13/viper" + + "git.kplus.net.ua/yevhen/resource-scraper/types/constant" + "git.kplus.net.ua/yevhen/resource-scraper/types/resource" +) + +// srrdbPageSegment locates the page:N segment so a run can continue from N into +// the following pages. +var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`) + +// srrdbCollectAll walks every page before anything is written, so the caller +// only ever receives a complete set or an error. +func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) { + endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/") + + // The flag binds to its own key and wins over the config value; an unused + // flag yields zero. + sleep := viper.GetDuration(constant.FlagSleep) + if sleep <= 0 { + sleep = viper.GetDuration("srrdb.sleep") + } + if sleep <= 0 { + sleep = 2 * time.Second + } + + slog.Info("srrdb pacing", "uri", uri, "sleep", sleep) + + page, paginated := srrdbStartPage(uri) + collected := &resource.SrrdbSearchResponse{} + + for { + url := endpoint + "/" + srrdbWithPage(uri, page, paginated) + + response, err := srrdbFetch(url) + if err != nil { + return nil, err + } + + srrdbCollect(collected, response) + + slog.Info("srrdb page fetched", + "url", url, + "page", page, + "fetched", len(response.Results), + "reported", response.ResultsCount) + + if !paginated || len(response.Results) == 0 { + break + } + + // Everything the API reported is already collected: one more page would + // cost a request and a pause to receive an empty answer. + if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount { + break + } + + page++ + + time.Sleep(sleep) + } + + return collected, nil +} + +// srrdbCollect merges one page into the answer kept for the dump: results are +// appended, while resultsCount and query come from the first page. +func srrdbCollect(all, page *resource.SrrdbSearchResponse) { + if all.ResultsCount == 0 { + all.ResultsCount = page.ResultsCount + } + + if len(all.Query) == 0 { + all.Query = page.Query + } + + all.Results = append(all.Results, page.Results...) + all.Warnings = append(all.Warnings, page.Warnings...) +} + +// srrdbStartPage reads the page:N segment. Without it the path is a single +// request and no further pages are attempted. +func srrdbStartPage(uri string) (int, bool) { + match := srrdbPageSegment.FindStringSubmatch(uri) + if match == nil { + return 1, false + } + + page, err := strconv.Atoi(match[1]) + if err != nil || page < 1 { + return 1, true + } + + return page, true +} + +// srrdbWithPage swaps the page:N segment for the page requested now. +func srrdbWithPage(uri string, page int, paginated bool) string { + if !paginated { + return uri + } + + return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page)) +} + +// srrdbFetch performs one search request and decodes the JSON answer. +func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) { + client := &http.Client{Timeout: 30 * time.Second} + + request, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, fmt.Errorf("build srrdb request %q: %w", url, err) + } + + request.Header.Set("Accept", "application/json") + + response, err := client.Do(request) + if err != nil { + return nil, fmt.Errorf("request srrdb %q: %w", url, err) + } + + defer func() { + if cerr := response.Body.Close(); cerr != nil { + slog.Error("closing srrdb response body", "url", url, "err", cerr) + } + }() + + if response.StatusCode != http.StatusOK { + return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status) + } + + body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) + if err != nil { + return nil, fmt.Errorf("read srrdb body %q: %w", url, err) + } + + decoded := &resource.SrrdbSearchResponse{} + if err := json.Unmarshal(body, decoded); err != nil { + return nil, fmt.Errorf("decode srrdb body %q: %w", url, err) + } + + return decoded, nil +} diff --git a/pkg/repository/srrdb_date.go b/pkg/repository/srrdb_date.go new file mode 100644 index 0000000..721be48 --- /dev/null +++ b/pkg/repository/srrdb_date.go @@ -0,0 +1,189 @@ +package repository + +import ( + "errors" + "fmt" + "os" + "regexp" + "strings" + "time" + + "github.com/spf13/viper" +) + +// srrdbAutoSegment is the placeholder replaced by the date the run has to +// harvest, so cron can repeat the same command every day. +const srrdbAutoSegment = "date:auto" + +// srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for +// the neighbouring days, so cron can name a day without hard-coding a date. +const ( + srrdbYesterdaySegment = "date:yesterday" + srrdbTodaySegment = "date:today" + srrdbTomorrowSegment = "date:tomorrow" +) + +// srrdbDayLayout is the layout of a date segment in a search path. +const srrdbDayLayout = "2006-01-02" + +// errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is +// the archive being fully covered rather than a failure. +var errSrrdbFloor = errors.New("srrdb.floor reached") + +// srrdbResolveAutoDate replaces date:auto with the newest day in +// [srrdb.floor..yesterday] that has not been harvested yet. +// +// Scanning from the top-down means a new day is picked up the moment it appears, +// and a hole left in the middle is closed once the walk reaches it. The previous +// rule (the day before the earliest dump) treated the harvested days as one +// uninterrupted run, so a single dump below the floor made it declare the whole +// archive covered while everything above stayed missing. +// +// The current moment is passed in rather than read here, so the walk can be tested +// without depending on the wall clock. +func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) { + if !strings.Contains(rawURI, srrdbAutoSegment) { + return rawURI, nil + } + + floor, err := srrdbFloorDate() + if err != nil { + return "", err + } + + harvested, err := srrdbDumpedDays(rawURI) + if err != nil { + return "", err + } + + done := make(map[string]bool, len(harvested)) + for _, day := range harvested { + done[day.Format(srrdbDayLayout)] = true + } + + // The current day is never taken: it is still being filled, so it would have + // no dump and would be chosen by every single run, freezing the walk. cron + // refreshes today explicitly instead. + for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) { + if !done[day.Format(srrdbDayLayout)] { + return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil + } + } + + return "", errSrrdbFloor +} + +// srrdbDumpedDays lists the days already harvested for the same query, read back +// from the dump file names inside srrdb.dump_dir. +// +// A missing directory is an error rather than "nothing harvested yet": the dumps +// are the cursor of the walk, so a renamed, unmounted or mistyped directory would +// otherwise send the run silently back to the beginning. The directory is created +// once by hand, exactly like the database file. +func srrdbDumpedDays(uri string) ([]time.Time, error) { + pattern, err := srrdbDumpPattern(uri) + if err != nil { + return nil, err + } + + dir := viper.GetString("srrdb.dump_dir") + + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err) + } + + days := make([]time.Time, 0, len(entries)) + + for _, entry := range entries { + match := pattern.FindStringSubmatch(entry.Name()) + if match == nil { + continue + } + + day, err := time.Parse(srrdbDayLayout, match[1]) + if err != nil { + continue + } + + days = append(days, day) + } + + return days, nil +} + +// srrdbDumpPattern builds the regular expression matching the dump names of the +// same query, capturing the day segment so it can be read back. The literal part +// is quoted first, therefore the capture group is the only pattern syntax left in +// the result. +func srrdbDumpPattern(uri string) (*regexp.Regexp, error) { + const normalizedAuto = "date：auto" + + escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri)) + + head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto)) + if !ok { + return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri) + } + + return regexp.Compile("^" + head + `date：(\d{4}-\d{2}-\d{2})` + tail + "$") +} + +// srrdbFloorDate is the earliest day the automatic walk may ask for, kept in +// configuration because it is a property of the archive, not of the code. +// +// The value is read through viper.Get rather than viper.GetString: YAML decodes +// an unquoted date as a timestamp, and asking for its string form yields +// "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the +// quoted and the unquoted spelling are accepted here. +func srrdbFloorDate() (time.Time, error) { + switch raw := viper.Get("srrdb.floor").(type) { + case nil: + return time.Time{}, errors.New("srrdb.floor is not configured") + case time.Time: + return raw, nil + case string: + floor, err := time.Parse(srrdbDayLayout, raw) + if err != nil { + return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err) + } + + return floor, nil + default: + return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw) + } +} + +// srrdbReplaceDate swaps a date placeholder for a concrete day. +func srrdbReplaceDate(uri, segment string, day time.Time) string { + return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1) +} + +// srrdbRelativeDays holds the date placeholders that stand for a day counted +// from now. date:auto is deliberately absent: it means "the next day the walk +// has to cover", which only srrdbResolveAutoDate can decide. +var srrdbRelativeDays = []struct { + segment string + offset int +}{ + {segment: srrdbYesterdaySegment, offset: -1}, + {segment: srrdbTodaySegment, offset: 0}, + {segment: srrdbTomorrowSegment, offset: 1}, +} + +// srrdbResolveRelativeDays expands every relative date placeholder present in the +// path. The current moment is passed in for the same reason as in +// srrdbResolveAutoDate: a test has to be able to pin it. +func srrdbResolveRelativeDays(rawURI string, now time.Time) string { + uri := rawURI + + for _, relative := range srrdbRelativeDays { + if !strings.Contains(uri, relative.segment) { + continue + } + + uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset)) + } + + return uri +} diff --git a/pkg/repository/srrdb_date_test.go b/pkg/repository/srrdb_date_test.go new file mode 100644 index 0000000..6a3669c --- /dev/null +++ b/pkg/repository/srrdb_date_test.go @@ -0,0 +1,256 @@ +package repository + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/spf13/viper" +) + +// srrdbTestNow pins the moment the date logic is tested against, so no assertion +// depends on the wall clock. +var srrdbTestNow = time.Date(2026, 9, 30, 12, 0, 0, 0, time.UTC) + +// srrdbTestDir gives every test a clean viper and an empty dump directory, because +// the package reads srrdb.dump_dir from the global configuration instance. +func srrdbTestDir(t *testing.T) string { + t.Helper() + + viper.Reset() + + dir := t.TempDir() + viper.Set("srrdb.dump_dir", dir) + + return dir +} + +// srrdbTestFloor points srrdb.floor at the given day. +func srrdbTestFloor(t *testing.T, day time.Time) { + t.Helper() + + viper.Set("srrdb.floor", day.Format(srrdbDayLayout)) +} + +// srrdbTestDump writes a dump file for a query, which is how the walk learns that +// a day is done. +func srrdbTestDump(t *testing.T, uri string) { + t.Helper() + + path := srrdbDumpPath(uri) + if err := os.WriteFile(path, []byte("{}"), 0o644); err != nil { + t.Fatalf("write dump %q: %v", path, err) + } +} + +func TestSrrdbFloorDateFromString(t *testing.T) { + srrdbTestDir(t) + viper.Set("srrdb.floor", "2013-03-16") + + floor, err := srrdbFloorDate() + if err != nil { + t.Fatalf("srrdbFloorDate: %v", err) + } + + want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC) + if !floor.Equal(want) { + t.Errorf("srrdbFloorDate = %s, want %s", floor, want) + } +} + +// The regression this test exists for: YAML decodes an unquoted date as a +// timestamp, so the value reaches the code as a time.Time and not as a string. It +// used to fail with "extra text: 00:00:00 +0000 UTC". +func TestSrrdbFloorDateFromTimestamp(t *testing.T) { + srrdbTestDir(t) + + want := time.Date(2013, 3, 16, 0, 0, 0, 0, time.UTC) + viper.Set("srrdb.floor", want) + + floor, err := srrdbFloorDate() + if err != nil { + t.Fatalf("srrdbFloorDate: %v", err) + } + + if !floor.Equal(want) { + t.Errorf("srrdbFloorDate = %s, want %s", floor, want) + } +} + +func TestSrrdbFloorDateNotConfigured(t *testing.T) { + srrdbTestDir(t) + + if _, err := srrdbFloorDate(); err == nil { + t.Error("srrdbFloorDate without srrdb.floor returned no error") + } +} + +func TestSrrdbDumpPattern(t *testing.T) { + pattern, err := srrdbDumpPattern("category:flac/date:auto/page:1") + if err != nil { + t.Fatalf("srrdbDumpPattern: %v", err) + } + + match := pattern.FindStringSubmatch(srrdbNormalizedPath("category:flac/date:2026-09-28/page:1")) + if match == nil { + t.Fatal("the pattern did not match the dump name of the same query") + } + + if match[1] != "2026-09-28" { + t.Errorf("captured day = %q, want 2026-09-28", match[1]) + } +} + +func TestSrrdbDumpPatternNeedsAutoSegment(t *testing.T) { + if _, err := srrdbDumpPattern("category:flac/date:2026-09-28/page:1"); err == nil { + t.Error("srrdbDumpPattern without date:auto returned no error") + } +} + +func TestSrrdbDumpedDaysIgnoresForeignQueries(t *testing.T) { + srrdbTestDir(t) + + srrdbTestDump(t, "category:flac/date:2026-09-28/page:1") + srrdbTestDump(t, "category:flac/date:2026-09-27/page:1") + srrdbTestDump(t, "category:mp3/date:2026-09-26/page:1") + + days, err := srrdbDumpedDays("category:flac/date:auto/page:1") + if err != nil { + t.Fatalf("srrdbDumpedDays: %v", err) + } + + if len(days) != 2 { + t.Fatalf("len(days) = %d, want 2: only dumps of the same query may count", len(days)) + } +} + +// An absent directory must not read as "nothing harvested yet": the dumps are the +// cursor of the walk, so a renamed or unmounted directory would otherwise send the +// run silently back to the beginning. +func TestSrrdbDumpedDaysMissingDirectory(t *testing.T) { + viper.Reset() + viper.Set("srrdb.dump_dir", filepath.Join(t.TempDir(), "absent")) + + if _, err := srrdbDumpedDays("category:flac/date:auto/page:1"); err == nil { + t.Error("srrdbDumpedDays on an absent directory returned no error") + } +} + +func TestSrrdbResolveAutoDateStartsFromYesterday(t *testing.T) { + srrdbTestDir(t) + srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10)) + + got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow) + if err != nil { + t.Fatalf("srrdbResolveAutoDate: %v", err) + } + + want := "category:flac/date:2026-09-29/page:1" + if got != want { + t.Errorf("srrdbResolveAutoDate = %q, want %q: the current day is still being filled", got, want) + } +} + +func TestSrrdbResolveAutoDatePicksTheNewestMissingDay(t *testing.T) { + srrdbTestDir(t) + srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10)) + + srrdbTestDump(t, "category:flac/date:2026-09-29/page:1") + srrdbTestDump(t, "category:flac/date:2026-09-28/page:1") + + got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow) + if err != nil { + t.Fatalf("srrdbResolveAutoDate: %v", err) + } + + want := "category:flac/date:2026-09-27/page:1" + if got != want { + t.Errorf("srrdbResolveAutoDate = %q, want %q", got, want) + } +} + +// A hole in the middle must be closed once the walk reaches it, not declared +// covered: this is the defect that made the previous cursor stop at the floor. +func TestSrrdbResolveAutoDateClosesHoles(t *testing.T) { + srrdbTestDir(t) + srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -10)) + + srrdbTestDump(t, "category:flac/date:2026-09-29/page:1") + srrdbTestDump(t, "category:flac/date:2026-09-27/page:1") + + got, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow) + if err != nil { + t.Fatalf("srrdbResolveAutoDate: %v", err) + } + + want := "category:flac/date:2026-09-28/page:1" + if got != want { + t.Errorf("srrdbResolveAutoDate = %q, want %q: the gap above the floor must be filled first", got, want) + } +} + +func TestSrrdbResolveAutoDateStopsAtFloor(t *testing.T) { + srrdbTestDir(t) + srrdbTestFloor(t, srrdbTestNow.AddDate(0, 0, -1)) + + srrdbTestDump(t, "category:flac/date:2026-09-29/page:1") + + _, err := srrdbResolveAutoDate("category:flac/date:auto/page:1", srrdbTestNow) + if !errors.Is(err, errSrrdbFloor) { + t.Errorf("srrdbResolveAutoDate = %v, want errSrrdbFloor", err) + } +} + +func TestSrrdbResolveAutoDateIgnoresConcreteDates(t *testing.T) { + srrdbTestDir(t) + + uri := "category:flac/date:2026-09-28/page:1" + + got, err := srrdbResolveAutoDate(uri, srrdbTestNow) + if err != nil { + t.Fatalf("srrdbResolveAutoDate: %v", err) + } + + if got != uri { + t.Errorf("srrdbResolveAutoDate = %q, want %q left untouched", got, uri) + } +} + +func TestSrrdbResolveRelativeDays(t *testing.T) { + tests := []struct { + name string + uri string + want string + }{ + { + name: "today", + uri: "category:flac/date:today/page:1", + want: "category:flac/date:2026-09-30/page:1", + }, + { + name: "yesterday", + uri: "category:flac/date:yesterday/page:1", + want: "category:flac/date:2026-09-29/page:1", + }, + { + name: "tomorrow", + uri: "category:flac/date:tomorrow/page:1", + want: "category:flac/date:2026-10-01/page:1", + }, + { + name: "auto belongs to the walk, not here", + uri: "category:flac/date:auto/page:1", + want: "category:flac/date:auto/page:1", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := srrdbResolveRelativeDays(test.uri, srrdbTestNow); got != test.want { + t.Errorf("srrdbResolveRelativeDays(%q) = %q, want %q", test.uri, got, test.want) + } + }) + } +} diff --git a/pkg/repository/srrdb_dump.go b/pkg/repository/srrdb_dump.go new file mode 100644 index 0000000..4a9b8b7 --- /dev/null +++ b/pkg/repository/srrdb_dump.go @@ -0,0 +1,62 @@ +package repository + +import ( + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/spf13/viper" + + "git.kplus.net.ua/yevhen/resource-scraper/types/resource" +) + +// srrdbNormalizedPath turns a search path into a file name. The two characters +// that carry structure in a path are swapped for look-alikes, so the name still +// reads as the query it came from and can be reversed character by character. +func srrdbNormalizedPath(uri string) string { + return strings.NewReplacer(":", "：", "/", "∕").Replace(uri) + ".json" +} + +// srrdbDumpPath is the guard file of a search path inside srrdb.dump_dir. +func srrdbDumpPath(uri string) string { + return filepath.Join(viper.GetString("srrdb.dump_dir"), srrdbNormalizedPath(uri)) +} + +// srrdbExists reports whether a dump file is present. Any stat failure other than +// "not found" is returned as an error: treating it as absent would let the run +// overwrite a dump it has no right to read. +func srrdbExists(path string) (bool, error) { + _, err := os.Stat(path) + + switch { + case err == nil: + return true, nil + case errors.Is(err, fs.ErrNotExist): + return false, nil + default: + return false, fmt.Errorf("stat srrdb dump %q: %w", path, err) + } +} + +// srrdbWriteDump stores the merged answer as indented JSON. It runs only after +// the database is updated, therefore the presence of the file means the whole +// run completed. +// +// The directory is never created here: it holds the cursor of the walk and is made +// once by hand, like the database file. +func srrdbWriteDump(path string, all *resource.SrrdbSearchResponse) error { + data, err := json.MarshalIndent(all, "", " ") + if err != nil { + return fmt.Errorf("encode srrdb dump %q: %w", path, err) + } + + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write srrdb dump %q: %w", path, err) + } + + return nil +} diff --git a/pkg/repository/srrdb_dump_test.go b/pkg/repository/srrdb_dump_test.go new file mode 100644 index 0000000..6199360 --- /dev/null +++ b/pkg/repository/srrdb_dump_test.go @@ -0,0 +1,45 @@ +package repository + +import ( + "os" + "path/filepath" + "testing" +) + +func TestSrrdbDumpPath(t *testing.T) { + dir := srrdbTestDir(t) + + got := srrdbDumpPath("category:flac/date:2026-09-28/page:1") + want := filepath.Join(dir, "category：flac∕date：2026-09-28∕page：1.json") + + if got != want { + t.Errorf("srrdbDumpPath = %q, want %q", got, want) + } +} + +func TestSrrdbExists(t *testing.T) { + dir := srrdbTestDir(t) + + present := filepath.Join(dir, "present.json") + if err := os.WriteFile(present, []byte("{}"), 0o644); err != nil { + t.Fatalf("write %q: %v", present, err) + } + + exists, err := srrdbExists(present) + if err != nil { + t.Fatalf("srrdbExists(%q) returned an error: %v", present, err) + } + + if !exists { + t.Errorf("srrdbExists(%q) = false, want true", present) + } + + exists, err = srrdbExists(filepath.Join(dir, "absent.json")) + if err != nil { + t.Fatalf("srrdbExists on a missing file returned an error: %v", err) + } + + if exists { + t.Error("srrdbExists on a missing file = true, want false") + } +} diff --git a/pkg/repository/srrdb_log.go b/pkg/repository/srrdb_log.go new file mode 100644 index 0000000..0e026d1 --- /dev/null +++ b/pkg/repository/srrdb_log.go @@ -0,0 +1,77 @@ +package repository + +import ( + "encoding/json" + "log/slog" + "os" + "strings" + "time" + + "github.com/spf13/viper" +) + +// srrdbAppendJSONL adds one object as a single line to a .jsonl log. A missing +// or failing log never changes the outcome of the run itself. +func srrdbAppendJSONL(key string, record map[string]any) { + path := viper.GetString(key) + if path == "" { + return + } + + data, err := json.Marshal(record) + if err != nil { + slog.Error("encode srrdb log record", "key", key, "err", err) + + return + } + + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + slog.Error("open srrdb log", "path", path, "err", err) + + return + } + + defer func() { + if cerr := file.Close(); cerr != nil { + slog.Error("close srrdb log", "path", path, "err", cerr) + } + }() + + if _, err := file.Write(append(data, '\n')); err != nil { + slog.Error("write srrdb log", "path", path, "err", err) + } +} + +// srrdbLogStatus appends the outcome of one finished run to srrdb.log. +func srrdbLogStatus(uri string, stored, reported int) { + srrdbAppendJSONL("srrdb.log", map[string]any{ + "time": time.Now().Format(srrdbTimestamp), + "uri": uri, + "stored": stored, + "reported": reported, + }) +} + +// srrdbLogWarning records warnings reported by the API. They share the failure +// file but carry their own kind, because the endpoint answers an unrecognized +// keyword with HTTP 200 and an empty result set: the text is the only trace of a +// bad path, and it must stay distinguishable from a real breakdown. +func srrdbLogWarning(uri string, warnings []string) { + srrdbAppendJSONL("srrdb.error_log", map[string]any{ + "kind": "warning", + "time": time.Now().Format(srrdbTimestamp), + "uri": uri, + "warnings": strings.Join(warnings, "; "), + }) +} + +// srrdbLogError appends one failure to srrdb.error_log. +func srrdbLogError(uri string, err error) { + srrdbAppendJSONL("srrdb.error_log", map[string]any{ + "kind": "error", + "time": time.Now().Format(srrdbTimestamp), + "uri": uri, + "error": err.Error(), + }) +} diff --git a/pkg/repository/srrdb_row.go b/pkg/repository/srrdb_row.go new file mode 100644 index 0000000..45763cb --- /dev/null +++ b/pkg/repository/srrdb_row.go @@ -0,0 +1,61 @@ +package repository + +import ( + "regexp" + "strings" + + "git.kplus.net.ua/yevhen/resource-scraper/types/model" + "git.kplus.net.ua/yevhen/resource-scraper/types/resource" +) + +// srrdbGroupSegment cuts the group off the tail of a release line, which always +// sits last. Compiled once at package level instead of per record. +// +// The pattern takes the trailing word, so a group whose own name contains a +// hyphen or a dot would lose everything before that character. Nothing in the +// line tells such a name apart from the tokens in front of it, so widening the +// character class without a list of real group names would only start eating the +// title. +var srrdbGroupSegment = regexp.MustCompile(`-(\w+)$`) + +// srrdbCategory reads the value of the category: segment of a search path, which +// is guaranteed to be present by the time this is called. +func srrdbCategory(uri string) string { + _, rest, ok := strings.Cut(uri, "category:") + if !ok { + return "" + } + + value, _, _ := strings.Cut(rest, "/") + + return value +} + +// srrdbRow maps one API record onto a table row. status, nfo, content and +// synced_at stay nil and are stored as NULL. +func srrdbRow(result resource.SrrdbResult, run, category string) model.SrrdbRelease { + return model.SrrdbRelease{ + Release: result.Release, + Date: result.Date, + HasNfo: result.HasNFO, + HasSrs: result.HasSRS, + IsForeign: result.IsForeign, + ImdbID: result.ImdbID, + Size: result.Size, + GroupName: srrdbGroupName(result.Release), + Category: category, + CreatedAt: run, + } +} + +// srrdbGroupName returns the trailing group of a release line, or an empty +// string when the tail is not a plain word. The nil check is deliberate: +// FindStringSubmatch returns nil on no match, and indexing it directly panics. +func srrdbGroupName(release string) string { + match := srrdbGroupSegment.FindStringSubmatch(release) + if match == nil { + return "" + } + + return match[1] +} diff --git a/pkg/repository/srrdb_test.go b/pkg/repository/srrdb_test.go new file mode 100644 index 0000000..0c878b9 --- /dev/null +++ b/pkg/repository/srrdb_test.go @@ -0,0 +1,269 @@ +package repository + +import ( + "encoding/json" + "testing" + + "git.kplus.net.ua/yevhen/resource-scraper/types/resource" +) + +func TestSrrdbNormalizedPath(t *testing.T) { + tests := []struct { + name string + uri string + want string + }{ + { + name: "full search path", + uri: "category:flac/date:2026-09-28/page:1", + want: "category：flac∕date：2026-09-28∕page：1.json", + }, + { + name: "single segment", + uri: "category:flac", + want: "category：flac.json", + }, + { + name: "empty path", + uri: "", + want: ".json", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := srrdbNormalizedPath(test.uri); got != test.want { + t.Errorf("srrdbNormalizedPath(%q) = %q, want %q", test.uri, got, test.want) + } + }) + } +} + +func TestSrrdbGroupName(t *testing.T) { + tests := []struct { + name string + release string + want string + }{ + { + name: "music release", + release: "A-Reis-Aetherial_Flow-16BIT-WEB-FLAC-2025-NeonDreams", + want: "NeonDreams", + }, + { + name: "group containing digits", + release: "Old_Man-Grim_Tales-16BIT-WEB-FLAC-2020-86D", + want: "86D", + }, + { + name: "two tokens only", + release: "Artist-Title", + want: "Title", + }, + { + // The pattern takes the trailing word, so a group that has a hyphen + // in its own name loses everything before it. Pinned here on purpose: + // it is the known limitation, and this test will fail the moment the + // pattern is improved. + name: "hyphen inside the group", + release: "Artist-Title-16BIT-WEB-FLAC-2024-MOON-BLOOD", + want: "BLOOD", + }, + { + name: "trailing hyphen", + release: "Artist-Title-FLAC-2024-", + want: "", + }, + { + name: "empty line", + release: "", + want: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := srrdbGroupName(test.release); got != test.want { + t.Errorf("srrdbGroupName(%q) = %q, want %q", test.release, got, test.want) + } + }) + } +} + +func TestSrrdbCategory(t *testing.T) { + tests := []struct { + name string + uri string + want string + }{ + { + name: "flac with further segments", + uri: "category:flac/date:2026-09-28/page:1", + want: "flac", + }, + { + name: "category alone", + uri: "category:mp3", + want: "mp3", + }, + { + name: "no category segment", + uri: "date:2026-09-28/page:1", + want: "", + }, + { + name: "empty path", + uri: "", + want: "", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := srrdbCategory(test.uri); got != test.want { + t.Errorf("srrdbCategory(%q) = %q, want %q", test.uri, got, test.want) + } + }) + } +} + +func TestSrrdbStartPage(t *testing.T) { + tests := []struct { + name string + uri string + wantPage int + wantPaginated bool + }{ + { + name: "first page", + uri: "category:flac/date:2026-09-28/page:1", + wantPage: 1, + wantPaginated: true, + }, + { + name: "later page", + uri: "category:flac/date:2026-09-28/page:7", + wantPage: 7, + wantPaginated: true, + }, + { + name: "no page segment", + uri: "category:flac/date:2026-09-28", + wantPage: 1, + wantPaginated: false, + }, + { + name: "zero page falls back to one", + uri: "category:flac/date:2026-09-28/page:0", + wantPage: 1, + wantPaginated: true, + }, + { + name: "non numeric page is not a page segment", + uri: "category:flac/date:2026-09-28/page:abc", + wantPage: 1, + wantPaginated: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + page, paginated := srrdbStartPage(test.uri) + if page != test.wantPage || paginated != test.wantPaginated { + t.Errorf("srrdbStartPage(%q) = %d, %t, want %d, %t", + test.uri, page, paginated, test.wantPage, test.wantPaginated) + } + }) + } +} + +func TestSrrdbWithPage(t *testing.T) { + tests := []struct { + name string + uri string + page int + paginated bool + want string + }{ + { + name: "swaps the page segment", + uri: "category:flac/date:2026-09-28/page:3", + page: 4, + paginated: true, + want: "category:flac/date:2026-09-28/page:4", + }, + { + name: "single request keeps the path untouched", + uri: "category:flac/date:2026-09-28/page:3", + page: 4, + paginated: false, + want: "category:flac/date:2026-09-28/page:3", + }, + { + name: "no page segment to swap", + uri: "category:flac/date:2026-09-28", + page: 2, + paginated: true, + want: "category:flac/date:2026-09-28", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := srrdbWithPage(test.uri, test.page, test.paginated); got != test.want { + t.Errorf("srrdbWithPage(%q, %d, %t) = %q, want %q", + test.uri, test.page, test.paginated, got, test.want) + } + }) + } +} + +func TestSrrdbCollectKeepsFirstPageEnvelope(t *testing.T) { + collected := &resource.SrrdbSearchResponse{} + + srrdbCollect(collected, &resource.SrrdbSearchResponse{ + ResultsCount: 155, + Query: json.RawMessage(`[21]`), + Results: []resource.SrrdbResult{ + {Release: "First-Release-16BIT-WEB-FLAC-2026-GROUP"}, + {Release: "Second-Release-16BIT-WEB-FLAC-2026-GROUP"}, + }, + Warnings: []string{"first warning"}, + }) + + srrdbCollect(collected, &resource.SrrdbSearchResponse{ + ResultsCount: 999, + Query: json.RawMessage(`[22]`), + Results: []resource.SrrdbResult{ + {Release: "Third-Release-16BIT-WEB-FLAC-2026-GROUP"}, + }, + Warnings: []string{"second warning"}, + }) + + if collected.ResultsCount != 155 { + t.Errorf("ResultsCount = %d, want 155: only the first page may set it", collected.ResultsCount) + } + + if got := string(collected.Query); got != "[21]" { + t.Errorf("Query = %s, want [21]: only the first page may set it", got) + } + + if len(collected.Results) != 3 { + t.Errorf("len(Results) = %d, want 3", len(collected.Results)) + } + + if len(collected.Warnings) != 2 { + t.Errorf("len(Warnings) = %d, want 2", len(collected.Warnings)) + } +} + +func TestSrrdbCollectZeroCountOnFirstPageIsNotSticky(t *testing.T) { + collected := &resource.SrrdbSearchResponse{} + + srrdbCollect(collected, &resource.SrrdbSearchResponse{}) + srrdbCollect(collected, &resource.SrrdbSearchResponse{ResultsCount: 42}) + + if collected.ResultsCount != 42 { + t.Errorf("ResultsCount = %d, want 42: a zero count must not freeze the envelope", collected.ResultsCount) + } +} diff --git a/pkg/repository/table/srrdb.go b/pkg/repository/table/srrdb.go new file mode 100644 index 0000000..8a4d8ee --- /dev/null +++ b/pkg/repository/table/srrdb.go @@ -0,0 +1,59 @@ +package table + +import ( + "fmt" + + "github.com/jmoiron/sqlx" + + "git.kplus.net.ua/yevhen/resource-scraper/types/model" +) + +// srrdbUpsert keys on the "release" primary key: a repeated search refreshes the +// existing row instead of inserting a copy. synced_at is stamped with the same +// run timestamp as created_at, so the two differ only on a release that was +// already stored by an earlier run. +const srrdbUpsert = ` + INSERT INTO srrdb_releases + ("release", date, has_nfo, has_srs, is_foreign, imdb_id, size, group_name, category, created_at) + VALUES + (:release, :date, :has_nfo, :has_srs, :is_foreign, :imdb_id, :size, :group_name, :category, :created_at) + ON CONFLICT("release") DO UPDATE SET + date = excluded.date, + has_nfo = excluded.has_nfo, + has_srs = excluded.has_srs, + is_foreign = excluded.is_foreign, + imdb_id = excluded.imdb_id, + size = excluded.size, + group_name = excluded.group_name, + category = excluded.category, + synced_at = excluded.created_at` + +// UpsertAllSrrdb writes a whole set in one transaction: either every release +// lands or none does, so a failure halfway through cannot leave a day half +// harvested. +func UpsertAllSrrdb(db *sqlx.DB, rows []model.SrrdbRelease) error { + if len(rows) == 0 { + return nil + } + + tx, err := db.Beginx() + if err != nil { + return fmt.Errorf("begin srrdb transaction: %w", err) + } + + for _, row := range rows { + if _, err := tx.NamedExec(srrdbUpsert, &row); err != nil { + // Rollback failure is not reported: the insert error is the one that + // matters to the caller, and the transaction is gone either way. + _ = tx.Rollback() + + return fmt.Errorf("upsert srrdb release %q: %w", row.Release, err) + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit srrdb transaction: %w", err) + } + + return nil +} diff --git a/pkg/service/service.go b/pkg/service/service.go index 4abb540..83ad9a9 100644 --- a/pkg/service/service.go +++ b/pkg/service/service.go @@ -10,6 +10,7 @@ type Service struct { _interface.Prescene _interface.Info _interface.ShareTheBrutality + _interface.Srrdb } func New(repos *repository.Repository) *Service { @@ -18,5 +19,6 @@ func New(repos *repository.Repository) *Service { Prescene: NewPresceneService(repos.Prescene), Info: NewInfoService(), ShareTheBrutality: NewShareTheBrutalityService(repos.ShareTheBrutality), + Srrdb: NewSrrdbService(repos.Srrdb), } } diff --git a/pkg/service/srrdb.go b/pkg/service/srrdb.go new file mode 100644 index 0000000..9220105 --- /dev/null +++ b/pkg/service/srrdb.go @@ -0,0 +1,17 @@ +package service + +import ( + "git.kplus.net.ua/yevhen/resource-scraper/types/interface" +) + +type SrrdbService struct { + repo _interface.Srrdb +} + +func NewSrrdbService(repo _interface.Srrdb) *SrrdbService { + return &SrrdbService{repo: repo} +} + +func (s *SrrdbService) Search(uri string) (int, error) { + return s.repo.Search(uri) +} diff --git a/types/constant/constants.go b/types/constant/constants.go index 25b3650..a83c32a 100644 --- a/types/constant/constants.go +++ b/types/constant/constants.go @@ -6,6 +6,7 @@ const ( ScopeWeb string = "web" ScopeInfo string = "info" ScopeShareTheBrutality string = "stb" + ScopeSrrdb string = "srrdb" ) const ( diff --git a/types/constant/flag.go b/types/constant/flag.go index 8a37cf7..96820e5 100644 --- a/types/constant/flag.go +++ b/types/constant/flag.go @@ -10,6 +10,8 @@ const ( FlagScopeEnable = "scope-enable" FlagSingleUri = "single-uri" FlagEnv = "env" + FlagCreate = "create" + FlagSleep = "sleep" DefaultEnvProd = "prod" DefaultEnvDev = "devel" diff --git a/types/interface/interfaces.go b/types/interface/interfaces.go index 39b4c97..946e7f6 100644 --- a/types/interface/interfaces.go +++ b/types/interface/interfaces.go @@ -22,4 +22,8 @@ type MetalArchiveInterface interface { GetSearchResults(searchResults []model.ExternalSources) ([]model.ExternalSources, error) } +type Srrdb interface { + Search(uri string) (int, error) +} + type Info interface{} diff --git a/types/model/srrdb.go b/types/model/srrdb.go new file mode 100644 index 0000000..6bd8e44 --- /dev/null +++ b/types/model/srrdb.go @@ -0,0 +1,22 @@ +package model + +// SrrdbRelease is one row of the srrdb_releases table described by +// internal/db/srrdb.sql. Release is the primary key, therefore a repeated +// search rewrites the previous row of the same release instead of inserting +// a copy. Columns left nil are stored as NULL. +type SrrdbRelease struct { + Release string `db:"release"` + Date string `db:"date"` + HasNfo string `db:"has_nfo"` + HasSrs string `db:"has_srs"` + IsForeign string `db:"is_foreign"` + ImdbID *string `db:"imdb_id"` + Size int64 `db:"size"` + GroupName string `db:"group_name"` + Category string `db:"category"` + Status *string `db:"status"` + Nfo []byte `db:"nfo"` + Content *string `db:"content"` + CreatedAt string `db:"created_at"` + SyncedAt *string `db:"synced_at"` +} diff --git a/types/resource/srrdb.go b/types/resource/srrdb.go new file mode 100644 index 0000000..903b7d5 --- /dev/null +++ b/types/resource/srrdb.go @@ -0,0 +1,26 @@ +package resource + +import "encoding/json" + +// SrrdbSearchResponse is the payload of GET https://api.srrdb.com/v1/search/{path}. +// Query stays raw because the API returns a heterogeneous array there: category +// ids as numbers, search terms as strings. +type SrrdbSearchResponse struct { + Results []SrrdbResult `json:"results"` + ResultsCount int `json:"resultsCount"` + Warnings []string `json:"warnings"` + Query json.RawMessage `json:"query"` +} + +// SrrdbResult is a single release as reported by the search endpoint. +// HasNFO, HasSRS and IsForeign arrive as the literal strings "yes" and "no"; +// ImdbID is null for music releases. +type SrrdbResult struct { + Release string `json:"release"` + Date string `json:"date"` + HasNFO string `json:"hasNFO"` + HasSRS string `json:"hasSRS"` + IsForeign string `json:"isForeign"` + ImdbID *string `json:"imdbId"` + Size int64 `json:"size"` +}