diff --git a/.idea/harvester.iml b/.idea/harvester.iml
index 9c408cc..c50f1aa 100644
--- a/.idea/harvester.iml
+++ b/.idea/harvester.iml
@@ -3,7 +3,9 @@
-
+
+
+
diff --git a/cmd/srrdb/main.go b/cmd/srrdb/main.go
index d7aef6b..6b9557a 100644
--- a/cmd/srrdb/main.go
+++ b/cmd/srrdb/main.go
@@ -49,7 +49,7 @@ func main() {
err = dbase.Select(&titles,
`SELECT title FROM theamok_mdb.external_sources
- WHERE type = 'prescene' AND releaser = ? limit 2000`, releaser)
+ WHERE type = 'prescene' AND releaser = ?`, releaser)
if err != nil {
log.Fatal(err)
}
diff --git a/go.mod b/go.mod
index 66d25d4..3684682 100644
--- a/go.mod
+++ b/go.mod
@@ -1,43 +1,42 @@
module git.kplus.net.ua/yevhen/resource-scraper
-go 1.26.2
+go 1.27
require (
github.com/davecgh/go-spew v1.1.1
- github.com/dromara/carbon/v2 v2.6.16
+ github.com/dromara/carbon/v2 v2.6.17
github.com/emersion/go-imap/v2 v2.0.0-beta.8
- github.com/go-chi/chi/v5 v5.2.5
+ github.com/go-chi/chi/v5 v5.3.2
github.com/go-chi/cors v1.2.2
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
- github.com/go-sql-driver/mysql v1.9.3
+ github.com/go-sql-driver/mysql v1.10.1
github.com/iancoleman/strcase v0.3.0
github.com/jmoiron/sqlx v1.4.0
github.com/logrusorgru/aurora/v4 v4.0.0
- github.com/mewkiz/flac v1.0.13
+ github.com/mewkiz/flac v1.0.14
github.com/spf13/pflag v1.0.10
github.com/spf13/viper v1.21.0
- golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
- golang.org/x/net v0.49.0
+ golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
+ golang.org/x/net v0.59.0
)
require (
filippo.io/edwards25519 v1.2.0 // indirect
- github.com/andybalholm/cascadia v1.3.3 // indirect
+ github.com/andybalholm/cascadia v1.3.5 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
- github.com/fsnotify/fsnotify v1.9.0 // indirect
+ github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/icza/bitio v1.1.0 // indirect
- github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de // indirect
+ github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 // indirect
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
- github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+ github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
- go.yaml.in/yaml/v3 v3.0.4 // indirect
- golang.org/x/sys v0.40.0 // indirect
- golang.org/x/text v0.33.0 // indirect
- gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+ go.yaml.in/yaml/v3 v3.0.5 // indirect
+ golang.org/x/sys v0.48.0 // indirect
+ golang.org/x/text v0.42.0 // indirect
)
diff --git a/go.sum b/go.sum
index a89feb1..cc4afb5 100644
--- a/go.sum
+++ b/go.sum
@@ -1,12 +1,12 @@
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
-github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
-github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/andybalholm/cascadia v1.3.5 h1:RLjq12WJy58dN6eCIQrz0bAGZkztHWsEPFxP53Y7Ms8=
+github.com/andybalholm/cascadia v1.3.5/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
-github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
+github.com/dromara/carbon/v2 v2.6.17 h1:QL1LVcp5A/NQtIPbc1vrL9qIXxPNWnaDOBaFlmP1iDg=
+github.com/dromara/carbon/v2 v2.6.17/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -15,17 +15,17 @@ github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
-github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
-github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
-github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
+github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
+github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
+github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
+github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
-github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
-github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
+github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
+github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
@@ -50,14 +50,14 @@ github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUp
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
-github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
-github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
-github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
+github.com/mewkiz/flac v1.0.14 h1:hyRGAM8NCKznoPmIi9zz2jyO+nfmxY2ErqBnHZ+gxh4=
+github.com/mewkiz/flac v1.0.14/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
+github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 h1:fpWNLIg7nX2f+D+6ZveQ5hfkTjq5WgMBtYbz5Nb1w0Y=
+github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87/go.mod h1:wohiqwTwXA5twjDES74xIJLGjAxo3/0pHSbTXLbSDW8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
-github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
-github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
+github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
@@ -77,81 +77,45 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
+go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
-golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
-golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
+golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
+golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
-golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
-golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
-golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
+golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
-golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
+golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
-golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
-golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
-golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
-golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
-golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
-gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
diff --git a/internal/releaseparser/groups/parser.go b/internal/releaseparser/groups/parser.go
index a49c8ef..aa95688 100644
--- a/internal/releaseparser/groups/parser.go
+++ b/internal/releaseparser/groups/parser.go
@@ -5,8 +5,6 @@ import (
"strconv"
"strings"
- "golang.org/x/exp/slices"
-
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
@@ -58,31 +56,31 @@ func Parse(raw string) releaseparser.ParsedRelease {
return result
}
- // Where the dictionary-driven tail zone begins: after the resolved
- // Artist/Title core (2 tokens by default, 3 with an artist dictionary hit).
+ // Left core boundary: after the resolved Artist (2 tokens by default,
+ // 3 with an artist dictionary hit, or fixed by an override).
start := 2
// Level 1: exact-lookup override table fixes Artist/Title only;
- // the tail zone is parsed as usual below.
+ // the tail zone is parsed as usual below. The override is
+ // authoritative about the core: leftover tokens are plain unknowns,
+ // not part of the title.
if o, ok := Profile.Overrides[result.Raw]; ok {
result.Artist, result.Title = o.Artist, o.Title
start = overrideCoreLength(tokens, o)
}
- // Level 2: artist dictionary — "[0]-[1]" whitelisted as a hyphenated
- // artist shifts the core by one token.
- switch {
- case result.Artist != "":
- // An override already fixed the core — leave it alone.
- case len(tokens) >= 3 &&
- Profile.ArtistTokens[strings.ToUpper(tokens[0]+"-"+tokens[1])]:
- result.Artist = tokens[0] + "-" + tokens[1]
- result.Title = tokens[2]
- start = 3
- default:
- result.Artist = tokens[0]
- if len(tokens) > 1 {
- result.Title = tokens[1]
+ // Level 2: artist dictionary — a hyphenated artist may span ANY number
+ // of leading tokens ("Shadu-Nar-Mattaru" = 3). The LONGEST whitelisted
+ // prefix wins; the title is the token right after it. At least one
+ // title token and one tail token must remain.
+ artistDict := false
+ if result.Artist == "" {
+ for k := len(tokens) - 2; k >= 2; k-- {
+ if Profile.ArtistTokens[strings.ToUpper(strings.Join(tokens[:k], "-"))] {
+ artistDict = true
+ start = k + 1
+ break
+ }
}
}
@@ -93,8 +91,41 @@ func Parse(raw string) releaseparser.ParsedRelease {
end = len(tokens) - 2
}
+ // Right core boundary: the first dictionary-known token after the
+ // resolved artist. The boundary is contextual, but the title itself
+ // stays a single token ([1], or [2] with an artist dictionary hit).
+ // Unknown tokens inside the core region ([start .. tailStart)) are
+ // REPORTED in UnknownTokens — never eaten, never glued into the title.
+ tailStart := end
+ for i := start; i < end; i++ {
+ if isKnownToken(tokens[i]) {
+ tailStart = i
+ break
+ }
+ }
+
+ // Resolve Artist/Title from the core unless the override already did.
+ if result.Artist == "" {
+ if artistDict {
+ // The artist spans tokens [0 .. start-2]; re-join them.
+ result.Artist = strings.Join(tokens[:start-1], "-")
+ } else {
+ result.Artist = tokens[0]
+ }
+ if len(tokens) > start-1 {
+ result.Title = tokens[start-1]
+ }
+ }
+
+ // Core-region leftovers (e.g. the rest of a wildcards title after an
+ // override, or a hyphenated title fragment without an artist entry):
+ // visible in UnknownTokens for review.
+ for _, tok := range tokens[start:tailStart] {
+ result.UnknownTokens = append(result.UnknownTokens, tok)
+ }
+
// Level 3: dictionary-driven tail zone, starting after the resolved core.
- parseTailZone(start, tokens[start:end], &result)
+ parseTailZone(tokens[tailStart:end], &result)
score(&result)
return result
@@ -160,12 +191,24 @@ func catalogNumber(token string) string {
return token[1 : len(token)-1]
}
+// isKnownToken reports whether the token belongs to the tail zone: it
+// matches one of the profile dictionaries or is a parenthesized catalogue
+// number. The first such token after the core marks the right core
+// boundary.
+func isKnownToken(tok string) bool {
+ up := strings.ToUpper(tok)
+ return Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
+ Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
+ Profile.EditionTokens[up] || Profile.LanguageMarkers[up] ||
+ catalogNumber(tok) != ""
+}
+
// parseTailZone resolves middle-zone tokens against the dictionaries.
// The first match wins for each field; catalogue numbers are recognized
// from parenthesized tokens, and anything unrecognized is appended to
// UnknownTokens so that it can be reviewed and curated into a dictionary.
-func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease) {
- for i, tok := range zone {
+func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
+ for _, tok := range zone {
up := strings.ToUpper(tok)
switch {
case Profile.FormatTokens[up]:
@@ -197,13 +240,6 @@ func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease
default:
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
result.CatalogNo = cat
- } else if hasLaterKnownToken(zone, i) {
- // suspected title continuation: the core is probably longer
- // than 2 tokens. Not eaten, not unknown — review section.
- result.TitleExtra = append(result.TitleExtra, tok)
- if !slices.Contains(result.Warnings, "suspected title continuation") {
- result.Warnings = append(result.Warnings, "suspected title continuation")
- }
} else {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
@@ -220,20 +256,6 @@ func applyType(tok string, result *releaseparser.ParsedRelease) {
}
}
-// hasLaterKnownToken reports whether any token after index i in the zone
-// matches one of the profile dictionaries.
-func hasLaterKnownToken(zone []string, i int) bool {
- for _, later := range zone[i+1:] {
- up := strings.ToUpper(later)
- if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
- Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
- Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
- return true
- }
- }
- return false
-}
-
// overrideCoreLength returns the number of leading tokens covered by the
// override's Artist and Title (each may span several hyphen-separated
// tokens), falling back to the default 2-token core when the raw tokens
diff --git a/internal/releaseparser/groups/parser_test.go b/internal/releaseparser/groups/parser_test.go
index 966a64a..f237dce 100644
--- a/internal/releaseparser/groups/parser_test.go
+++ b/internal/releaseparser/groups/parser_test.go
@@ -38,23 +38,20 @@ func TestParseEditionAndDemo(t *testing.T) {
func TestParseArtistWithHyphen(t *testing.T) {
// Artist сам містить дефіс, але Ultra-Violence ще не в словнику:
- // модель [0]/[1] лишається, а All_My_Life (за ним ідуть словникові
- // маркери) — suspected title continuation, не unknown.
+ // core контекстуальний — все між артистом і першим словниковим
+ // маркером (EP) є назвою, зібраною назад через "-".
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
- if r.Artist != "Ultra" || r.Title != "Violence" {
- t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
+ if r.Artist != "Ultra" {
+ t.Errorf("Artist = %q, want Ultra", r.Artist)
+ }
+ if r.Title != "Violence" {
+ t.Errorf("Title = %q, want Violence", r.Title)
}
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
- if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "All_My_Life" {
- t.Errorf("TitleExtra = %v, want [All_My_Life]", r.TitleExtra)
- }
- if !contains(r.Warnings, "suspected title continuation") {
- t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings)
- }
- if len(r.UnknownTokens) != 0 {
- t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+ if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
+ t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
}
}
@@ -147,22 +144,23 @@ func TestParseUnknownTailTokenReported(t *testing.T) {
}
}
-func TestParseSuspectedTitleContinuation(t *testing.T) {
- // Невідомий токен МІЖ title-зоною і хвостом (за ним далі йдуть
- // словникові маркери) → не UnknownTokens, а TitleExtra + warning.
- r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
- if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "FOO" {
- t.Errorf("TitleExtra = %v, want [FOO]", r.TitleExtra)
+func TestParseContextualCoreAbsorbsTitleTail(t *testing.T) {
+ // Права межа core контекстуальна (перший словниковий маркер), але
+ // невідомі токени всередині core-зони — в UnknownTokens, не з'їдаються.
+ r := Parse("Artist-Some_Title-Part_Two-Part_Three-16BIT-WEB-FLAC-2025-MOONBLOOD")
+ if r.Artist != "Artist" {
+ t.Errorf("Artist = %q, want Artist", r.Artist)
}
- if len(r.UnknownTokens) != 0 {
- t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+ if r.Title != "Some_Title" {
+ t.Errorf("Title = %q, want Some_Title", r.Title)
}
- if !contains(r.Warnings, "suspected title continuation") {
- t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings)
- }
- if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
+ if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2025 {
t.Errorf("tail fields wrong: %+v", r)
}
+ if len(r.UnknownTokens) != 2 || r.UnknownTokens[0] != "Part_Two" ||
+ r.UnknownTokens[1] != "Part_Three" {
+ t.Errorf("UnknownTokens = %v, want [Part_Two Part_Three]", r.UnknownTokens)
+ }
}
func TestParseCoreTokensNeverReported(t *testing.T) {
@@ -282,9 +280,8 @@ func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
}
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
- // [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
- // за ним ідуть словникові маркери → suspected title continuation.
- // (SMG409) — катало, залишається каталом.
+ // Evolution — не словниковий токен у core-зоні → в UnknownTokens,
+ // не з'їдається. (SMG409) — катало, перший відомий маркер.
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
if r.CatalogNo != "SMG409" {
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
@@ -292,11 +289,8 @@ func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
if r.Artist != "Ferosity" || r.Title != "De" {
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
}
- if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "Evolution" {
- t.Errorf("TitleExtra = %v, want [Evolution]", r.TitleExtra)
- }
- if len(r.UnknownTokens) != 0 {
- t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+ if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
+ t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
}
}
@@ -363,18 +357,14 @@ func TestParseCompoundEditionRecognized(t *testing.T) {
}
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
- // [1]=Title (один токен). Хвіст назви (_ і Sky_Burial) — не словник,
- // але за ним ідуть словникові маркери → suspected title continuation,
- // не UnknownTokens.
+ // Хвіст назви (_Sky_Burial) — не словник, але він ДО першого маркера
+ // (катало (RTM103)) → частина назви, не UnknownTokens.
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
- if len(r.UnknownTokens) != 0 {
- t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
- }
- if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "_Sky_Burial" {
- t.Errorf("TitleExtra = %v, want [_Sky_Burial]", r.TitleExtra)
+ if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
+ t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
}
if r.CatalogNo != "RTM103" {
t.Errorf("CatalogNo = %q, want RTM103", r.CatalogNo)
@@ -426,10 +416,30 @@ func TestParseArtistDictionaryCoversReleases(t *testing.T) {
}
}
+func TestParseArtistDictionaryMultiTokenArtist(t *testing.T) {
+ // artists.txt: "Shadu-Nar-Mattaru" — артист з ТРЬОХ токенів.
+ // Найдовший словниковий префікс виграє: Artist = 3 токени,
+ // Title = наступний токен, хвіст парситься як завжди.
+ r := Parse("Shadu-Nar-Mattaru-On_Pathways_Macabre-24BIT-WEB-FLAC-2023-MOONBLOOD")
+ if r.Artist != "Shadu-Nar-Mattaru" {
+ t.Errorf("Artist = %q, want Shadu-Nar-Mattaru", r.Artist)
+ }
+ if r.Title != "On_Pathways_Macabre" {
+ t.Errorf("Title = %q, want On_Pathways_Macabre", r.Title)
+ }
+ if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2023 {
+ t.Errorf("tail fields wrong: %+v", r)
+ }
+ if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
+ t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
+ r.UnknownTokens, r.TitleExtra, r.Warnings)
+ }
+}
+
func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
// overrides.txt (найвищий пріоритет): точний lookup по Raw
// фіксує Artist/Title; хвіст парситься як завжди.
- raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2024-GRAVEWAVE"
+ raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2013-MOONBLOOD"
r := Parse(raw)
if r.Artist != "Milicia_Oscura" {
t.Errorf("Artist = %q, want Milicia_Oscura", r.Artist)
@@ -438,7 +448,7 @@ func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
t.Errorf("Title = %q, want Anti-Cristo", r.Title)
}
if r.Language != "ES" || r.BitDepth != "16BIT" || r.Source != "WEB" ||
- r.Format != "FLAC" || r.Year != 2024 || r.Group != "GRAVEWAVE" {
+ r.Format != "FLAC" || r.Year != 2013 || r.Group != "MOONBLOOD" {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
diff --git a/pkg/repository/prescene.go b/pkg/repository/prescene.go
index a728cc7..889e22b 100644
--- a/pkg/repository/prescene.go
+++ b/pkg/repository/prescene.go
@@ -85,16 +85,25 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
}
var validID = regexp.MustCompile(`-\d+\/$`)
- for i, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
+ for i, item := range dom.QuerySelectorAll(doc, "li.row") {
var es model.ExternalSources
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
+ // TypeId тепер напряму з data-id на , без regex по class
+ idAttr := dom.GetAttribute(item, "data-id")
+ es.TypeId, _ = strconv.Atoi(idAttr)
+
title := dom.QuerySelector(item, ".title")
if title != nil {
- anchor := dom.QuerySelector(title, "h1 > a")
+ anchor := dom.QuerySelector(title, "a")
if anchor != nil {
es.Type = constant.ScopePrescene
- es.Title = dom.GetAttribute(anchor, "title")
+
+ titleSpan := dom.QuerySelector(anchor, "span.title-start")
+ if titleSpan != nil {
+ es.Title = dom.TextContent(titleSpan)
+ }
+
if es.Title == "Auto Draft" {
slog.Info("Skipped", "title", es.Title)
continue
@@ -123,17 +132,13 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
continue
}
- localtime := dom.QuerySelector(title, "small > span.localtime")
+ localtime := dom.QuerySelector(item, "span.localtime")
if localtime != nil {
lc := dom.GetAttribute(localtime, "data-lttime")
es.Created = carbon.Parse(lc, "Europe/Kyiv")
}
}
- cls := dom.GetAttribute(item, "class")
- pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
- es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
-
esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, db)
entries = append(entries, entry)