Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 39aa8e2e81 | |||
| 347f525801 | |||
| f2629f002c |
+1
-1
@@ -14,7 +14,7 @@ vendor/
|
||||
config/secret.yaml
|
||||
*.md
|
||||
mdb
|
||||
internal/releaseparser/resources/*
|
||||
internal/releaseparser/groups/resources/*
|
||||
review_unknown_tokens.txt
|
||||
srrdb.exe
|
||||
**/*.exe
|
||||
|
||||
Generated
+3
-1
@@ -3,7 +3,9 @@
|
||||
<component name="Go" enabled="true" />
|
||||
<component name="GoModuleSettings" enabled="true" />
|
||||
<component name="NewModuleRootManager">
|
||||
<content url="file://$MODULE_DIR$" />
|
||||
<content url="file://$MODULE_DIR$">
|
||||
<excludeFolder url="file://$MODULE_DIR$/tmp" />
|
||||
</content>
|
||||
<orderEntry type="inheritedJdk" />
|
||||
<orderEntry type="sourceFolder" forTests="false" />
|
||||
</component>
|
||||
|
||||
+49
-3
@@ -5,13 +5,15 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -35,7 +37,12 @@ func main() {
|
||||
}
|
||||
defer dbase.Close()
|
||||
|
||||
out := make(map[string]moonblood.GroupStats, len(releasers))
|
||||
const outDir = "tmp/releaseparser"
|
||||
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
out := make(map[string]groups.GroupStats, len(releasers))
|
||||
|
||||
for _, releaser := range releasers {
|
||||
var titles []string
|
||||
@@ -47,7 +54,16 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
out[releaser] = moonblood.AnalyzeTitles(titles)
|
||||
out[releaser] = groups.AnalyzeTitles(titles)
|
||||
|
||||
// Per-group parsed records keyed by the actual group read from each
|
||||
// line: every release with all resolved fields for manual validation.
|
||||
for group, parsed := range groups.ParseByGroup(titles) {
|
||||
writeJSON(filepath.Join(outDir, safeName(group)+".json"), parsed)
|
||||
}
|
||||
|
||||
// Per-request report identical to what is printed to the console.
|
||||
writeJSON(filepath.Join(outDir, safeName(releaser)+".report.json"), out[releaser])
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(out, "", " ")
|
||||
@@ -77,3 +93,33 @@ func main() {
|
||||
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
|
||||
// writeJSON writes value as an indented JSON document to the given path.
|
||||
func writeJSON(path string, value any) {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// safeName strips characters that are unsafe in file names.
|
||||
func safeName(s string) string {
|
||||
repl := func(r rune) rune {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
return r
|
||||
case r >= 'A' && r <= 'Z':
|
||||
return r
|
||||
case r >= '0' && r <= '9':
|
||||
return r
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
return r
|
||||
default:
|
||||
return '_'
|
||||
}
|
||||
}
|
||||
return strings.Map(repl, s)
|
||||
}
|
||||
|
||||
@@ -1,43 +1,42 @@
|
||||
module git.kplus.net.ua/yevhen/resource-scraper
|
||||
|
||||
go 1.26.2
|
||||
go 1.27
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/dromara/carbon/v2 v2.6.16
|
||||
github.com/dromara/carbon/v2 v2.6.17
|
||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8
|
||||
github.com/go-chi/chi/v5 v5.2.5
|
||||
github.com/go-chi/chi/v5 v5.3.2
|
||||
github.com/go-chi/cors v1.2.2
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
|
||||
github.com/go-sql-driver/mysql v1.9.3
|
||||
github.com/go-sql-driver/mysql v1.10.1
|
||||
github.com/iancoleman/strcase v0.3.0
|
||||
github.com/jmoiron/sqlx v1.4.0
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0
|
||||
github.com/mewkiz/flac v1.0.13
|
||||
github.com/mewkiz/flac v1.0.14
|
||||
github.com/spf13/pflag v1.0.10
|
||||
github.com/spf13/viper v1.21.0
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
|
||||
golang.org/x/net v0.49.0
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
|
||||
golang.org/x/net v0.59.0
|
||||
)
|
||||
|
||||
require (
|
||||
filippo.io/edwards25519 v1.2.0 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.3 // indirect
|
||||
github.com/andybalholm/cascadia v1.3.5 // indirect
|
||||
github.com/emersion/go-message v0.18.2 // indirect
|
||||
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
|
||||
github.com/fsnotify/fsnotify v1.9.0 // indirect
|
||||
github.com/fsnotify/fsnotify v1.10.1 // indirect
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
|
||||
github.com/icza/bitio v1.1.0 // indirect
|
||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de // indirect
|
||||
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 // indirect
|
||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
|
||||
github.com/sagikazarmark/locafero v0.12.0 // indirect
|
||||
github.com/spf13/afero v1.15.0 // indirect
|
||||
github.com/spf13/cast v1.10.0 // indirect
|
||||
github.com/subosito/gotenv v1.6.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/sys v0.40.0 // indirect
|
||||
golang.org/x/text v0.33.0 // indirect
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.5 // indirect
|
||||
golang.org/x/sys v0.48.0 // indirect
|
||||
golang.org/x/text v0.42.0 // indirect
|
||||
)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
|
||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||
github.com/andybalholm/cascadia v1.3.5 h1:RLjq12WJy58dN6eCIQrz0bAGZkztHWsEPFxP53Y7Ms8=
|
||||
github.com/andybalholm/cascadia v1.3.5/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
|
||||
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
||||
github.com/dromara/carbon/v2 v2.6.17 h1:QL1LVcp5A/NQtIPbc1vrL9qIXxPNWnaDOBaFlmP1iDg=
|
||||
github.com/dromara/carbon/v2 v2.6.17/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
|
||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
|
||||
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
||||
@@ -15,17 +15,17 @@ github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3
|
||||
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
|
||||
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
|
||||
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
|
||||
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
|
||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
|
||||
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
|
||||
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
|
||||
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
|
||||
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
|
||||
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
|
||||
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
|
||||
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
|
||||
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
|
||||
@@ -50,14 +50,14 @@ github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUp
|
||||
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
|
||||
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
|
||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
|
||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
|
||||
github.com/mewkiz/flac v1.0.14 h1:hyRGAM8NCKznoPmIi9zz2jyO+nfmxY2ErqBnHZ+gxh4=
|
||||
github.com/mewkiz/flac v1.0.14/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
|
||||
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 h1:fpWNLIg7nX2f+D+6ZveQ5hfkTjq5WgMBtYbz5Nb1w0Y=
|
||||
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87/go.mod h1:wohiqwTwXA5twjDES74xIJLGjAxo3/0pHSbTXLbSDW8=
|
||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
|
||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
|
||||
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
|
||||
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
|
||||
@@ -77,81 +77,45 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD
|
||||
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
|
||||
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
|
||||
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
|
||||
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
|
||||
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
|
||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
|
||||
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
|
||||
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
|
||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
|
||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
|
||||
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
|
||||
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
|
||||
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
|
||||
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
|
||||
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
|
||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
|
||||
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
|
||||
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
|
||||
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
|
||||
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
|
||||
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
|
||||
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
|
||||
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
|
||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
||||
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
|
||||
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
|
||||
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
|
||||
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
|
||||
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
|
||||
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
+38
-29
@@ -1,8 +1,6 @@
|
||||
package moonblood
|
||||
package groups
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
@@ -18,6 +16,7 @@ type GroupStats struct {
|
||||
SourceDistribution map[string]int
|
||||
BitDepthDistribution map[string]int
|
||||
ResolutionDistribution map[string]int
|
||||
TypeDistribution map[string]int
|
||||
LanguageDistribution map[string]int
|
||||
EditionDistribution map[string]int
|
||||
CommentDistribution map[string]int
|
||||
@@ -26,37 +25,13 @@ type GroupStats struct {
|
||||
AvgConfidence float64
|
||||
}
|
||||
|
||||
// AnalyzeGroup parses the release lines from the local resource file. It is
|
||||
// used for a quick local check of the dictionaries without touching a
|
||||
// database.
|
||||
func AnalyzeGroup() (GroupStats, error) {
|
||||
file, err := os.Open("../../resources/moonblood.txt")
|
||||
if err != nil {
|
||||
return GroupStats{}, err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
var titles []string
|
||||
scanner := bufio.NewScanner(file)
|
||||
for scanner.Scan() {
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
if line != "" {
|
||||
titles = append(titles, line)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return GroupStats{}, err
|
||||
}
|
||||
|
||||
return AnalyzeTitles(titles), nil
|
||||
}
|
||||
|
||||
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
||||
// database by a particular releaser key) and aggregates the results into a
|
||||
// GroupStats summary.
|
||||
func AnalyzeTitles(titles []string) GroupStats {
|
||||
stats := GroupStats{
|
||||
YearDistribution: make(map[int]int),
|
||||
TypeDistribution: make(map[string]int),
|
||||
SourceDistribution: make(map[string]int),
|
||||
BitDepthDistribution: make(map[string]int),
|
||||
ResolutionDistribution: make(map[string]int),
|
||||
@@ -99,6 +74,10 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
}
|
||||
}
|
||||
|
||||
if result.Type != "" {
|
||||
stats.TypeDistribution[result.Type]++
|
||||
}
|
||||
|
||||
for _, unknown := range result.UnknownTokens {
|
||||
if _, exists := unknownMap[unknown]; !exists {
|
||||
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
|
||||
@@ -131,7 +110,6 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
for _, tokenStats := range unknownMap {
|
||||
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
||||
}
|
||||
|
||||
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
|
||||
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
||||
})
|
||||
@@ -149,3 +127,34 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
// AnalyzeByGroup parses a set of release lines and returns per-group
|
||||
// statistics keyed by the actual group read from each line (not from the
|
||||
// query key, which may cover several groups). It is used to validate parsing
|
||||
// quality against the original data, grouped by the real releaser name.
|
||||
func AnalyzeByGroup(titles []string) map[string]GroupStats {
|
||||
byGroup := make(map[string][]string)
|
||||
for _, title := range titles {
|
||||
group := Parse(title).Group
|
||||
byGroup[group] = append(byGroup[group], title)
|
||||
}
|
||||
|
||||
out := make(map[string]GroupStats, len(byGroup))
|
||||
for group, lines := range byGroup {
|
||||
out[group] = AnalyzeTitles(lines)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ParseByGroup parses a set of release lines into a per-group list of fully
|
||||
// populated ParsedRelease records, keyed by the actual group read from each
|
||||
// line. It is used to validate parsing quality field-by-field against the
|
||||
// original data.
|
||||
func ParseByGroup(titles []string) map[string][]releaseparser.ParsedRelease {
|
||||
byGroup := make(map[string][]releaseparser.ParsedRelease)
|
||||
for _, title := range titles {
|
||||
result := Parse(title)
|
||||
byGroup[result.Group] = append(byGroup[result.Group], result)
|
||||
}
|
||||
return byGroup
|
||||
}
|
||||
@@ -1,5 +0,0 @@
|
||||
# {bit_depth}
|
||||
16BIT
|
||||
24BIT
|
||||
32BIT
|
||||
DE16BIT
|
||||
@@ -1,6 +0,0 @@
|
||||
REPACK
|
||||
REAL_REPACK
|
||||
READNFO
|
||||
PROPER
|
||||
RERIP
|
||||
199X
|
||||
@@ -1,70 +0,0 @@
|
||||
# {edition} — тип видання
|
||||
EP
|
||||
DEMO
|
||||
SPLIT
|
||||
REISSUE
|
||||
DELUXE_EDITION
|
||||
REMASTERED
|
||||
BOXSET
|
||||
SINGLE
|
||||
PROMO
|
||||
REISSUE_REMASTERED
|
||||
REMASTERED_25TH_ANNIVERSARY_EDITION
|
||||
REMASTERED_20TH_ANNIVERSARY_EDITION
|
||||
20TH_ANNIVERSARY_EDITION_REMASTERED
|
||||
Deluxe_Edition
|
||||
LIMITED_EDITION_BOXSET
|
||||
REMASTERED_EP
|
||||
10TH_ANNIVERSARY_EDITION
|
||||
OST
|
||||
XX_Legacy_Edition
|
||||
REISSUE_DEMO
|
||||
JP_Deluxe_Edition
|
||||
Remastered_Edition
|
||||
Anniversary_Edition
|
||||
Limited_Edition
|
||||
CDS
|
||||
CDM
|
||||
RETAIL
|
||||
JP_RETAIL
|
||||
JP_REISSUE
|
||||
DEMO_REMASTERED
|
||||
EU_RETAIL
|
||||
DE_RETAIL
|
||||
REISSUE_DELUXE_EDITION
|
||||
RETAIL_REISSUE
|
||||
SPECIAL_EDITION
|
||||
SPECIAL_COLLECTORS_EDITION_REMASTERED
|
||||
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
|
||||
LIMITED_EDITION_REMASTERED
|
||||
25th_Anniversary_Edition
|
||||
DEMO_REISSUE
|
||||
Ultimate_Edition
|
||||
Issue_255
|
||||
Issue_254
|
||||
Issue_198
|
||||
7INCH_VINYL
|
||||
REMASTERED_REISSUE
|
||||
PROMO_DEMO
|
||||
ES_RETAIL
|
||||
BOX_SET
|
||||
BONUS
|
||||
REISSUE_EP
|
||||
30th_Anniversary
|
||||
REDUX
|
||||
REMASTERED_BOXSET
|
||||
MAG
|
||||
REISSUE_BOXSET
|
||||
30TH_ANNIVERSARY_EDITION_REMASTERED
|
||||
DELUXE
|
||||
Deluxe_Edition_Reissue
|
||||
INSTRUMENTAL
|
||||
REISSUE_EXPANDED_EDITION
|
||||
BOOTLEG
|
||||
EXPANDED_EDITION
|
||||
ADVANCE
|
||||
15TH_ANNIVERSARY_DELUXE_EDITION
|
||||
TOUR_EDITION
|
||||
REMASTERED_DEMO
|
||||
REMASTERED_DELUXE_EDITION
|
||||
30TH_ANNIVERSARY_EDITION
|
||||
@@ -1,3 +0,0 @@
|
||||
# {format} — аудіоформат
|
||||
# один токен на рядок, регістр не має значення
|
||||
FLAC
|
||||
@@ -1,38 +0,0 @@
|
||||
# {language} — мовні маркери
|
||||
DE
|
||||
FR
|
||||
ES
|
||||
SK
|
||||
FI
|
||||
SE
|
||||
BR
|
||||
PL
|
||||
NL
|
||||
NO
|
||||
PT
|
||||
IT
|
||||
HU
|
||||
DK
|
||||
AT
|
||||
IS
|
||||
UA
|
||||
JP
|
||||
GR
|
||||
CN
|
||||
RU
|
||||
ID
|
||||
PH
|
||||
RO
|
||||
LT
|
||||
BY
|
||||
CZ
|
||||
SP
|
||||
IR
|
||||
FO
|
||||
RS
|
||||
DA
|
||||
HI
|
||||
CH
|
||||
LU
|
||||
CY
|
||||
EE
|
||||
@@ -1,8 +0,0 @@
|
||||
# {resolution} — частота дискретизації
|
||||
44KHZ
|
||||
48KHZ
|
||||
48.0kHz
|
||||
88KHZ
|
||||
96KHZ
|
||||
176KHZ
|
||||
192KHZ
|
||||
@@ -1,36 +0,0 @@
|
||||
# {source} — джерело релізу
|
||||
# VINYL сюди входить як source (за домовленістю, не edition)
|
||||
WEB
|
||||
CD
|
||||
2CD
|
||||
CDEP
|
||||
VINYL
|
||||
TAPE
|
||||
CDR
|
||||
17CD
|
||||
3CD
|
||||
4CD
|
||||
18CD
|
||||
3CDEP
|
||||
DVD
|
||||
7CD
|
||||
2CDR
|
||||
10CD
|
||||
6CD
|
||||
2CDEP
|
||||
2VINYL
|
||||
CDREP
|
||||
LP
|
||||
2LP
|
||||
3VINYL
|
||||
3TAPE
|
||||
13CD
|
||||
13CDR
|
||||
10INCH_VINYL
|
||||
25CD
|
||||
WEBFLAC
|
||||
2TAPE
|
||||
BLURAY
|
||||
VLS
|
||||
MCD
|
||||
24BIT_WEB
|
||||
@@ -1,42 +0,0 @@
|
||||
package moonblood
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"strings"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||
)
|
||||
|
||||
// Словники токенів лежать у dicts/*.txt — один токен на рядок.
|
||||
// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці).
|
||||
|
||||
//go:embed dicts/*.txt
|
||||
var dictFS embed.FS
|
||||
|
||||
func loadDict(name string) map[string]bool {
|
||||
data, err := dictFS.ReadFile("dicts/" + name + ".txt")
|
||||
if err != nil {
|
||||
panic("moonblood: cannot load dictionary " + name + ": " + err.Error())
|
||||
}
|
||||
out := make(map[string]bool)
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.ToUpper(strings.TrimSpace(line))
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var Profile = releaseparser.GroupProfile{
|
||||
Name: "MOONBLOOD",
|
||||
ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"},
|
||||
FormatTokens: loadDict("format"),
|
||||
SourceTokens: loadDict("source"),
|
||||
BitDepthTokens: loadDict("bitdepth"),
|
||||
ResolutionTokens: loadDict("resolution"),
|
||||
LanguageMarkers: loadDict("language"),
|
||||
EditionTokens: loadDict("edition"),
|
||||
CommentTokens: loadDict("comment"),
|
||||
}
|
||||
+108
-11
@@ -1,4 +1,4 @@
|
||||
package moonblood
|
||||
package groups
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
@@ -56,20 +56,76 @@ func Parse(raw string) releaseparser.ParsedRelease {
|
||||
return result
|
||||
}
|
||||
|
||||
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
|
||||
// [len-2]=Year (when present).
|
||||
result.Artist = tokens[0]
|
||||
if len(tokens) > 1 {
|
||||
result.Title = tokens[1]
|
||||
// Left core boundary: after the resolved Artist (2 tokens by default,
|
||||
// 3 with an artist dictionary hit, or fixed by an override).
|
||||
start := 2
|
||||
|
||||
// Level 1: exact-lookup override table fixes Artist/Title only;
|
||||
// the tail zone is parsed as usual below. The override is
|
||||
// authoritative about the core: leftover tokens are plain unknowns,
|
||||
// not part of the title.
|
||||
if o, ok := Profile.Overrides[result.Raw]; ok {
|
||||
result.Artist, result.Title = o.Artist, o.Title
|
||||
start = overrideCoreLength(tokens, o)
|
||||
}
|
||||
|
||||
// Level 2: artist dictionary — a hyphenated artist may span ANY number
|
||||
// of leading tokens ("Shadu-Nar-Mattaru" = 3). The LONGEST whitelisted
|
||||
// prefix wins; the title is the token right after it. At least one
|
||||
// title token and one tail token must remain.
|
||||
artistDict := false
|
||||
if result.Artist == "" {
|
||||
for k := len(tokens) - 2; k >= 2; k-- {
|
||||
if Profile.ArtistTokens[strings.ToUpper(strings.Join(tokens[:k], "-"))] {
|
||||
artistDict = true
|
||||
start = k + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
result.Group = tokens[len(tokens)-1]
|
||||
end := len(tokens) - 1
|
||||
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
|
||||
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
|
||||
end = len(tokens) - 2
|
||||
}
|
||||
// Resolve the middle zone tokens via the dictionaries.
|
||||
parseTailZone(tokens[2:end], &result)
|
||||
|
||||
// Right core boundary: the first dictionary-known token after the
|
||||
// resolved artist. The boundary is contextual, but the title itself
|
||||
// stays a single token ([1], or [2] with an artist dictionary hit).
|
||||
// Unknown tokens inside the core region ([start .. tailStart)) are
|
||||
// REPORTED in UnknownTokens — never eaten, never glued into the title.
|
||||
tailStart := end
|
||||
for i := start; i < end; i++ {
|
||||
if isKnownToken(tokens[i]) {
|
||||
tailStart = i
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve Artist/Title from the core unless the override already did.
|
||||
if result.Artist == "" {
|
||||
if artistDict {
|
||||
// The artist spans tokens [0 .. start-2]; re-join them.
|
||||
result.Artist = strings.Join(tokens[:start-1], "-")
|
||||
} else {
|
||||
result.Artist = tokens[0]
|
||||
}
|
||||
if len(tokens) > start-1 {
|
||||
result.Title = tokens[start-1]
|
||||
}
|
||||
}
|
||||
|
||||
// Core-region leftovers (e.g. the rest of a wildcards title after an
|
||||
// override, or a hyphenated title fragment without an artist entry):
|
||||
// visible in UnknownTokens for review.
|
||||
for _, tok := range tokens[start:tailStart] {
|
||||
result.UnknownTokens = append(result.UnknownTokens, tok)
|
||||
}
|
||||
|
||||
// Level 3: dictionary-driven tail zone, starting after the resolved core.
|
||||
parseTailZone(tokens[tailStart:end], &result)
|
||||
|
||||
score(&result)
|
||||
return result
|
||||
@@ -110,7 +166,7 @@ func split(raw string) []string {
|
||||
// list and returns the cleaned list together with the removed comments.
|
||||
func extractComments(tokens []string) ([]string, []string) {
|
||||
out := make([]string, 0, len(tokens))
|
||||
comments := []string{}
|
||||
var comments []string
|
||||
for _, tok := range tokens {
|
||||
if Profile.CommentTokens[strings.ToUpper(tok)] {
|
||||
comments = append(comments, tok)
|
||||
@@ -135,9 +191,21 @@ func catalogNumber(token string) string {
|
||||
return token[1 : len(token)-1]
|
||||
}
|
||||
|
||||
// isKnownToken reports whether the token belongs to the tail zone: it
|
||||
// matches one of the profile dictionaries or is a parenthesized catalogue
|
||||
// number. The first such token after the core marks the right core
|
||||
// boundary.
|
||||
func isKnownToken(tok string) bool {
|
||||
up := strings.ToUpper(tok)
|
||||
return Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
|
||||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
|
||||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] ||
|
||||
catalogNumber(tok) != ""
|
||||
}
|
||||
|
||||
// parseTailZone resolves middle-zone tokens against the dictionaries.
|
||||
// The first match wins for each field; catalogue numbers are recognised
|
||||
// from parenthesized tokens, and anything unrecognised is appended to
|
||||
// The first match wins for each field; catalogue numbers are recognized
|
||||
// from parenthesized tokens, and anything unrecognized is appended to
|
||||
// UnknownTokens so that it can be reviewed and curated into a dictionary.
|
||||
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||
for _, tok := range zone {
|
||||
@@ -151,6 +219,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||
if result.Source == "" {
|
||||
result.Source = tok
|
||||
}
|
||||
applyType(tok, result)
|
||||
case Profile.BitDepthTokens[up]:
|
||||
if result.BitDepth == "" {
|
||||
result.BitDepth = tok
|
||||
@@ -163,6 +232,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||
if result.Edition == "" {
|
||||
result.Edition = tok
|
||||
}
|
||||
applyType(tok, result)
|
||||
case Profile.LanguageMarkers[up]:
|
||||
if result.Language == "" {
|
||||
result.Language = tok
|
||||
@@ -177,6 +247,33 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||
}
|
||||
}
|
||||
|
||||
// applyType normalizes a matched edition/source token into ParsedRelease.Type
|
||||
// via the types.txt variant map; first match wins. The token still fills its
|
||||
// own raw field (Source stays raw).
|
||||
func applyType(tok string, result *releaseparser.ParsedRelease) {
|
||||
if base, ok := Profile.TypeVariants[strings.ToUpper(tok)]; ok && result.Type == "" {
|
||||
result.Type = base
|
||||
}
|
||||
}
|
||||
|
||||
// overrideCoreLength returns the number of leading tokens covered by the
|
||||
// override's Artist and Title (each may span several hyphen-separated
|
||||
// tokens), falling back to the default 2-token core when the raw tokens
|
||||
// do not reconstruct the override fields.
|
||||
func overrideCoreLength(tokens []string, o releaseparser.Override) int {
|
||||
for a := 1; a <= len(tokens); a++ {
|
||||
if strings.Join(tokens[:a], "-") != o.Artist {
|
||||
continue
|
||||
}
|
||||
for t := a + 1; t <= len(tokens); t++ {
|
||||
if strings.Join(tokens[a:t], "-") == o.Title {
|
||||
return t
|
||||
}
|
||||
}
|
||||
}
|
||||
return 2
|
||||
}
|
||||
|
||||
// score computes a heuristic confidence in [0,1] for a parsed release based
|
||||
// on how many core fields (group, year, format, source, artist) were resolved
|
||||
// and whether any warnings or unknown tokens are present.
|
||||
+138
-45
@@ -1,43 +1,9 @@
|
||||
package moonblood
|
||||
package groups
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"os"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestParseMoonblood(t *testing.T) {
|
||||
file, err := os.Open("../../resources/moonblood.txt")
|
||||
if err != nil {
|
||||
t.Skipf("resource file not found (DB mode), skipping: %v", err)
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
total, success := 0, 0
|
||||
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
result := Parse(line)
|
||||
total++
|
||||
|
||||
if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" {
|
||||
success++
|
||||
}
|
||||
|
||||
if result.Confidence < 0.7 {
|
||||
t.Logf("Low confidence: %s (%.2f)", line, result.Confidence)
|
||||
}
|
||||
}
|
||||
|
||||
t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100)
|
||||
}
|
||||
|
||||
func TestParseFullTail(t *testing.T) {
|
||||
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||
if r.Artist != "Fane" {
|
||||
@@ -71,11 +37,15 @@ func TestParseEditionAndDemo(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseArtistWithHyphen(t *testing.T) {
|
||||
// [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
|
||||
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
|
||||
// Artist сам містить дефіс, але Ultra-Violence ще не в словнику:
|
||||
// core контекстуальний — все між артистом і першим словниковим
|
||||
// маркером (EP) є назвою, зібраною назад через "-".
|
||||
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
|
||||
if r.Artist != "Ultra" || r.Title != "Violence" {
|
||||
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
|
||||
if r.Artist != "Ultra" {
|
||||
t.Errorf("Artist = %q, want Ultra", r.Artist)
|
||||
}
|
||||
if r.Title != "Violence" {
|
||||
t.Errorf("Title = %q, want Violence", r.Title)
|
||||
}
|
||||
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
|
||||
t.Errorf("tail fields wrong: %+v", r)
|
||||
@@ -157,12 +127,16 @@ func TestParseCaseInsensitiveDictionaries(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseUnknownTailTokenReported(t *testing.T) {
|
||||
// Невідомий токен між двома відомими маркерами → у UnknownTokens.
|
||||
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||
// Невідомий токен ПІСЛЯ останнього словникового маркера (за ним уже
|
||||
// нічого відомого немає) → у UnknownTokens.
|
||||
r := Parse("Artist-Some_Title-16BIT-WEB-FLAC-FOO-2025-MOONBLOOD")
|
||||
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
|
||||
t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
|
||||
}
|
||||
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
|
||||
if len(r.TitleExtra) != 0 {
|
||||
t.Errorf("TitleExtra = %v, want empty", r.TitleExtra)
|
||||
}
|
||||
if r.Edition != "" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
|
||||
t.Errorf("tail fields wrong: %+v", r)
|
||||
}
|
||||
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
||||
@@ -170,6 +144,25 @@ func TestParseUnknownTailTokenReported(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseContextualCoreAbsorbsTitleTail(t *testing.T) {
|
||||
// Права межа core контекстуальна (перший словниковий маркер), але
|
||||
// невідомі токени всередині core-зони — в UnknownTokens, не з'їдаються.
|
||||
r := Parse("Artist-Some_Title-Part_Two-Part_Three-16BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||
if r.Artist != "Artist" {
|
||||
t.Errorf("Artist = %q, want Artist", r.Artist)
|
||||
}
|
||||
if r.Title != "Some_Title" {
|
||||
t.Errorf("Title = %q, want Some_Title", r.Title)
|
||||
}
|
||||
if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2025 {
|
||||
t.Errorf("tail fields wrong: %+v", r)
|
||||
}
|
||||
if len(r.UnknownTokens) != 2 || r.UnknownTokens[0] != "Part_Two" ||
|
||||
r.UnknownTokens[1] != "Part_Three" {
|
||||
t.Errorf("UnknownTokens = %v, want [Part_Two Part_Three]", r.UnknownTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseCoreTokensNeverReported(t *testing.T) {
|
||||
// Артист/тайтл ніколи не потрапляють в UnknownTokens.
|
||||
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||
@@ -287,8 +280,8 @@ func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
|
||||
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
|
||||
// потрапляє в UnknownTokens. (SMG409) — катало.
|
||||
// Evolution — не словниковий токен у core-зоні → в UnknownTokens,
|
||||
// не з'їдається. (SMG409) — катало, перший відомий маркер.
|
||||
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
|
||||
if r.CatalogNo != "SMG409" {
|
||||
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
|
||||
@@ -364,7 +357,8 @@ func TestParseCompoundEditionRecognized(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
|
||||
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
|
||||
// Хвіст назви (_Sky_Burial) — не словник, але він ДО першого маркера
|
||||
// (катало (RTM103)) → частина назви, не UnknownTokens.
|
||||
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
|
||||
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
|
||||
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||
@@ -372,6 +366,9 @@ func TestParseMixedCaseTitleNeverReported(t *testing.T) {
|
||||
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
|
||||
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
|
||||
}
|
||||
if r.CatalogNo != "RTM103" {
|
||||
t.Errorf("CatalogNo = %q, want RTM103", r.CatalogNo)
|
||||
}
|
||||
}
|
||||
|
||||
func contains(list []string, s string) bool {
|
||||
@@ -399,3 +396,99 @@ func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
|
||||
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArtistDictionaryCoversReleases(t *testing.T) {
|
||||
// artists.txt: "Iku-Turso" у словнику → Artist = 2 токени,
|
||||
// Title = [2], хвостова зона зсувається і парситься як завжди.
|
||||
r := Parse("Iku-Turso-Ikuinen_Steelfest_MMXXII_Live-24BIT-WEB-FLAC-2016-MOONBLOOD")
|
||||
if r.Artist != "Iku-Turso" {
|
||||
t.Errorf("Artist = %q, want Iku-Turso", r.Artist)
|
||||
}
|
||||
if r.Title != "Ikuinen_Steelfest_MMXXII_Live" {
|
||||
t.Errorf("Title = %q, want Ikuinen_Steelfest_MMXXII_Live", r.Title)
|
||||
}
|
||||
if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2016 {
|
||||
t.Errorf("tail fields wrong: %+v", r)
|
||||
}
|
||||
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
|
||||
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
|
||||
r.UnknownTokens, r.TitleExtra, r.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseArtistDictionaryMultiTokenArtist(t *testing.T) {
|
||||
// artists.txt: "Shadu-Nar-Mattaru" — артист з ТРЬОХ токенів.
|
||||
// Найдовший словниковий префікс виграє: Artist = 3 токени,
|
||||
// Title = наступний токен, хвіст парситься як завжди.
|
||||
r := Parse("Shadu-Nar-Mattaru-On_Pathways_Macabre-24BIT-WEB-FLAC-2023-MOONBLOOD")
|
||||
if r.Artist != "Shadu-Nar-Mattaru" {
|
||||
t.Errorf("Artist = %q, want Shadu-Nar-Mattaru", r.Artist)
|
||||
}
|
||||
if r.Title != "On_Pathways_Macabre" {
|
||||
t.Errorf("Title = %q, want On_Pathways_Macabre", r.Title)
|
||||
}
|
||||
if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2023 {
|
||||
t.Errorf("tail fields wrong: %+v", r)
|
||||
}
|
||||
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
|
||||
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
|
||||
r.UnknownTokens, r.TitleExtra, r.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
|
||||
// overrides.txt (найвищий пріоритет): точний lookup по Raw
|
||||
// фіксує Artist/Title; хвіст парситься як завжди.
|
||||
raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2013-MOONBLOOD"
|
||||
r := Parse(raw)
|
||||
if r.Artist != "Milicia_Oscura" {
|
||||
t.Errorf("Artist = %q, want Milicia_Oscura", r.Artist)
|
||||
}
|
||||
if r.Title != "Anti-Cristo" {
|
||||
t.Errorf("Title = %q, want Anti-Cristo", r.Title)
|
||||
}
|
||||
if r.Language != "ES" || r.BitDepth != "16BIT" || r.Source != "WEB" ||
|
||||
r.Format != "FLAC" || r.Year != 2013 || r.Group != "MOONBLOOD" {
|
||||
t.Errorf("tail fields wrong: %+v", r)
|
||||
}
|
||||
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
|
||||
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
|
||||
r.UnknownTokens, r.TitleExtra, r.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTypeNormalizedSourceStaysRaw(t *testing.T) {
|
||||
// CDEP у source.txt: Source лишається сирим, Type нормалізується в EP.
|
||||
r := Parse("Black_Ceremonial_Kult-Crowned_in_Chaos-CDEP-FLAC-2021-MOONBLOOD")
|
||||
if r.Type != "EP" {
|
||||
t.Errorf("Type = %q, want EP", r.Type)
|
||||
}
|
||||
if r.Source != "CDEP" {
|
||||
t.Errorf("Source = %q, want raw CDEP", r.Source)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseTypeFirstMatchWins(t *testing.T) {
|
||||
// REMASTERED_EP → EP; наступний типовий токен не перезаписує.
|
||||
r := Parse("Some_Artist-Some_Title-REMASTERED_EP-SINGLE-FLAC-2021-MOONBLOOD")
|
||||
if r.Type != "EP" {
|
||||
t.Errorf("Type = %q, want EP (first match wins)", r.Type)
|
||||
}
|
||||
if r.Edition != "REMASTERED_EP" {
|
||||
t.Errorf("Edition = %q, want REMASTERED_EP", r.Edition)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyzeTypeDistribution(t *testing.T) {
|
||||
stats := AnalyzeTitles([]string{
|
||||
"Some_Artist-Some_Title-CDEP-FLAC-2021-MOONBLOOD",
|
||||
"Other_Artist-Other_Title-EP-FLAC-2022-MOONBLOOD",
|
||||
"Third_Artist-Third_Title-SPLIT-FLAC-2023-MOONBLOOD",
|
||||
})
|
||||
if stats.TypeDistribution["EP"] != 2 {
|
||||
t.Errorf("TypeDistribution[EP] = %d, want 2", stats.TypeDistribution["EP"])
|
||||
}
|
||||
if stats.TypeDistribution["Split"] != 1 {
|
||||
t.Errorf("TypeDistribution[Split] = %d, want 1", stats.TypeDistribution["Split"])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package groups
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"strings"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||
)
|
||||
|
||||
// The token dictionaries live under resources/dicts/*.txt — one token (in any
|
||||
// case) per line; blank lines and lines starting with "#" are ignored. They
|
||||
// can be edited without recompiling the parsing logic: go:embed pulls them
|
||||
// into the binary at build time.
|
||||
|
||||
//go:embed resources/dicts/*.txt
|
||||
var dictFS embed.FS
|
||||
|
||||
func loadDict(name string) map[string]bool {
|
||||
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
||||
if err != nil {
|
||||
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
||||
}
|
||||
out := make(map[string]bool)
|
||||
for line := range strings.SplitSeq(string(data), "\n") {
|
||||
line = strings.ToUpper(strings.TrimSpace(line))
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
out[line] = true
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// loadOverrides parses the override table: "<raw line> => artist | title".
|
||||
func loadOverrides(name string) map[string]releaseparser.Override {
|
||||
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
||||
if err != nil {
|
||||
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
||||
}
|
||||
out := make(map[string]releaseparser.Override)
|
||||
for line := range strings.SplitSeq(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
head, rest, ok := strings.Cut(line, "=>")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
artist, title, ok := strings.Cut(strings.TrimSpace(rest), "|")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
out[strings.TrimSpace(head)] = releaseparser.Override{
|
||||
Artist: strings.TrimSpace(artist),
|
||||
Title: strings.TrimSpace(title),
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// loadTypeVariants parses types.txt: "<Base> = variant, variant, ..." and
|
||||
// returns variant (uppercased) -> base as written in the file.
|
||||
func loadTypeVariants(name string) map[string]string {
|
||||
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
||||
if err != nil {
|
||||
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
||||
}
|
||||
out := make(map[string]string)
|
||||
for line := range strings.SplitSeq(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
base, rest, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
base = strings.TrimSpace(base)
|
||||
for v := range strings.SplitSeq(rest, ",") {
|
||||
v = strings.ToUpper(strings.TrimSpace(v))
|
||||
if v != "" {
|
||||
out[v] = base
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Profile holds the shared dictionaries for the release parser. The parser is
|
||||
// group-agnostic: the group name is read from each release line, never from
|
||||
// this profile, so there is a single set of dictionaries covering all groups.
|
||||
var Profile = releaseparser.GroupProfile{
|
||||
FormatTokens: loadDict("format"),
|
||||
ArtistTokens: loadDict("artists"),
|
||||
TypeVariants: loadTypeVariants("types"),
|
||||
SourceTokens: loadDict("source"),
|
||||
BitDepthTokens: loadDict("bitdepth"),
|
||||
ResolutionTokens: loadDict("resolution"),
|
||||
LanguageMarkers: loadDict("language"),
|
||||
EditionTokens: loadDict("edition"),
|
||||
Overrides: loadOverrides("overrides"),
|
||||
CommentTokens: loadDict("comment"),
|
||||
}
|
||||
@@ -1,8 +1,14 @@
|
||||
package releaseparser
|
||||
|
||||
type Override struct {
|
||||
Artist string
|
||||
Title string
|
||||
}
|
||||
|
||||
type GroupProfile struct {
|
||||
Name string
|
||||
ExpectedTailOrder []string
|
||||
ArtistTokens map[string]bool // hyphenated artist whitelist
|
||||
TypeVariants map[string]string // variant (upper) -> normalized Type
|
||||
Overrides map[string]Override
|
||||
FormatTokens map[string]bool
|
||||
SourceTokens map[string]bool
|
||||
BitDepthTokens map[string]bool
|
||||
@@ -15,18 +21,21 @@ type GroupProfile struct {
|
||||
type ParsedRelease struct {
|
||||
Raw string
|
||||
Group string
|
||||
Artist string
|
||||
Year int
|
||||
Title string
|
||||
TitleExtra []string
|
||||
Type string
|
||||
Source string
|
||||
Edition string
|
||||
CatalogNo string
|
||||
Language string
|
||||
BitDepth string
|
||||
Resolution string
|
||||
Comment string
|
||||
CatalogNo string
|
||||
Source string
|
||||
Format string
|
||||
Language string
|
||||
Edition string
|
||||
Core string
|
||||
Artist string
|
||||
Title string
|
||||
Override Override
|
||||
UnknownTokens []string
|
||||
Warnings []string
|
||||
Confidence float64
|
||||
@@ -40,6 +49,7 @@ type UnknownTokenStats struct {
|
||||
|
||||
type GroupStats struct {
|
||||
Total int
|
||||
TypeDistribution map[string]int
|
||||
YearDistribution map[int]int
|
||||
SourceDistribution map[string]int
|
||||
BitDepthDistribution map[string]int
|
||||
|
||||
@@ -85,16 +85,25 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
|
||||
}
|
||||
var validID = regexp.MustCompile(`-\d+\/$`)
|
||||
|
||||
for i, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
|
||||
for i, item := range dom.QuerySelectorAll(doc, "li.row") {
|
||||
var es model.ExternalSources
|
||||
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
|
||||
|
||||
// TypeId тепер напряму з data-id на <li>, без regex по class
|
||||
idAttr := dom.GetAttribute(item, "data-id")
|
||||
es.TypeId, _ = strconv.Atoi(idAttr)
|
||||
|
||||
title := dom.QuerySelector(item, ".title")
|
||||
if title != nil {
|
||||
anchor := dom.QuerySelector(title, "h1 > a")
|
||||
anchor := dom.QuerySelector(title, "a")
|
||||
if anchor != nil {
|
||||
es.Type = constant.ScopePrescene
|
||||
es.Title = dom.GetAttribute(anchor, "title")
|
||||
|
||||
titleSpan := dom.QuerySelector(anchor, "span.title-start")
|
||||
if titleSpan != nil {
|
||||
es.Title = dom.TextContent(titleSpan)
|
||||
}
|
||||
|
||||
if es.Title == "Auto Draft" {
|
||||
slog.Info("Skipped", "title", es.Title)
|
||||
continue
|
||||
@@ -123,17 +132,13 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
|
||||
continue
|
||||
}
|
||||
|
||||
localtime := dom.QuerySelector(title, "small > span.localtime")
|
||||
localtime := dom.QuerySelector(item, "span.localtime")
|
||||
if localtime != nil {
|
||||
lc := dom.GetAttribute(localtime, "data-lttime")
|
||||
es.Created = carbon.Parse(lc, "Europe/Kyiv")
|
||||
}
|
||||
}
|
||||
|
||||
cls := dom.GetAttribute(item, "class")
|
||||
pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
|
||||
es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
|
||||
|
||||
esModel := table.ExternalSources{Columns: columns}
|
||||
entry := esModel.InsertOnDuplicate(es, db)
|
||||
entries = append(entries, entry)
|
||||
|
||||
Reference in New Issue
Block a user