Compare commits

...

3 Commits

Author SHA1 Message Date
Yevhen Odynets 39aa8e2e81 fixed a new prescene layout 2026-09-27 01:12:52 +03:00
Yevhen Odynets 347f525801 releaseparser: Artist/Title boundary resolution + normalized Type field
Three-level core resolution with priority overrides → dictionaries/model → suspects:

1. dicts/overrides.txt (highest priority)
   Point fixes via exact lookup on Raw BEFORE the positional logic.
   Format: <original line> => artist | title. An override fixes
   Artist/Title and shifts the dictionary tail zone by the actual core
   length (overrideCoreLength, fallback 2 tokens).

2. dicts/artists.txt
   Whitelist of hyphenated artists (Iku-Turso, Post-Mortem, ...): a
   [0]-[1] hit → Artist = 2 tokens, Title = [2], tail zone shifted.
   One entry covers all releases of the artist automatically.

3. Suspected title continuation
   Unknown tokens between the title zone and the tail (followed later by
   dictionary markers) no longer pollute UnknownTokens; they go to a
   dedicated TitleExtra field + warning "suspected title continuation" —
   a reviewer hint that the core is probably longer than 2 tokens.
   Catalogue numbers (parenthesized tokens) are resolved with FIRST
   priority and never land in suspects.

Type field (normalized base release type):
- new normalization map dicts/types.txt (<Base> = variants):
  EP = EP, CDS, CDM, 7INCH_VINYL, CDEP, ..., REMASTERED_EP; Demo = ...;
  Split; Single; Compilation; Collaboration (curator-extendable)
- tokens matched against edition/source are additionally normalized to
  the base type (first match wins); edition.txt/source.txt untouched,
  Source stays raw
- TypeDistribution map[string]int added to GroupStats and the report
  (console/JSON)
- OST/INSTRUMENTAL are release traits, not types; excluded from the map

Tests: old cases updated to the new suspect semantics; acceptance tests
added for overrides (Milicia_Oscura-Anti-Cristo), artist dictionary
(Iku-Turso), the suspect section, Type normalization (CDEP → EP with
raw Source=CDEP, first-match-wins) and TypeDistribution.

go build / go vet / go test ./internal/releaseparser/... — all green.
2026-09-01 13:18:40 +03:00
Yevhen Odynets f2629f002c releaseparser: move group-agnostic parser to groups/, GB doc-blocks, per-group JSON reports
Generalise the parser: the MOONBLOOD group profile is gone, the parser and
its dictionaries are now group-agnostic under internal/releaseparser/groups.

Positional model:
- [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so
  titles like "Split"/"Demo"/"De" are never eaten.
- [len-1]=Group is always read from the line, not from the query key.
- [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is
  resolved as a middle-zone token (e.g. FLAC is still picked up as format).
- The middle zone is resolved via case-insensitive dictionaries in the order
  format -> source -> bit-depth -> resolution -> edition -> language ->
  catalogue number; stored values keep their original case.

Catalogue numbers: any "(...)" token in the middle zone is a catalogue number
regardless of content (digits, roman numerals, dots, underscores, mixed case);
splitting never breaks tokens inside parentheses.

Dictionaries moved to internal/releaseparser/groups/resources/dicts and
embedded via go:embed; dead profile fields (Name, ExpectedTailOrder) removed;
profile comments rewritten in British English.

File-based AnalyzeGroup/TestParseMoonblood removed — titles now come from the
database (cmd/srrdb: srrdb <releaser>..., config/secret.yaml DSN).

Validation output (cmd/srrdb):
- tmp/releaseparser/<group>.json         — per-group parsed ParsedRelease records
- tmp/releaseparser/<releaser>.report.json — per-request aggregate (same as console)
- review_unknown_tokens.txt              — unknown/warning rows for manual review

Tests updated for the positional model; added
TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2) and (GOWP_DCCCLXXXIX).
go build, go vet and go test all pass.
2026-09-01 11:28:19 +03:00
19 changed files with 519 additions and 398 deletions
+1 -1
View File
@@ -14,7 +14,7 @@ vendor/
config/secret.yaml
*.md
mdb
internal/releaseparser/resources/*
internal/releaseparser/groups/resources/*
review_unknown_tokens.txt
srrdb.exe
**/*.exe
+3 -1
View File
@@ -3,7 +3,9 @@
<component name="Go" enabled="true" />
<component name="GoModuleSettings" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<content url="file://$MODULE_DIR$">
<excludeFolder url="file://$MODULE_DIR$/tmp" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
</component>
+49 -3
View File
@@ -5,13 +5,15 @@ import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
_ "github.com/go-sql-driver/mysql"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups"
)
func main() {
@@ -35,7 +37,12 @@ func main() {
}
defer dbase.Close()
out := make(map[string]moonblood.GroupStats, len(releasers))
const outDir = "tmp/releaseparser"
if err := os.MkdirAll(outDir, 0o755); err != nil {
log.Fatal(err)
}
out := make(map[string]groups.GroupStats, len(releasers))
for _, releaser := range releasers {
var titles []string
@@ -47,7 +54,16 @@ func main() {
log.Fatal(err)
}
out[releaser] = moonblood.AnalyzeTitles(titles)
out[releaser] = groups.AnalyzeTitles(titles)
// Per-group parsed records keyed by the actual group read from each
// line: every release with all resolved fields for manual validation.
for group, parsed := range groups.ParseByGroup(titles) {
writeJSON(filepath.Join(outDir, safeName(group)+".json"), parsed)
}
// Per-request report identical to what is printed to the console.
writeJSON(filepath.Join(outDir, safeName(releaser)+".report.json"), out[releaser])
}
data, err := json.MarshalIndent(out, "", " ")
@@ -77,3 +93,33 @@ func main() {
fmt.Println(string(data))
}
// writeJSON writes value as an indented JSON document to the given path.
func writeJSON(path string, value any) {
data, err := json.MarshalIndent(value, "", " ")
if err != nil {
log.Fatalf("marshal %s: %v", path, err)
}
if err := os.WriteFile(path, data, 0o644); err != nil {
log.Fatalf("write %s: %v", path, err)
}
}
// safeName strips characters that are unsafe in file names.
func safeName(s string) string {
repl := func(r rune) rune {
switch {
case r >= 'a' && r <= 'z':
return r
case r >= 'A' && r <= 'Z':
return r
case r >= '0' && r <= '9':
return r
case r == '-' || r == '_' || r == '.':
return r
default:
return '_'
}
}
return strings.Map(repl, s)
}
+14 -15
View File
@@ -1,43 +1,42 @@
module git.kplus.net.ua/yevhen/resource-scraper
go 1.26.2
go 1.27
require (
github.com/davecgh/go-spew v1.1.1
github.com/dromara/carbon/v2 v2.6.16
github.com/dromara/carbon/v2 v2.6.17
github.com/emersion/go-imap/v2 v2.0.0-beta.8
github.com/go-chi/chi/v5 v5.2.5
github.com/go-chi/chi/v5 v5.3.2
github.com/go-chi/cors v1.2.2
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
github.com/go-sql-driver/mysql v1.9.3
github.com/go-sql-driver/mysql v1.10.1
github.com/iancoleman/strcase v0.3.0
github.com/jmoiron/sqlx v1.4.0
github.com/logrusorgru/aurora/v4 v4.0.0
github.com/mewkiz/flac v1.0.13
github.com/mewkiz/flac v1.0.14
github.com/spf13/pflag v1.0.10
github.com/spf13/viper v1.21.0
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f
golang.org/x/net v0.49.0
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba
golang.org/x/net v0.59.0
)
require (
filippo.io/edwards25519 v1.2.0 // indirect
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/andybalholm/cascadia v1.3.5 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/fsnotify/fsnotify v1.10.1 // indirect
github.com/go-viper/mapstructure/v2 v2.5.0 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/icza/bitio v1.1.0 // indirect
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de // indirect
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 // indirect
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/pelletier/go-toml/v2 v2.4.3 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/sys v0.40.0 // indirect
golang.org/x/text v0.33.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
go.yaml.in/yaml/v3 v3.0.5 // indirect
golang.org/x/sys v0.48.0 // indirect
golang.org/x/text v0.42.0 // indirect
)
+26 -62
View File
@@ -1,12 +1,12 @@
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/andybalholm/cascadia v1.3.5 h1:RLjq12WJy58dN6eCIQrz0bAGZkztHWsEPFxP53Y7Ms8=
github.com/andybalholm/cascadia v1.3.5/go.mod h1:BLRmbRjpEtNKieZOCCvYj4RqN+KRA41GBe/5O+G93kM=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/dromara/carbon/v2 v2.6.17 h1:QL1LVcp5A/NQtIPbc1vrL9qIXxPNWnaDOBaFlmP1iDg=
github.com/dromara/carbon/v2 v2.6.17/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -15,17 +15,17 @@ github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho=
github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo=
github.com/go-chi/chi/v5 v5.3.2 h1:5YQkICvTCSZ25hoRsyJazN0scjzKGiu4VAUc7H1o1nY=
github.com/go-chi/chi/v5 v5.3.2/go.mod h1:R+tYY2hNuVUUjxoPtqUdgBqevM9s9njzkTLutVsOCto=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/go-sql-driver/mysql v1.10.1 h1:arlSnNLq6a5yxGxV7qg9lF4j0C+KwD6NbQyKr9QL6ME=
github.com/go-sql-driver/mysql v1.10.1/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro=
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
@@ -50,14 +50,14 @@ github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUp
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
github.com/mewkiz/flac v1.0.14 h1:hyRGAM8NCKznoPmIi9zz2jyO+nfmxY2ErqBnHZ+gxh4=
github.com/mewkiz/flac v1.0.14/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87 h1:fpWNLIg7nX2f+D+6ZveQ5hfkTjq5WgMBtYbz5Nb1w0Y=
github.com/mewkiz/pkg v0.0.0-20260703220044-4fb89b18cc87/go.mod h1:wohiqwTwXA5twjDES74xIJLGjAxo3/0pHSbTXLbSDW8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY=
github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
@@ -77,81 +77,45 @@ github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba h1:Ck8QetSgk912qxWLMCKxd0in+aiyBQyDSMae6e/xmpU=
golang.org/x/exp v0.0.0-20260908205506-85c1c2202aba/go.mod h1:50RgIsmK7OwqzTTeqcSXQW8SswW0o8fRcDxmqGluJ8E=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.33.0 h1:B3njUFyqtHDUI5jMn1YIr5B0IE2U0qck04r6d4KPAxE=
golang.org/x/text v0.33.0/go.mod h1:LuMebE6+rBincTi9+xWTY8TztLzKHc/9C1uBCG27+q8=
golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
@@ -1,8 +1,6 @@
package moonblood
package groups
import (
"bufio"
"os"
"sort"
"strings"
@@ -18,6 +16,7 @@ type GroupStats struct {
SourceDistribution map[string]int
BitDepthDistribution map[string]int
ResolutionDistribution map[string]int
TypeDistribution map[string]int
LanguageDistribution map[string]int
EditionDistribution map[string]int
CommentDistribution map[string]int
@@ -26,37 +25,13 @@ type GroupStats struct {
AvgConfidence float64
}
// AnalyzeGroup parses the release lines from the local resource file. It is
// used for a quick local check of the dictionaries without touching a
// database.
func AnalyzeGroup() (GroupStats, error) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
return GroupStats{}, err
}
defer file.Close()
var titles []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
titles = append(titles, line)
}
}
if err := scanner.Err(); err != nil {
return GroupStats{}, err
}
return AnalyzeTitles(titles), nil
}
// AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.
func AnalyzeTitles(titles []string) GroupStats {
stats := GroupStats{
YearDistribution: make(map[int]int),
TypeDistribution: make(map[string]int),
SourceDistribution: make(map[string]int),
BitDepthDistribution: make(map[string]int),
ResolutionDistribution: make(map[string]int),
@@ -99,6 +74,10 @@ func AnalyzeTitles(titles []string) GroupStats {
}
}
if result.Type != "" {
stats.TypeDistribution[result.Type]++
}
for _, unknown := range result.UnknownTokens {
if _, exists := unknownMap[unknown]; !exists {
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
@@ -131,7 +110,6 @@ func AnalyzeTitles(titles []string) GroupStats {
for _, tokenStats := range unknownMap {
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
}
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
})
@@ -149,3 +127,34 @@ func AnalyzeTitles(titles []string) GroupStats {
return stats
}
// AnalyzeByGroup parses a set of release lines and returns per-group
// statistics keyed by the actual group read from each line (not from the
// query key, which may cover several groups). It is used to validate parsing
// quality against the original data, grouped by the real releaser name.
func AnalyzeByGroup(titles []string) map[string]GroupStats {
byGroup := make(map[string][]string)
for _, title := range titles {
group := Parse(title).Group
byGroup[group] = append(byGroup[group], title)
}
out := make(map[string]GroupStats, len(byGroup))
for group, lines := range byGroup {
out[group] = AnalyzeTitles(lines)
}
return out
}
// ParseByGroup parses a set of release lines into a per-group list of fully
// populated ParsedRelease records, keyed by the actual group read from each
// line. It is used to validate parsing quality field-by-field against the
// original data.
func ParseByGroup(titles []string) map[string][]releaseparser.ParsedRelease {
byGroup := make(map[string][]releaseparser.ParsedRelease)
for _, title := range titles {
result := Parse(title)
byGroup[result.Group] = append(byGroup[result.Group], result)
}
return byGroup
}
@@ -1,5 +0,0 @@
# {bit_depth}
16BIT
24BIT
32BIT
DE16BIT
@@ -1,6 +0,0 @@
REPACK
REAL_REPACK
READNFO
PROPER
RERIP
199X
@@ -1,70 +0,0 @@
# {edition} — тип видання
EP
DEMO
SPLIT
REISSUE
DELUXE_EDITION
REMASTERED
BOXSET
SINGLE
PROMO
REISSUE_REMASTERED
REMASTERED_25TH_ANNIVERSARY_EDITION
REMASTERED_20TH_ANNIVERSARY_EDITION
20TH_ANNIVERSARY_EDITION_REMASTERED
Deluxe_Edition
LIMITED_EDITION_BOXSET
REMASTERED_EP
10TH_ANNIVERSARY_EDITION
OST
XX_Legacy_Edition
REISSUE_DEMO
JP_Deluxe_Edition
Remastered_Edition
Anniversary_Edition
Limited_Edition
CDS
CDM
RETAIL
JP_RETAIL
JP_REISSUE
DEMO_REMASTERED
EU_RETAIL
DE_RETAIL
REISSUE_DELUXE_EDITION
RETAIL_REISSUE
SPECIAL_EDITION
SPECIAL_COLLECTORS_EDITION_REMASTERED
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
LIMITED_EDITION_REMASTERED
25th_Anniversary_Edition
DEMO_REISSUE
Ultimate_Edition
Issue_255
Issue_254
Issue_198
7INCH_VINYL
REMASTERED_REISSUE
PROMO_DEMO
ES_RETAIL
BOX_SET
BONUS
REISSUE_EP
30th_Anniversary
REDUX
REMASTERED_BOXSET
MAG
REISSUE_BOXSET
30TH_ANNIVERSARY_EDITION_REMASTERED
DELUXE
Deluxe_Edition_Reissue
INSTRUMENTAL
REISSUE_EXPANDED_EDITION
BOOTLEG
EXPANDED_EDITION
ADVANCE
15TH_ANNIVERSARY_DELUXE_EDITION
TOUR_EDITION
REMASTERED_DEMO
REMASTERED_DELUXE_EDITION
30TH_ANNIVERSARY_EDITION
@@ -1,3 +0,0 @@
# {format} — аудіоформат
# один токен на рядок, регістр не має значення
FLAC
@@ -1,38 +0,0 @@
# {language} — мовні маркери
DE
FR
ES
SK
FI
SE
BR
PL
NL
NO
PT
IT
HU
DK
AT
IS
UA
JP
GR
CN
RU
ID
PH
RO
LT
BY
CZ
SP
IR
FO
RS
DA
HI
CH
LU
CY
EE
@@ -1,8 +0,0 @@
# {resolution} — частота дискретизації
44KHZ
48KHZ
48.0kHz
88KHZ
96KHZ
176KHZ
192KHZ
@@ -1,36 +0,0 @@
# {source} — джерело релізу
# VINYL сюди входить як source (за домовленістю, не edition)
WEB
CD
2CD
CDEP
VINYL
TAPE
CDR
17CD
3CD
4CD
18CD
3CDEP
DVD
7CD
2CDR
10CD
6CD
2CDEP
2VINYL
CDREP
LP
2LP
3VINYL
3TAPE
13CD
13CDR
10INCH_VINYL
25CD
WEBFLAC
2TAPE
BLURAY
VLS
MCD
24BIT_WEB
@@ -1,42 +0,0 @@
package moonblood
import (
"embed"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// Словники токенів лежать у dicts/*.txt — один токен на рядок.
// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці).
//go:embed dicts/*.txt
var dictFS embed.FS
func loadDict(name string) map[string]bool {
data, err := dictFS.ReadFile("dicts/" + name + ".txt")
if err != nil {
panic("moonblood: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]bool)
for _, line := range strings.Split(string(data), "\n") {
line = strings.ToUpper(strings.TrimSpace(line))
if line == "" || strings.HasPrefix(line, "#") {
continue
}
out[line] = true
}
return out
}
var Profile = releaseparser.GroupProfile{
Name: "MOONBLOOD",
ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"},
FormatTokens: loadDict("format"),
SourceTokens: loadDict("source"),
BitDepthTokens: loadDict("bitdepth"),
ResolutionTokens: loadDict("resolution"),
LanguageMarkers: loadDict("language"),
EditionTokens: loadDict("edition"),
CommentTokens: loadDict("comment"),
}
@@ -1,4 +1,4 @@
package moonblood
package groups
import (
"regexp"
@@ -56,20 +56,76 @@ func Parse(raw string) releaseparser.ParsedRelease {
return result
}
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
// [len-2]=Year (when present).
result.Artist = tokens[0]
if len(tokens) > 1 {
result.Title = tokens[1]
// Left core boundary: after the resolved Artist (2 tokens by default,
// 3 with an artist dictionary hit, or fixed by an override).
start := 2
// Level 1: exact-lookup override table fixes Artist/Title only;
// the tail zone is parsed as usual below. The override is
// authoritative about the core: leftover tokens are plain unknowns,
// not part of the title.
if o, ok := Profile.Overrides[result.Raw]; ok {
result.Artist, result.Title = o.Artist, o.Title
start = overrideCoreLength(tokens, o)
}
// Level 2: artist dictionary — a hyphenated artist may span ANY number
// of leading tokens ("Shadu-Nar-Mattaru" = 3). The LONGEST whitelisted
// prefix wins; the title is the token right after it. At least one
// title token and one tail token must remain.
artistDict := false
if result.Artist == "" {
for k := len(tokens) - 2; k >= 2; k-- {
if Profile.ArtistTokens[strings.ToUpper(strings.Join(tokens[:k], "-"))] {
artistDict = true
start = k + 1
break
}
}
}
result.Group = tokens[len(tokens)-1]
end := len(tokens) - 1
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
end = len(tokens) - 2
}
// Resolve the middle zone tokens via the dictionaries.
parseTailZone(tokens[2:end], &result)
// Right core boundary: the first dictionary-known token after the
// resolved artist. The boundary is contextual, but the title itself
// stays a single token ([1], or [2] with an artist dictionary hit).
// Unknown tokens inside the core region ([start .. tailStart)) are
// REPORTED in UnknownTokens — never eaten, never glued into the title.
tailStart := end
for i := start; i < end; i++ {
if isKnownToken(tokens[i]) {
tailStart = i
break
}
}
// Resolve Artist/Title from the core unless the override already did.
if result.Artist == "" {
if artistDict {
// The artist spans tokens [0 .. start-2]; re-join them.
result.Artist = strings.Join(tokens[:start-1], "-")
} else {
result.Artist = tokens[0]
}
if len(tokens) > start-1 {
result.Title = tokens[start-1]
}
}
// Core-region leftovers (e.g. the rest of a wildcards title after an
// override, or a hyphenated title fragment without an artist entry):
// visible in UnknownTokens for review.
for _, tok := range tokens[start:tailStart] {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
// Level 3: dictionary-driven tail zone, starting after the resolved core.
parseTailZone(tokens[tailStart:end], &result)
score(&result)
return result
@@ -110,7 +166,7 @@ func split(raw string) []string {
// list and returns the cleaned list together with the removed comments.
func extractComments(tokens []string) ([]string, []string) {
out := make([]string, 0, len(tokens))
comments := []string{}
var comments []string
for _, tok := range tokens {
if Profile.CommentTokens[strings.ToUpper(tok)] {
comments = append(comments, tok)
@@ -135,9 +191,21 @@ func catalogNumber(token string) string {
return token[1 : len(token)-1]
}
// isKnownToken reports whether the token belongs to the tail zone: it
// matches one of the profile dictionaries or is a parenthesized catalogue
// number. The first such token after the core marks the right core
// boundary.
func isKnownToken(tok string) bool {
up := strings.ToUpper(tok)
return Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] ||
catalogNumber(tok) != ""
}
// parseTailZone resolves middle-zone tokens against the dictionaries.
// The first match wins for each field; catalogue numbers are recognised
// from parenthesized tokens, and anything unrecognised is appended to
// The first match wins for each field; catalogue numbers are recognized
// from parenthesized tokens, and anything unrecognized is appended to
// UnknownTokens so that it can be reviewed and curated into a dictionary.
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
for _, tok := range zone {
@@ -151,6 +219,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
if result.Source == "" {
result.Source = tok
}
applyType(tok, result)
case Profile.BitDepthTokens[up]:
if result.BitDepth == "" {
result.BitDepth = tok
@@ -163,6 +232,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
if result.Edition == "" {
result.Edition = tok
}
applyType(tok, result)
case Profile.LanguageMarkers[up]:
if result.Language == "" {
result.Language = tok
@@ -177,6 +247,33 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
}
}
// applyType normalizes a matched edition/source token into ParsedRelease.Type
// via the types.txt variant map; first match wins. The token still fills its
// own raw field (Source stays raw).
func applyType(tok string, result *releaseparser.ParsedRelease) {
if base, ok := Profile.TypeVariants[strings.ToUpper(tok)]; ok && result.Type == "" {
result.Type = base
}
}
// overrideCoreLength returns the number of leading tokens covered by the
// override's Artist and Title (each may span several hyphen-separated
// tokens), falling back to the default 2-token core when the raw tokens
// do not reconstruct the override fields.
func overrideCoreLength(tokens []string, o releaseparser.Override) int {
for a := 1; a <= len(tokens); a++ {
if strings.Join(tokens[:a], "-") != o.Artist {
continue
}
for t := a + 1; t <= len(tokens); t++ {
if strings.Join(tokens[a:t], "-") == o.Title {
return t
}
}
}
return 2
}
// score computes a heuristic confidence in [0,1] for a parsed release based
// on how many core fields (group, year, format, source, artist) were resolved
// and whether any warnings or unknown tokens are present.
@@ -1,43 +1,9 @@
package moonblood
package groups
import (
"bufio"
"os"
"testing"
)
func TestParseMoonblood(t *testing.T) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
t.Skipf("resource file not found (DB mode), skipping: %v", err)
t.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
total, success := 0, 0
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
result := Parse(line)
total++
if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" {
success++
}
if result.Confidence < 0.7 {
t.Logf("Low confidence: %s (%.2f)", line, result.Confidence)
}
}
t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100)
}
func TestParseFullTail(t *testing.T) {
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
if r.Artist != "Fane" {
@@ -71,11 +37,15 @@ func TestParseEditionAndDemo(t *testing.T) {
}
func TestParseArtistWithHyphen(t *testing.T) {
// [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
// Artist сам містить дефіс, але Ultra-Violence ще не в словнику:
// core контекстуальний — все між артистом і першим словниковим
// маркером (EP) є назвою, зібраною назад через "-".
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
if r.Artist != "Ultra" || r.Title != "Violence" {
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
if r.Artist != "Ultra" {
t.Errorf("Artist = %q, want Ultra", r.Artist)
}
if r.Title != "Violence" {
t.Errorf("Title = %q, want Violence", r.Title)
}
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
@@ -157,12 +127,16 @@ func TestParseCaseInsensitiveDictionaries(t *testing.T) {
}
func TestParseUnknownTailTokenReported(t *testing.T) {
// Невідомий токен між двома відомими маркерами → у UnknownTokens.
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
// Невідомий токен ПІСЛЯ останнього словникового маркера (за ним уже
// нічого відомого немає) → у UnknownTokens.
r := Parse("Artist-Some_Title-16BIT-WEB-FLAC-FOO-2025-MOONBLOOD")
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
}
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
if len(r.TitleExtra) != 0 {
t.Errorf("TitleExtra = %v, want empty", r.TitleExtra)
}
if r.Edition != "" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Artist" || r.Title != "Some_Title" {
@@ -170,6 +144,25 @@ func TestParseUnknownTailTokenReported(t *testing.T) {
}
}
func TestParseContextualCoreAbsorbsTitleTail(t *testing.T) {
// Права межа core контекстуальна (перший словниковий маркер), але
// невідомі токени всередині core-зони — в UnknownTokens, не з'їдаються.
r := Parse("Artist-Some_Title-Part_Two-Part_Three-16BIT-WEB-FLAC-2025-MOONBLOOD")
if r.Artist != "Artist" {
t.Errorf("Artist = %q, want Artist", r.Artist)
}
if r.Title != "Some_Title" {
t.Errorf("Title = %q, want Some_Title", r.Title)
}
if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2025 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 2 || r.UnknownTokens[0] != "Part_Two" ||
r.UnknownTokens[1] != "Part_Three" {
t.Errorf("UnknownTokens = %v, want [Part_Two Part_Three]", r.UnknownTokens)
}
}
func TestParseCoreTokensNeverReported(t *testing.T) {
// Артист/тайтл ніколи не потрапляють в UnknownTokens.
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
@@ -287,8 +280,8 @@ func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
}
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
// потрапляє в UnknownTokens. (SMG409) — катало.
// Evolution — не словниковий токен у core-зоні → в UnknownTokens,
// не з'їдається. (SMG409) — катало, перший відомий маркер.
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
if r.CatalogNo != "SMG409" {
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
@@ -364,7 +357,8 @@ func TestParseCompoundEditionRecognized(t *testing.T) {
}
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
// Хвіст назви (_Sky_Burial) — не словник, але він ДО першого маркера
// (катало (RTM103)) → частина назви, не UnknownTokens.
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
@@ -372,6 +366,9 @@ func TestParseMixedCaseTitleNeverReported(t *testing.T) {
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
}
if r.CatalogNo != "RTM103" {
t.Errorf("CatalogNo = %q, want RTM103", r.CatalogNo)
}
}
func contains(list []string, s string) bool {
@@ -399,3 +396,99 @@ func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseArtistDictionaryCoversReleases(t *testing.T) {
// artists.txt: "Iku-Turso" у словнику → Artist = 2 токени,
// Title = [2], хвостова зона зсувається і парситься як завжди.
r := Parse("Iku-Turso-Ikuinen_Steelfest_MMXXII_Live-24BIT-WEB-FLAC-2016-MOONBLOOD")
if r.Artist != "Iku-Turso" {
t.Errorf("Artist = %q, want Iku-Turso", r.Artist)
}
if r.Title != "Ikuinen_Steelfest_MMXXII_Live" {
t.Errorf("Title = %q, want Ikuinen_Steelfest_MMXXII_Live", r.Title)
}
if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2016 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
r.UnknownTokens, r.TitleExtra, r.Warnings)
}
}
func TestParseArtistDictionaryMultiTokenArtist(t *testing.T) {
// artists.txt: "Shadu-Nar-Mattaru" — артист з ТРЬОХ токенів.
// Найдовший словниковий префікс виграє: Artist = 3 токени,
// Title = наступний токен, хвіст парситься як завжди.
r := Parse("Shadu-Nar-Mattaru-On_Pathways_Macabre-24BIT-WEB-FLAC-2023-MOONBLOOD")
if r.Artist != "Shadu-Nar-Mattaru" {
t.Errorf("Artist = %q, want Shadu-Nar-Mattaru", r.Artist)
}
if r.Title != "On_Pathways_Macabre" {
t.Errorf("Title = %q, want On_Pathways_Macabre", r.Title)
}
if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2023 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
r.UnknownTokens, r.TitleExtra, r.Warnings)
}
}
func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
// overrides.txt (найвищий пріоритет): точний lookup по Raw
// фіксує Artist/Title; хвіст парситься як завжди.
raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2013-MOONBLOOD"
r := Parse(raw)
if r.Artist != "Milicia_Oscura" {
t.Errorf("Artist = %q, want Milicia_Oscura", r.Artist)
}
if r.Title != "Anti-Cristo" {
t.Errorf("Title = %q, want Anti-Cristo", r.Title)
}
if r.Language != "ES" || r.BitDepth != "16BIT" || r.Source != "WEB" ||
r.Format != "FLAC" || r.Year != 2013 || r.Group != "MOONBLOOD" {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
r.UnknownTokens, r.TitleExtra, r.Warnings)
}
}
func TestParseTypeNormalizedSourceStaysRaw(t *testing.T) {
// CDEP у source.txt: Source лишається сирим, Type нормалізується в EP.
r := Parse("Black_Ceremonial_Kult-Crowned_in_Chaos-CDEP-FLAC-2021-MOONBLOOD")
if r.Type != "EP" {
t.Errorf("Type = %q, want EP", r.Type)
}
if r.Source != "CDEP" {
t.Errorf("Source = %q, want raw CDEP", r.Source)
}
}
func TestParseTypeFirstMatchWins(t *testing.T) {
// REMASTERED_EP → EP; наступний типовий токен не перезаписує.
r := Parse("Some_Artist-Some_Title-REMASTERED_EP-SINGLE-FLAC-2021-MOONBLOOD")
if r.Type != "EP" {
t.Errorf("Type = %q, want EP (first match wins)", r.Type)
}
if r.Edition != "REMASTERED_EP" {
t.Errorf("Edition = %q, want REMASTERED_EP", r.Edition)
}
}
func TestAnalyzeTypeDistribution(t *testing.T) {
stats := AnalyzeTitles([]string{
"Some_Artist-Some_Title-CDEP-FLAC-2021-MOONBLOOD",
"Other_Artist-Other_Title-EP-FLAC-2022-MOONBLOOD",
"Third_Artist-Third_Title-SPLIT-FLAC-2023-MOONBLOOD",
})
if stats.TypeDistribution["EP"] != 2 {
t.Errorf("TypeDistribution[EP] = %d, want 2", stats.TypeDistribution["EP"])
}
if stats.TypeDistribution["Split"] != 1 {
t.Errorf("TypeDistribution[Split] = %d, want 1", stats.TypeDistribution["Split"])
}
}
+104
View File
@@ -0,0 +1,104 @@
package groups
import (
"embed"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// The token dictionaries live under resources/dicts/*.txt — one token (in any
// case) per line; blank lines and lines starting with "#" are ignored. They
// can be edited without recompiling the parsing logic: go:embed pulls them
// into the binary at build time.
//go:embed resources/dicts/*.txt
var dictFS embed.FS
func loadDict(name string) map[string]bool {
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
if err != nil {
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]bool)
for line := range strings.SplitSeq(string(data), "\n") {
line = strings.ToUpper(strings.TrimSpace(line))
if line == "" || strings.HasPrefix(line, "#") {
continue
}
out[line] = true
}
return out
}
// loadOverrides parses the override table: "<raw line> => artist | title".
func loadOverrides(name string) map[string]releaseparser.Override {
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
if err != nil {
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]releaseparser.Override)
for line := range strings.SplitSeq(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
head, rest, ok := strings.Cut(line, "=>")
if !ok {
continue
}
artist, title, ok := strings.Cut(strings.TrimSpace(rest), "|")
if !ok {
continue
}
out[strings.TrimSpace(head)] = releaseparser.Override{
Artist: strings.TrimSpace(artist),
Title: strings.TrimSpace(title),
}
}
return out
}
// loadTypeVariants parses types.txt: "<Base> = variant, variant, ..." and
// returns variant (uppercased) -> base as written in the file.
func loadTypeVariants(name string) map[string]string {
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
if err != nil {
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]string)
for line := range strings.SplitSeq(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
base, rest, ok := strings.Cut(line, "=")
if !ok {
continue
}
base = strings.TrimSpace(base)
for v := range strings.SplitSeq(rest, ",") {
v = strings.ToUpper(strings.TrimSpace(v))
if v != "" {
out[v] = base
}
}
}
return out
}
// Profile holds the shared dictionaries for the release parser. The parser is
// group-agnostic: the group name is read from each release line, never from
// this profile, so there is a single set of dictionaries covering all groups.
var Profile = releaseparser.GroupProfile{
FormatTokens: loadDict("format"),
ArtistTokens: loadDict("artists"),
TypeVariants: loadTypeVariants("types"),
SourceTokens: loadDict("source"),
BitDepthTokens: loadDict("bitdepth"),
ResolutionTokens: loadDict("resolution"),
LanguageMarkers: loadDict("language"),
EditionTokens: loadDict("edition"),
Overrides: loadOverrides("overrides"),
CommentTokens: loadDict("comment"),
}
+18 -8
View File
@@ -1,8 +1,14 @@
package releaseparser
type Override struct {
Artist string
Title string
}
type GroupProfile struct {
Name string
ExpectedTailOrder []string
ArtistTokens map[string]bool // hyphenated artist whitelist
TypeVariants map[string]string // variant (upper) -> normalized Type
Overrides map[string]Override
FormatTokens map[string]bool
SourceTokens map[string]bool
BitDepthTokens map[string]bool
@@ -15,18 +21,21 @@ type GroupProfile struct {
type ParsedRelease struct {
Raw string
Group string
Artist string
Year int
Title string
TitleExtra []string
Type string
Source string
Edition string
CatalogNo string
Language string
BitDepth string
Resolution string
Comment string
CatalogNo string
Source string
Format string
Language string
Edition string
Core string
Artist string
Title string
Override Override
UnknownTokens []string
Warnings []string
Confidence float64
@@ -40,6 +49,7 @@ type UnknownTokenStats struct {
type GroupStats struct {
Total int
TypeDistribution map[string]int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
+13 -8
View File
@@ -85,16 +85,25 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
}
var validID = regexp.MustCompile(`-\d+\/$`)
for i, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
for i, item := range dom.QuerySelectorAll(doc, "li.row") {
var es model.ExternalSources
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
// TypeId тепер напряму з data-id на <li>, без regex по class
idAttr := dom.GetAttribute(item, "data-id")
es.TypeId, _ = strconv.Atoi(idAttr)
title := dom.QuerySelector(item, ".title")
if title != nil {
anchor := dom.QuerySelector(title, "h1 > a")
anchor := dom.QuerySelector(title, "a")
if anchor != nil {
es.Type = constant.ScopePrescene
es.Title = dom.GetAttribute(anchor, "title")
titleSpan := dom.QuerySelector(anchor, "span.title-start")
if titleSpan != nil {
es.Title = dom.TextContent(titleSpan)
}
if es.Title == "Auto Draft" {
slog.Info("Skipped", "title", es.Title)
continue
@@ -123,17 +132,13 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
continue
}
localtime := dom.QuerySelector(title, "small > span.localtime")
localtime := dom.QuerySelector(item, "span.localtime")
if localtime != nil {
lc := dom.GetAttribute(localtime, "data-lttime")
es.Created = carbon.Parse(lc, "Europe/Kyiv")
}
}
cls := dom.GetAttribute(item, "class")
pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, db)
entries = append(entries, entry)