Yevhen Odynets
347f525801
Three-level core resolution with priority overrides → dictionaries/model → suspects: 1. dicts/overrides.txt (highest priority) Point fixes via exact lookup on Raw BEFORE the positional logic. Format: <original line> => artist | title. An override fixes Artist/Title and shifts the dictionary tail zone by the actual core length (overrideCoreLength, fallback 2 tokens). 2. dicts/artists.txt Whitelist of hyphenated artists (Iku-Turso, Post-Mortem, ...): a [0]-[1] hit → Artist = 2 tokens, Title = [2], tail zone shifted. One entry covers all releases of the artist automatically. 3. Suspected title continuation Unknown tokens between the title zone and the tail (followed later by dictionary markers) no longer pollute UnknownTokens; they go to a dedicated TitleExtra field + warning "suspected title continuation" — a reviewer hint that the core is probably longer than 2 tokens. Catalogue numbers (parenthesized tokens) are resolved with FIRST priority and never land in suspects. Type field (normalized base release type): - new normalization map dicts/types.txt (<Base> = variants): EP = EP, CDS, CDM, 7INCH_VINYL, CDEP, ..., REMASTERED_EP; Demo = ...; Split; Single; Compilation; Collaboration (curator-extendable) - tokens matched against edition/source are additionally normalized to the base type (first match wins); edition.txt/source.txt untouched, Source stays raw - TypeDistribution map[string]int added to GroupStats and the report (console/JSON) - OST/INSTRUMENTAL are release traits, not types; excluded from the map Tests: old cases updated to the new suspect semantics; acceptance tests added for overrides (Milicia_Oscura-Anti-Cristo), artist dictionary (Iku-Turso), the suspect section, Type normalization (CDEP → EP with raw Source=CDEP, first-match-wins) and TypeDistribution. go build / go vet / go test ./internal/releaseparser/... — all green.
105 lines
3.1 KiB
Go
105 lines
3.1 KiB
Go
package groups
|
|
|
|
import (
|
|
"embed"
|
|
"strings"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
|
)
|
|
|
|
// The token dictionaries live under resources/dicts/*.txt — one token (in any
|
|
// case) per line; blank lines and lines starting with "#" are ignored. They
|
|
// can be edited without recompiling the parsing logic: go:embed pulls them
|
|
// into the binary at build time.
|
|
|
|
//go:embed resources/dicts/*.txt
|
|
var dictFS embed.FS
|
|
|
|
func loadDict(name string) map[string]bool {
|
|
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
|
if err != nil {
|
|
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
|
}
|
|
out := make(map[string]bool)
|
|
for line := range strings.SplitSeq(string(data), "\n") {
|
|
line = strings.ToUpper(strings.TrimSpace(line))
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
out[line] = true
|
|
}
|
|
return out
|
|
}
|
|
|
|
// loadOverrides parses the override table: "<raw line> => artist | title".
|
|
func loadOverrides(name string) map[string]releaseparser.Override {
|
|
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
|
if err != nil {
|
|
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
|
}
|
|
out := make(map[string]releaseparser.Override)
|
|
for line := range strings.SplitSeq(string(data), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
head, rest, ok := strings.Cut(line, "=>")
|
|
if !ok {
|
|
continue
|
|
}
|
|
artist, title, ok := strings.Cut(strings.TrimSpace(rest), "|")
|
|
if !ok {
|
|
continue
|
|
}
|
|
out[strings.TrimSpace(head)] = releaseparser.Override{
|
|
Artist: strings.TrimSpace(artist),
|
|
Title: strings.TrimSpace(title),
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// loadTypeVariants parses types.txt: "<Base> = variant, variant, ..." and
|
|
// returns variant (uppercased) -> base as written in the file.
|
|
func loadTypeVariants(name string) map[string]string {
|
|
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
|
if err != nil {
|
|
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
|
}
|
|
out := make(map[string]string)
|
|
for line := range strings.SplitSeq(string(data), "\n") {
|
|
line = strings.TrimSpace(line)
|
|
if line == "" || strings.HasPrefix(line, "#") {
|
|
continue
|
|
}
|
|
base, rest, ok := strings.Cut(line, "=")
|
|
if !ok {
|
|
continue
|
|
}
|
|
base = strings.TrimSpace(base)
|
|
for v := range strings.SplitSeq(rest, ",") {
|
|
v = strings.ToUpper(strings.TrimSpace(v))
|
|
if v != "" {
|
|
out[v] = base
|
|
}
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Profile holds the shared dictionaries for the release parser. The parser is
|
|
// group-agnostic: the group name is read from each release line, never from
|
|
// this profile, so there is a single set of dictionaries covering all groups.
|
|
var Profile = releaseparser.GroupProfile{
|
|
FormatTokens: loadDict("format"),
|
|
ArtistTokens: loadDict("artists"),
|
|
TypeVariants: loadTypeVariants("types"),
|
|
SourceTokens: loadDict("source"),
|
|
BitDepthTokens: loadDict("bitdepth"),
|
|
ResolutionTokens: loadDict("resolution"),
|
|
LanguageMarkers: loadDict("language"),
|
|
EditionTokens: loadDict("edition"),
|
|
Overrides: loadOverrides("overrides"),
|
|
CommentTokens: loadDict("comment"),
|
|
}
|