releaseparser: Artist/Title boundary resolution + normalized Type field

Three-level core resolution with priority overrides → dictionaries/model → suspects:

1. dicts/overrides.txt (highest priority)
   Point fixes via exact lookup on Raw BEFORE the positional logic.
   Format: <original line> => artist | title. An override fixes
   Artist/Title and shifts the dictionary tail zone by the actual core
   length (overrideCoreLength, fallback 2 tokens).

2. dicts/artists.txt
   Whitelist of hyphenated artists (Iku-Turso, Post-Mortem, ...): a
   [0]-[1] hit → Artist = 2 tokens, Title = [2], tail zone shifted.
   One entry covers all releases of the artist automatically.

3. Suspected title continuation
   Unknown tokens between the title zone and the tail (followed later by
   dictionary markers) no longer pollute UnknownTokens; they go to a
   dedicated TitleExtra field + warning "suspected title continuation" —
   a reviewer hint that the core is probably longer than 2 tokens.
   Catalogue numbers (parenthesized tokens) are resolved with FIRST
   priority and never land in suspects.

Type field (normalized base release type):
- new normalization map dicts/types.txt (<Base> = variants):
  EP = EP, CDS, CDM, 7INCH_VINYL, CDEP, ..., REMASTERED_EP; Demo = ...;
  Split; Single; Compilation; Collaboration (curator-extendable)
- tokens matched against edition/source are additionally normalized to
  the base type (first match wins); edition.txt/source.txt untouched,
  Source stays raw
- TypeDistribution map[string]int added to GroupStats and the report
  (console/JSON)
- OST/INSTRUMENTAL are release traits, not types; excluded from the map

Tests: old cases updated to the new suspect semantics; acceptance tests
added for overrides (Milicia_Oscura-Anti-Cristo), artist dictionary
(Iku-Turso), the suspect section, Type normalization (CDEP → EP with
raw Source=CDEP, first-match-wins) and TypeDistribution.

go build / go vet / go test ./internal/releaseparser/... — all green.
This commit is contained in:
Yevhen Odynets
2026-09-01 13:18:40 +03:00
parent f2629f002c
commit 347f525801
6 changed files with 294 additions and 32 deletions
+86 -11
View File
@@ -5,6 +5,8 @@ import (
"strconv"
"strings"
"golang.org/x/exp/slices"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
@@ -56,20 +58,43 @@ func Parse(raw string) releaseparser.ParsedRelease {
return result
}
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
// [len-2]=Year (when present).
result.Artist = tokens[0]
if len(tokens) > 1 {
result.Title = tokens[1]
// Where the dictionary-driven tail zone begins: after the resolved
// Artist/Title core (2 tokens by default, 3 with an artist dictionary hit).
start := 2
// Level 1: exact-lookup override table fixes Artist/Title only;
// the tail zone is parsed as usual below.
if o, ok := Profile.Overrides[result.Raw]; ok {
result.Artist, result.Title = o.Artist, o.Title
start = overrideCoreLength(tokens, o)
}
// Level 2: artist dictionary — "[0]-[1]" whitelisted as a hyphenated
// artist shifts the core by one token.
switch {
case result.Artist != "":
// An override already fixed the core — leave it alone.
case len(tokens) >= 3 &&
Profile.ArtistTokens[strings.ToUpper(tokens[0]+"-"+tokens[1])]:
result.Artist = tokens[0] + "-" + tokens[1]
result.Title = tokens[2]
start = 3
default:
result.Artist = tokens[0]
if len(tokens) > 1 {
result.Title = tokens[1]
}
}
result.Group = tokens[len(tokens)-1]
end := len(tokens) - 1
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
end = len(tokens) - 2
}
// Resolve the middle zone tokens via the dictionaries.
parseTailZone(tokens[2:end], &result)
// Level 3: dictionary-driven tail zone, starting after the resolved core.
parseTailZone(start, tokens[start:end], &result)
score(&result)
return result
@@ -136,11 +161,11 @@ func catalogNumber(token string) string {
}
// parseTailZone resolves middle-zone tokens against the dictionaries.
// The first match wins for each field; catalogue numbers are recognised
// from parenthesized tokens, and anything unrecognised is appended to
// The first match wins for each field; catalogue numbers are recognized
// from parenthesized tokens, and anything unrecognized is appended to
// UnknownTokens so that it can be reviewed and curated into a dictionary.
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
for _, tok := range zone {
func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease) {
for i, tok := range zone {
up := strings.ToUpper(tok)
switch {
case Profile.FormatTokens[up]:
@@ -151,6 +176,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
if result.Source == "" {
result.Source = tok
}
applyType(tok, result)
case Profile.BitDepthTokens[up]:
if result.BitDepth == "" {
result.BitDepth = tok
@@ -163,6 +189,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
if result.Edition == "" {
result.Edition = tok
}
applyType(tok, result)
case Profile.LanguageMarkers[up]:
if result.Language == "" {
result.Language = tok
@@ -170,6 +197,13 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
default:
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
result.CatalogNo = cat
} else if hasLaterKnownToken(zone, i) {
// suspected title continuation: the core is probably longer
// than 2 tokens. Not eaten, not unknown — review section.
result.TitleExtra = append(result.TitleExtra, tok)
if !slices.Contains(result.Warnings, "suspected title continuation") {
result.Warnings = append(result.Warnings, "suspected title continuation")
}
} else {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
@@ -177,6 +211,47 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
}
}
// applyType normalizes a matched edition/source token into ParsedRelease.Type
// via the types.txt variant map; first match wins. The token still fills its
// own raw field (Source stays raw).
func applyType(tok string, result *releaseparser.ParsedRelease) {
if base, ok := Profile.TypeVariants[strings.ToUpper(tok)]; ok && result.Type == "" {
result.Type = base
}
}
// hasLaterKnownToken reports whether any token after index i in the zone
// matches one of the profile dictionaries.
func hasLaterKnownToken(zone []string, i int) bool {
for _, later := range zone[i+1:] {
up := strings.ToUpper(later)
if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
return true
}
}
return false
}
// overrideCoreLength returns the number of leading tokens covered by the
// override's Artist and Title (each may span several hyphen-separated
// tokens), falling back to the default 2-token core when the raw tokens
// do not reconstruct the override fields.
func overrideCoreLength(tokens []string, o releaseparser.Override) int {
for a := 1; a <= len(tokens); a++ {
if strings.Join(tokens[:a], "-") != o.Artist {
continue
}
for t := a + 1; t <= len(tokens); t++ {
if strings.Join(tokens[a:t], "-") == o.Title {
return t
}
}
}
return 2
}
// score computes a heuristic confidence in [0,1] for a parsed release based
// on how many core fields (group, year, format, source, artist) were resolved
// and whether any warnings or unknown tokens are present.