releaseparser: Artist/Title boundary resolution + normalized Type field
Three-level core resolution with priority overrides → dictionaries/model → suspects: 1. dicts/overrides.txt (highest priority) Point fixes via exact lookup on Raw BEFORE the positional logic. Format: <original line> => artist | title. An override fixes Artist/Title and shifts the dictionary tail zone by the actual core length (overrideCoreLength, fallback 2 tokens). 2. dicts/artists.txt Whitelist of hyphenated artists (Iku-Turso, Post-Mortem, ...): a [0]-[1] hit → Artist = 2 tokens, Title = [2], tail zone shifted. One entry covers all releases of the artist automatically. 3. Suspected title continuation Unknown tokens between the title zone and the tail (followed later by dictionary markers) no longer pollute UnknownTokens; they go to a dedicated TitleExtra field + warning "suspected title continuation" — a reviewer hint that the core is probably longer than 2 tokens. Catalogue numbers (parenthesized tokens) are resolved with FIRST priority and never land in suspects. Type field (normalized base release type): - new normalization map dicts/types.txt (<Base> = variants): EP = EP, CDS, CDM, 7INCH_VINYL, CDEP, ..., REMASTERED_EP; Demo = ...; Split; Single; Compilation; Collaboration (curator-extendable) - tokens matched against edition/source are additionally normalized to the base type (first match wins); edition.txt/source.txt untouched, Source stays raw - TypeDistribution map[string]int added to GroupStats and the report (console/JSON) - OST/INSTRUMENTAL are release traits, not types; excluded from the map Tests: old cases updated to the new suspect semantics; acceptance tests added for overrides (Milicia_Oscura-Anti-Cristo), artist dictionary (Iku-Turso), the suspect section, Type normalization (CDEP → EP with raw Source=CDEP, first-match-wins) and TypeDistribution. go build / go vet / go test ./internal/releaseparser/... — all green.
This commit is contained in:
+1
-1
@@ -49,7 +49,7 @@ func main() {
|
|||||||
|
|
||||||
err = dbase.Select(&titles,
|
err = dbase.Select(&titles,
|
||||||
`SELECT title FROM theamok_mdb.external_sources
|
`SELECT title FROM theamok_mdb.external_sources
|
||||||
WHERE type = 'prescene' AND releaser = ? limit 1000`, releaser)
|
WHERE type = 'prescene' AND releaser = ? limit 2000`, releaser)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,13 +10,13 @@ import (
|
|||||||
// GroupStats summarises how a set of release lines parsed: field
|
// GroupStats summarises how a set of release lines parsed: field
|
||||||
// distributions, unknown tokens and warnings aggregated by occurrence, and
|
// distributions, unknown tokens and warnings aggregated by occurrence, and
|
||||||
// the average parsing confidence. The slices are sorted by descending count.
|
// the average parsing confidence. The slices are sorted by descending count.
|
||||||
|
|
||||||
type GroupStats struct {
|
type GroupStats struct {
|
||||||
Total int
|
Total int
|
||||||
YearDistribution map[int]int
|
YearDistribution map[int]int
|
||||||
SourceDistribution map[string]int
|
SourceDistribution map[string]int
|
||||||
BitDepthDistribution map[string]int
|
BitDepthDistribution map[string]int
|
||||||
ResolutionDistribution map[string]int
|
ResolutionDistribution map[string]int
|
||||||
|
TypeDistribution map[string]int
|
||||||
LanguageDistribution map[string]int
|
LanguageDistribution map[string]int
|
||||||
EditionDistribution map[string]int
|
EditionDistribution map[string]int
|
||||||
CommentDistribution map[string]int
|
CommentDistribution map[string]int
|
||||||
@@ -25,14 +25,13 @@ type GroupStats struct {
|
|||||||
AvgConfidence float64
|
AvgConfidence float64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
||||||
// database by a particular releaser key) and aggregates the results into a
|
// database by a particular releaser key) and aggregates the results into a
|
||||||
// GroupStats summary.
|
// GroupStats summary.
|
||||||
|
|
||||||
func AnalyzeTitles(titles []string) GroupStats {
|
func AnalyzeTitles(titles []string) GroupStats {
|
||||||
stats := GroupStats{
|
stats := GroupStats{
|
||||||
YearDistribution: make(map[int]int),
|
YearDistribution: make(map[int]int),
|
||||||
|
TypeDistribution: make(map[string]int),
|
||||||
SourceDistribution: make(map[string]int),
|
SourceDistribution: make(map[string]int),
|
||||||
BitDepthDistribution: make(map[string]int),
|
BitDepthDistribution: make(map[string]int),
|
||||||
ResolutionDistribution: make(map[string]int),
|
ResolutionDistribution: make(map[string]int),
|
||||||
@@ -43,12 +42,10 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
WarningStats: []releaseparser.UnknownTokenStats{},
|
WarningStats: []releaseparser.UnknownTokenStats{},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
totalConfidence := 0.0
|
totalConfidence := 0.0
|
||||||
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||||
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
|
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||||
|
|
||||||
|
|
||||||
for _, title := range titles {
|
for _, title := range titles {
|
||||||
result := Parse(title)
|
result := Parse(title)
|
||||||
stats.Total++
|
stats.Total++
|
||||||
@@ -77,6 +74,10 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if result.Type != "" {
|
||||||
|
stats.TypeDistribution[result.Type]++
|
||||||
|
}
|
||||||
|
|
||||||
for _, unknown := range result.UnknownTokens {
|
for _, unknown := range result.UnknownTokens {
|
||||||
if _, exists := unknownMap[unknown]; !exists {
|
if _, exists := unknownMap[unknown]; !exists {
|
||||||
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
|
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
|
||||||
@@ -127,7 +128,6 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
return stats
|
return stats
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// AnalyzeByGroup parses a set of release lines and returns per-group
|
// AnalyzeByGroup parses a set of release lines and returns per-group
|
||||||
// statistics keyed by the actual group read from each line (not from the
|
// statistics keyed by the actual group read from each line (not from the
|
||||||
// query key, which may cover several groups). It is used to validate parsing
|
// query key, which may cover several groups). It is used to validate parsing
|
||||||
@@ -146,7 +146,6 @@ func AnalyzeByGroup(titles []string) map[string]GroupStats {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// ParseByGroup parses a set of release lines into a per-group list of fully
|
// ParseByGroup parses a set of release lines into a per-group list of fully
|
||||||
// populated ParsedRelease records, keyed by the actual group read from each
|
// populated ParsedRelease records, keyed by the actual group read from each
|
||||||
// line. It is used to validate parsing quality field-by-field against the
|
// line. It is used to validate parsing quality field-by-field against the
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"golang.org/x/exp/slices"
|
||||||
|
|
||||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -56,20 +58,43 @@ func Parse(raw string) releaseparser.ParsedRelease {
|
|||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
|
// Where the dictionary-driven tail zone begins: after the resolved
|
||||||
// [len-2]=Year (when present).
|
// Artist/Title core (2 tokens by default, 3 with an artist dictionary hit).
|
||||||
|
start := 2
|
||||||
|
|
||||||
|
// Level 1: exact-lookup override table fixes Artist/Title only;
|
||||||
|
// the tail zone is parsed as usual below.
|
||||||
|
if o, ok := Profile.Overrides[result.Raw]; ok {
|
||||||
|
result.Artist, result.Title = o.Artist, o.Title
|
||||||
|
start = overrideCoreLength(tokens, o)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Level 2: artist dictionary — "[0]-[1]" whitelisted as a hyphenated
|
||||||
|
// artist shifts the core by one token.
|
||||||
|
switch {
|
||||||
|
case result.Artist != "":
|
||||||
|
// An override already fixed the core — leave it alone.
|
||||||
|
case len(tokens) >= 3 &&
|
||||||
|
Profile.ArtistTokens[strings.ToUpper(tokens[0]+"-"+tokens[1])]:
|
||||||
|
result.Artist = tokens[0] + "-" + tokens[1]
|
||||||
|
result.Title = tokens[2]
|
||||||
|
start = 3
|
||||||
|
default:
|
||||||
result.Artist = tokens[0]
|
result.Artist = tokens[0]
|
||||||
if len(tokens) > 1 {
|
if len(tokens) > 1 {
|
||||||
result.Title = tokens[1]
|
result.Title = tokens[1]
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
result.Group = tokens[len(tokens)-1]
|
result.Group = tokens[len(tokens)-1]
|
||||||
end := len(tokens) - 1
|
end := len(tokens) - 1
|
||||||
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
|
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
|
||||||
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
|
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
|
||||||
end = len(tokens) - 2
|
end = len(tokens) - 2
|
||||||
}
|
}
|
||||||
// Resolve the middle zone tokens via the dictionaries.
|
|
||||||
parseTailZone(tokens[2:end], &result)
|
// Level 3: dictionary-driven tail zone, starting after the resolved core.
|
||||||
|
parseTailZone(start, tokens[start:end], &result)
|
||||||
|
|
||||||
score(&result)
|
score(&result)
|
||||||
return result
|
return result
|
||||||
@@ -136,11 +161,11 @@ func catalogNumber(token string) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// parseTailZone resolves middle-zone tokens against the dictionaries.
|
// parseTailZone resolves middle-zone tokens against the dictionaries.
|
||||||
// The first match wins for each field; catalogue numbers are recognised
|
// The first match wins for each field; catalogue numbers are recognized
|
||||||
// from parenthesized tokens, and anything unrecognised is appended to
|
// from parenthesized tokens, and anything unrecognized is appended to
|
||||||
// UnknownTokens so that it can be reviewed and curated into a dictionary.
|
// UnknownTokens so that it can be reviewed and curated into a dictionary.
|
||||||
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease) {
|
||||||
for _, tok := range zone {
|
for i, tok := range zone {
|
||||||
up := strings.ToUpper(tok)
|
up := strings.ToUpper(tok)
|
||||||
switch {
|
switch {
|
||||||
case Profile.FormatTokens[up]:
|
case Profile.FormatTokens[up]:
|
||||||
@@ -151,6 +176,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
|||||||
if result.Source == "" {
|
if result.Source == "" {
|
||||||
result.Source = tok
|
result.Source = tok
|
||||||
}
|
}
|
||||||
|
applyType(tok, result)
|
||||||
case Profile.BitDepthTokens[up]:
|
case Profile.BitDepthTokens[up]:
|
||||||
if result.BitDepth == "" {
|
if result.BitDepth == "" {
|
||||||
result.BitDepth = tok
|
result.BitDepth = tok
|
||||||
@@ -163,6 +189,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
|||||||
if result.Edition == "" {
|
if result.Edition == "" {
|
||||||
result.Edition = tok
|
result.Edition = tok
|
||||||
}
|
}
|
||||||
|
applyType(tok, result)
|
||||||
case Profile.LanguageMarkers[up]:
|
case Profile.LanguageMarkers[up]:
|
||||||
if result.Language == "" {
|
if result.Language == "" {
|
||||||
result.Language = tok
|
result.Language = tok
|
||||||
@@ -170,6 +197,13 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
|||||||
default:
|
default:
|
||||||
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
|
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
|
||||||
result.CatalogNo = cat
|
result.CatalogNo = cat
|
||||||
|
} else if hasLaterKnownToken(zone, i) {
|
||||||
|
// suspected title continuation: the core is probably longer
|
||||||
|
// than 2 tokens. Not eaten, not unknown — review section.
|
||||||
|
result.TitleExtra = append(result.TitleExtra, tok)
|
||||||
|
if !slices.Contains(result.Warnings, "suspected title continuation") {
|
||||||
|
result.Warnings = append(result.Warnings, "suspected title continuation")
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
result.UnknownTokens = append(result.UnknownTokens, tok)
|
result.UnknownTokens = append(result.UnknownTokens, tok)
|
||||||
}
|
}
|
||||||
@@ -177,6 +211,47 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// applyType normalizes a matched edition/source token into ParsedRelease.Type
|
||||||
|
// via the types.txt variant map; first match wins. The token still fills its
|
||||||
|
// own raw field (Source stays raw).
|
||||||
|
func applyType(tok string, result *releaseparser.ParsedRelease) {
|
||||||
|
if base, ok := Profile.TypeVariants[strings.ToUpper(tok)]; ok && result.Type == "" {
|
||||||
|
result.Type = base
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// hasLaterKnownToken reports whether any token after index i in the zone
|
||||||
|
// matches one of the profile dictionaries.
|
||||||
|
func hasLaterKnownToken(zone []string, i int) bool {
|
||||||
|
for _, later := range zone[i+1:] {
|
||||||
|
up := strings.ToUpper(later)
|
||||||
|
if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
|
||||||
|
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
|
||||||
|
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// overrideCoreLength returns the number of leading tokens covered by the
|
||||||
|
// override's Artist and Title (each may span several hyphen-separated
|
||||||
|
// tokens), falling back to the default 2-token core when the raw tokens
|
||||||
|
// do not reconstruct the override fields.
|
||||||
|
func overrideCoreLength(tokens []string, o releaseparser.Override) int {
|
||||||
|
for a := 1; a <= len(tokens); a++ {
|
||||||
|
if strings.Join(tokens[:a], "-") != o.Artist {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for t := a + 1; t <= len(tokens); t++ {
|
||||||
|
if strings.Join(tokens[a:t], "-") == o.Title {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 2
|
||||||
|
}
|
||||||
|
|
||||||
// score computes a heuristic confidence in [0,1] for a parsed release based
|
// score computes a heuristic confidence in [0,1] for a parsed release based
|
||||||
// on how many core fields (group, year, format, source, artist) were resolved
|
// on how many core fields (group, year, format, source, artist) were resolved
|
||||||
// and whether any warnings or unknown tokens are present.
|
// and whether any warnings or unknown tokens are present.
|
||||||
|
|||||||
@@ -37,8 +37,9 @@ func TestParseEditionAndDemo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseArtistWithHyphen(t *testing.T) {
|
func TestParseArtistWithHyphen(t *testing.T) {
|
||||||
// [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
|
// Artist сам містить дефіс, але Ultra-Violence ще не в словнику:
|
||||||
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
|
// модель [0]/[1] лишається, а All_My_Life (за ним ідуть словникові
|
||||||
|
// маркери) — suspected title continuation, не unknown.
|
||||||
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
|
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
|
||||||
if r.Artist != "Ultra" || r.Title != "Violence" {
|
if r.Artist != "Ultra" || r.Title != "Violence" {
|
||||||
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
|
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
|
||||||
@@ -46,8 +47,14 @@ func TestParseArtistWithHyphen(t *testing.T) {
|
|||||||
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
|
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
|
||||||
t.Errorf("tail fields wrong: %+v", r)
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
}
|
}
|
||||||
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
|
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "All_My_Life" {
|
||||||
t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
|
t.Errorf("TitleExtra = %v, want [All_My_Life]", r.TitleExtra)
|
||||||
|
}
|
||||||
|
if !contains(r.Warnings, "suspected title continuation") {
|
||||||
|
t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -123,12 +130,16 @@ func TestParseCaseInsensitiveDictionaries(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseUnknownTailTokenReported(t *testing.T) {
|
func TestParseUnknownTailTokenReported(t *testing.T) {
|
||||||
// Невідомий токен між двома відомими маркерами → у UnknownTokens.
|
// Невідомий токен ПІСЛЯ останнього словникового маркера (за ним уже
|
||||||
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
|
// нічого відомого немає) → у UnknownTokens.
|
||||||
|
r := Parse("Artist-Some_Title-16BIT-WEB-FLAC-FOO-2025-MOONBLOOD")
|
||||||
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
|
||||||
t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
|
t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
|
||||||
}
|
}
|
||||||
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
|
if len(r.TitleExtra) != 0 {
|
||||||
|
t.Errorf("TitleExtra = %v, want empty", r.TitleExtra)
|
||||||
|
}
|
||||||
|
if r.Edition != "" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
|
||||||
t.Errorf("tail fields wrong: %+v", r)
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
}
|
}
|
||||||
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
||||||
@@ -136,6 +147,24 @@ func TestParseUnknownTailTokenReported(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseSuspectedTitleContinuation(t *testing.T) {
|
||||||
|
// Невідомий токен МІЖ title-зоною і хвостом (за ним далі йдуть
|
||||||
|
// словникові маркери) → не UnknownTokens, а TitleExtra + warning.
|
||||||
|
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||||
|
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "FOO" {
|
||||||
|
t.Errorf("TitleExtra = %v, want [FOO]", r.TitleExtra)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if !contains(r.Warnings, "suspected title continuation") {
|
||||||
|
t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings)
|
||||||
|
}
|
||||||
|
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseCoreTokensNeverReported(t *testing.T) {
|
func TestParseCoreTokensNeverReported(t *testing.T) {
|
||||||
// Артист/тайтл ніколи не потрапляють в UnknownTokens.
|
// Артист/тайтл ніколи не потрапляють в UnknownTokens.
|
||||||
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
|
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||||
@@ -254,7 +283,8 @@ func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
|
|||||||
|
|
||||||
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
|
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
|
||||||
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
|
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
|
||||||
// потрапляє в UnknownTokens. (SMG409) — катало.
|
// за ним ідуть словникові маркери → suspected title continuation.
|
||||||
|
// (SMG409) — катало, залишається каталом.
|
||||||
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
|
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
|
||||||
if r.CatalogNo != "SMG409" {
|
if r.CatalogNo != "SMG409" {
|
||||||
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
|
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
|
||||||
@@ -262,8 +292,11 @@ func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
|
|||||||
if r.Artist != "Ferosity" || r.Title != "De" {
|
if r.Artist != "Ferosity" || r.Title != "De" {
|
||||||
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
|
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
|
||||||
}
|
}
|
||||||
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
|
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "Evolution" {
|
||||||
t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
|
t.Errorf("TitleExtra = %v, want [Evolution]", r.TitleExtra)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -330,13 +363,21 @@ func TestParseCompoundEditionRecognized(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
|
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
|
||||||
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
|
// [1]=Title (один токен). Хвіст назви (_ і Sky_Burial) — не словник,
|
||||||
|
// але за ним ідуть словникові маркери → suspected title continuation,
|
||||||
|
// не UnknownTokens.
|
||||||
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
|
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
|
||||||
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
|
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
|
||||||
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
}
|
}
|
||||||
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
|
if len(r.UnknownTokens) != 0 {
|
||||||
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "_Sky_Burial" {
|
||||||
|
t.Errorf("TitleExtra = %v, want [_Sky_Burial]", r.TitleExtra)
|
||||||
|
}
|
||||||
|
if r.CatalogNo != "RTM103" {
|
||||||
|
t.Errorf("CatalogNo = %q, want RTM103", r.CatalogNo)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -365,3 +406,79 @@ func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
|
|||||||
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseArtistDictionaryCoversReleases(t *testing.T) {
|
||||||
|
// artists.txt: "Iku-Turso" у словнику → Artist = 2 токени,
|
||||||
|
// Title = [2], хвостова зона зсувається і парситься як завжди.
|
||||||
|
r := Parse("Iku-Turso-Ikuinen_Steelfest_MMXXII_Live-24BIT-WEB-FLAC-2016-MOONBLOOD")
|
||||||
|
if r.Artist != "Iku-Turso" {
|
||||||
|
t.Errorf("Artist = %q, want Iku-Turso", r.Artist)
|
||||||
|
}
|
||||||
|
if r.Title != "Ikuinen_Steelfest_MMXXII_Live" {
|
||||||
|
t.Errorf("Title = %q, want Ikuinen_Steelfest_MMXXII_Live", r.Title)
|
||||||
|
}
|
||||||
|
if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2016 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
|
||||||
|
r.UnknownTokens, r.TitleExtra, r.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
|
||||||
|
// overrides.txt (найвищий пріоритет): точний lookup по Raw
|
||||||
|
// фіксує Artist/Title; хвіст парситься як завжди.
|
||||||
|
raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2024-GRAVEWAVE"
|
||||||
|
r := Parse(raw)
|
||||||
|
if r.Artist != "Milicia_Oscura" {
|
||||||
|
t.Errorf("Artist = %q, want Milicia_Oscura", r.Artist)
|
||||||
|
}
|
||||||
|
if r.Title != "Anti-Cristo" {
|
||||||
|
t.Errorf("Title = %q, want Anti-Cristo", r.Title)
|
||||||
|
}
|
||||||
|
if r.Language != "ES" || r.BitDepth != "16BIT" || r.Source != "WEB" ||
|
||||||
|
r.Format != "FLAC" || r.Year != 2024 || r.Group != "GRAVEWAVE" {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
|
||||||
|
r.UnknownTokens, r.TitleExtra, r.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTypeNormalizedSourceStaysRaw(t *testing.T) {
|
||||||
|
// CDEP у source.txt: Source лишається сирим, Type нормалізується в EP.
|
||||||
|
r := Parse("Black_Ceremonial_Kult-Crowned_in_Chaos-CDEP-FLAC-2021-MOONBLOOD")
|
||||||
|
if r.Type != "EP" {
|
||||||
|
t.Errorf("Type = %q, want EP", r.Type)
|
||||||
|
}
|
||||||
|
if r.Source != "CDEP" {
|
||||||
|
t.Errorf("Source = %q, want raw CDEP", r.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseTypeFirstMatchWins(t *testing.T) {
|
||||||
|
// REMASTERED_EP → EP; наступний типовий токен не перезаписує.
|
||||||
|
r := Parse("Some_Artist-Some_Title-REMASTERED_EP-SINGLE-FLAC-2021-MOONBLOOD")
|
||||||
|
if r.Type != "EP" {
|
||||||
|
t.Errorf("Type = %q, want EP (first match wins)", r.Type)
|
||||||
|
}
|
||||||
|
if r.Edition != "REMASTERED_EP" {
|
||||||
|
t.Errorf("Edition = %q, want REMASTERED_EP", r.Edition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAnalyzeTypeDistribution(t *testing.T) {
|
||||||
|
stats := AnalyzeTitles([]string{
|
||||||
|
"Some_Artist-Some_Title-CDEP-FLAC-2021-MOONBLOOD",
|
||||||
|
"Other_Artist-Other_Title-EP-FLAC-2022-MOONBLOOD",
|
||||||
|
"Third_Artist-Third_Title-SPLIT-FLAC-2023-MOONBLOOD",
|
||||||
|
})
|
||||||
|
if stats.TypeDistribution["EP"] != 2 {
|
||||||
|
t.Errorf("TypeDistribution[EP] = %d, want 2", stats.TypeDistribution["EP"])
|
||||||
|
}
|
||||||
|
if stats.TypeDistribution["Split"] != 1 {
|
||||||
|
t.Errorf("TypeDistribution[Split] = %d, want 1", stats.TypeDistribution["Split"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -31,15 +31,74 @@ func loadDict(name string) map[string]bool {
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// loadOverrides parses the override table: "<raw line> => artist | title".
|
||||||
|
func loadOverrides(name string) map[string]releaseparser.Override {
|
||||||
|
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
||||||
|
if err != nil {
|
||||||
|
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
||||||
|
}
|
||||||
|
out := make(map[string]releaseparser.Override)
|
||||||
|
for line := range strings.SplitSeq(string(data), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
head, rest, ok := strings.Cut(line, "=>")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
artist, title, ok := strings.Cut(strings.TrimSpace(rest), "|")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[strings.TrimSpace(head)] = releaseparser.Override{
|
||||||
|
Artist: strings.TrimSpace(artist),
|
||||||
|
Title: strings.TrimSpace(title),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadTypeVariants parses types.txt: "<Base> = variant, variant, ..." and
|
||||||
|
// returns variant (uppercased) -> base as written in the file.
|
||||||
|
func loadTypeVariants(name string) map[string]string {
|
||||||
|
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
|
||||||
|
if err != nil {
|
||||||
|
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
|
||||||
|
}
|
||||||
|
out := make(map[string]string)
|
||||||
|
for line := range strings.SplitSeq(string(data), "\n") {
|
||||||
|
line = strings.TrimSpace(line)
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
base, rest, ok := strings.Cut(line, "=")
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
base = strings.TrimSpace(base)
|
||||||
|
for v := range strings.SplitSeq(rest, ",") {
|
||||||
|
v = strings.ToUpper(strings.TrimSpace(v))
|
||||||
|
if v != "" {
|
||||||
|
out[v] = base
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
// Profile holds the shared dictionaries for the release parser. The parser is
|
// Profile holds the shared dictionaries for the release parser. The parser is
|
||||||
// group-agnostic: the group name is read from each release line, never from
|
// group-agnostic: the group name is read from each release line, never from
|
||||||
// this profile, so there is a single set of dictionaries covering all groups.
|
// this profile, so there is a single set of dictionaries covering all groups.
|
||||||
var Profile = releaseparser.GroupProfile{
|
var Profile = releaseparser.GroupProfile{
|
||||||
FormatTokens: loadDict("format"),
|
FormatTokens: loadDict("format"),
|
||||||
|
ArtistTokens: loadDict("artists"),
|
||||||
|
TypeVariants: loadTypeVariants("types"),
|
||||||
SourceTokens: loadDict("source"),
|
SourceTokens: loadDict("source"),
|
||||||
BitDepthTokens: loadDict("bitdepth"),
|
BitDepthTokens: loadDict("bitdepth"),
|
||||||
ResolutionTokens: loadDict("resolution"),
|
ResolutionTokens: loadDict("resolution"),
|
||||||
LanguageMarkers: loadDict("language"),
|
LanguageMarkers: loadDict("language"),
|
||||||
EditionTokens: loadDict("edition"),
|
EditionTokens: loadDict("edition"),
|
||||||
|
Overrides: loadOverrides("overrides"),
|
||||||
CommentTokens: loadDict("comment"),
|
CommentTokens: loadDict("comment"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,14 @@
|
|||||||
package releaseparser
|
package releaseparser
|
||||||
|
|
||||||
|
type Override struct {
|
||||||
|
Artist string
|
||||||
|
Title string
|
||||||
|
}
|
||||||
|
|
||||||
type GroupProfile struct {
|
type GroupProfile struct {
|
||||||
|
ArtistTokens map[string]bool // hyphenated artist whitelist
|
||||||
|
TypeVariants map[string]string // variant (upper) -> normalized Type
|
||||||
|
Overrides map[string]Override
|
||||||
FormatTokens map[string]bool
|
FormatTokens map[string]bool
|
||||||
SourceTokens map[string]bool
|
SourceTokens map[string]bool
|
||||||
BitDepthTokens map[string]bool
|
BitDepthTokens map[string]bool
|
||||||
@@ -16,6 +24,8 @@ type ParsedRelease struct {
|
|||||||
Artist string
|
Artist string
|
||||||
Year int
|
Year int
|
||||||
Title string
|
Title string
|
||||||
|
TitleExtra []string
|
||||||
|
Type string
|
||||||
Source string
|
Source string
|
||||||
Edition string
|
Edition string
|
||||||
CatalogNo string
|
CatalogNo string
|
||||||
@@ -25,6 +35,7 @@ type ParsedRelease struct {
|
|||||||
Comment string
|
Comment string
|
||||||
Format string
|
Format string
|
||||||
Core string
|
Core string
|
||||||
|
Override Override
|
||||||
UnknownTokens []string
|
UnknownTokens []string
|
||||||
Warnings []string
|
Warnings []string
|
||||||
Confidence float64
|
Confidence float64
|
||||||
@@ -38,6 +49,7 @@ type UnknownTokenStats struct {
|
|||||||
|
|
||||||
type GroupStats struct {
|
type GroupStats struct {
|
||||||
Total int
|
Total int
|
||||||
|
TypeDistribution map[string]int
|
||||||
YearDistribution map[int]int
|
YearDistribution map[int]int
|
||||||
SourceDistribution map[string]int
|
SourceDistribution map[string]int
|
||||||
BitDepthDistribution map[string]int
|
BitDepthDistribution map[string]int
|
||||||
|
|||||||
Reference in New Issue
Block a user