diff --git a/cmd/srrdb/main.go b/cmd/srrdb/main.go index ab24350..d7aef6b 100644 --- a/cmd/srrdb/main.go +++ b/cmd/srrdb/main.go @@ -49,7 +49,7 @@ func main() { err = dbase.Select(&titles, `SELECT title FROM theamok_mdb.external_sources - WHERE type = 'prescene' AND releaser = ? limit 1000`, releaser) + WHERE type = 'prescene' AND releaser = ? limit 2000`, releaser) if err != nil { log.Fatal(err) } diff --git a/internal/releaseparser/groups/analyzer.go b/internal/releaseparser/groups/analyzer.go index 585e0a7..4c126c9 100644 --- a/internal/releaseparser/groups/analyzer.go +++ b/internal/releaseparser/groups/analyzer.go @@ -10,13 +10,13 @@ import ( // GroupStats summarises how a set of release lines parsed: field // distributions, unknown tokens and warnings aggregated by occurrence, and // the average parsing confidence. The slices are sorted by descending count. - type GroupStats struct { Total int YearDistribution map[int]int SourceDistribution map[string]int BitDepthDistribution map[string]int ResolutionDistribution map[string]int + TypeDistribution map[string]int LanguageDistribution map[string]int EditionDistribution map[string]int CommentDistribution map[string]int @@ -25,14 +25,13 @@ type GroupStats struct { AvgConfidence float64 } - // AnalyzeTitles parses a set of release lines (for example fetched from the // database by a particular releaser key) and aggregates the results into a // GroupStats summary. - func AnalyzeTitles(titles []string) GroupStats { stats := GroupStats{ YearDistribution: make(map[int]int), + TypeDistribution: make(map[string]int), SourceDistribution: make(map[string]int), BitDepthDistribution: make(map[string]int), ResolutionDistribution: make(map[string]int), @@ -43,12 +42,10 @@ func AnalyzeTitles(titles []string) GroupStats { WarningStats: []releaseparser.UnknownTokenStats{}, } - totalConfidence := 0.0 unknownMap := make(map[string]*releaseparser.UnknownTokenStats) warningMap := make(map[string]*releaseparser.UnknownTokenStats) - for _, title := range titles { result := Parse(title) stats.Total++ @@ -77,6 +74,10 @@ func AnalyzeTitles(titles []string) GroupStats { } } + if result.Type != "" { + stats.TypeDistribution[result.Type]++ + } + for _, unknown := range result.UnknownTokens { if _, exists := unknownMap[unknown]; !exists { unknownMap[unknown] = &releaseparser.UnknownTokenStats{ @@ -127,7 +128,6 @@ func AnalyzeTitles(titles []string) GroupStats { return stats } - // AnalyzeByGroup parses a set of release lines and returns per-group // statistics keyed by the actual group read from each line (not from the // query key, which may cover several groups). It is used to validate parsing @@ -146,7 +146,6 @@ func AnalyzeByGroup(titles []string) map[string]GroupStats { return out } - // ParseByGroup parses a set of release lines into a per-group list of fully // populated ParsedRelease records, keyed by the actual group read from each // line. It is used to validate parsing quality field-by-field against the diff --git a/internal/releaseparser/groups/parser.go b/internal/releaseparser/groups/parser.go index 38b888b..a49c8ef 100644 --- a/internal/releaseparser/groups/parser.go +++ b/internal/releaseparser/groups/parser.go @@ -5,6 +5,8 @@ import ( "strconv" "strings" + "golang.org/x/exp/slices" + "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" ) @@ -56,20 +58,43 @@ func Parse(raw string) releaseparser.ParsedRelease { return result } - // Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group, - // [len-2]=Year (when present). - result.Artist = tokens[0] - if len(tokens) > 1 { - result.Title = tokens[1] + // Where the dictionary-driven tail zone begins: after the resolved + // Artist/Title core (2 tokens by default, 3 with an artist dictionary hit). + start := 2 + + // Level 1: exact-lookup override table fixes Artist/Title only; + // the tail zone is parsed as usual below. + if o, ok := Profile.Overrides[result.Raw]; ok { + result.Artist, result.Title = o.Artist, o.Title + start = overrideCoreLength(tokens, o) } + + // Level 2: artist dictionary — "[0]-[1]" whitelisted as a hyphenated + // artist shifts the core by one token. + switch { + case result.Artist != "": + // An override already fixed the core — leave it alone. + case len(tokens) >= 3 && + Profile.ArtistTokens[strings.ToUpper(tokens[0]+"-"+tokens[1])]: + result.Artist = tokens[0] + "-" + tokens[1] + result.Title = tokens[2] + start = 3 + default: + result.Artist = tokens[0] + if len(tokens) > 1 { + result.Title = tokens[1] + } + } + result.Group = tokens[len(tokens)-1] end := len(tokens) - 1 if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) { result.Year, _ = strconv.Atoi(tokens[len(tokens)-2]) end = len(tokens) - 2 } - // Resolve the middle zone tokens via the dictionaries. - parseTailZone(tokens[2:end], &result) + + // Level 3: dictionary-driven tail zone, starting after the resolved core. + parseTailZone(start, tokens[start:end], &result) score(&result) return result @@ -136,11 +161,11 @@ func catalogNumber(token string) string { } // parseTailZone resolves middle-zone tokens against the dictionaries. -// The first match wins for each field; catalogue numbers are recognised -// from parenthesized tokens, and anything unrecognised is appended to +// The first match wins for each field; catalogue numbers are recognized +// from parenthesized tokens, and anything unrecognized is appended to // UnknownTokens so that it can be reviewed and curated into a dictionary. -func parseTailZone(zone []string, result *releaseparser.ParsedRelease) { - for _, tok := range zone { +func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease) { + for i, tok := range zone { up := strings.ToUpper(tok) switch { case Profile.FormatTokens[up]: @@ -151,6 +176,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) { if result.Source == "" { result.Source = tok } + applyType(tok, result) case Profile.BitDepthTokens[up]: if result.BitDepth == "" { result.BitDepth = tok @@ -163,6 +189,7 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) { if result.Edition == "" { result.Edition = tok } + applyType(tok, result) case Profile.LanguageMarkers[up]: if result.Language == "" { result.Language = tok @@ -170,6 +197,13 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) { default: if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" { result.CatalogNo = cat + } else if hasLaterKnownToken(zone, i) { + // suspected title continuation: the core is probably longer + // than 2 tokens. Not eaten, not unknown — review section. + result.TitleExtra = append(result.TitleExtra, tok) + if !slices.Contains(result.Warnings, "suspected title continuation") { + result.Warnings = append(result.Warnings, "suspected title continuation") + } } else { result.UnknownTokens = append(result.UnknownTokens, tok) } @@ -177,6 +211,47 @@ func parseTailZone(zone []string, result *releaseparser.ParsedRelease) { } } +// applyType normalizes a matched edition/source token into ParsedRelease.Type +// via the types.txt variant map; first match wins. The token still fills its +// own raw field (Source stays raw). +func applyType(tok string, result *releaseparser.ParsedRelease) { + if base, ok := Profile.TypeVariants[strings.ToUpper(tok)]; ok && result.Type == "" { + result.Type = base + } +} + +// hasLaterKnownToken reports whether any token after index i in the zone +// matches one of the profile dictionaries. +func hasLaterKnownToken(zone []string, i int) bool { + for _, later := range zone[i+1:] { + up := strings.ToUpper(later) + if Profile.FormatTokens[up] || Profile.SourceTokens[up] || + Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] || + Profile.EditionTokens[up] || Profile.LanguageMarkers[up] { + return true + } + } + return false +} + +// overrideCoreLength returns the number of leading tokens covered by the +// override's Artist and Title (each may span several hyphen-separated +// tokens), falling back to the default 2-token core when the raw tokens +// do not reconstruct the override fields. +func overrideCoreLength(tokens []string, o releaseparser.Override) int { + for a := 1; a <= len(tokens); a++ { + if strings.Join(tokens[:a], "-") != o.Artist { + continue + } + for t := a + 1; t <= len(tokens); t++ { + if strings.Join(tokens[a:t], "-") == o.Title { + return t + } + } + } + return 2 +} + // score computes a heuristic confidence in [0,1] for a parsed release based // on how many core fields (group, year, format, source, artist) were resolved // and whether any warnings or unknown tokens are present. diff --git a/internal/releaseparser/groups/parser_test.go b/internal/releaseparser/groups/parser_test.go index 7d12a76..966a64a 100644 --- a/internal/releaseparser/groups/parser_test.go +++ b/internal/releaseparser/groups/parser_test.go @@ -37,8 +37,9 @@ func TestParseEditionAndDemo(t *testing.T) { } func TestParseArtistWithHyphen(t *testing.T) { - // [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл - // стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown. + // Artist сам містить дефіс, але Ultra-Violence ще не в словнику: + // модель [0]/[1] лишається, а All_My_Life (за ним ідуть словникові + // маркери) — suspected title continuation, не unknown. r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD") if r.Artist != "Ultra" || r.Title != "Violence" { t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title) @@ -46,8 +47,14 @@ func TestParseArtistWithHyphen(t *testing.T) { if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 { t.Errorf("tail fields wrong: %+v", r) } - if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" { - t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens) + if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "All_My_Life" { + t.Errorf("TitleExtra = %v, want [All_My_Life]", r.TitleExtra) + } + if !contains(r.Warnings, "suspected title continuation") { + t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) } } @@ -123,12 +130,16 @@ func TestParseCaseInsensitiveDictionaries(t *testing.T) { } func TestParseUnknownTailTokenReported(t *testing.T) { - // Невідомий токен між двома відомими маркерами → у UnknownTokens. - r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD") + // Невідомий токен ПІСЛЯ останнього словникового маркера (за ним уже + // нічого відомого немає) → у UnknownTokens. + r := Parse("Artist-Some_Title-16BIT-WEB-FLAC-FOO-2025-MOONBLOOD") if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" { t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens) } - if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 { + if len(r.TitleExtra) != 0 { + t.Errorf("TitleExtra = %v, want empty", r.TitleExtra) + } + if r.Edition != "" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 { t.Errorf("tail fields wrong: %+v", r) } if r.Artist != "Artist" || r.Title != "Some_Title" { @@ -136,6 +147,24 @@ func TestParseUnknownTailTokenReported(t *testing.T) { } } +func TestParseSuspectedTitleContinuation(t *testing.T) { + // Невідомий токен МІЖ title-зоною і хвостом (за ним далі йдуть + // словникові маркери) → не UnknownTokens, а TitleExtra + warning. + r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD") + if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "FOO" { + t.Errorf("TitleExtra = %v, want [FOO]", r.TitleExtra) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } + if !contains(r.Warnings, "suspected title continuation") { + t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings) + } + if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 { + t.Errorf("tail fields wrong: %+v", r) + } +} + func TestParseCoreTokensNeverReported(t *testing.T) { // Артист/тайтл ніколи не потрапляють в UnknownTokens. r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD") @@ -254,7 +283,8 @@ func TestParseCatalogNumberWithDotAndRoman(t *testing.T) { func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) { // [1]=Title (фіксовано), а Evolution — не словниковий токен у середині: - // потрапляє в UnknownTokens. (SMG409) — катало. + // за ним ідуть словникові маркери → suspected title continuation. + // (SMG409) — катало, залишається каталом. r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D") if r.CatalogNo != "SMG409" { t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo) @@ -262,8 +292,11 @@ func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) { if r.Artist != "Ferosity" || r.Title != "De" { t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title) } - if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" { - t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens) + if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "Evolution" { + t.Errorf("TitleExtra = %v, want [Evolution]", r.TitleExtra) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) } } @@ -330,13 +363,21 @@ func TestParseCompoundEditionRecognized(t *testing.T) { } func TestParseMixedCaseTitleNeverReported(t *testing.T) { - // [1]=Title (один токен), решта, що не словник — UnknownTokens. + // [1]=Title (один токен). Хвіст назви (_ і Sky_Burial) — не словник, + // але за ним ідуть словникові маркери → suspected title continuation, + // не UnknownTokens. r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D") if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" { t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) } - if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" { - t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens) + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } + if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "_Sky_Burial" { + t.Errorf("TitleExtra = %v, want [_Sky_Burial]", r.TitleExtra) + } + if r.CatalogNo != "RTM103" { + t.Errorf("CatalogNo = %q, want RTM103", r.CatalogNo) } } @@ -365,3 +406,79 @@ func TestParseParenthesizedTitleNotCatNo(t *testing.T) { t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) } } + +func TestParseArtistDictionaryCoversReleases(t *testing.T) { + // artists.txt: "Iku-Turso" у словнику → Artist = 2 токени, + // Title = [2], хвостова зона зсувається і парситься як завжди. + r := Parse("Iku-Turso-Ikuinen_Steelfest_MMXXII_Live-24BIT-WEB-FLAC-2016-MOONBLOOD") + if r.Artist != "Iku-Turso" { + t.Errorf("Artist = %q, want Iku-Turso", r.Artist) + } + if r.Title != "Ikuinen_Steelfest_MMXXII_Live" { + t.Errorf("Title = %q, want Ikuinen_Steelfest_MMXXII_Live", r.Title) + } + if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2016 { + t.Errorf("tail fields wrong: %+v", r) + } + if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 { + t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty", + r.UnknownTokens, r.TitleExtra, r.Warnings) + } +} + +func TestParseOverrideWinsOverPositionalModel(t *testing.T) { + // overrides.txt (найвищий пріоритет): точний lookup по Raw + // фіксує Artist/Title; хвіст парситься як завжди. + raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2024-GRAVEWAVE" + r := Parse(raw) + if r.Artist != "Milicia_Oscura" { + t.Errorf("Artist = %q, want Milicia_Oscura", r.Artist) + } + if r.Title != "Anti-Cristo" { + t.Errorf("Title = %q, want Anti-Cristo", r.Title) + } + if r.Language != "ES" || r.BitDepth != "16BIT" || r.Source != "WEB" || + r.Format != "FLAC" || r.Year != 2024 || r.Group != "GRAVEWAVE" { + t.Errorf("tail fields wrong: %+v", r) + } + if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 { + t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty", + r.UnknownTokens, r.TitleExtra, r.Warnings) + } +} + +func TestParseTypeNormalizedSourceStaysRaw(t *testing.T) { + // CDEP у source.txt: Source лишається сирим, Type нормалізується в EP. + r := Parse("Black_Ceremonial_Kult-Crowned_in_Chaos-CDEP-FLAC-2021-MOONBLOOD") + if r.Type != "EP" { + t.Errorf("Type = %q, want EP", r.Type) + } + if r.Source != "CDEP" { + t.Errorf("Source = %q, want raw CDEP", r.Source) + } +} + +func TestParseTypeFirstMatchWins(t *testing.T) { + // REMASTERED_EP → EP; наступний типовий токен не перезаписує. + r := Parse("Some_Artist-Some_Title-REMASTERED_EP-SINGLE-FLAC-2021-MOONBLOOD") + if r.Type != "EP" { + t.Errorf("Type = %q, want EP (first match wins)", r.Type) + } + if r.Edition != "REMASTERED_EP" { + t.Errorf("Edition = %q, want REMASTERED_EP", r.Edition) + } +} + +func TestAnalyzeTypeDistribution(t *testing.T) { + stats := AnalyzeTitles([]string{ + "Some_Artist-Some_Title-CDEP-FLAC-2021-MOONBLOOD", + "Other_Artist-Other_Title-EP-FLAC-2022-MOONBLOOD", + "Third_Artist-Third_Title-SPLIT-FLAC-2023-MOONBLOOD", + }) + if stats.TypeDistribution["EP"] != 2 { + t.Errorf("TypeDistribution[EP] = %d, want 2", stats.TypeDistribution["EP"]) + } + if stats.TypeDistribution["Split"] != 1 { + t.Errorf("TypeDistribution[Split] = %d, want 1", stats.TypeDistribution["Split"]) + } +} diff --git a/internal/releaseparser/groups/profile.go b/internal/releaseparser/groups/profile.go index 1db32c1..5e98cd9 100644 --- a/internal/releaseparser/groups/profile.go +++ b/internal/releaseparser/groups/profile.go @@ -31,15 +31,74 @@ func loadDict(name string) map[string]bool { return out } +// loadOverrides parses the override table: " => artist | title". +func loadOverrides(name string) map[string]releaseparser.Override { + data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt") + if err != nil { + panic("release parser: cannot load dictionary " + name + ": " + err.Error()) + } + out := make(map[string]releaseparser.Override) + for line := range strings.SplitSeq(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + head, rest, ok := strings.Cut(line, "=>") + if !ok { + continue + } + artist, title, ok := strings.Cut(strings.TrimSpace(rest), "|") + if !ok { + continue + } + out[strings.TrimSpace(head)] = releaseparser.Override{ + Artist: strings.TrimSpace(artist), + Title: strings.TrimSpace(title), + } + } + return out +} + +// loadTypeVariants parses types.txt: " = variant, variant, ..." and +// returns variant (uppercased) -> base as written in the file. +func loadTypeVariants(name string) map[string]string { + data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt") + if err != nil { + panic("release parser: cannot load dictionary " + name + ": " + err.Error()) + } + out := make(map[string]string) + for line := range strings.SplitSeq(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + base, rest, ok := strings.Cut(line, "=") + if !ok { + continue + } + base = strings.TrimSpace(base) + for v := range strings.SplitSeq(rest, ",") { + v = strings.ToUpper(strings.TrimSpace(v)) + if v != "" { + out[v] = base + } + } + } + return out +} + // Profile holds the shared dictionaries for the release parser. The parser is // group-agnostic: the group name is read from each release line, never from // this profile, so there is a single set of dictionaries covering all groups. var Profile = releaseparser.GroupProfile{ FormatTokens: loadDict("format"), + ArtistTokens: loadDict("artists"), + TypeVariants: loadTypeVariants("types"), SourceTokens: loadDict("source"), BitDepthTokens: loadDict("bitdepth"), ResolutionTokens: loadDict("resolution"), LanguageMarkers: loadDict("language"), EditionTokens: loadDict("edition"), + Overrides: loadOverrides("overrides"), CommentTokens: loadDict("comment"), } diff --git a/internal/releaseparser/types.go b/internal/releaseparser/types.go index f15efa0..b8fb5b2 100644 --- a/internal/releaseparser/types.go +++ b/internal/releaseparser/types.go @@ -1,6 +1,14 @@ package releaseparser +type Override struct { + Artist string + Title string +} + type GroupProfile struct { + ArtistTokens map[string]bool // hyphenated artist whitelist + TypeVariants map[string]string // variant (upper) -> normalized Type + Overrides map[string]Override FormatTokens map[string]bool SourceTokens map[string]bool BitDepthTokens map[string]bool @@ -16,6 +24,8 @@ type ParsedRelease struct { Artist string Year int Title string + TitleExtra []string + Type string Source string Edition string CatalogNo string @@ -25,6 +35,7 @@ type ParsedRelease struct { Comment string Format string Core string + Override Override UnknownTokens []string Warnings []string Confidence float64 @@ -38,6 +49,7 @@ type UnknownTokenStats struct { type GroupStats struct { Total int + TypeDistribution map[string]int YearDistribution map[int]int SourceDistribution map[string]int BitDepthDistribution map[string]int