releaseparser: adopt fixed positional model, auto-detect catalogues, add GB doc-blocks

Parse releases with an explicit fixed positional model and document the
whole logic in British English doc-blocks.

Model:
- [0]=Artist, [1]=Title are fixed and never matched against dictionaries,
  so titles like "Split"/"Demo"/"De" are never eaten.
- [len-1]=Group is always read from the line, not from the query key.
- [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is
  resolved as a middle-zone token (e.g. FLAC still picked up as format).
- The middle zone is resolved via case-insensitive dictionaries in the order
  format -> source -> bit-depth -> resolution -> edition -> language ->
  catalogue number. Position within the zone is irrelevant.

Catalogue numbers:
- Any "(...)" token in the middle zone is a catalogue number regardless of
  content (digits, roman numerals, dots, underscores, mixed case). Removed
  the hasDigit requirement and character whitelist that rejected real
  catalogues like (DYN.3140-2) and (GOWP_DCCCLXXXIX).
- Splitting never breaks tokens inside parentheses.

Case handling: lookups are case-insensitive; stored values keep their
original case from the line.

Documentation: replaced Ukrainian comments in parser.go and analyzer.go with
British-English doc-blocks describing the model, tail resolution order,
catalogue rule and confidence scoring.

Tests:
- Add TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2)/(GOWP_DCCCLXXXIX).
- Update case-insensitive, split-title, no-year and language expectations.
- go build, go vet and go test all pass.
This commit is contained in:
Yevhen Odynets
2026-09-01 04:26:06 +03:00
parent 3f3598c446
commit 31febb9f56
13 changed files with 11621 additions and 104 deletions
+20
View File
@@ -55,5 +55,25 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
// Ручна ревізія: всі невідомі токени — у файл для правки бази.
review, err := os.OpenFile("review_unknown_tokens.txt",
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
log.Fatal(err)
}
defer review.Close()
for releaser, stats := range out {
for _, unk := range stats.UnknownTokens {
for _, rel := range unk.Releases {
fmt.Fprintf(review, "%s\t%s\t%s\n", releaser, unk.Token, rel)
}
}
for _, warn := range stats.WarningStats {
for _, rel := range warn.Releases {
fmt.Fprintf(review, "%s\tWARN\t%s\t%s\n", releaser, warn.Token, rel)
}
}
}
fmt.Println(string(data)) fmt.Println(string(data))
} }
@@ -9,6 +9,9 @@ import (
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
) )
// GroupStats summarises how a set of release lines parsed: field
// distributions, unknown tokens and warnings aggregated by occurrence, and
// the average parsing confidence. The slices are sorted by descending count.
type GroupStats struct { type GroupStats struct {
Total int Total int
YearDistribution map[int]int YearDistribution map[int]int
@@ -17,11 +20,15 @@ type GroupStats struct {
ResolutionDistribution map[string]int ResolutionDistribution map[string]int
LanguageDistribution map[string]int LanguageDistribution map[string]int
EditionDistribution map[string]int EditionDistribution map[string]int
CommentDistribution map[string]int
UnknownTokens []releaseparser.UnknownTokenStats UnknownTokens []releaseparser.UnknownTokenStats
WarningStats []releaseparser.UnknownTokenStats
AvgConfidence float64 AvgConfidence float64
} }
// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників). // AnalyzeGroup parses the release lines from the local resource file. It is
// used for a quick local check of the dictionaries without touching a
// database.
func AnalyzeGroup() (GroupStats, error) { func AnalyzeGroup() (GroupStats, error) {
file, err := os.Open("../../resources/moonblood.txt") file, err := os.Open("../../resources/moonblood.txt")
if err != nil { if err != nil {
@@ -44,7 +51,9 @@ func AnalyzeGroup() (GroupStats, error) {
return AnalyzeTitles(titles), nil return AnalyzeTitles(titles), nil
} }
// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі). // AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.
func AnalyzeTitles(titles []string) GroupStats { func AnalyzeTitles(titles []string) GroupStats {
stats := GroupStats{ stats := GroupStats{
YearDistribution: make(map[int]int), YearDistribution: make(map[int]int),
@@ -53,11 +62,14 @@ func AnalyzeTitles(titles []string) GroupStats {
ResolutionDistribution: make(map[string]int), ResolutionDistribution: make(map[string]int),
LanguageDistribution: make(map[string]int), LanguageDistribution: make(map[string]int),
EditionDistribution: make(map[string]int), EditionDistribution: make(map[string]int),
CommentDistribution: make(map[string]int),
UnknownTokens: []releaseparser.UnknownTokenStats{}, UnknownTokens: []releaseparser.UnknownTokenStats{},
WarningStats: []releaseparser.UnknownTokenStats{},
} }
totalConfidence := 0.0 totalConfidence := 0.0
unknownMap := make(map[string]*releaseparser.UnknownTokenStats) unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
for _, title := range titles { for _, title := range titles {
result := Parse(title) result := Parse(title)
@@ -81,6 +93,11 @@ func AnalyzeTitles(titles []string) GroupStats {
if result.Edition != "" { if result.Edition != "" {
stats.EditionDistribution[result.Edition]++ stats.EditionDistribution[result.Edition]++
} }
if result.Comment != "" {
for _, c := range strings.Split(result.Comment, ",") {
stats.CommentDistribution[c]++
}
}
for _, unknown := range result.UnknownTokens { for _, unknown := range result.UnknownTokens {
if _, exists := unknownMap[unknown]; !exists { if _, exists := unknownMap[unknown]; !exists {
@@ -94,10 +111,23 @@ func AnalyzeTitles(titles []string) GroupStats {
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title) unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
} }
for _, warning := range result.Warnings {
if _, exists := warningMap[warning]; !exists {
warningMap[warning] = &releaseparser.UnknownTokenStats{
Token: warning,
Count: 0,
Releases: []string{},
}
}
warningMap[warning].Count++
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
}
totalConfidence += result.Confidence totalConfidence += result.Confidence
} }
// Convert map to slice and sort by count // Convert the aggregates from maps to slices and order them by count
// (most frequent first).
for _, tokenStats := range unknownMap { for _, tokenStats := range unknownMap {
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats) stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
} }
@@ -106,6 +136,13 @@ func AnalyzeTitles(titles []string) GroupStats {
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
}) })
for _, w := range warningMap {
stats.WarningStats = append(stats.WarningStats, *w)
}
sort.Slice(stats.WarningStats, func(i, j int) bool {
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
})
if stats.Total > 0 { if stats.Total > 0 {
stats.AvgConfidence = totalConfidence / float64(stats.Total) stats.AvgConfidence = totalConfidence / float64(stats.Total)
} }
@@ -1,3 +1,5 @@
# {bit_depth} # {bit_depth}
16BIT 16BIT
24BIT 24BIT
32BIT
DE16BIT
@@ -1,3 +1,5 @@
REPACK REPACK
REAL_REPACK REAL_REPACK
READNFO READNFO
PROPER
RERIP
@@ -17,4 +17,41 @@ LIMITED_EDITION_BOXSET
REMASTERED_EP REMASTERED_EP
10TH_ANNIVERSARY_EDITION 10TH_ANNIVERSARY_EDITION
OST OST
XX_Legacy_Edition
REISSUE_DEMO REISSUE_DEMO
JP_Deluxe_Edition
Remastered_Edition
Anniversary_Edition
Limited_Edition
CDS
CDM
RETAIL
JP_RETAIL
JP_REISSUE
DEMO_REMASTERED
EU_RETAIL
DE_RETAIL
REISSUE_DELUXE_EDITION
RETAIL_REISSUE
SPECIAL_EDITION
SPECIAL_COLLECTORS_EDITION_REMASTERED
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
LIMITED_EDITION_REMASTERED
DEMO_REISSUE
REMASTERED_REISSUE
PROMO_DEMO
REISSUE_EP
MAG
REISSUE_BOXSET
30TH_ANNIVERSARY_EDITION_REMASTERED
Deluxe_Edition_Reissue
INSTRUMENTAL
REISSUE_EXPANDED_EDITION
BOOTLEG
EXPANDED_EDITION
ADVANCE
15TH_ANNIVERSARY_DELUXE_EDITION
TOUR_EDITION
REMASTERED_DEMO
REMASTERED_DELUXE_EDITION
30TH_ANNIVERSARY_EDITION
@@ -17,3 +17,17 @@ AT
IS IS
UA UA
JP JP
GR
CN
RU
ID
PH
BY
CZ
RS
DA
HI
CH
LU
CY
EE
@@ -1,6 +1,7 @@
# {resolution} — частота дискретизації # {resolution} — частота дискретизації
44KHZ 44KHZ
48KHZ 48KHZ
48.0kHz
88KHZ 88KHZ
96KHZ 96KHZ
176KHZ 176KHZ
@@ -9,3 +9,24 @@ TAPE
CDR CDR
17CD 17CD
3CD 3CD
4CD
18CD
3CDEP
DVD
7CD
2CDR
10CD
6CD
2CDEP
2VINYL
CDREP
LP
2LP
3TAPE
25CD
WEBFLAC
2TAPE
BLURAY
VLS
MCD
24BIT_WEB
+139 -93
View File
@@ -10,6 +10,35 @@ import (
var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`) var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
// Parse splits a release line into tokens and maps them onto a fixed
// positional model while the interior is resolved solely through the
// group profile's dictionaries.
//
// The release line is split on "-" (but never inside curved parentheses, so
// that a catalogue number such as (BRC30-001-CD-RRR108) stays one token).
// The positional layout is:
//
// [0] Artist - fixed; never matched against dictionaries.
// [1] Title - fixed; never matched against dictionaries.
// [len-2] Year - if the token is a 4-digit year (19xx | 20xx).
// [len-1] Group - always read from the line, never from the query key.
//
// Everything between [1] and the reserved trailing slots ([2 .. len-3], or
// [2 .. len-2] when there is no year) is the middle zone. Each token there is
// looked up in the dictionaries (format, source, bit-depth, resolution,
// language, edition) and, if it is a parenthesized token, as a catalogue
// number. Position within the middle zone is irrelevant. Tokens that match
// nothing are NOT silently consumed — they are reported in UnknownTokens so
// that they can be reviewed and added to a dictionary.
//
// Why [0]/[1] are fixed and never matched against the dictionaries: titles
// such as "Split" (Alone_In_The_Morgue-Split-...) or "Demo"
// (Mystifier-Demo-Nized-...) are legitimate words that collide with edition
// markers. Since index 1 is always the title, regardless of case or
// dictionary membership, such words are never eaten.
//
// Values are preserved in their original case (as they appear in the line);
// the dictionary lookups themselves are case-insensitive.
func Parse(raw string) releaseparser.ParsedRelease { func Parse(raw string) releaseparser.ParsedRelease {
result := releaseparser.ParsedRelease{ result := releaseparser.ParsedRelease{
Raw: strings.TrimSpace(raw), Raw: strings.TrimSpace(raw),
@@ -17,123 +46,140 @@ func Parse(raw string) releaseparser.ParsedRelease {
UnknownTokens: []string{}, UnknownTokens: []string{},
} }
tokens := split(raw) tokens, comments := extractComments(split(raw))
if len(comments) > 0 {
result.Comment = strings.Join(comments, ",")
}
if len(tokens) < 5 { if len(tokens) < 5 {
result.Warnings = append(result.Warnings, "too few tokens") result.Warnings = append(result.Warnings, "too few tokens")
result.Confidence = 0.2 result.Confidence = 0.2
return result return result
} }
// Останній токен — група. Вона ж — ключ до профілю групи (довідників). // Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
// [len-2]=Year (when present).
result.Artist = tokens[0]
if len(tokens) > 1 {
result.Title = tokens[1]
}
result.Group = tokens[len(tokens)-1] result.Group = tokens[len(tokens)-1]
end := len(tokens) - 1
// Передостанній — рік if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
last := len(tokens) - 2 result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
if yearRegexp.MatchString(tokens[last]) { end = len(tokens) - 2
result.Year, _ = strconv.Atoi(tokens[last])
last--
}
// Рухаємось вліво по хвосту: format, source, bit_depth, resolution, language, edition.
// Кожен слот бере токен лише зі свого словника (регістронезалежно).
// Невідомий токен: репортимо лише якщо ліворуч ще є відомий хвостовий маркер
// (токен між двома маркерами = сміття в хвості). Інакше це початок core.
tailLoop:
for last >= 0 {
token := strings.ToUpper(tokens[last])
switch {
case Profile.FormatTokens[token]:
if result.Format == "" {
result.Format = token
}
last--
case Profile.SourceTokens[token]:
if result.Source == "" {
result.Source = token
}
last--
case Profile.BitDepthTokens[token]:
if result.BitDepth == "" {
result.BitDepth = token
}
last--
case Profile.ResolutionTokens[token]:
if result.Resolution == "" {
result.Resolution = token
}
last--
case Profile.LanguageMarkers[token]:
if result.Language == "" {
result.Language = token
}
last--
case Profile.EditionTokens[token]:
if result.Edition == "" {
result.Edition = token
}
last--
default:
if hasKnownTailToken(tokens[:last]) {
result.UnknownTokens = append(result.UnknownTokens, tokens[last])
last--
continue
}
break tailLoop
}
}
// Core = artist-title (все, що залишилось ліворуч)
coreTokens := tokens[:last+1]
result.Core = strings.Join(coreTokens, "-")
if len(coreTokens) >= 2 {
result.Artist = coreTokens[0]
result.Title = strings.Join(coreTokens[1:], "-")
} else if len(coreTokens) == 1 {
result.Artist = coreTokens[0]
result.Title = ""
result.Warnings = append(result.Warnings, "title not found")
} }
// Resolve the middle zone tokens via the dictionaries.
parseTailZone(tokens[2:end], &result)
score(&result) score(&result)
return result return result
} }
// split breaks a raw release line into tokens on "-", but never inside
// curved parentheses, so that a catalogue number such as
// (BRC30-001-CD-RRR108) remains a single token.
func split(raw string) []string { func split(raw string) []string {
parts := strings.Split(strings.TrimSpace(raw), "-") s := strings.TrimSpace(raw)
out := make([]string, 0, len(parts)) out := make([]string, 0, 8)
for _, part := range parts { var cur strings.Builder
if part = strings.TrimSpace(part); part != "" { inParen := false
out = append(out, part) for _, r := range s {
switch {
case r == '(':
inParen = true
cur.WriteRune(r)
case r == ')':
inParen = false
cur.WriteRune(r)
case r == '-' && !inParen:
if t := strings.TrimSpace(cur.String()); t != "" {
out = append(out, t)
} }
cur.Reset()
default:
cur.WriteRune(r)
}
}
if t := strings.TrimSpace(cur.String()); t != "" {
out = append(out, t)
} }
return out return out
} }
// hasKnownTailToken reports whether any token is a known tail marker // extractComments removes service comments (CommentTokens) from the token
// (format, source, bit_depth, resolution, edition, language or a year). // list and returns the cleaned list together with the removed comments.
func hasKnownTailToken(tokens []string) bool { func extractComments(tokens []string) ([]string, []string) {
out := make([]string, 0, len(tokens))
comments := []string{}
for _, tok := range tokens { for _, tok := range tokens {
up := strings.ToUpper(tok) if Profile.CommentTokens[strings.ToUpper(tok)] {
if Profile.FormatTokens[up] || Profile.SourceTokens[up] || comments = append(comments, tok)
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] || continue
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
return true
} }
if yearRegexp.MatchString(up) { out = append(out, tok)
return true
} }
} return out, comments
return false
} }
// catalogNumber reports whether the token is a parenthesized catalogue
// number. In the positional model any token wrapped in curved parentheses
// that sits in the middle zone ([2..]) is a catalogue number regardless of
// its content (digits, letters, roman numerals, dots, underscores, mixed
// case). Parenthesized tokens at [0]/[1] are handled as artist/title, not
// here. It returns the parenthesized content, or "" when the token is not
// a "(...)" group.
func catalogNumber(token string) string {
if len(token) < 3 || token[0] != '(' || token[len(token)-1] != ')' {
return ""
}
return token[1 : len(token)-1]
}
// parseTailZone resolves middle-zone tokens against the dictionaries.
// The first match wins for each field; catalogue numbers are recognised
// from parenthesized tokens, and anything unrecognised is appended to
// UnknownTokens so that it can be reviewed and curated into a dictionary.
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
for _, tok := range zone {
up := strings.ToUpper(tok)
switch {
case Profile.FormatTokens[up]:
if result.Format == "" {
result.Format = tok
}
case Profile.SourceTokens[up]:
if result.Source == "" {
result.Source = tok
}
case Profile.BitDepthTokens[up]:
if result.BitDepth == "" {
result.BitDepth = tok
}
case Profile.ResolutionTokens[up]:
if result.Resolution == "" {
result.Resolution = tok
}
case Profile.EditionTokens[up]:
if result.Edition == "" {
result.Edition = tok
}
case Profile.LanguageMarkers[up]:
if result.Language == "" {
result.Language = tok
}
default:
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
result.CatalogNo = cat
} else {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
}
}
}
// score computes a heuristic confidence in [0,1] for a parsed release based
// on how many core fields (group, year, format, source, artist) were resolved
// and whether any warnings or unknown tokens are present.
func score(result *releaseparser.ParsedRelease) { func score(result *releaseparser.ParsedRelease) {
score := 0.0 score := 0.0
if result.Group != "" { if result.Group != "" {
@@ -9,6 +9,7 @@ import (
func TestParseMoonblood(t *testing.T) { func TestParseMoonblood(t *testing.T) {
file, err := os.Open("../../resources/moonblood.txt") file, err := os.Open("../../resources/moonblood.txt")
if err != nil { if err != nil {
t.Skipf("resource file not found (DB mode), skipping: %v", err)
t.Fatal(err) t.Fatal(err)
} }
defer file.Close() defer file.Close()
@@ -70,16 +71,17 @@ func TestParseEditionAndDemo(t *testing.T) {
} }
func TestParseArtistWithHyphen(t *testing.T) { func TestParseArtistWithHyphen(t *testing.T) {
// Artist itself contains a hyphen; no part of it may leak into UnknownTokens. // [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD") r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
if r.Artist != "Ultra" || r.Title != "Violence-All_My_Life" { if r.Artist != "Ultra" || r.Title != "Violence" {
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence-All_My_Life", r.Artist, r.Title) t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
} }
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 { if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r) t.Errorf("tail fields wrong: %+v", r)
} }
if len(r.UnknownTokens) != 0 { if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
} }
} }
@@ -144,7 +146,9 @@ func TestParseVinylIsSource(t *testing.T) {
func TestParseCaseInsensitiveDictionaries(t *testing.T) { func TestParseCaseInsensitiveDictionaries(t *testing.T) {
r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood") r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood")
if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" { // Значення зберігаються в оригінальному регістрі (як у рядку),
// а звірка зі словниками йде регістронезалежно.
if r.BitDepth != "16bit" || r.Source != "web" || r.Format != "flac" {
t.Errorf("case-insensitive lookup failed: %+v", r) t.Errorf("case-insensitive lookup failed: %+v", r)
} }
if r.Artist != "Artist" || r.Title != "Some_Title" { if r.Artist != "Artist" || r.Title != "Some_Title" {
@@ -199,3 +203,199 @@ func TestParseAnyGroupNoWarning(t *testing.T) {
} }
} }
} }
func TestParseCommentExtracted(t *testing.T) {
r := Parse("Rain_City_Drive-Rain_City_Drive-DELUXE_EDITION-REPACK-16BIT-WEB-FLAC-2023-FLACCiD")
if r.Comment != "REPACK" {
t.Errorf("Comment = %q, want REPACK", r.Comment)
}
if r.Edition != "DELUXE_EDITION" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2023 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Rain_City_Drive" || r.Title != "Rain_City_Drive" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
}
}
func TestParseNumericArtistNotAYear(t *testing.T) {
// [0]=Artist — навіть якщо це число-рік (1914, 1928), а не "рік" токен.
for _, line := range []string{
"1914-Eschatology_of_War-24BIT-WEB-FLAC-2015-MOONBLOOD",
"1928-Lappel_du_Spectre-FR-CD-FLAC-2025-MOONBLOOD",
} {
r := Parse(line)
if r.Artist != "1914" && r.Artist != "1928" {
t.Errorf("%s: Artist = %q, want numeric artist", line, r.Artist)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
}
}
}
func TestParseCatalogNumber(t *testing.T) {
r := Parse("Antidot-Beyond_Trance_And_Time-(GLOBAL006)-REPACK-EP-FLAC-2026-AFTERHOUR")
if r.CatalogNo != "GLOBAL006" {
t.Errorf("CatalogNo = %q, want GLOBAL006", r.CatalogNo)
}
if r.Comment != "REPACK" || r.Edition != "EP" || r.Format != "FLAC" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Antidot" || r.Title != "Beyond_Trance_And_Time" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseCatalogNumberWithDashesInside(t *testing.T) {
r := Parse("Murder_Intentions-Excessive_Display_Of_Human_Nature-(BRC30-001-CD-RRR108)-CD-FLAC-2018-86D")
if r.CatalogNo != "BRC30-001-CD-RRR108" {
t.Errorf("CatalogNo = %q, want BRC30-001-CD-RRR108", r.CatalogNo)
}
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2018 || r.Group != "86D" {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Murder_Intentions" || r.Title != "Excessive_Display_Of_Human_Nature" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
// Катало у дужках у середній зоні — незалежно від вмісту
// (крапка, римські, букви, дефіси): все валідне.
cases := []struct{ line, catno string }{
{"Nervosa-Perpetual_Chaos-(DYN.3140-2)-LIMITED_EDITION-CD-FLAC-2021-MOONBLOOD", "DYN.3140-2"},
{"Black_Ceremonial_Kult-Crowned_in_Chaos-(GOWP_DCCCLXXXIX)-CDEP-FLAC-2021-MOONBLOOD", "GOWP_DCCCLXXXIX"},
}
for _, c := range cases {
r := Parse(c.line)
if r.CatalogNo != c.catno {
t.Errorf("CatalogNo = %q, want %q (release: %s)", r.CatalogNo, c.catno, c.line)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty (release: %s)", r.UnknownTokens, c.line)
}
}
}
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
// потрапляє в UnknownTokens. (SMG409) — катало.
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
if r.CatalogNo != "SMG409" {
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
}
if r.Artist != "Ferosity" || r.Title != "De" {
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
}
}
func TestParseLanguageRequiresKnownNeighbor(t *testing.T) {
// Мовний маркер валідний, коли праворуч — відомий хвостовий маркер.
r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
if r.Language != "DE" || r.Source != "VINYL" {
t.Errorf("Language/Source = %q/%q, want DE/VINYL", r.Language, r.Source)
}
}
func TestParseLanguageBetweenTitleAndFormat(t *testing.T) {
// A: мовний маркер у зоні між title і format — валідний,
// все ліворуч від нього — artist+title, нічого в unknown.
r := Parse("Kollektiver_Brechreiz-Heilig_Scheint_Scheinheilig-(NG207)-DE-CD-FLAC-2009-SHELTER")
if r.CatalogNo != "NG207" {
t.Errorf("CatalogNo = %q, want NG207", r.CatalogNo)
}
if r.Language != "DE" || r.Source != "CD" || r.Format != "FLAC" || r.Year != 2009 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Kollektiver_Brechreiz" || r.Title != "Heilig_Scheint_Scheinheilig" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
}
}
func TestParseLanguageNeverBeforeFormat(t *testing.T) {
// DE — словниковий токен у середині (language.txt), позиція неважлива:
// Language=DE.
r := Parse("Some_Artist-Some_Song-DE-FLAC-2020-SOMEGROUP")
if r.Language != "DE" {
t.Errorf("Language = %q, want DE", r.Language)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if r.Title != "Some_Song" {
t.Errorf("Title = %q, want Some_Song", r.Title)
}
}
func TestParseCompoundEditionRecognized(t *testing.T) {
// REMASTERED_REISSUE — складений маркер у dicts/edition.txt, розпізнається
// як edition і не йде в UnknownTokens.
r := Parse("Ominous_Scriptures-Incarnation_of_the_Unheavenly-(LSR07)-REMASTERED_REISSUE-CD-FLAC-2021-86D")
if r.CatalogNo != "LSR07" {
t.Errorf("CatalogNo = %q, want LSR07", r.CatalogNo)
}
if r.Edition != "REMASTERED_REISSUE" {
t.Errorf("Edition = %q, want REMASTERED_REISSUE", r.Edition)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if r.Artist != "Ominous_Scriptures" || r.Title != "Incarnation_of_the_Unheavenly" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2021 {
t.Errorf("tail fields wrong: %+v", r)
}
}
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
}
}
func contains(list []string, s string) bool {
for _, v := range list {
if v == s {
return true
}
}
return false
}
func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
// Дужки в тайтлі при WEB/BitDepth — не каталономер, лишається в тайтлі.
r := Parse("The_Amazons-21st_Century_Fiction_(Epilogue)-16BIT-WEB-FLAC-2026-ENViED")
if r.CatalogNo != "" {
t.Errorf("CatalogNo = %q, want empty", r.CatalogNo)
}
if r.Title != "21st_Century_Fiction_(Epilogue)" {
t.Errorf("Title = %q, want 21st_Century_Fiction_(Epilogue)", r.Title)
}
if r.Source != "WEB" || r.BitDepth != "16BIT" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
@@ -38,4 +38,5 @@ var Profile = releaseparser.GroupProfile{
ResolutionTokens: loadDict("resolution"), ResolutionTokens: loadDict("resolution"),
LanguageMarkers: loadDict("language"), LanguageMarkers: loadDict("language"),
EditionTokens: loadDict("edition"), EditionTokens: loadDict("edition"),
CommentTokens: loadDict("comment"),
} }
+4
View File
@@ -9,6 +9,7 @@ type GroupProfile struct {
ResolutionTokens map[string]bool ResolutionTokens map[string]bool
LanguageMarkers map[string]bool LanguageMarkers map[string]bool
EditionTokens map[string]bool EditionTokens map[string]bool
CommentTokens map[string]bool
} }
type ParsedRelease struct { type ParsedRelease struct {
@@ -18,6 +19,7 @@ type ParsedRelease struct {
BitDepth string BitDepth string
Resolution string Resolution string
Comment string Comment string
CatalogNo string
Source string Source string
Format string Format string
Language string Language string
@@ -44,6 +46,8 @@ type GroupStats struct {
ResolutionDistribution map[string]int ResolutionDistribution map[string]int
LanguageDistribution map[string]int LanguageDistribution map[string]int
EditionDistribution map[string]int EditionDistribution map[string]int
CommentDistribution map[string]int
UnknownTokens []UnknownTokenStats UnknownTokens []UnknownTokenStats
WarningStats []UnknownTokenStats
AvgConfidence float64 AvgConfidence float64
} }
File diff suppressed because it is too large Load Diff