releaseparser: adopt fixed positional model, auto-detect catalogues, add GB doc-blocks

Parse releases with an explicit fixed positional model and document the
whole logic in British English doc-blocks.

Model:
- [0]=Artist, [1]=Title are fixed and never matched against dictionaries,
  so titles like "Split"/"Demo"/"De" are never eaten.
- [len-1]=Group is always read from the line, not from the query key.
- [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is
  resolved as a middle-zone token (e.g. FLAC still picked up as format).
- The middle zone is resolved via case-insensitive dictionaries in the order
  format -> source -> bit-depth -> resolution -> edition -> language ->
  catalogue number. Position within the zone is irrelevant.

Catalogue numbers:
- Any "(...)" token in the middle zone is a catalogue number regardless of
  content (digits, roman numerals, dots, underscores, mixed case). Removed
  the hasDigit requirement and character whitelist that rejected real
  catalogues like (DYN.3140-2) and (GOWP_DCCCLXXXIX).
- Splitting never breaks tokens inside parentheses.

Case handling: lookups are case-insensitive; stored values keep their
original case from the line.

Documentation: replaced Ukrainian comments in parser.go and analyzer.go with
British-English doc-blocks describing the model, tail resolution order,
catalogue rule and confidence scoring.

Tests:
- Add TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2)/(GOWP_DCCCLXXXIX).
- Update case-insensitive, split-title, no-year and language expectations.
- go build, go vet and go test all pass.
This commit is contained in:
Yevhen Odynets
2026-09-01 04:26:06 +03:00
parent 3f3598c446
commit 31febb9f56
13 changed files with 11621 additions and 104 deletions
@@ -9,6 +9,9 @@ import (
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// GroupStats summarises how a set of release lines parsed: field
// distributions, unknown tokens and warnings aggregated by occurrence, and
// the average parsing confidence. The slices are sorted by descending count.
type GroupStats struct {
Total int
YearDistribution map[int]int
@@ -17,11 +20,15 @@ type GroupStats struct {
ResolutionDistribution map[string]int
LanguageDistribution map[string]int
EditionDistribution map[string]int
CommentDistribution map[string]int
UnknownTokens []releaseparser.UnknownTokenStats
WarningStats []releaseparser.UnknownTokenStats
AvgConfidence float64
}
// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників).
// AnalyzeGroup parses the release lines from the local resource file. It is
// used for a quick local check of the dictionaries without touching a
// database.
func AnalyzeGroup() (GroupStats, error) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
@@ -44,7 +51,9 @@ func AnalyzeGroup() (GroupStats, error) {
return AnalyzeTitles(titles), nil
}
// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі).
// AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.
func AnalyzeTitles(titles []string) GroupStats {
stats := GroupStats{
YearDistribution: make(map[int]int),
@@ -53,11 +62,14 @@ func AnalyzeTitles(titles []string) GroupStats {
ResolutionDistribution: make(map[string]int),
LanguageDistribution: make(map[string]int),
EditionDistribution: make(map[string]int),
CommentDistribution: make(map[string]int),
UnknownTokens: []releaseparser.UnknownTokenStats{},
WarningStats: []releaseparser.UnknownTokenStats{},
}
totalConfidence := 0.0
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
for _, title := range titles {
result := Parse(title)
@@ -81,6 +93,11 @@ func AnalyzeTitles(titles []string) GroupStats {
if result.Edition != "" {
stats.EditionDistribution[result.Edition]++
}
if result.Comment != "" {
for _, c := range strings.Split(result.Comment, ",") {
stats.CommentDistribution[c]++
}
}
for _, unknown := range result.UnknownTokens {
if _, exists := unknownMap[unknown]; !exists {
@@ -94,10 +111,23 @@ func AnalyzeTitles(titles []string) GroupStats {
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
}
for _, warning := range result.Warnings {
if _, exists := warningMap[warning]; !exists {
warningMap[warning] = &releaseparser.UnknownTokenStats{
Token: warning,
Count: 0,
Releases: []string{},
}
}
warningMap[warning].Count++
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
}
totalConfidence += result.Confidence
}
// Convert map to slice and sort by count
// Convert the aggregates from maps to slices and order them by count
// (most frequent first).
for _, tokenStats := range unknownMap {
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
}
@@ -106,6 +136,13 @@ func AnalyzeTitles(titles []string) GroupStats {
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
})
for _, w := range warningMap {
stats.WarningStats = append(stats.WarningStats, *w)
}
sort.Slice(stats.WarningStats, func(i, j int) bool {
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
})
if stats.Total > 0 {
stats.AvgConfidence = totalConfidence / float64(stats.Total)
}