releaseparser: adopt fixed positional model, auto-detect catalogues, add GB doc-blocks
Parse releases with an explicit fixed positional model and document the whole logic in British English doc-blocks. Model: - [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so titles like "Split"/"Demo"/"De" are never eaten. - [len-1]=Group is always read from the line, not from the query key. - [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is resolved as a middle-zone token (e.g. FLAC still picked up as format). - The middle zone is resolved via case-insensitive dictionaries in the order format -> source -> bit-depth -> resolution -> edition -> language -> catalogue number. Position within the zone is irrelevant. Catalogue numbers: - Any "(...)" token in the middle zone is a catalogue number regardless of content (digits, roman numerals, dots, underscores, mixed case). Removed the hasDigit requirement and character whitelist that rejected real catalogues like (DYN.3140-2) and (GOWP_DCCCLXXXIX). - Splitting never breaks tokens inside parentheses. Case handling: lookups are case-insensitive; stored values keep their original case from the line. Documentation: replaced Ukrainian comments in parser.go and analyzer.go with British-English doc-blocks describing the model, tail resolution order, catalogue rule and confidence scoring. Tests: - Add TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2)/(GOWP_DCCCLXXXIX). - Update case-insensitive, split-title, no-year and language expectations. - go build, go vet and go test all pass.
This commit is contained in:
@@ -9,6 +9,9 @@ import (
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||
)
|
||||
|
||||
// GroupStats summarises how a set of release lines parsed: field
|
||||
// distributions, unknown tokens and warnings aggregated by occurrence, and
|
||||
// the average parsing confidence. The slices are sorted by descending count.
|
||||
type GroupStats struct {
|
||||
Total int
|
||||
YearDistribution map[int]int
|
||||
@@ -17,11 +20,15 @@ type GroupStats struct {
|
||||
ResolutionDistribution map[string]int
|
||||
LanguageDistribution map[string]int
|
||||
EditionDistribution map[string]int
|
||||
CommentDistribution map[string]int
|
||||
UnknownTokens []releaseparser.UnknownTokenStats
|
||||
WarningStats []releaseparser.UnknownTokenStats
|
||||
AvgConfidence float64
|
||||
}
|
||||
|
||||
// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників).
|
||||
// AnalyzeGroup parses the release lines from the local resource file. It is
|
||||
// used for a quick local check of the dictionaries without touching a
|
||||
// database.
|
||||
func AnalyzeGroup() (GroupStats, error) {
|
||||
file, err := os.Open("../../resources/moonblood.txt")
|
||||
if err != nil {
|
||||
@@ -44,7 +51,9 @@ func AnalyzeGroup() (GroupStats, error) {
|
||||
return AnalyzeTitles(titles), nil
|
||||
}
|
||||
|
||||
// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі).
|
||||
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
||||
// database by a particular releaser key) and aggregates the results into a
|
||||
// GroupStats summary.
|
||||
func AnalyzeTitles(titles []string) GroupStats {
|
||||
stats := GroupStats{
|
||||
YearDistribution: make(map[int]int),
|
||||
@@ -53,11 +62,14 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
ResolutionDistribution: make(map[string]int),
|
||||
LanguageDistribution: make(map[string]int),
|
||||
EditionDistribution: make(map[string]int),
|
||||
CommentDistribution: make(map[string]int),
|
||||
UnknownTokens: []releaseparser.UnknownTokenStats{},
|
||||
WarningStats: []releaseparser.UnknownTokenStats{},
|
||||
}
|
||||
|
||||
totalConfidence := 0.0
|
||||
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||
|
||||
for _, title := range titles {
|
||||
result := Parse(title)
|
||||
@@ -81,6 +93,11 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
if result.Edition != "" {
|
||||
stats.EditionDistribution[result.Edition]++
|
||||
}
|
||||
if result.Comment != "" {
|
||||
for _, c := range strings.Split(result.Comment, ",") {
|
||||
stats.CommentDistribution[c]++
|
||||
}
|
||||
}
|
||||
|
||||
for _, unknown := range result.UnknownTokens {
|
||||
if _, exists := unknownMap[unknown]; !exists {
|
||||
@@ -94,10 +111,23 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
|
||||
}
|
||||
|
||||
for _, warning := range result.Warnings {
|
||||
if _, exists := warningMap[warning]; !exists {
|
||||
warningMap[warning] = &releaseparser.UnknownTokenStats{
|
||||
Token: warning,
|
||||
Count: 0,
|
||||
Releases: []string{},
|
||||
}
|
||||
}
|
||||
warningMap[warning].Count++
|
||||
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
|
||||
}
|
||||
|
||||
totalConfidence += result.Confidence
|
||||
}
|
||||
|
||||
// Convert map to slice and sort by count
|
||||
// Convert the aggregates from maps to slices and order them by count
|
||||
// (most frequent first).
|
||||
for _, tokenStats := range unknownMap {
|
||||
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
||||
}
|
||||
@@ -106,6 +136,13 @@ func AnalyzeTitles(titles []string) GroupStats {
|
||||
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
||||
})
|
||||
|
||||
for _, w := range warningMap {
|
||||
stats.WarningStats = append(stats.WarningStats, *w)
|
||||
}
|
||||
sort.Slice(stats.WarningStats, func(i, j int) bool {
|
||||
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
|
||||
})
|
||||
|
||||
if stats.Total > 0 {
|
||||
stats.AvgConfidence = totalConfidence / float64(stats.Total)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user