releaseparser: move group-agnostic parser to groups/, GB doc-blocks, per-group JSON reports
Generalise the parser: the MOONBLOOD group profile is gone, the parser and its dictionaries are now group-agnostic under internal/releaseparser/groups. Positional model: - [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so titles like "Split"/"Demo"/"De" are never eaten. - [len-1]=Group is always read from the line, not from the query key. - [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is resolved as a middle-zone token (e.g. FLAC is still picked up as format). - The middle zone is resolved via case-insensitive dictionaries in the order format -> source -> bit-depth -> resolution -> edition -> language -> catalogue number; stored values keep their original case. Catalogue numbers: any "(...)" token in the middle zone is a catalogue number regardless of content (digits, roman numerals, dots, underscores, mixed case); splitting never breaks tokens inside parentheses. Dictionaries moved to internal/releaseparser/groups/resources/dicts and embedded via go:embed; dead profile fields (Name, ExpectedTailOrder) removed; profile comments rewritten in British English. File-based AnalyzeGroup/TestParseMoonblood removed — titles now come from the database (cmd/srrdb: srrdb <releaser>..., config/secret.yaml DSN). Validation output (cmd/srrdb): - tmp/releaseparser/<group>.json — per-group parsed ParsedRelease records - tmp/releaseparser/<releaser>.report.json — per-request aggregate (same as console) - review_unknown_tokens.txt — unknown/warning rows for manual review Tests updated for the positional model; added TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2) and (GOWP_DCCCLXXXIX). go build, go vet and go test all pass.
This commit is contained in:
@@ -0,0 +1,161 @@
|
||||
package groups
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||
)
|
||||
|
||||
// GroupStats summarises how a set of release lines parsed: field
|
||||
// distributions, unknown tokens and warnings aggregated by occurrence, and
|
||||
// the average parsing confidence. The slices are sorted by descending count.
|
||||
|
||||
type GroupStats struct {
|
||||
Total int
|
||||
YearDistribution map[int]int
|
||||
SourceDistribution map[string]int
|
||||
BitDepthDistribution map[string]int
|
||||
ResolutionDistribution map[string]int
|
||||
LanguageDistribution map[string]int
|
||||
EditionDistribution map[string]int
|
||||
CommentDistribution map[string]int
|
||||
UnknownTokens []releaseparser.UnknownTokenStats
|
||||
WarningStats []releaseparser.UnknownTokenStats
|
||||
AvgConfidence float64
|
||||
}
|
||||
|
||||
|
||||
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
||||
// database by a particular releaser key) and aggregates the results into a
|
||||
// GroupStats summary.
|
||||
|
||||
func AnalyzeTitles(titles []string) GroupStats {
|
||||
stats := GroupStats{
|
||||
YearDistribution: make(map[int]int),
|
||||
SourceDistribution: make(map[string]int),
|
||||
BitDepthDistribution: make(map[string]int),
|
||||
ResolutionDistribution: make(map[string]int),
|
||||
LanguageDistribution: make(map[string]int),
|
||||
EditionDistribution: make(map[string]int),
|
||||
CommentDistribution: make(map[string]int),
|
||||
UnknownTokens: []releaseparser.UnknownTokenStats{},
|
||||
WarningStats: []releaseparser.UnknownTokenStats{},
|
||||
}
|
||||
|
||||
|
||||
totalConfidence := 0.0
|
||||
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||
|
||||
|
||||
for _, title := range titles {
|
||||
result := Parse(title)
|
||||
stats.Total++
|
||||
|
||||
if result.Year != 0 {
|
||||
stats.YearDistribution[result.Year]++
|
||||
}
|
||||
if result.Source != "" {
|
||||
stats.SourceDistribution[result.Source]++
|
||||
}
|
||||
if result.BitDepth != "" {
|
||||
stats.BitDepthDistribution[result.BitDepth]++
|
||||
}
|
||||
if result.Resolution != "" {
|
||||
stats.ResolutionDistribution[result.Resolution]++
|
||||
}
|
||||
if result.Language != "" {
|
||||
stats.LanguageDistribution[result.Language]++
|
||||
}
|
||||
if result.Edition != "" {
|
||||
stats.EditionDistribution[result.Edition]++
|
||||
}
|
||||
if result.Comment != "" {
|
||||
for _, c := range strings.Split(result.Comment, ",") {
|
||||
stats.CommentDistribution[c]++
|
||||
}
|
||||
}
|
||||
|
||||
for _, unknown := range result.UnknownTokens {
|
||||
if _, exists := unknownMap[unknown]; !exists {
|
||||
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
|
||||
Token: unknown,
|
||||
Count: 0,
|
||||
Releases: []string{},
|
||||
}
|
||||
}
|
||||
unknownMap[unknown].Count++
|
||||
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
|
||||
}
|
||||
|
||||
for _, warning := range result.Warnings {
|
||||
if _, exists := warningMap[warning]; !exists {
|
||||
warningMap[warning] = &releaseparser.UnknownTokenStats{
|
||||
Token: warning,
|
||||
Count: 0,
|
||||
Releases: []string{},
|
||||
}
|
||||
}
|
||||
warningMap[warning].Count++
|
||||
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
|
||||
}
|
||||
|
||||
totalConfidence += result.Confidence
|
||||
}
|
||||
|
||||
// Convert the aggregates from maps to slices and order them by count
|
||||
// (most frequent first).
|
||||
for _, tokenStats := range unknownMap {
|
||||
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
||||
}
|
||||
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
|
||||
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
||||
})
|
||||
|
||||
for _, w := range warningMap {
|
||||
stats.WarningStats = append(stats.WarningStats, *w)
|
||||
}
|
||||
sort.Slice(stats.WarningStats, func(i, j int) bool {
|
||||
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
|
||||
})
|
||||
|
||||
if stats.Total > 0 {
|
||||
stats.AvgConfidence = totalConfidence / float64(stats.Total)
|
||||
}
|
||||
|
||||
return stats
|
||||
}
|
||||
|
||||
|
||||
// AnalyzeByGroup parses a set of release lines and returns per-group
|
||||
// statistics keyed by the actual group read from each line (not from the
|
||||
// query key, which may cover several groups). It is used to validate parsing
|
||||
// quality against the original data, grouped by the real releaser name.
|
||||
func AnalyzeByGroup(titles []string) map[string]GroupStats {
|
||||
byGroup := make(map[string][]string)
|
||||
for _, title := range titles {
|
||||
group := Parse(title).Group
|
||||
byGroup[group] = append(byGroup[group], title)
|
||||
}
|
||||
|
||||
out := make(map[string]GroupStats, len(byGroup))
|
||||
for group, lines := range byGroup {
|
||||
out[group] = AnalyzeTitles(lines)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
|
||||
// ParseByGroup parses a set of release lines into a per-group list of fully
|
||||
// populated ParsedRelease records, keyed by the actual group read from each
|
||||
// line. It is used to validate parsing quality field-by-field against the
|
||||
// original data.
|
||||
func ParseByGroup(titles []string) map[string][]releaseparser.ParsedRelease {
|
||||
byGroup := make(map[string][]releaseparser.ParsedRelease)
|
||||
for _, title := range titles {
|
||||
result := Parse(title)
|
||||
byGroup[result.Group] = append(byGroup[result.Group], result)
|
||||
}
|
||||
return byGroup
|
||||
}
|
||||
Reference in New Issue
Block a user