package groups import ( "sort" "strings" "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" ) // GroupStats summarises how a set of release lines parsed: field // distributions, unknown tokens and warnings aggregated by occurrence, and // the average parsing confidence. The slices are sorted by descending count. type GroupStats struct { Total int YearDistribution map[int]int SourceDistribution map[string]int BitDepthDistribution map[string]int ResolutionDistribution map[string]int LanguageDistribution map[string]int EditionDistribution map[string]int CommentDistribution map[string]int UnknownTokens []releaseparser.UnknownTokenStats WarningStats []releaseparser.UnknownTokenStats AvgConfidence float64 } // AnalyzeTitles parses a set of release lines (for example fetched from the // database by a particular releaser key) and aggregates the results into a // GroupStats summary. func AnalyzeTitles(titles []string) GroupStats { stats := GroupStats{ YearDistribution: make(map[int]int), SourceDistribution: make(map[string]int), BitDepthDistribution: make(map[string]int), ResolutionDistribution: make(map[string]int), LanguageDistribution: make(map[string]int), EditionDistribution: make(map[string]int), CommentDistribution: make(map[string]int), UnknownTokens: []releaseparser.UnknownTokenStats{}, WarningStats: []releaseparser.UnknownTokenStats{}, } totalConfidence := 0.0 unknownMap := make(map[string]*releaseparser.UnknownTokenStats) warningMap := make(map[string]*releaseparser.UnknownTokenStats) for _, title := range titles { result := Parse(title) stats.Total++ if result.Year != 0 { stats.YearDistribution[result.Year]++ } if result.Source != "" { stats.SourceDistribution[result.Source]++ } if result.BitDepth != "" { stats.BitDepthDistribution[result.BitDepth]++ } if result.Resolution != "" { stats.ResolutionDistribution[result.Resolution]++ } if result.Language != "" { stats.LanguageDistribution[result.Language]++ } if result.Edition != "" { stats.EditionDistribution[result.Edition]++ } if result.Comment != "" { for _, c := range strings.Split(result.Comment, ",") { stats.CommentDistribution[c]++ } } for _, unknown := range result.UnknownTokens { if _, exists := unknownMap[unknown]; !exists { unknownMap[unknown] = &releaseparser.UnknownTokenStats{ Token: unknown, Count: 0, Releases: []string{}, } } unknownMap[unknown].Count++ unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title) } for _, warning := range result.Warnings { if _, exists := warningMap[warning]; !exists { warningMap[warning] = &releaseparser.UnknownTokenStats{ Token: warning, Count: 0, Releases: []string{}, } } warningMap[warning].Count++ warningMap[warning].Releases = append(warningMap[warning].Releases, title) } totalConfidence += result.Confidence } // Convert the aggregates from maps to slices and order them by count // (most frequent first). for _, tokenStats := range unknownMap { stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats) } sort.Slice(stats.UnknownTokens, func(i, j int) bool { return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count }) for _, w := range warningMap { stats.WarningStats = append(stats.WarningStats, *w) } sort.Slice(stats.WarningStats, func(i, j int) bool { return stats.WarningStats[i].Count > stats.WarningStats[j].Count }) if stats.Total > 0 { stats.AvgConfidence = totalConfidence / float64(stats.Total) } return stats } // AnalyzeByGroup parses a set of release lines and returns per-group // statistics keyed by the actual group read from each line (not from the // query key, which may cover several groups). It is used to validate parsing // quality against the original data, grouped by the real releaser name. func AnalyzeByGroup(titles []string) map[string]GroupStats { byGroup := make(map[string][]string) for _, title := range titles { group := Parse(title).Group byGroup[group] = append(byGroup[group], title) } out := make(map[string]GroupStats, len(byGroup)) for group, lines := range byGroup { out[group] = AnalyzeTitles(lines) } return out } // ParseByGroup parses a set of release lines into a per-group list of fully // populated ParsedRelease records, keyed by the actual group read from each // line. It is used to validate parsing quality field-by-field against the // original data. func ParseByGroup(titles []string) map[string][]releaseparser.ParsedRelease { byGroup := make(map[string][]releaseparser.ParsedRelease) for _, title := range titles { result := Parse(title) byGroup[result.Group] = append(byGroup[result.Group], result) } return byGroup }