package moonblood import ( "bufio" "os" "sort" "strings" "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" ) // GroupStats summarises how a set of release lines parsed: field // distributions, unknown tokens and warnings aggregated by occurrence, and // the average parsing confidence. The slices are sorted by descending count. type GroupStats struct { Total int YearDistribution map[int]int SourceDistribution map[string]int BitDepthDistribution map[string]int ResolutionDistribution map[string]int LanguageDistribution map[string]int EditionDistribution map[string]int CommentDistribution map[string]int UnknownTokens []releaseparser.UnknownTokenStats WarningStats []releaseparser.UnknownTokenStats AvgConfidence float64 } // AnalyzeGroup parses the release lines from the local resource file. It is // used for a quick local check of the dictionaries without touching a // database. func AnalyzeGroup() (GroupStats, error) { file, err := os.Open("../../resources/moonblood.txt") if err != nil { return GroupStats{}, err } defer file.Close() var titles []string scanner := bufio.NewScanner(file) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) if line != "" { titles = append(titles, line) } } if err := scanner.Err(); err != nil { return GroupStats{}, err } return AnalyzeTitles(titles), nil } // AnalyzeTitles parses a set of release lines (for example fetched from the // database by a particular releaser key) and aggregates the results into a // GroupStats summary. func AnalyzeTitles(titles []string) GroupStats { stats := GroupStats{ YearDistribution: make(map[int]int), SourceDistribution: make(map[string]int), BitDepthDistribution: make(map[string]int), ResolutionDistribution: make(map[string]int), LanguageDistribution: make(map[string]int), EditionDistribution: make(map[string]int), CommentDistribution: make(map[string]int), UnknownTokens: []releaseparser.UnknownTokenStats{}, WarningStats: []releaseparser.UnknownTokenStats{}, } totalConfidence := 0.0 unknownMap := make(map[string]*releaseparser.UnknownTokenStats) warningMap := make(map[string]*releaseparser.UnknownTokenStats) for _, title := range titles { result := Parse(title) stats.Total++ if result.Year != 0 { stats.YearDistribution[result.Year]++ } if result.Source != "" { stats.SourceDistribution[result.Source]++ } if result.BitDepth != "" { stats.BitDepthDistribution[result.BitDepth]++ } if result.Resolution != "" { stats.ResolutionDistribution[result.Resolution]++ } if result.Language != "" { stats.LanguageDistribution[result.Language]++ } if result.Edition != "" { stats.EditionDistribution[result.Edition]++ } if result.Comment != "" { for _, c := range strings.Split(result.Comment, ",") { stats.CommentDistribution[c]++ } } for _, unknown := range result.UnknownTokens { if _, exists := unknownMap[unknown]; !exists { unknownMap[unknown] = &releaseparser.UnknownTokenStats{ Token: unknown, Count: 0, Releases: []string{}, } } unknownMap[unknown].Count++ unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title) } for _, warning := range result.Warnings { if _, exists := warningMap[warning]; !exists { warningMap[warning] = &releaseparser.UnknownTokenStats{ Token: warning, Count: 0, Releases: []string{}, } } warningMap[warning].Count++ warningMap[warning].Releases = append(warningMap[warning].Releases, title) } totalConfidence += result.Confidence } // Convert the aggregates from maps to slices and order them by count // (most frequent first). for _, tokenStats := range unknownMap { stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats) } sort.Slice(stats.UnknownTokens, func(i, j int) bool { return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count }) for _, w := range warningMap { stats.WarningStats = append(stats.WarningStats, *w) } sort.Slice(stats.WarningStats, func(i, j int) bool { return stats.WarningStats[i].Count > stats.WarningStats[j].Count }) if stats.Total > 0 { stats.AvgConfidence = totalConfidence / float64(stats.Total) } return stats }