resource-scraper/internal/releaseparser/groups/moonblood/analyzer.go

package moonblood

import (
	"bufio"
	"os"
	"sort"
	"strings"

	"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)

// GroupStats summarises how a set of release lines parsed: field
// distributions, unknown tokens and warnings aggregated by occurrence, and
// the average parsing confidence. The slices are sorted by descending count.
type GroupStats struct {
	Total                  int
	YearDistribution       map[int]int
	SourceDistribution     map[string]int
	BitDepthDistribution   map[string]int
	ResolutionDistribution map[string]int
	LanguageDistribution   map[string]int
	EditionDistribution    map[string]int
	CommentDistribution    map[string]int
	UnknownTokens          []releaseparser.UnknownTokenStats
	WarningStats           []releaseparser.UnknownTokenStats
	AvgConfidence          float64
}

// AnalyzeGroup parses the release lines from the local resource file. It is
// used for a quick local check of the dictionaries without touching a
// database.
func AnalyzeGroup() (GroupStats, error) {
	file, err := os.Open("../../resources/moonblood.txt")
	if err != nil {
		return GroupStats{}, err
	}
	defer file.Close()

	var titles []string
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if line != "" {
			titles = append(titles, line)
		}
	}
	if err := scanner.Err(); err != nil {
		return GroupStats{}, err
	}

	return AnalyzeTitles(titles), nil
}

// AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.
func AnalyzeTitles(titles []string) GroupStats {
	stats := GroupStats{
		YearDistribution:       make(map[int]int),
		SourceDistribution:     make(map[string]int),
		BitDepthDistribution:   make(map[string]int),
		ResolutionDistribution: make(map[string]int),
		LanguageDistribution:   make(map[string]int),
		EditionDistribution:    make(map[string]int),
		CommentDistribution:    make(map[string]int),
		UnknownTokens:          []releaseparser.UnknownTokenStats{},
		WarningStats:           []releaseparser.UnknownTokenStats{},
	}

	totalConfidence := 0.0
	unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
	warningMap := make(map[string]*releaseparser.UnknownTokenStats)

	for _, title := range titles {
		result := Parse(title)
		stats.Total++

		if result.Year != 0 {
			stats.YearDistribution[result.Year]++
		}
		if result.Source != "" {
			stats.SourceDistribution[result.Source]++
		}
		if result.BitDepth != "" {
			stats.BitDepthDistribution[result.BitDepth]++
		}
		if result.Resolution != "" {
			stats.ResolutionDistribution[result.Resolution]++
		}
		if result.Language != "" {
			stats.LanguageDistribution[result.Language]++
		}
		if result.Edition != "" {
			stats.EditionDistribution[result.Edition]++
		}
		if result.Comment != "" {
			for _, c := range strings.Split(result.Comment, ",") {
				stats.CommentDistribution[c]++
			}
		}

		for _, unknown := range result.UnknownTokens {
			if _, exists := unknownMap[unknown]; !exists {
				unknownMap[unknown] = &releaseparser.UnknownTokenStats{
					Token:    unknown,
					Count:    0,
					Releases: []string{},
				}
			}
			unknownMap[unknown].Count++
			unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
		}

		for _, warning := range result.Warnings {
			if _, exists := warningMap[warning]; !exists {
				warningMap[warning] = &releaseparser.UnknownTokenStats{
					Token:    warning,
					Count:    0,
					Releases: []string{},
				}
			}
			warningMap[warning].Count++
			warningMap[warning].Releases = append(warningMap[warning].Releases, title)
		}

		totalConfidence += result.Confidence
	}

	// Convert the aggregates from maps to slices and order them by count
	// (most frequent first).
	for _, tokenStats := range unknownMap {
		stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
	}

	sort.Slice(stats.UnknownTokens, func(i, j int) bool {
		return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
	})

	for _, w := range warningMap {
		stats.WarningStats = append(stats.WarningStats, *w)
	}
	sort.Slice(stats.WarningStats, func(i, j int) bool {
		return stats.WarningStats[i].Count > stats.WarningStats[j].Count
	})

	if stats.Total > 0 {
		stats.AvgConfidence = totalConfidence / float64(stats.Total)
	}

	return stats
}