resource-scraper/internal/releaseparser/groups/analyzer.go

package groups

import (
	"sort"
	"strings"

	"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)

// GroupStats summarises how a set of release lines parsed: field
// distributions, unknown tokens and warnings aggregated by occurrence, and
// the average parsing confidence. The slices are sorted by descending count.

type GroupStats struct {
	Total                  int
	YearDistribution       map[int]int
	SourceDistribution     map[string]int
	BitDepthDistribution   map[string]int
	ResolutionDistribution map[string]int
	LanguageDistribution   map[string]int
	EditionDistribution    map[string]int
	CommentDistribution    map[string]int
	UnknownTokens          []releaseparser.UnknownTokenStats
	WarningStats           []releaseparser.UnknownTokenStats
	AvgConfidence          float64
}


// AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.

func AnalyzeTitles(titles []string) GroupStats {
	stats := GroupStats{
		YearDistribution:       make(map[int]int),
		SourceDistribution:     make(map[string]int),
		BitDepthDistribution:   make(map[string]int),
		ResolutionDistribution: make(map[string]int),
		LanguageDistribution:   make(map[string]int),
		EditionDistribution:    make(map[string]int),
		CommentDistribution:    make(map[string]int),
		UnknownTokens:          []releaseparser.UnknownTokenStats{},
		WarningStats:           []releaseparser.UnknownTokenStats{},
	}


	totalConfidence := 0.0
	unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
	warningMap := make(map[string]*releaseparser.UnknownTokenStats)


	for _, title := range titles {
		result := Parse(title)
		stats.Total++

		if result.Year != 0 {
			stats.YearDistribution[result.Year]++
		}
		if result.Source != "" {
			stats.SourceDistribution[result.Source]++
		}
		if result.BitDepth != "" {
			stats.BitDepthDistribution[result.BitDepth]++
		}
		if result.Resolution != "" {
			stats.ResolutionDistribution[result.Resolution]++
		}
		if result.Language != "" {
			stats.LanguageDistribution[result.Language]++
		}
		if result.Edition != "" {
			stats.EditionDistribution[result.Edition]++
		}
		if result.Comment != "" {
			for _, c := range strings.Split(result.Comment, ",") {
				stats.CommentDistribution[c]++
			}
		}

		for _, unknown := range result.UnknownTokens {
			if _, exists := unknownMap[unknown]; !exists {
				unknownMap[unknown] = &releaseparser.UnknownTokenStats{
					Token:    unknown,
					Count:    0,
					Releases: []string{},
				}
			}
			unknownMap[unknown].Count++
			unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
		}

		for _, warning := range result.Warnings {
			if _, exists := warningMap[warning]; !exists {
				warningMap[warning] = &releaseparser.UnknownTokenStats{
					Token:    warning,
					Count:    0,
					Releases: []string{},
				}
			}
			warningMap[warning].Count++
			warningMap[warning].Releases = append(warningMap[warning].Releases, title)
		}

		totalConfidence += result.Confidence
	}

	// Convert the aggregates from maps to slices and order them by count
	// (most frequent first).
	for _, tokenStats := range unknownMap {
		stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
	}
	sort.Slice(stats.UnknownTokens, func(i, j int) bool {
		return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
	})

	for _, w := range warningMap {
		stats.WarningStats = append(stats.WarningStats, *w)
	}
	sort.Slice(stats.WarningStats, func(i, j int) bool {
		return stats.WarningStats[i].Count > stats.WarningStats[j].Count
	})

	if stats.Total > 0 {
		stats.AvgConfidence = totalConfidence / float64(stats.Total)
	}

	return stats
}


// AnalyzeByGroup parses a set of release lines and returns per-group
// statistics keyed by the actual group read from each line (not from the
// query key, which may cover several groups). It is used to validate parsing
// quality against the original data, grouped by the real releaser name.
func AnalyzeByGroup(titles []string) map[string]GroupStats {
	byGroup := make(map[string][]string)
	for _, title := range titles {
		group := Parse(title).Group
		byGroup[group] = append(byGroup[group], title)
	}

	out := make(map[string]GroupStats, len(byGroup))
	for group, lines := range byGroup {
		out[group] = AnalyzeTitles(lines)
	}
	return out
}


// ParseByGroup parses a set of release lines into a per-group list of fully
// populated ParsedRelease records, keyed by the actual group read from each
// line. It is used to validate parsing quality field-by-field against the
// original data.
func ParseByGroup(titles []string) map[string][]releaseparser.ParsedRelease {
	byGroup := make(map[string][]releaseparser.ParsedRelease)
	for _, title := range titles {
		result := Parse(title)
		byGroup[result.Group] = append(byGroup[result.Group], result)
	}
	return byGroup
}