Yevhen Odynets
115 lines
3.0 KiB
Go
115 lines
3.0 KiB
Go
package moonblood
|
|
|
|
import (
|
|
"bufio"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
|
)
|
|
|
|
type GroupStats struct {
|
|
Total int
|
|
YearDistribution map[int]int
|
|
SourceDistribution map[string]int
|
|
BitDepthDistribution map[string]int
|
|
ResolutionDistribution map[string]int
|
|
LanguageDistribution map[string]int
|
|
EditionDistribution map[string]int
|
|
UnknownTokens []releaseparser.UnknownTokenStats
|
|
AvgConfidence float64
|
|
}
|
|
|
|
// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників).
|
|
func AnalyzeGroup() (GroupStats, error) {
|
|
file, err := os.Open("../../resources/moonblood.txt")
|
|
if err != nil {
|
|
return GroupStats{}, err
|
|
}
|
|
defer file.Close()
|
|
|
|
var titles []string
|
|
scanner := bufio.NewScanner(file)
|
|
for scanner.Scan() {
|
|
line := strings.TrimSpace(scanner.Text())
|
|
if line != "" {
|
|
titles = append(titles, line)
|
|
}
|
|
}
|
|
if err := scanner.Err(); err != nil {
|
|
return GroupStats{}, err
|
|
}
|
|
|
|
return AnalyzeTitles(titles), nil
|
|
}
|
|
|
|
// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі).
|
|
func AnalyzeTitles(titles []string) GroupStats {
|
|
stats := GroupStats{
|
|
YearDistribution: make(map[int]int),
|
|
SourceDistribution: make(map[string]int),
|
|
BitDepthDistribution: make(map[string]int),
|
|
ResolutionDistribution: make(map[string]int),
|
|
LanguageDistribution: make(map[string]int),
|
|
EditionDistribution: make(map[string]int),
|
|
UnknownTokens: []releaseparser.UnknownTokenStats{},
|
|
}
|
|
|
|
totalConfidence := 0.0
|
|
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
|
|
|
for _, title := range titles {
|
|
result := Parse(title)
|
|
stats.Total++
|
|
|
|
if result.Year != 0 {
|
|
stats.YearDistribution[result.Year]++
|
|
}
|
|
if result.Source != "" {
|
|
stats.SourceDistribution[result.Source]++
|
|
}
|
|
if result.BitDepth != "" {
|
|
stats.BitDepthDistribution[result.BitDepth]++
|
|
}
|
|
if result.Resolution != "" {
|
|
stats.ResolutionDistribution[result.Resolution]++
|
|
}
|
|
if result.Language != "" {
|
|
stats.LanguageDistribution[result.Language]++
|
|
}
|
|
if result.Edition != "" {
|
|
stats.EditionDistribution[result.Edition]++
|
|
}
|
|
|
|
for _, unknown := range result.UnknownTokens {
|
|
if _, exists := unknownMap[unknown]; !exists {
|
|
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
|
|
Token: unknown,
|
|
Count: 0,
|
|
Releases: []string{},
|
|
}
|
|
}
|
|
unknownMap[unknown].Count++
|
|
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
|
|
}
|
|
|
|
totalConfidence += result.Confidence
|
|
}
|
|
|
|
// Convert map to slice and sort by count
|
|
for _, tokenStats := range unknownMap {
|
|
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
|
}
|
|
|
|
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
|
|
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
|
})
|
|
|
|
if stats.Total > 0 {
|
|
stats.AvgConfidence = totalConfidence / float64(stats.Total)
|
|
}
|
|
|
|
return stats
|
|
}
|