refactored & audited, real analizer added

2026-08-31 22:24:38 +03:00
parent e11c82bd76
commit 1b2521d4ac
15 changed files with 629 additions and 1 deletions
@@ -14,3 +14,4 @@ vendor/
 config/secret.yaml
 *.md
 mdb
+internal/releaseparser/resources/*
@@ -0,0 +1,23 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
+)
+
+func main() {
+	stats, err := moonblood.AnalyzeGroup()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	data, err := json.MarshalIndent(stats, "", "  ")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Println(string(data))
+}
@@ -0,0 +1,103 @@
+package moonblood
+
+import (
+	"bufio"
+	"os"
+	"sort"
+	"strings"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
+)
+
+type GroupStats struct {
+	Total                int
+	YearDistribution     map[int]int
+	SourceDistribution   map[string]int
+	BitDepthDistribution map[string]int
+	ResolutionDistribution map[string]int
+	LanguageDistribution map[string]int
+	EditionDistribution  map[string]int
+	UnknownTokens        []releaseparser.UnknownTokenStats
+	AvgConfidence        float64
+}
+
+func AnalyzeGroup() (GroupStats, error) {
+	stats := GroupStats{
+		YearDistribution:     make(map[int]int),
+		SourceDistribution:   make(map[string]int),
+		BitDepthDistribution: make(map[string]int),
+		ResolutionDistribution: make(map[string]int),
+		LanguageDistribution: make(map[string]int),
+		EditionDistribution:  make(map[string]int),
+		UnknownTokens:        []releaseparser.UnknownTokenStats{},
+	}
+
+	file, err := os.Open("internal/releaseparser/resources/moonblood.txt")
+	if err != nil {
+		return stats, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	totalConfidence := 0.0
+
+	unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
+
+	for scanner.Scan() {
+		line := strings.TrimSpace(scanner.Text())
+		if line == "" {
+			continue
+		}
+
+		result := Parse(line)
+		stats.Total++
+
+		if result.Year != 0 {
+			stats.YearDistribution[result.Year]++
+		}
+		if result.Source != "" {
+			stats.SourceDistribution[result.Source]++
+		}
+		if result.BitDepth != "" {
+			stats.BitDepthDistribution[result.BitDepth]++
+		}
+		if result.Resolution != "" {
+			stats.ResolutionDistribution[result.Resolution]++
+		}
+		if result.Language != "" {
+			stats.LanguageDistribution[result.Language]++
+		}
+		if result.Edition != "" {
+			stats.EditionDistribution[result.Edition]++
+		}
+
+		for _, unknown := range result.UnknownTokens {
+			if _, exists := unknownMap[unknown]; !exists {
+				unknownMap[unknown] = &releaseparser.UnknownTokenStats{
+					Token:    unknown,
+					Count:    0,
+					Releases: []string{},
+				}
+			}
+			unknownMap[unknown].Count++
+			unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, line)
+		}
+
+		totalConfidence += result.Confidence
+	}
+
+	// Convert map to slice and sort by count
+	for _, tokenStats := range unknownMap {
+		stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
+	}
+
+	sort.Slice(stats.UnknownTokens, func(i, j int) bool {
+		return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
+	})
+
+	if stats.Total > 0 {
+		stats.AvgConfidence = totalConfidence / float64(stats.Total)
+	}
+
+	return stats, nil
+}
@@ -0,0 +1,3 @@
+# {bit_depth}
+16BIT
+24BIT
@@ -0,0 +1,3 @@
+REPACK
+REAL_REPACK
+READNFO
@@ -0,0 +1,20 @@
+# {edition} — тип видання
+EP
+DEMO
+SPLIT
+REISSUE
+DELUXE_EDITION
+REMASTERED
+BOXSET
+SINGLE
+PROMO
+REISSUE_REMASTERED
+REMASTERED_25TH_ANNIVERSARY_EDITION
+REMASTERED_20TH_ANNIVERSARY_EDITION
+20TH_ANNIVERSARY_EDITION_REMASTERED
+Deluxe_Edition
+LIMITED_EDITION_BOXSET
+REMASTERED_EP
+10TH_ANNIVERSARY_EDITION
+OST
+REISSUE_DEMO
@@ -0,0 +1,3 @@
+# {format} — аудіоформат
+# один токен на рядок, регістр не має значення
+FLAC
@@ -0,0 +1,18 @@
+# {language} — мовні маркери
+DE
+FR
+ES
+SK
+FI
+SE
+BR
+PL
+NL
+NO
+PT
+IT
+HU
+DK
+AT
+IS
+UA
@@ -0,0 +1,6 @@
+# {resolution} — частота дискретизації
+44KHZ
+44.1KHZ
+48KHZ
+88.2KHZ
+96KHZ
@@ -0,0 +1,11 @@
+# {source} — джерело релізу
+# VINYL сюди входить як source (за домовленістю, не edition)
+WEB
+CD
+2CD
+CDEP
+VINYL
+TAPE
+CDR
+17CD
+3CD
@@ -0,0 +1,170 @@
+package moonblood
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
+)
+
+var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
+
+func Parse(raw string) releaseparser.ParsedRelease {
+	result := releaseparser.ParsedRelease{
+		Raw:           strings.TrimSpace(raw),
+		Warnings:      []string{},
+		UnknownTokens: []string{},
+	}
+
+	tokens := split(raw)
+	if len(tokens) < 5 {
+		result.Warnings = append(result.Warnings, "too few tokens")
+		result.Confidence = 0.2
+		return result
+	}
+
+	// Останній токен — група
+	result.Group = tokens[len(tokens)-1]
+	if result.Group != "MOONBLOOD" {
+		result.Warnings = append(result.Warnings, "unexpected group")
+	}
+
+	// Передостанній — рік
+	last := len(tokens) - 2
+	if yearRegexp.MatchString(tokens[last]) {
+		result.Year, _ = strconv.Atoi(tokens[last])
+		last--
+	}
+
+	// Рухаємось вліво по хвосту: format, source, bit_depth, resolution, language, edition.
+	// Кожен слот бере токен лише зі свого словника (регістронезалежно).
+	// Невідомий токен: репортимо лише якщо ліворуч ще є відомий хвостовий маркер
+	// (токен між двома маркерами = сміття в хвості). Інакше це початок core.
+tailLoop:
+	for last >= 0 {
+		token := strings.ToUpper(tokens[last])
+
+		switch {
+		case Profile.FormatTokens[token]:
+			if result.Format == "" {
+				result.Format = token
+			}
+			last--
+
+		case Profile.SourceTokens[token]:
+			if result.Source == "" {
+				result.Source = token
+			}
+			last--
+
+		case Profile.BitDepthTokens[token]:
+			if result.BitDepth == "" {
+				result.BitDepth = token
+			}
+			last--
+
+		case Profile.ResolutionTokens[token]:
+			if result.Resolution == "" {
+				result.Resolution = token
+			}
+			last--
+
+		case Profile.LanguageMarkers[token]:
+			if result.Language == "" {
+				result.Language = token
+			}
+			last--
+
+		case Profile.EditionTokens[token]:
+			if result.Edition == "" {
+				result.Edition = token
+			}
+			last--
+
+		default:
+			if hasKnownTailToken(tokens[:last]) {
+				result.UnknownTokens = append(result.UnknownTokens, tokens[last])
+				last--
+				continue
+			}
+			break tailLoop
+		}
+	}
+
+	// Core = artist-title (все, що залишилось ліворуч)
+	coreTokens := tokens[:last+1]
+	result.Core = strings.Join(coreTokens, "-")
+
+	if len(coreTokens) >= 2 {
+		result.Artist = coreTokens[0]
+		result.Title = strings.Join(coreTokens[1:], "-")
+	} else if len(coreTokens) == 1 {
+		result.Artist = coreTokens[0]
+		result.Title = ""
+		result.Warnings = append(result.Warnings, "title not found")
+	}
+
+	score(&result)
+	return result
+}
+
+func split(raw string) []string {
+	parts := strings.Split(strings.TrimSpace(raw), "-")
+	out := make([]string, 0, len(parts))
+	for _, part := range parts {
+		if part = strings.TrimSpace(part); part != "" {
+			out = append(out, part)
+		}
+	}
+	return out
+}
+
+// hasKnownTailToken reports whether any token is a known tail marker
+// (format, source, bit_depth, resolution, edition, language or a year).
+func hasKnownTailToken(tokens []string) bool {
+	for _, tok := range tokens {
+		up := strings.ToUpper(tok)
+		if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
+			Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
+			Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
+			return true
+		}
+		if yearRegexp.MatchString(up) {
+			return true
+		}
+	}
+	return false
+}
+
+func score(result *releaseparser.ParsedRelease) {
+	score := 0.0
+	if result.Group == "MOONBLOOD" {
+		score += 0.25
+	}
+	if result.Year != 0 {
+		score += 0.20
+	}
+	if result.Format != "" {
+		score += 0.15
+	}
+	if result.Source != "" {
+		score += 0.15
+	}
+	if result.Artist != "" {
+		score += 0.15
+	}
+	if len(result.Warnings) == 0 {
+		score += 0.10
+	}
+	if len(result.UnknownTokens) > 0 {
+		score -= 0.10
+	}
+	if score > 1 {
+		score = 1
+	}
+	if score < 0 {
+		score = 0
+	}
+	result.Confidence = score
+}
@@ -0,0 +1,178 @@
+package moonblood
+
+import (
+	"bufio"
+	"os"
+	"testing"
+)
+
+func TestParseMoonblood(t *testing.T) {
+	file, err := os.Open("../../resources/moonblood.txt")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	total, success := 0, 0
+
+	for scanner.Scan() {
+		line := scanner.Text()
+		if line == "" {
+			continue
+		}
+
+		result := Parse(line)
+		total++
+
+		if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" {
+			success++
+		}
+
+		if result.Confidence < 0.7 {
+			t.Logf("Low confidence: %s (%.2f)", line, result.Confidence)
+		}
+	}
+
+	t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100)
+}
+
+func TestParseFullTail(t *testing.T) {
+	r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
+	if r.Artist != "Fane" {
+		t.Errorf("Artist = %q, want %q", r.Artist, "Fane")
+	}
+	if r.Title != "Dragged_Through_the_Unending" {
+		t.Errorf("Title = %q, want %q", r.Title, "Dragged_Through_the_Unending")
+	}
+	if r.Year != 2025 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth != "24BIT" {
+		t.Errorf("tail fields wrong: %+v", r)
+	}
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+	if r.Confidence < 0.99 {
+		t.Errorf("Confidence = %.2f, want ~1.0", r.Confidence)
+	}
+}
+
+func TestParseEditionAndDemo(t *testing.T) {
+	r := Parse("Satans_Grind-Rotten_Demo-DEMO-16BIT-WEB-FLAC-2016-MOONBLOOD")
+	if r.Edition != "DEMO" {
+		t.Errorf("Edition = %q, want %q", r.Edition, "DEMO")
+	}
+	if r.Artist != "Satans_Grind" || r.Title != "Rotten_Demo" {
+		t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
+	}
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+}
+
+func TestParseArtistWithHyphen(t *testing.T) {
+	// Artist itself contains a hyphen; no part of it may leak into UnknownTokens.
+	r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
+	if r.Artist != "Ultra" || r.Title != "Violence-All_My_Life" {
+		t.Errorf("Artist/Title = %q/%q, want Ultra / Violence-All_My_Life", r.Artist, r.Title)
+	}
+	if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
+		t.Errorf("tail fields wrong: %+v", r)
+	}
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+}
+
+func TestParseLanguageAndCDSource(t *testing.T) {
+	r := Parse("Burn_Thou_in_Effigy-Memorias_Postumas_do_Imperador-BR-CD-FLAC-2026-MOONBLOOD")
+	if r.Language != "BR" || r.Source != "CD" || r.Year != 2026 || r.Format != "FLAC" {
+		t.Errorf("tail fields wrong: %+v", r)
+	}
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+}
+
+func TestParseNoYear(t *testing.T) {
+	// Without a year the tail still parses; group stays the last token.
+	r := Parse("Fane-Gallant_Shroud-EP-16BIT-WEB-FLAC-MOONBLOOD")
+	if r.Group != "MOONBLOOD" || r.Year != 0 {
+		t.Errorf("Group/Year = %q/%d", r.Group, r.Year)
+	}
+	if r.Edition != "EP" || r.Source != "WEB" || r.Format != "FLAC" {
+		t.Errorf("tail fields wrong: %+v", r)
+	}
+	if r.Artist != "Fane" || r.Title != "Gallant_Shroud" {
+		t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
+	}
+}
+
+func TestParseResolutionSlot(t *testing.T) {
+	r := Parse("Ice_Nine_Kills-Last_Chance_To_Make_Amends-REMASTERED-24BIT-44KHZ-WEB-FLAC-2024-FLACCiD")
+	if r.Group != "FLACCiD" {
+		t.Errorf("Group = %q, want FLACCiD", r.Group)
+	}
+	if r.Resolution != "44KHZ" {
+		t.Errorf("Resolution = %q, want 44KHZ", r.Resolution)
+	}
+	if r.BitDepth != "24BIT" || r.Edition != "REMASTERED" || r.Source != "WEB" || r.Year != 2024 {
+		t.Errorf("tail fields wrong: %+v", r)
+	}
+	if r.Artist != "Ice_Nine_Kills" || r.Title != "Last_Chance_To_Make_Amends" {
+		t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
+	}
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+}
+
+func TestParseVinylIsSource(t *testing.T) {
+	r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
+	if r.Language != "DE" {
+		t.Errorf("Language = %q, want DE", r.Language)
+	}
+	if r.Source != "VINYL" {
+		t.Errorf("Source = %q, want VINYL", r.Source)
+	}
+	if r.Edition != "" {
+		t.Errorf("Edition = %q, want empty (VINYL is source, not edition)", r.Edition)
+	}
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+}
+
+func TestParseCaseInsensitiveDictionaries(t *testing.T) {
+	r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood")
+	if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" {
+		t.Errorf("case-insensitive lookup failed: %+v", r)
+	}
+	if r.Artist != "Artist" || r.Title != "Some_Title" {
+		t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
+	}
+}
+
+func TestParseUnknownTailTokenReported(t *testing.T) {
+	// Невідомий токен між двома відомими маркерами → у UnknownTokens.
+	r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
+	if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
+		t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
+	}
+	if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
+		t.Errorf("tail fields wrong: %+v", r)
+	}
+	if r.Artist != "Artist" || r.Title != "Some_Title" {
+		t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
+	}
+}
+
+func TestParseCoreTokensNeverReported(t *testing.T) {
+	// Артист/тайтл ніколи не потрапляють в UnknownTokens.
+	r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
+	if len(r.UnknownTokens) != 0 {
+		t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
+	}
+	if r.Artist != "Fane" || r.Title != "Dragged_Through_the_Unending" {
+		t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
+	}
+}
@@ -0,0 +1,41 @@
+package moonblood
+
+import (
+	"embed"
+	"strings"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
+)
+
+// Словники токенів лежать у dicts/*.txt — один токен на рядок.
+// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці).
+
+//go:embed dicts/*.txt
+var dictFS embed.FS
+
+func loadDict(name string) map[string]bool {
+	data, err := dictFS.ReadFile("dicts/" + name + ".txt")
+	if err != nil {
+		panic("moonblood: cannot load dictionary " + name + ": " + err.Error())
+	}
+	out := make(map[string]bool)
+	for _, line := range strings.Split(string(data), "\n") {
+		line = strings.ToUpper(strings.TrimSpace(line))
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		out[line] = true
+	}
+	return out
+}
+
+var Profile = releaseparser.GroupProfile{
+	Name:              "MOONBLOOD",
+	ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"},
+	FormatTokens:      loadDict("format"),
+	SourceTokens:      loadDict("source"),
+	BitDepthTokens:    loadDict("bitdepth"),
+	ResolutionTokens:  loadDict("resolution"),
+	LanguageMarkers:   loadDict("language"),
+	EditionTokens:     loadDict("edition"),
+}
@@ -0,0 +1,48 @@
+package releaseparser
+
+type GroupProfile struct {
+	Name             string
+	ExpectedTailOrder []string
+	FormatTokens     map[string]bool
+	SourceTokens     map[string]bool
+	BitDepthTokens   map[string]bool
+	ResolutionTokens map[string]bool
+	LanguageMarkers  map[string]bool
+	EditionTokens    map[string]bool
+}
+
+type ParsedRelease struct {
+	Raw           string
+	Group         string
+	Year          int
+	BitDepth      string
+	Resolution    string
+	Source        string
+	Format        string
+	Language      string
+	Edition       string
+	Core          string
+	Artist        string
+	Title         string
+	UnknownTokens []string
+	Warnings      []string
+	Confidence    float64
+}
+
+type UnknownTokenStats struct {
+	Token    string
+	Count    int
+	Releases []string
+}
+
+type GroupStats struct {
+	Total                int
+	YearDistribution     map[int]int
+	SourceDistribution   map[string]int
+	BitDepthDistribution map[string]int
+	ResolutionDistribution map[string]int
+	LanguageDistribution map[string]int
+	EditionDistribution  map[string]int
+	UnknownTokens        []UnknownTokenStats
+	AvgConfidence        float64
+}