diff --git a/.gitignore b/.gitignore index 58ce334..ebe3296 100644 --- a/.gitignore +++ b/.gitignore @@ -13,4 +13,5 @@ tmp/ vendor/ config/secret.yaml *.md -mdb \ No newline at end of file +mdb +internal/releaseparser/resources/* \ No newline at end of file diff --git a/cmd/srrdb/main.go b/cmd/srrdb/main.go new file mode 100644 index 0000000..3c00451 --- /dev/null +++ b/cmd/srrdb/main.go @@ -0,0 +1,23 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + + "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood" +) + +func main() { + stats, err := moonblood.AnalyzeGroup() + if err != nil { + log.Fatal(err) + } + + data, err := json.MarshalIndent(stats, "", " ") + if err != nil { + log.Fatal(err) + } + + fmt.Println(string(data)) +} diff --git a/config/releaseparser.yaml b/config/releaseparser.yaml new file mode 100644 index 0000000..e69de29 diff --git a/internal/releaseparser/groups/moonblood/analyzer.go b/internal/releaseparser/groups/moonblood/analyzer.go new file mode 100644 index 0000000..e2670b4 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/analyzer.go @@ -0,0 +1,103 @@ +package moonblood + +import ( + "bufio" + "os" + "sort" + "strings" + + "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" +) + +type GroupStats struct { + Total int + YearDistribution map[int]int + SourceDistribution map[string]int + BitDepthDistribution map[string]int + ResolutionDistribution map[string]int + LanguageDistribution map[string]int + EditionDistribution map[string]int + UnknownTokens []releaseparser.UnknownTokenStats + AvgConfidence float64 +} + +func AnalyzeGroup() (GroupStats, error) { + stats := GroupStats{ + YearDistribution: make(map[int]int), + SourceDistribution: make(map[string]int), + BitDepthDistribution: make(map[string]int), + ResolutionDistribution: make(map[string]int), + LanguageDistribution: make(map[string]int), + EditionDistribution: make(map[string]int), + UnknownTokens: []releaseparser.UnknownTokenStats{}, + } + + file, err := os.Open("internal/releaseparser/resources/moonblood.txt") + if err != nil { + return stats, err + } + defer file.Close() + + scanner := bufio.NewScanner(file) + totalConfidence := 0.0 + + unknownMap := make(map[string]*releaseparser.UnknownTokenStats) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + + result := Parse(line) + stats.Total++ + + if result.Year != 0 { + stats.YearDistribution[result.Year]++ + } + if result.Source != "" { + stats.SourceDistribution[result.Source]++ + } + if result.BitDepth != "" { + stats.BitDepthDistribution[result.BitDepth]++ + } + if result.Resolution != "" { + stats.ResolutionDistribution[result.Resolution]++ + } + if result.Language != "" { + stats.LanguageDistribution[result.Language]++ + } + if result.Edition != "" { + stats.EditionDistribution[result.Edition]++ + } + + for _, unknown := range result.UnknownTokens { + if _, exists := unknownMap[unknown]; !exists { + unknownMap[unknown] = &releaseparser.UnknownTokenStats{ + Token: unknown, + Count: 0, + Releases: []string{}, + } + } + unknownMap[unknown].Count++ + unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, line) + } + + totalConfidence += result.Confidence + } + + // Convert map to slice and sort by count + for _, tokenStats := range unknownMap { + stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats) + } + + sort.Slice(stats.UnknownTokens, func(i, j int) bool { + return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count + }) + + if stats.Total > 0 { + stats.AvgConfidence = totalConfidence / float64(stats.Total) + } + + return stats, nil +} diff --git a/internal/releaseparser/groups/moonblood/dicts/bitdepth.txt b/internal/releaseparser/groups/moonblood/dicts/bitdepth.txt new file mode 100644 index 0000000..cd9226c --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/bitdepth.txt @@ -0,0 +1,3 @@ +# {bit_depth} +16BIT +24BIT diff --git a/internal/releaseparser/groups/moonblood/dicts/comment.txt b/internal/releaseparser/groups/moonblood/dicts/comment.txt new file mode 100644 index 0000000..645c5d1 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/comment.txt @@ -0,0 +1,3 @@ +REPACK +REAL_REPACK +READNFO \ No newline at end of file diff --git a/internal/releaseparser/groups/moonblood/dicts/edition.txt b/internal/releaseparser/groups/moonblood/dicts/edition.txt new file mode 100644 index 0000000..d13607c --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/edition.txt @@ -0,0 +1,20 @@ +# {edition} — тип видання +EP +DEMO +SPLIT +REISSUE +DELUXE_EDITION +REMASTERED +BOXSET +SINGLE +PROMO +REISSUE_REMASTERED +REMASTERED_25TH_ANNIVERSARY_EDITION +REMASTERED_20TH_ANNIVERSARY_EDITION +20TH_ANNIVERSARY_EDITION_REMASTERED +Deluxe_Edition +LIMITED_EDITION_BOXSET +REMASTERED_EP +10TH_ANNIVERSARY_EDITION +OST +REISSUE_DEMO \ No newline at end of file diff --git a/internal/releaseparser/groups/moonblood/dicts/format.txt b/internal/releaseparser/groups/moonblood/dicts/format.txt new file mode 100644 index 0000000..b7c21f2 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/format.txt @@ -0,0 +1,3 @@ +# {format} — аудіоформат +# один токен на рядок, регістр не має значення +FLAC diff --git a/internal/releaseparser/groups/moonblood/dicts/language.txt b/internal/releaseparser/groups/moonblood/dicts/language.txt new file mode 100644 index 0000000..5d43261 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/language.txt @@ -0,0 +1,18 @@ +# {language} — мовні маркери +DE +FR +ES +SK +FI +SE +BR +PL +NL +NO +PT +IT +HU +DK +AT +IS +UA diff --git a/internal/releaseparser/groups/moonblood/dicts/resolution.txt b/internal/releaseparser/groups/moonblood/dicts/resolution.txt new file mode 100644 index 0000000..bb8e616 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/resolution.txt @@ -0,0 +1,6 @@ +# {resolution} — частота дискретизації +44KHZ +44.1KHZ +48KHZ +88.2KHZ +96KHZ diff --git a/internal/releaseparser/groups/moonblood/dicts/source.txt b/internal/releaseparser/groups/moonblood/dicts/source.txt new file mode 100644 index 0000000..e5da59c --- /dev/null +++ b/internal/releaseparser/groups/moonblood/dicts/source.txt @@ -0,0 +1,11 @@ +# {source} — джерело релізу +# VINYL сюди входить як source (за домовленістю, не edition) +WEB +CD +2CD +CDEP +VINYL +TAPE +CDR +17CD +3CD \ No newline at end of file diff --git a/internal/releaseparser/groups/moonblood/parser.go b/internal/releaseparser/groups/moonblood/parser.go new file mode 100644 index 0000000..42bdf24 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/parser.go @@ -0,0 +1,170 @@ +package moonblood + +import ( + "regexp" + "strconv" + "strings" + + "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" +) + +var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`) + +func Parse(raw string) releaseparser.ParsedRelease { + result := releaseparser.ParsedRelease{ + Raw: strings.TrimSpace(raw), + Warnings: []string{}, + UnknownTokens: []string{}, + } + + tokens := split(raw) + if len(tokens) < 5 { + result.Warnings = append(result.Warnings, "too few tokens") + result.Confidence = 0.2 + return result + } + + // Останній токен — група + result.Group = tokens[len(tokens)-1] + if result.Group != "MOONBLOOD" { + result.Warnings = append(result.Warnings, "unexpected group") + } + + // Передостанній — рік + last := len(tokens) - 2 + if yearRegexp.MatchString(tokens[last]) { + result.Year, _ = strconv.Atoi(tokens[last]) + last-- + } + + // Рухаємось вліво по хвосту: format, source, bit_depth, resolution, language, edition. + // Кожен слот бере токен лише зі свого словника (регістронезалежно). + // Невідомий токен: репортимо лише якщо ліворуч ще є відомий хвостовий маркер + // (токен між двома маркерами = сміття в хвості). Інакше це початок core. +tailLoop: + for last >= 0 { + token := strings.ToUpper(tokens[last]) + + switch { + case Profile.FormatTokens[token]: + if result.Format == "" { + result.Format = token + } + last-- + + case Profile.SourceTokens[token]: + if result.Source == "" { + result.Source = token + } + last-- + + case Profile.BitDepthTokens[token]: + if result.BitDepth == "" { + result.BitDepth = token + } + last-- + + case Profile.ResolutionTokens[token]: + if result.Resolution == "" { + result.Resolution = token + } + last-- + + case Profile.LanguageMarkers[token]: + if result.Language == "" { + result.Language = token + } + last-- + + case Profile.EditionTokens[token]: + if result.Edition == "" { + result.Edition = token + } + last-- + + default: + if hasKnownTailToken(tokens[:last]) { + result.UnknownTokens = append(result.UnknownTokens, tokens[last]) + last-- + continue + } + break tailLoop + } + } + + // Core = artist-title (все, що залишилось ліворуч) + coreTokens := tokens[:last+1] + result.Core = strings.Join(coreTokens, "-") + + if len(coreTokens) >= 2 { + result.Artist = coreTokens[0] + result.Title = strings.Join(coreTokens[1:], "-") + } else if len(coreTokens) == 1 { + result.Artist = coreTokens[0] + result.Title = "" + result.Warnings = append(result.Warnings, "title not found") + } + + score(&result) + return result +} + +func split(raw string) []string { + parts := strings.Split(strings.TrimSpace(raw), "-") + out := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out +} + +// hasKnownTailToken reports whether any token is a known tail marker +// (format, source, bit_depth, resolution, edition, language or a year). +func hasKnownTailToken(tokens []string) bool { + for _, tok := range tokens { + up := strings.ToUpper(tok) + if Profile.FormatTokens[up] || Profile.SourceTokens[up] || + Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] || + Profile.EditionTokens[up] || Profile.LanguageMarkers[up] { + return true + } + if yearRegexp.MatchString(up) { + return true + } + } + return false +} + +func score(result *releaseparser.ParsedRelease) { + score := 0.0 + if result.Group == "MOONBLOOD" { + score += 0.25 + } + if result.Year != 0 { + score += 0.20 + } + if result.Format != "" { + score += 0.15 + } + if result.Source != "" { + score += 0.15 + } + if result.Artist != "" { + score += 0.15 + } + if len(result.Warnings) == 0 { + score += 0.10 + } + if len(result.UnknownTokens) > 0 { + score -= 0.10 + } + if score > 1 { + score = 1 + } + if score < 0 { + score = 0 + } + result.Confidence = score +} diff --git a/internal/releaseparser/groups/moonblood/parser_test.go b/internal/releaseparser/groups/moonblood/parser_test.go new file mode 100644 index 0000000..4ab5f2c --- /dev/null +++ b/internal/releaseparser/groups/moonblood/parser_test.go @@ -0,0 +1,178 @@ +package moonblood + +import ( + "bufio" + "os" + "testing" +) + +func TestParseMoonblood(t *testing.T) { + file, err := os.Open("../../resources/moonblood.txt") + if err != nil { + t.Fatal(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + total, success := 0, 0 + + for scanner.Scan() { + line := scanner.Text() + if line == "" { + continue + } + + result := Parse(line) + total++ + + if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" { + success++ + } + + if result.Confidence < 0.7 { + t.Logf("Low confidence: %s (%.2f)", line, result.Confidence) + } + } + + t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100) +} + +func TestParseFullTail(t *testing.T) { + r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD") + if r.Artist != "Fane" { + t.Errorf("Artist = %q, want %q", r.Artist, "Fane") + } + if r.Title != "Dragged_Through_the_Unending" { + t.Errorf("Title = %q, want %q", r.Title, "Dragged_Through_the_Unending") + } + if r.Year != 2025 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth != "24BIT" { + t.Errorf("tail fields wrong: %+v", r) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } + if r.Confidence < 0.99 { + t.Errorf("Confidence = %.2f, want ~1.0", r.Confidence) + } +} + +func TestParseEditionAndDemo(t *testing.T) { + r := Parse("Satans_Grind-Rotten_Demo-DEMO-16BIT-WEB-FLAC-2016-MOONBLOOD") + if r.Edition != "DEMO" { + t.Errorf("Edition = %q, want %q", r.Edition, "DEMO") + } + if r.Artist != "Satans_Grind" || r.Title != "Rotten_Demo" { + t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } +} + +func TestParseArtistWithHyphen(t *testing.T) { + // Artist itself contains a hyphen; no part of it may leak into UnknownTokens. + r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD") + if r.Artist != "Ultra" || r.Title != "Violence-All_My_Life" { + t.Errorf("Artist/Title = %q/%q, want Ultra / Violence-All_My_Life", r.Artist, r.Title) + } + if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 { + t.Errorf("tail fields wrong: %+v", r) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } +} + +func TestParseLanguageAndCDSource(t *testing.T) { + r := Parse("Burn_Thou_in_Effigy-Memorias_Postumas_do_Imperador-BR-CD-FLAC-2026-MOONBLOOD") + if r.Language != "BR" || r.Source != "CD" || r.Year != 2026 || r.Format != "FLAC" { + t.Errorf("tail fields wrong: %+v", r) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } +} + +func TestParseNoYear(t *testing.T) { + // Without a year the tail still parses; group stays the last token. + r := Parse("Fane-Gallant_Shroud-EP-16BIT-WEB-FLAC-MOONBLOOD") + if r.Group != "MOONBLOOD" || r.Year != 0 { + t.Errorf("Group/Year = %q/%d", r.Group, r.Year) + } + if r.Edition != "EP" || r.Source != "WEB" || r.Format != "FLAC" { + t.Errorf("tail fields wrong: %+v", r) + } + if r.Artist != "Fane" || r.Title != "Gallant_Shroud" { + t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) + } +} + +func TestParseResolutionSlot(t *testing.T) { + r := Parse("Ice_Nine_Kills-Last_Chance_To_Make_Amends-REMASTERED-24BIT-44KHZ-WEB-FLAC-2024-FLACCiD") + if r.Group != "FLACCiD" { + t.Errorf("Group = %q, want FLACCiD", r.Group) + } + if r.Resolution != "44KHZ" { + t.Errorf("Resolution = %q, want 44KHZ", r.Resolution) + } + if r.BitDepth != "24BIT" || r.Edition != "REMASTERED" || r.Source != "WEB" || r.Year != 2024 { + t.Errorf("tail fields wrong: %+v", r) + } + if r.Artist != "Ice_Nine_Kills" || r.Title != "Last_Chance_To_Make_Amends" { + t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } +} + +func TestParseVinylIsSource(t *testing.T) { + r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT") + if r.Language != "DE" { + t.Errorf("Language = %q, want DE", r.Language) + } + if r.Source != "VINYL" { + t.Errorf("Source = %q, want VINYL", r.Source) + } + if r.Edition != "" { + t.Errorf("Edition = %q, want empty (VINYL is source, not edition)", r.Edition) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } +} + +func TestParseCaseInsensitiveDictionaries(t *testing.T) { + r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood") + if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" { + t.Errorf("case-insensitive lookup failed: %+v", r) + } + if r.Artist != "Artist" || r.Title != "Some_Title" { + t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) + } +} + +func TestParseUnknownTailTokenReported(t *testing.T) { + // Невідомий токен між двома відомими маркерами → у UnknownTokens. + r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD") + if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" { + t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens) + } + if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 { + t.Errorf("tail fields wrong: %+v", r) + } + if r.Artist != "Artist" || r.Title != "Some_Title" { + t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) + } +} + +func TestParseCoreTokensNeverReported(t *testing.T) { + // Артист/тайтл ніколи не потрапляють в UnknownTokens. + r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD") + if len(r.UnknownTokens) != 0 { + t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens) + } + if r.Artist != "Fane" || r.Title != "Dragged_Through_the_Unending" { + t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) + } +} diff --git a/internal/releaseparser/groups/moonblood/profile.go b/internal/releaseparser/groups/moonblood/profile.go new file mode 100644 index 0000000..357d919 --- /dev/null +++ b/internal/releaseparser/groups/moonblood/profile.go @@ -0,0 +1,41 @@ +package moonblood + +import ( + "embed" + "strings" + + "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser" +) + +// Словники токенів лежать у dicts/*.txt — один токен на рядок. +// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці). + +//go:embed dicts/*.txt +var dictFS embed.FS + +func loadDict(name string) map[string]bool { + data, err := dictFS.ReadFile("dicts/" + name + ".txt") + if err != nil { + panic("moonblood: cannot load dictionary " + name + ": " + err.Error()) + } + out := make(map[string]bool) + for _, line := range strings.Split(string(data), "\n") { + line = strings.ToUpper(strings.TrimSpace(line)) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + out[line] = true + } + return out +} + +var Profile = releaseparser.GroupProfile{ + Name: "MOONBLOOD", + ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"}, + FormatTokens: loadDict("format"), + SourceTokens: loadDict("source"), + BitDepthTokens: loadDict("bitdepth"), + ResolutionTokens: loadDict("resolution"), + LanguageMarkers: loadDict("language"), + EditionTokens: loadDict("edition"), +} diff --git a/internal/releaseparser/types.go b/internal/releaseparser/types.go new file mode 100644 index 0000000..c0d7b98 --- /dev/null +++ b/internal/releaseparser/types.go @@ -0,0 +1,48 @@ +package releaseparser + +type GroupProfile struct { + Name string + ExpectedTailOrder []string + FormatTokens map[string]bool + SourceTokens map[string]bool + BitDepthTokens map[string]bool + ResolutionTokens map[string]bool + LanguageMarkers map[string]bool + EditionTokens map[string]bool +} + +type ParsedRelease struct { + Raw string + Group string + Year int + BitDepth string + Resolution string + Source string + Format string + Language string + Edition string + Core string + Artist string + Title string + UnknownTokens []string + Warnings []string + Confidence float64 +} + +type UnknownTokenStats struct { + Token string + Count int + Releases []string +} + +type GroupStats struct { + Total int + YearDistribution map[int]int + SourceDistribution map[string]int + BitDepthDistribution map[string]int + ResolutionDistribution map[string]int + LanguageDistribution map[string]int + EditionDistribution map[string]int + UnknownTokens []UnknownTokenStats + AvgConfidence float64 +}