yevhen/resource-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

releaseparser: move group-agnostic parser to groups/, GB doc-blocks, per-group JSON reports

Browse Source 
Generalise the parser: the MOONBLOOD group profile is gone, the parser and
its dictionaries are now group-agnostic under internal/releaseparser/groups.

Positional model:
- [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so
  titles like "Split"/"Demo"/"De" are never eaten.
- [len-1]=Group is always read from the line, not from the query key.
- [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is
  resolved as a middle-zone token (e.g. FLAC is still picked up as format).
- The middle zone is resolved via case-insensitive dictionaries in the order
  format -> source -> bit-depth -> resolution -> edition -> language ->
  catalogue number; stored values keep their original case.

Catalogue numbers: any "(...)" token in the middle zone is a catalogue number
regardless of content (digits, roman numerals, dots, underscores, mixed case);
splitting never breaks tokens inside parentheses.

Dictionaries moved to internal/releaseparser/groups/resources/dicts and
embedded via go:embed; dead profile fields (Name, ExpectedTailOrder) removed;
profile comments rewritten in British English.

File-based AnalyzeGroup/TestParseMoonblood removed — titles now come from the
database (cmd/srrdb: srrdb <releaser>..., config/secret.yaml DSN).

Validation output (cmd/srrdb):
- tmp/releaseparser/<group>.json         — per-group parsed ParsedRelease records
- tmp/releaseparser/<releaser>.report.json — per-request aggregate (same as console)
- review_unknown_tokens.txt              — unknown/warning rows for manual review

Tests updated for the positional model; added
TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2) and (GOWP_DCCCLXXXIX).
go build, go vet and go test all pass.
This commit is contained in:
Yevhen Odynets
2026-09-01 11:28:19 +03:00
parent feef080d80
commit f2629f002c
15 changed files with 150 additions and 293 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/releaseparser/groups/moonblood/analyzer.go → internal/releaseparser/groups/analyzer.go
+38 -28
View File
@@ -1,8 +1,6 @@
package moonblood
package groups
import (
"bufio"
"os"
"sort"
"strings"
@@ -12,6 +10,7 @@ import (
// GroupStats summarises how a set of release lines parsed: field
// distributions, unknown tokens and warnings aggregated by occurrence, and
// the average parsing confidence. The slices are sorted by descending count.
type GroupStats struct {
Total int
YearDistribution map[int]int
@@ -26,34 +25,11 @@ type GroupStats struct {
AvgConfidence float64
}
// AnalyzeGroup parses the release lines from the local resource file. It is
// used for a quick local check of the dictionaries without touching a
// database.
func AnalyzeGroup() (GroupStats, error) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
return GroupStats{}, err
}
defer file.Close()
var titles []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
titles = append(titles, line)
}
}
if err := scanner.Err(); err != nil {
return GroupStats{}, err
}
return AnalyzeTitles(titles), nil
}
// AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.
func AnalyzeTitles(titles []string) GroupStats {
stats := GroupStats{
YearDistribution: make(map[int]int),
@@ -67,10 +43,12 @@ func AnalyzeTitles(titles []string) GroupStats {
WarningStats: []releaseparser.UnknownTokenStats{},
}
totalConfidence := 0.0
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
for _, title := range titles {
result := Parse(title)
stats.Total++
@@ -131,7 +109,6 @@ func AnalyzeTitles(titles []string) GroupStats {
for _, tokenStats := range unknownMap {
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
}
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
})
@@ -149,3 +126,36 @@ func AnalyzeTitles(titles []string) GroupStats {
return stats
}
// AnalyzeByGroup parses a set of release lines and returns per-group
// statistics keyed by the actual group read from each line (not from the
// query key, which may cover several groups). It is used to validate parsing
// quality against the original data, grouped by the real releaser name.
func AnalyzeByGroup(titles []string) map[string]GroupStats {
byGroup := make(map[string][]string)
for _, title := range titles {
group := Parse(title).Group
byGroup[group] = append(byGroup[group], title)
}
out := make(map[string]GroupStats, len(byGroup))
for group, lines := range byGroup {
out[group] = AnalyzeTitles(lines)
}
return out
}
// ParseByGroup parses a set of release lines into a per-group list of fully
// populated ParsedRelease records, keyed by the actual group read from each
// line. It is used to validate parsing quality field-by-field against the
// original data.
func ParseByGroup(titles []string) map[string][]releaseparser.ParsedRelease {
byGroup := make(map[string][]releaseparser.ParsedRelease)
for _, title := range titles {
result := Parse(title)
byGroup[result.Group] = append(byGroup[result.Group], result)
}
return byGroup
}
internal/releaseparser/groups/moonblood/dicts/bitdepth.txt
-5
View File
@@ -1,5 +0,0 @@
# {bit_depth}
16BIT
24BIT
32BIT
DE16BIT
internal/releaseparser/groups/moonblood/dicts/comment.txt
-6
View File
@@ -1,6 +0,0 @@
REPACK
REAL_REPACK
READNFO
PROPER
RERIP
199X
internal/releaseparser/groups/moonblood/dicts/edition.txt
-70
View File
@@ -1,70 +0,0 @@
# {edition} — тип видання
EP
DEMO
SPLIT
REISSUE
DELUXE_EDITION
REMASTERED
BOXSET
SINGLE
PROMO
REISSUE_REMASTERED
REMASTERED_25TH_ANNIVERSARY_EDITION
REMASTERED_20TH_ANNIVERSARY_EDITION
20TH_ANNIVERSARY_EDITION_REMASTERED
Deluxe_Edition
LIMITED_EDITION_BOXSET
REMASTERED_EP
10TH_ANNIVERSARY_EDITION
OST
XX_Legacy_Edition
REISSUE_DEMO
JP_Deluxe_Edition
Remastered_Edition
Anniversary_Edition
Limited_Edition
CDS
CDM
RETAIL
JP_RETAIL
JP_REISSUE
DEMO_REMASTERED
EU_RETAIL
DE_RETAIL
REISSUE_DELUXE_EDITION
RETAIL_REISSUE
SPECIAL_EDITION
SPECIAL_COLLECTORS_EDITION_REMASTERED
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
LIMITED_EDITION_REMASTERED
25th_Anniversary_Edition
DEMO_REISSUE
Ultimate_Edition
Issue_255
Issue_254
Issue_198
7INCH_VINYL
REMASTERED_REISSUE
PROMO_DEMO
ES_RETAIL
BOX_SET
BONUS
REISSUE_EP
30th_Anniversary
REDUX
REMASTERED_BOXSET
MAG
REISSUE_BOXSET
30TH_ANNIVERSARY_EDITION_REMASTERED
DELUXE
Deluxe_Edition_Reissue
INSTRUMENTAL
REISSUE_EXPANDED_EDITION
BOOTLEG
EXPANDED_EDITION
ADVANCE
15TH_ANNIVERSARY_DELUXE_EDITION
TOUR_EDITION
REMASTERED_DEMO
REMASTERED_DELUXE_EDITION
30TH_ANNIVERSARY_EDITION
internal/releaseparser/groups/moonblood/dicts/format.txt
-3
View File
@@ -1,3 +0,0 @@
# {format} — аудіоформат
# один токен на рядок, регістр не має значення
FLAC
internal/releaseparser/groups/moonblood/dicts/language.txt
-38
View File
@@ -1,38 +0,0 @@
# {language} — мовні маркери
DE
FR
ES
SK
FI
SE
BR
PL
NL
NO
PT
IT
HU
DK
AT
IS
UA
JP
GR
CN
RU
ID
PH
RO
LT
BY
CZ
SP
IR
FO
RS
DA
HI
CH
LU
CY
EE
internal/releaseparser/groups/moonblood/dicts/resolution.txt
-8
View File
@@ -1,8 +0,0 @@
# {resolution} — частота дискретизації
44KHZ
48KHZ
48.0kHz
88KHZ
96KHZ
176KHZ
192KHZ
internal/releaseparser/groups/moonblood/dicts/source.txt
-36
View File
@@ -1,36 +0,0 @@
# {source} — джерело релізу
# VINYL сюди входить як source (за домовленістю, не edition)
WEB
CD
2CD
CDEP
VINYL
TAPE
CDR
17CD
3CD
4CD
18CD
3CDEP
DVD
7CD
2CDR
10CD
6CD
2CDEP
2VINYL
CDREP
LP
2LP
3VINYL
3TAPE
13CD
13CDR
10INCH_VINYL
25CD
WEBFLAC
2TAPE
BLURAY
VLS
MCD
24BIT_WEB
internal/releaseparser/groups/moonblood/profile.go
-42
View File
@@ -1,42 +0,0 @@
package moonblood
import (
"embed"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// Словники токенів лежать у dicts/*.txt — один токен на рядок.
// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці).
//go:embed dicts/*.txt
var dictFS embed.FS
func loadDict(name string) map[string]bool {
data, err := dictFS.ReadFile("dicts/" + name + ".txt")
if err != nil {
panic("moonblood: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]bool)
for _, line := range strings.Split(string(data), "\n") {
line = strings.ToUpper(strings.TrimSpace(line))
if line == "" || strings.HasPrefix(line, "#") {
continue
}
out[line] = true
}
return out
}
var Profile = releaseparser.GroupProfile{
Name: "MOONBLOOD",
ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"},
FormatTokens: loadDict("format"),
SourceTokens: loadDict("source"),
BitDepthTokens: loadDict("bitdepth"),
ResolutionTokens: loadDict("resolution"),
LanguageMarkers: loadDict("language"),
EditionTokens: loadDict("edition"),
CommentTokens: loadDict("comment"),
}
internal/releaseparser/groups/moonblood/parser.go → internal/releaseparser/groups/parser.go
+2 -2
View File
@@ -1,4 +1,4 @@
package moonblood
package groups
import (
"regexp"
@@ -110,7 +110,7 @@ func split(raw string) []string {
// list and returns the cleaned list together with the removed comments.
func extractComments(tokens []string) ([]string, []string) {
out := make([]string, 0, len(tokens))
comments := []string{}
var comments []string
for _, tok := range tokens {
if Profile.CommentTokens[strings.ToUpper(tok)] {
comments = append(comments, tok)
internal/releaseparser/groups/moonblood/parser_test.go → internal/releaseparser/groups/parser_test.go
+1 -35
View File
@@ -1,43 +1,9 @@
package moonblood
package groups
import (
"bufio"
"os"
"testing"
)
func TestParseMoonblood(t *testing.T) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
t.Skipf("resource file not found (DB mode), skipping: %v", err)
t.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
total, success := 0, 0
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
result := Parse(line)
total++
if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" {
success++
}
if result.Confidence < 0.7 {
t.Logf("Low confidence: %s (%.2f)", line, result.Confidence)
}
}
t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100)
}
func TestParseFullTail(t *testing.T) {
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
if r.Artist != "Fane" {
internal/releaseparser/groups/profile.go
+45
View File
@@ -0,0 +1,45 @@
package groups
import (
"embed"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// The token dictionaries live under resources/dicts/*.txt — one token (in any
// case) per line; blank lines and lines starting with "#" are ignored. They
// can be edited without recompiling the parsing logic: go:embed pulls them
// into the binary at build time.
//go:embed resources/dicts/*.txt
var dictFS embed.FS
func loadDict(name string) map[string]bool {
data, err := dictFS.ReadFile("resources/dicts/" + name + ".txt")
if err != nil {
panic("release parser: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]bool)
for line := range strings.SplitSeq(string(data), "\n") {
line = strings.ToUpper(strings.TrimSpace(line))
if line == "" || strings.HasPrefix(line, "#") {
continue
}
out[line] = true
}
return out
}
// Profile holds the shared dictionaries for the release parser. The parser is
// group-agnostic: the group name is read from each release line, never from
// this profile, so there is a single set of dictionaries covering all groups.
var Profile = releaseparser.GroupProfile{
FormatTokens: loadDict("format"),
SourceTokens: loadDict("source"),
BitDepthTokens: loadDict("bitdepth"),
ResolutionTokens: loadDict("resolution"),
LanguageMarkers: loadDict("language"),
EditionTokens: loadDict("edition"),
CommentTokens: loadDict("comment"),
}
internal/releaseparser/types.go
+13 -15
View File
@@ -1,32 +1,30 @@
package releaseparser
type GroupProfile struct {
Name string
ExpectedTailOrder []string
FormatTokens map[string]bool
SourceTokens map[string]bool
BitDepthTokens map[string]bool
ResolutionTokens map[string]bool
LanguageMarkers map[string]bool
EditionTokens map[string]bool
CommentTokens map[string]bool
FormatTokens map[string]bool
SourceTokens map[string]bool
BitDepthTokens map[string]bool
ResolutionTokens map[string]bool
LanguageMarkers map[string]bool
EditionTokens map[string]bool
CommentTokens map[string]bool
}
type ParsedRelease struct {
Raw string
Group string
Artist string
Year int
Title string
Source string
Edition string
CatalogNo string
Language string
BitDepth string
Resolution string
Comment string
CatalogNo string
Source string
Format string
Language string
Edition string
Core string
Artist string
Title string
UnknownTokens []string
Warnings []string
Confidence float64