releaseparser: move group-agnostic parser to groups/, GB doc-blocks, per-group JSON reports
Generalise the parser: the MOONBLOOD group profile is gone, the parser and its dictionaries are now group-agnostic under internal/releaseparser/groups. Positional model: - [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so titles like "Split"/"Demo"/"De" are never eaten. - [len-1]=Group is always read from the line, not from the query key. - [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is resolved as a middle-zone token (e.g. FLAC is still picked up as format). - The middle zone is resolved via case-insensitive dictionaries in the order format -> source -> bit-depth -> resolution -> edition -> language -> catalogue number; stored values keep their original case. Catalogue numbers: any "(...)" token in the middle zone is a catalogue number regardless of content (digits, roman numerals, dots, underscores, mixed case); splitting never breaks tokens inside parentheses. Dictionaries moved to internal/releaseparser/groups/resources/dicts and embedded via go:embed; dead profile fields (Name, ExpectedTailOrder) removed; profile comments rewritten in British English. File-based AnalyzeGroup/TestParseMoonblood removed — titles now come from the database (cmd/srrdb: srrdb <releaser>..., config/secret.yaml DSN). Validation output (cmd/srrdb): - tmp/releaseparser/<group>.json — per-group parsed ParsedRelease records - tmp/releaseparser/<releaser>.report.json — per-request aggregate (same as console) - review_unknown_tokens.txt — unknown/warning rows for manual review Tests updated for the positional model; added TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2) and (GOWP_DCCCLXXXIX). go build, go vet and go test all pass.
This commit is contained in:
+50
-4
@@ -5,13 +5,15 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
|
||||
_ "github.com/go-sql-driver/mysql"
|
||||
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
|
||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups"
|
||||
)
|
||||
|
||||
func main() {
|
||||
@@ -35,19 +37,33 @@ func main() {
|
||||
}
|
||||
defer dbase.Close()
|
||||
|
||||
out := make(map[string]moonblood.GroupStats, len(releasers))
|
||||
const outDir = "tmp/releaseparser"
|
||||
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
out := make(map[string]groups.GroupStats, len(releasers))
|
||||
|
||||
for _, releaser := range releasers {
|
||||
var titles []string
|
||||
|
||||
err = dbase.Select(&titles,
|
||||
`SELECT title FROM theamok_mdb.external_sources
|
||||
WHERE type = 'prescene' AND releaser = ?`, releaser)
|
||||
WHERE type = 'prescene' AND releaser = ? limit 1000`, releaser)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
out[releaser] = moonblood.AnalyzeTitles(titles)
|
||||
out[releaser] = groups.AnalyzeTitles(titles)
|
||||
|
||||
// Per-group parsed records keyed by the actual group read from each
|
||||
// line: every release with all resolved fields for manual validation.
|
||||
for group, parsed := range groups.ParseByGroup(titles) {
|
||||
writeJSON(filepath.Join(outDir, safeName(group)+".json"), parsed)
|
||||
}
|
||||
|
||||
// Per-request report identical to what is printed to the console.
|
||||
writeJSON(filepath.Join(outDir, safeName(releaser)+".report.json"), out[releaser])
|
||||
}
|
||||
|
||||
data, err := json.MarshalIndent(out, "", " ")
|
||||
@@ -77,3 +93,33 @@ func main() {
|
||||
|
||||
fmt.Println(string(data))
|
||||
}
|
||||
|
||||
// writeJSON writes value as an indented JSON document to the given path.
|
||||
func writeJSON(path string, value any) {
|
||||
data, err := json.MarshalIndent(value, "", " ")
|
||||
if err != nil {
|
||||
log.Fatalf("marshal %s: %v", path, err)
|
||||
}
|
||||
if err := os.WriteFile(path, data, 0o644); err != nil {
|
||||
log.Fatalf("write %s: %v", path, err)
|
||||
}
|
||||
}
|
||||
|
||||
// safeName strips characters that are unsafe in file names.
|
||||
func safeName(s string) string {
|
||||
repl := func(r rune) rune {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z':
|
||||
return r
|
||||
case r >= 'A' && r <= 'Z':
|
||||
return r
|
||||
case r >= '0' && r <= '9':
|
||||
return r
|
||||
case r == '-' || r == '_' || r == '.':
|
||||
return r
|
||||
default:
|
||||
return '_'
|
||||
}
|
||||
}
|
||||
return strings.Map(repl, s)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user