yevhen/resource-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixed a new prescene layout

Browse Source
This commit is contained in:
Yevhen Odynets
2026-09-27 01:12:52 +03:00
parent 347f525801
commit 39aa8e2e81
7 changed files with 174 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/releaseparser/groups/parser.go
+65 -43
View File
@@ -5,8 +5,6 @@ import (
"strconv"
"strings"
"golang.org/x/exp/slices"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
@@ -58,31 +56,31 @@ func Parse(raw string) releaseparser.ParsedRelease {
return result
}
// Where the dictionary-driven tail zone begins: after the resolved
// Artist/Title core (2 tokens by default, 3 with an artist dictionary hit).
// Left core boundary: after the resolved Artist (2 tokens by default,
// 3 with an artist dictionary hit, or fixed by an override).
start := 2
// Level 1: exact-lookup override table fixes Artist/Title only;
// the tail zone is parsed as usual below.
// the tail zone is parsed as usual below. The override is
// authoritative about the core: leftover tokens are plain unknowns,
// not part of the title.
if o, ok := Profile.Overrides[result.Raw]; ok {
result.Artist, result.Title = o.Artist, o.Title
start = overrideCoreLength(tokens, o)
}
// Level 2: artist dictionary — "[0]-[1]" whitelisted as a hyphenated
// artist shifts the core by one token.
switch {
case result.Artist != "":
// An override already fixed the core — leave it alone.
case len(tokens) >= 3 &&
Profile.ArtistTokens[strings.ToUpper(tokens[0]+"-"+tokens[1])]:
result.Artist = tokens[0] + "-" + tokens[1]
result.Title = tokens[2]
start = 3
default:
result.Artist = tokens[0]
if len(tokens) > 1 {
result.Title = tokens[1]
// Level 2: artist dictionary — a hyphenated artist may span ANY number
// of leading tokens ("Shadu-Nar-Mattaru" = 3). The LONGEST whitelisted
// prefix wins; the title is the token right after it. At least one
// title token and one tail token must remain.
artistDict := false
if result.Artist == "" {
for k := len(tokens) - 2; k >= 2; k-- {
if Profile.ArtistTokens[strings.ToUpper(strings.Join(tokens[:k], "-"))] {
artistDict = true
start = k + 1
break
}
}
}
@@ -93,8 +91,41 @@ func Parse(raw string) releaseparser.ParsedRelease {
end = len(tokens) - 2
}
// Right core boundary: the first dictionary-known token after the
// resolved artist. The boundary is contextual, but the title itself
// stays a single token ([1], or [2] with an artist dictionary hit).
// Unknown tokens inside the core region ([start .. tailStart)) are
// REPORTED in UnknownTokens — never eaten, never glued into the title.
tailStart := end
for i := start; i < end; i++ {
if isKnownToken(tokens[i]) {
tailStart = i
break
}
}
// Resolve Artist/Title from the core unless the override already did.
if result.Artist == "" {
if artistDict {
// The artist spans tokens [0 .. start-2]; re-join them.
result.Artist = strings.Join(tokens[:start-1], "-")
} else {
result.Artist = tokens[0]
}
if len(tokens) > start-1 {
result.Title = tokens[start-1]
}
}
// Core-region leftovers (e.g. the rest of a wildcards title after an
// override, or a hyphenated title fragment without an artist entry):
// visible in UnknownTokens for review.
for _, tok := range tokens[start:tailStart] {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
// Level 3: dictionary-driven tail zone, starting after the resolved core.
parseTailZone(start, tokens[start:end], &result)
parseTailZone(tokens[tailStart:end], &result)
score(&result)
return result
@@ -160,12 +191,24 @@ func catalogNumber(token string) string {
return token[1 : len(token)-1]
}
// isKnownToken reports whether the token belongs to the tail zone: it
// matches one of the profile dictionaries or is a parenthesized catalogue
// number. The first such token after the core marks the right core
// boundary.
func isKnownToken(tok string) bool {
up := strings.ToUpper(tok)
return Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] ||
catalogNumber(tok) != ""
}
// parseTailZone resolves middle-zone tokens against the dictionaries.
// The first match wins for each field; catalogue numbers are recognized
// from parenthesized tokens, and anything unrecognized is appended to
// UnknownTokens so that it can be reviewed and curated into a dictionary.
func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease) {
for i, tok := range zone {
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
for _, tok := range zone {
up := strings.ToUpper(tok)
switch {
case Profile.FormatTokens[up]:
@@ -197,13 +240,6 @@ func parseTailZone(start int, zone []string, result *releaseparser.ParsedRelease
default:
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
result.CatalogNo = cat
} else if hasLaterKnownToken(zone, i) {
// suspected title continuation: the core is probably longer
// than 2 tokens. Not eaten, not unknown — review section.
result.TitleExtra = append(result.TitleExtra, tok)
if !slices.Contains(result.Warnings, "suspected title continuation") {
result.Warnings = append(result.Warnings, "suspected title continuation")
}
} else {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
@@ -220,20 +256,6 @@ func applyType(tok string, result *releaseparser.ParsedRelease) {
}
}
// hasLaterKnownToken reports whether any token after index i in the zone
// matches one of the profile dictionaries.
func hasLaterKnownToken(zone []string, i int) bool {
for _, later := range zone[i+1:] {
up := strings.ToUpper(later)
if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
return true
}
}
return false
}
// overrideCoreLength returns the number of leading tokens covered by the
// override's Artist and Title (each may span several hyphen-separated
// tokens), falling back to the default 2-token core when the raw tokens
internal/releaseparser/groups/parser_test.go
+52 -42
View File
@@ -38,23 +38,20 @@ func TestParseEditionAndDemo(t *testing.T) {
func TestParseArtistWithHyphen(t *testing.T) {
// Artist сам містить дефіс, але Ultra-Violence ще не в словнику:
// модель [0]/[1] лишається, а All_My_Life (за ним ідуть словникові
// маркери) — suspected title continuation, не unknown.
// core контекстуальний — все між артистом і першим словниковим
// маркером (EP) є назвою, зібраною назад через "-".
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
if r.Artist != "Ultra" || r.Title != "Violence" {
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
if r.Artist != "Ultra" {
t.Errorf("Artist = %q, want Ultra", r.Artist)
}
if r.Title != "Violence" {
t.Errorf("Title = %q, want Violence", r.Title)
}
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "All_My_Life" {
t.Errorf("TitleExtra = %v, want [All_My_Life]", r.TitleExtra)
}
if !contains(r.Warnings, "suspected title continuation") {
t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
}
}
@@ -147,22 +144,23 @@ func TestParseUnknownTailTokenReported(t *testing.T) {
}
}
func TestParseSuspectedTitleContinuation(t *testing.T) {
// Невідомий токен МІЖ title-зоною і хвостом (за ним далі йдуть
// словникові маркери) → не UnknownTokens, а TitleExtra + warning.
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "FOO" {
t.Errorf("TitleExtra = %v, want [FOO]", r.TitleExtra)
func TestParseContextualCoreAbsorbsTitleTail(t *testing.T) {
// Права межа core контекстуальна (перший словниковий маркер), але
// невідомі токени всередині core-зони — в UnknownTokens, не з'їдаються.
r := Parse("Artist-Some_Title-Part_Two-Part_Three-16BIT-WEB-FLAC-2025-MOONBLOOD")
if r.Artist != "Artist" {
t.Errorf("Artist = %q, want Artist", r.Artist)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
if r.Title != "Some_Title" {
t.Errorf("Title = %q, want Some_Title", r.Title)
}
if !contains(r.Warnings, "suspected title continuation") {
t.Errorf("Warnings = %v, want suspected title continuation", r.Warnings)
}
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2025 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 2 || r.UnknownTokens[0] != "Part_Two" ||
r.UnknownTokens[1] != "Part_Three" {
t.Errorf("UnknownTokens = %v, want [Part_Two Part_Three]", r.UnknownTokens)
}
}
func TestParseCoreTokensNeverReported(t *testing.T) {
@@ -282,9 +280,8 @@ func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
}
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
// за ним ідуть словникові маркери → suspected title continuation.
// (SMG409) — катало, залишається каталом.
// Evolution — не словниковий токен у core-зоні → в UnknownTokens,
// не з'їдається. (SMG409) — катало, перший відомий маркер.
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
if r.CatalogNo != "SMG409" {
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
@@ -292,11 +289,8 @@ func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
if r.Artist != "Ferosity" || r.Title != "De" {
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
}
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "Evolution" {
t.Errorf("TitleExtra = %v, want [Evolution]", r.TitleExtra)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
}
}
@@ -363,18 +357,14 @@ func TestParseCompoundEditionRecognized(t *testing.T) {
}
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
// [1]=Title (один токен). Хвіст назви (_ і Sky_Burial) — не словник,
// але за ним ідуть словникові маркери → suspected title continuation,
// не UnknownTokens.
// Хвіст назви (_Sky_Burial) — не словник, але він ДО першого маркера
// (катало (RTM103)) → частина назви, не UnknownTokens.
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if len(r.TitleExtra) != 1 || r.TitleExtra[0] != "_Sky_Burial" {
t.Errorf("TitleExtra = %v, want [_Sky_Burial]", r.TitleExtra)
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
}
if r.CatalogNo != "RTM103" {
t.Errorf("CatalogNo = %q, want RTM103", r.CatalogNo)
@@ -426,10 +416,30 @@ func TestParseArtistDictionaryCoversReleases(t *testing.T) {
}
}
func TestParseArtistDictionaryMultiTokenArtist(t *testing.T) {
// artists.txt: "Shadu-Nar-Mattaru" — артист з ТРЬОХ токенів.
// Найдовший словниковий префікс виграє: Artist = 3 токени,
// Title = наступний токен, хвіст парситься як завжди.
r := Parse("Shadu-Nar-Mattaru-On_Pathways_Macabre-24BIT-WEB-FLAC-2023-MOONBLOOD")
if r.Artist != "Shadu-Nar-Mattaru" {
t.Errorf("Artist = %q, want Shadu-Nar-Mattaru", r.Artist)
}
if r.Title != "On_Pathways_Macabre" {
t.Errorf("Title = %q, want On_Pathways_Macabre", r.Title)
}
if r.BitDepth != "24BIT" || r.Source != "WEB" || r.Format != "FLAC" || r.Year != 2023 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {
t.Errorf("Unknown=%v Extra=%v Warnings=%v, want empty",
r.UnknownTokens, r.TitleExtra, r.Warnings)
}
}
func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
// overrides.txt (найвищий пріоритет): точний lookup по Raw
// фіксує Artist/Title; хвіст парситься як завжди.
raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2024-GRAVEWAVE"
raw := "Milicia_Oscura-Anti-Cristo-ES-16BIT-WEB-FLAC-2013-MOONBLOOD"
r := Parse(raw)
if r.Artist != "Milicia_Oscura" {
t.Errorf("Artist = %q, want Milicia_Oscura", r.Artist)
@@ -438,7 +448,7 @@ func TestParseOverrideWinsOverPositionalModel(t *testing.T) {
t.Errorf("Title = %q, want Anti-Cristo", r.Title)
}
if r.Language != "ES" || r.BitDepth != "16BIT" || r.Source != "WEB" ||
r.Format != "FLAC" || r.Year != 2024 || r.Group != "GRAVEWAVE" {
r.Format != "FLAC" || r.Year != 2013 || r.Group != "MOONBLOOD" {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 || len(r.TitleExtra) != 0 || len(r.Warnings) != 0 {