releaseparser: adopt fixed positional model, auto-detect catalogues, add GB doc-blocks
Parse releases with an explicit fixed positional model and document the whole logic in British English doc-blocks. Model: - [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so titles like "Split"/"Demo"/"De" are never eaten. - [len-1]=Group is always read from the line, not from the query key. - [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is resolved as a middle-zone token (e.g. FLAC still picked up as format). - The middle zone is resolved via case-insensitive dictionaries in the order format -> source -> bit-depth -> resolution -> edition -> language -> catalogue number. Position within the zone is irrelevant. Catalogue numbers: - Any "(...)" token in the middle zone is a catalogue number regardless of content (digits, roman numerals, dots, underscores, mixed case). Removed the hasDigit requirement and character whitelist that rejected real catalogues like (DYN.3140-2) and (GOWP_DCCCLXXXIX). - Splitting never breaks tokens inside parentheses. Case handling: lookups are case-insensitive; stored values keep their original case from the line. Documentation: replaced Ukrainian comments in parser.go and analyzer.go with British-English doc-blocks describing the model, tail resolution order, catalogue rule and confidence scoring. Tests: - Add TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2)/(GOWP_DCCCLXXXIX). - Update case-insensitive, split-title, no-year and language expectations. - go build, go vet and go test all pass.
This commit is contained in:
@@ -55,5 +55,25 @@ func main() {
|
|||||||
log.Fatal(err)
|
log.Fatal(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Ручна ревізія: всі невідомі токени — у файл для правки бази.
|
||||||
|
review, err := os.OpenFile("review_unknown_tokens.txt",
|
||||||
|
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer review.Close()
|
||||||
|
for releaser, stats := range out {
|
||||||
|
for _, unk := range stats.UnknownTokens {
|
||||||
|
for _, rel := range unk.Releases {
|
||||||
|
fmt.Fprintf(review, "%s\t%s\t%s\n", releaser, unk.Token, rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, warn := range stats.WarningStats {
|
||||||
|
for _, rel := range warn.Releases {
|
||||||
|
fmt.Fprintf(review, "%s\tWARN\t%s\t%s\n", releaser, warn.Token, rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fmt.Println(string(data))
|
fmt.Println(string(data))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,9 @@ import (
|
|||||||
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// GroupStats summarises how a set of release lines parsed: field
|
||||||
|
// distributions, unknown tokens and warnings aggregated by occurrence, and
|
||||||
|
// the average parsing confidence. The slices are sorted by descending count.
|
||||||
type GroupStats struct {
|
type GroupStats struct {
|
||||||
Total int
|
Total int
|
||||||
YearDistribution map[int]int
|
YearDistribution map[int]int
|
||||||
@@ -17,11 +20,15 @@ type GroupStats struct {
|
|||||||
ResolutionDistribution map[string]int
|
ResolutionDistribution map[string]int
|
||||||
LanguageDistribution map[string]int
|
LanguageDistribution map[string]int
|
||||||
EditionDistribution map[string]int
|
EditionDistribution map[string]int
|
||||||
|
CommentDistribution map[string]int
|
||||||
UnknownTokens []releaseparser.UnknownTokenStats
|
UnknownTokens []releaseparser.UnknownTokenStats
|
||||||
|
WarningStats []releaseparser.UnknownTokenStats
|
||||||
AvgConfidence float64
|
AvgConfidence float64
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників).
|
// AnalyzeGroup parses the release lines from the local resource file. It is
|
||||||
|
// used for a quick local check of the dictionaries without touching a
|
||||||
|
// database.
|
||||||
func AnalyzeGroup() (GroupStats, error) {
|
func AnalyzeGroup() (GroupStats, error) {
|
||||||
file, err := os.Open("../../resources/moonblood.txt")
|
file, err := os.Open("../../resources/moonblood.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -44,7 +51,9 @@ func AnalyzeGroup() (GroupStats, error) {
|
|||||||
return AnalyzeTitles(titles), nil
|
return AnalyzeTitles(titles), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі).
|
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
||||||
|
// database by a particular releaser key) and aggregates the results into a
|
||||||
|
// GroupStats summary.
|
||||||
func AnalyzeTitles(titles []string) GroupStats {
|
func AnalyzeTitles(titles []string) GroupStats {
|
||||||
stats := GroupStats{
|
stats := GroupStats{
|
||||||
YearDistribution: make(map[int]int),
|
YearDistribution: make(map[int]int),
|
||||||
@@ -53,11 +62,14 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
ResolutionDistribution: make(map[string]int),
|
ResolutionDistribution: make(map[string]int),
|
||||||
LanguageDistribution: make(map[string]int),
|
LanguageDistribution: make(map[string]int),
|
||||||
EditionDistribution: make(map[string]int),
|
EditionDistribution: make(map[string]int),
|
||||||
|
CommentDistribution: make(map[string]int),
|
||||||
UnknownTokens: []releaseparser.UnknownTokenStats{},
|
UnknownTokens: []releaseparser.UnknownTokenStats{},
|
||||||
|
WarningStats: []releaseparser.UnknownTokenStats{},
|
||||||
}
|
}
|
||||||
|
|
||||||
totalConfidence := 0.0
|
totalConfidence := 0.0
|
||||||
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||||
|
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||||
|
|
||||||
for _, title := range titles {
|
for _, title := range titles {
|
||||||
result := Parse(title)
|
result := Parse(title)
|
||||||
@@ -81,6 +93,11 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
if result.Edition != "" {
|
if result.Edition != "" {
|
||||||
stats.EditionDistribution[result.Edition]++
|
stats.EditionDistribution[result.Edition]++
|
||||||
}
|
}
|
||||||
|
if result.Comment != "" {
|
||||||
|
for _, c := range strings.Split(result.Comment, ",") {
|
||||||
|
stats.CommentDistribution[c]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
for _, unknown := range result.UnknownTokens {
|
for _, unknown := range result.UnknownTokens {
|
||||||
if _, exists := unknownMap[unknown]; !exists {
|
if _, exists := unknownMap[unknown]; !exists {
|
||||||
@@ -94,10 +111,23 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
|
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
for _, warning := range result.Warnings {
|
||||||
|
if _, exists := warningMap[warning]; !exists {
|
||||||
|
warningMap[warning] = &releaseparser.UnknownTokenStats{
|
||||||
|
Token: warning,
|
||||||
|
Count: 0,
|
||||||
|
Releases: []string{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
warningMap[warning].Count++
|
||||||
|
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
|
||||||
|
}
|
||||||
|
|
||||||
totalConfidence += result.Confidence
|
totalConfidence += result.Confidence
|
||||||
}
|
}
|
||||||
|
|
||||||
// Convert map to slice and sort by count
|
// Convert the aggregates from maps to slices and order them by count
|
||||||
|
// (most frequent first).
|
||||||
for _, tokenStats := range unknownMap {
|
for _, tokenStats := range unknownMap {
|
||||||
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
||||||
}
|
}
|
||||||
@@ -106,6 +136,13 @@ func AnalyzeTitles(titles []string) GroupStats {
|
|||||||
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
||||||
})
|
})
|
||||||
|
|
||||||
|
for _, w := range warningMap {
|
||||||
|
stats.WarningStats = append(stats.WarningStats, *w)
|
||||||
|
}
|
||||||
|
sort.Slice(stats.WarningStats, func(i, j int) bool {
|
||||||
|
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
|
||||||
|
})
|
||||||
|
|
||||||
if stats.Total > 0 {
|
if stats.Total > 0 {
|
||||||
stats.AvgConfidence = totalConfidence / float64(stats.Total)
|
stats.AvgConfidence = totalConfidence / float64(stats.Total)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
# {bit_depth}
|
# {bit_depth}
|
||||||
16BIT
|
16BIT
|
||||||
24BIT
|
24BIT
|
||||||
|
32BIT
|
||||||
|
DE16BIT
|
||||||
|
|||||||
@@ -1,3 +1,5 @@
|
|||||||
REPACK
|
REPACK
|
||||||
REAL_REPACK
|
REAL_REPACK
|
||||||
READNFO
|
READNFO
|
||||||
|
PROPER
|
||||||
|
RERIP
|
||||||
|
|||||||
@@ -17,4 +17,41 @@ LIMITED_EDITION_BOXSET
|
|||||||
REMASTERED_EP
|
REMASTERED_EP
|
||||||
10TH_ANNIVERSARY_EDITION
|
10TH_ANNIVERSARY_EDITION
|
||||||
OST
|
OST
|
||||||
|
XX_Legacy_Edition
|
||||||
REISSUE_DEMO
|
REISSUE_DEMO
|
||||||
|
JP_Deluxe_Edition
|
||||||
|
Remastered_Edition
|
||||||
|
Anniversary_Edition
|
||||||
|
Limited_Edition
|
||||||
|
CDS
|
||||||
|
CDM
|
||||||
|
RETAIL
|
||||||
|
JP_RETAIL
|
||||||
|
JP_REISSUE
|
||||||
|
DEMO_REMASTERED
|
||||||
|
EU_RETAIL
|
||||||
|
DE_RETAIL
|
||||||
|
REISSUE_DELUXE_EDITION
|
||||||
|
RETAIL_REISSUE
|
||||||
|
SPECIAL_EDITION
|
||||||
|
SPECIAL_COLLECTORS_EDITION_REMASTERED
|
||||||
|
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
|
||||||
|
LIMITED_EDITION_REMASTERED
|
||||||
|
DEMO_REISSUE
|
||||||
|
REMASTERED_REISSUE
|
||||||
|
PROMO_DEMO
|
||||||
|
REISSUE_EP
|
||||||
|
MAG
|
||||||
|
REISSUE_BOXSET
|
||||||
|
30TH_ANNIVERSARY_EDITION_REMASTERED
|
||||||
|
Deluxe_Edition_Reissue
|
||||||
|
INSTRUMENTAL
|
||||||
|
REISSUE_EXPANDED_EDITION
|
||||||
|
BOOTLEG
|
||||||
|
EXPANDED_EDITION
|
||||||
|
ADVANCE
|
||||||
|
15TH_ANNIVERSARY_DELUXE_EDITION
|
||||||
|
TOUR_EDITION
|
||||||
|
REMASTERED_DEMO
|
||||||
|
REMASTERED_DELUXE_EDITION
|
||||||
|
30TH_ANNIVERSARY_EDITION
|
||||||
@@ -17,3 +17,17 @@ AT
|
|||||||
IS
|
IS
|
||||||
UA
|
UA
|
||||||
JP
|
JP
|
||||||
|
GR
|
||||||
|
CN
|
||||||
|
RU
|
||||||
|
ID
|
||||||
|
PH
|
||||||
|
BY
|
||||||
|
CZ
|
||||||
|
RS
|
||||||
|
DA
|
||||||
|
HI
|
||||||
|
CH
|
||||||
|
LU
|
||||||
|
CY
|
||||||
|
EE
|
||||||
@@ -1,6 +1,7 @@
|
|||||||
# {resolution} — частота дискретизації
|
# {resolution} — частота дискретизації
|
||||||
44KHZ
|
44KHZ
|
||||||
48KHZ
|
48KHZ
|
||||||
|
48.0kHz
|
||||||
88KHZ
|
88KHZ
|
||||||
96KHZ
|
96KHZ
|
||||||
176KHZ
|
176KHZ
|
||||||
|
|||||||
@@ -9,3 +9,24 @@ TAPE
|
|||||||
CDR
|
CDR
|
||||||
17CD
|
17CD
|
||||||
3CD
|
3CD
|
||||||
|
4CD
|
||||||
|
18CD
|
||||||
|
3CDEP
|
||||||
|
DVD
|
||||||
|
7CD
|
||||||
|
2CDR
|
||||||
|
10CD
|
||||||
|
6CD
|
||||||
|
2CDEP
|
||||||
|
2VINYL
|
||||||
|
CDREP
|
||||||
|
LP
|
||||||
|
2LP
|
||||||
|
3TAPE
|
||||||
|
25CD
|
||||||
|
WEBFLAC
|
||||||
|
2TAPE
|
||||||
|
BLURAY
|
||||||
|
VLS
|
||||||
|
MCD
|
||||||
|
24BIT_WEB
|
||||||
@@ -10,6 +10,35 @@ import (
|
|||||||
|
|
||||||
var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
|
var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
|
||||||
|
|
||||||
|
// Parse splits a release line into tokens and maps them onto a fixed
|
||||||
|
// positional model while the interior is resolved solely through the
|
||||||
|
// group profile's dictionaries.
|
||||||
|
//
|
||||||
|
// The release line is split on "-" (but never inside curved parentheses, so
|
||||||
|
// that a catalogue number such as (BRC30-001-CD-RRR108) stays one token).
|
||||||
|
// The positional layout is:
|
||||||
|
//
|
||||||
|
// [0] Artist - fixed; never matched against dictionaries.
|
||||||
|
// [1] Title - fixed; never matched against dictionaries.
|
||||||
|
// [len-2] Year - if the token is a 4-digit year (19xx | 20xx).
|
||||||
|
// [len-1] Group - always read from the line, never from the query key.
|
||||||
|
//
|
||||||
|
// Everything between [1] and the reserved trailing slots ([2 .. len-3], or
|
||||||
|
// [2 .. len-2] when there is no year) is the middle zone. Each token there is
|
||||||
|
// looked up in the dictionaries (format, source, bit-depth, resolution,
|
||||||
|
// language, edition) and, if it is a parenthesized token, as a catalogue
|
||||||
|
// number. Position within the middle zone is irrelevant. Tokens that match
|
||||||
|
// nothing are NOT silently consumed — they are reported in UnknownTokens so
|
||||||
|
// that they can be reviewed and added to a dictionary.
|
||||||
|
//
|
||||||
|
// Why [0]/[1] are fixed and never matched against the dictionaries: titles
|
||||||
|
// such as "Split" (Alone_In_The_Morgue-Split-...) or "Demo"
|
||||||
|
// (Mystifier-Demo-Nized-...) are legitimate words that collide with edition
|
||||||
|
// markers. Since index 1 is always the title, regardless of case or
|
||||||
|
// dictionary membership, such words are never eaten.
|
||||||
|
//
|
||||||
|
// Values are preserved in their original case (as they appear in the line);
|
||||||
|
// the dictionary lookups themselves are case-insensitive.
|
||||||
func Parse(raw string) releaseparser.ParsedRelease {
|
func Parse(raw string) releaseparser.ParsedRelease {
|
||||||
result := releaseparser.ParsedRelease{
|
result := releaseparser.ParsedRelease{
|
||||||
Raw: strings.TrimSpace(raw),
|
Raw: strings.TrimSpace(raw),
|
||||||
@@ -17,123 +46,140 @@ func Parse(raw string) releaseparser.ParsedRelease {
|
|||||||
UnknownTokens: []string{},
|
UnknownTokens: []string{},
|
||||||
}
|
}
|
||||||
|
|
||||||
tokens := split(raw)
|
tokens, comments := extractComments(split(raw))
|
||||||
|
if len(comments) > 0 {
|
||||||
|
result.Comment = strings.Join(comments, ",")
|
||||||
|
}
|
||||||
if len(tokens) < 5 {
|
if len(tokens) < 5 {
|
||||||
result.Warnings = append(result.Warnings, "too few tokens")
|
result.Warnings = append(result.Warnings, "too few tokens")
|
||||||
result.Confidence = 0.2
|
result.Confidence = 0.2
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
// Останній токен — група. Вона ж — ключ до профілю групи (довідників).
|
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
|
||||||
|
// [len-2]=Year (when present).
|
||||||
|
result.Artist = tokens[0]
|
||||||
|
if len(tokens) > 1 {
|
||||||
|
result.Title = tokens[1]
|
||||||
|
}
|
||||||
result.Group = tokens[len(tokens)-1]
|
result.Group = tokens[len(tokens)-1]
|
||||||
|
end := len(tokens) - 1
|
||||||
// Передостанній — рік
|
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
|
||||||
last := len(tokens) - 2
|
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
|
||||||
if yearRegexp.MatchString(tokens[last]) {
|
end = len(tokens) - 2
|
||||||
result.Year, _ = strconv.Atoi(tokens[last])
|
|
||||||
last--
|
|
||||||
}
|
|
||||||
|
|
||||||
// Рухаємось вліво по хвосту: format, source, bit_depth, resolution, language, edition.
|
|
||||||
// Кожен слот бере токен лише зі свого словника (регістронезалежно).
|
|
||||||
// Невідомий токен: репортимо лише якщо ліворуч ще є відомий хвостовий маркер
|
|
||||||
// (токен між двома маркерами = сміття в хвості). Інакше це початок core.
|
|
||||||
tailLoop:
|
|
||||||
for last >= 0 {
|
|
||||||
token := strings.ToUpper(tokens[last])
|
|
||||||
|
|
||||||
switch {
|
|
||||||
case Profile.FormatTokens[token]:
|
|
||||||
if result.Format == "" {
|
|
||||||
result.Format = token
|
|
||||||
}
|
|
||||||
last--
|
|
||||||
|
|
||||||
case Profile.SourceTokens[token]:
|
|
||||||
if result.Source == "" {
|
|
||||||
result.Source = token
|
|
||||||
}
|
|
||||||
last--
|
|
||||||
|
|
||||||
case Profile.BitDepthTokens[token]:
|
|
||||||
if result.BitDepth == "" {
|
|
||||||
result.BitDepth = token
|
|
||||||
}
|
|
||||||
last--
|
|
||||||
|
|
||||||
case Profile.ResolutionTokens[token]:
|
|
||||||
if result.Resolution == "" {
|
|
||||||
result.Resolution = token
|
|
||||||
}
|
|
||||||
last--
|
|
||||||
|
|
||||||
case Profile.LanguageMarkers[token]:
|
|
||||||
if result.Language == "" {
|
|
||||||
result.Language = token
|
|
||||||
}
|
|
||||||
last--
|
|
||||||
|
|
||||||
case Profile.EditionTokens[token]:
|
|
||||||
if result.Edition == "" {
|
|
||||||
result.Edition = token
|
|
||||||
}
|
|
||||||
last--
|
|
||||||
|
|
||||||
default:
|
|
||||||
if hasKnownTailToken(tokens[:last]) {
|
|
||||||
result.UnknownTokens = append(result.UnknownTokens, tokens[last])
|
|
||||||
last--
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
break tailLoop
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Core = artist-title (все, що залишилось ліворуч)
|
|
||||||
coreTokens := tokens[:last+1]
|
|
||||||
result.Core = strings.Join(coreTokens, "-")
|
|
||||||
|
|
||||||
if len(coreTokens) >= 2 {
|
|
||||||
result.Artist = coreTokens[0]
|
|
||||||
result.Title = strings.Join(coreTokens[1:], "-")
|
|
||||||
} else if len(coreTokens) == 1 {
|
|
||||||
result.Artist = coreTokens[0]
|
|
||||||
result.Title = ""
|
|
||||||
result.Warnings = append(result.Warnings, "title not found")
|
|
||||||
}
|
}
|
||||||
|
// Resolve the middle zone tokens via the dictionaries.
|
||||||
|
parseTailZone(tokens[2:end], &result)
|
||||||
|
|
||||||
score(&result)
|
score(&result)
|
||||||
return result
|
return result
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// split breaks a raw release line into tokens on "-", but never inside
|
||||||
|
// curved parentheses, so that a catalogue number such as
|
||||||
|
// (BRC30-001-CD-RRR108) remains a single token.
|
||||||
func split(raw string) []string {
|
func split(raw string) []string {
|
||||||
parts := strings.Split(strings.TrimSpace(raw), "-")
|
s := strings.TrimSpace(raw)
|
||||||
out := make([]string, 0, len(parts))
|
out := make([]string, 0, 8)
|
||||||
for _, part := range parts {
|
var cur strings.Builder
|
||||||
if part = strings.TrimSpace(part); part != "" {
|
inParen := false
|
||||||
out = append(out, part)
|
for _, r := range s {
|
||||||
|
switch {
|
||||||
|
case r == '(':
|
||||||
|
inParen = true
|
||||||
|
cur.WriteRune(r)
|
||||||
|
case r == ')':
|
||||||
|
inParen = false
|
||||||
|
cur.WriteRune(r)
|
||||||
|
case r == '-' && !inParen:
|
||||||
|
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
|
cur.Reset()
|
||||||
|
default:
|
||||||
|
cur.WriteRune(r)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
// hasKnownTailToken reports whether any token is a known tail marker
|
// extractComments removes service comments (CommentTokens) from the token
|
||||||
// (format, source, bit_depth, resolution, edition, language or a year).
|
// list and returns the cleaned list together with the removed comments.
|
||||||
func hasKnownTailToken(tokens []string) bool {
|
func extractComments(tokens []string) ([]string, []string) {
|
||||||
|
out := make([]string, 0, len(tokens))
|
||||||
|
comments := []string{}
|
||||||
for _, tok := range tokens {
|
for _, tok := range tokens {
|
||||||
up := strings.ToUpper(tok)
|
if Profile.CommentTokens[strings.ToUpper(tok)] {
|
||||||
if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
|
comments = append(comments, tok)
|
||||||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
|
continue
|
||||||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
|
|
||||||
return true
|
|
||||||
}
|
|
||||||
if yearRegexp.MatchString(up) {
|
|
||||||
return true
|
|
||||||
}
|
}
|
||||||
|
out = append(out, tok)
|
||||||
}
|
}
|
||||||
return false
|
return out, comments
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// catalogNumber reports whether the token is a parenthesized catalogue
|
||||||
|
// number. In the positional model any token wrapped in curved parentheses
|
||||||
|
// that sits in the middle zone ([2..]) is a catalogue number regardless of
|
||||||
|
// its content (digits, letters, roman numerals, dots, underscores, mixed
|
||||||
|
// case). Parenthesized tokens at [0]/[1] are handled as artist/title, not
|
||||||
|
// here. It returns the parenthesized content, or "" when the token is not
|
||||||
|
// a "(...)" group.
|
||||||
|
func catalogNumber(token string) string {
|
||||||
|
if len(token) < 3 || token[0] != '(' || token[len(token)-1] != ')' {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return token[1 : len(token)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTailZone resolves middle-zone tokens against the dictionaries.
|
||||||
|
// The first match wins for each field; catalogue numbers are recognised
|
||||||
|
// from parenthesized tokens, and anything unrecognised is appended to
|
||||||
|
// UnknownTokens so that it can be reviewed and curated into a dictionary.
|
||||||
|
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||||
|
for _, tok := range zone {
|
||||||
|
up := strings.ToUpper(tok)
|
||||||
|
switch {
|
||||||
|
case Profile.FormatTokens[up]:
|
||||||
|
if result.Format == "" {
|
||||||
|
result.Format = tok
|
||||||
|
}
|
||||||
|
case Profile.SourceTokens[up]:
|
||||||
|
if result.Source == "" {
|
||||||
|
result.Source = tok
|
||||||
|
}
|
||||||
|
case Profile.BitDepthTokens[up]:
|
||||||
|
if result.BitDepth == "" {
|
||||||
|
result.BitDepth = tok
|
||||||
|
}
|
||||||
|
case Profile.ResolutionTokens[up]:
|
||||||
|
if result.Resolution == "" {
|
||||||
|
result.Resolution = tok
|
||||||
|
}
|
||||||
|
case Profile.EditionTokens[up]:
|
||||||
|
if result.Edition == "" {
|
||||||
|
result.Edition = tok
|
||||||
|
}
|
||||||
|
case Profile.LanguageMarkers[up]:
|
||||||
|
if result.Language == "" {
|
||||||
|
result.Language = tok
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
|
||||||
|
result.CatalogNo = cat
|
||||||
|
} else {
|
||||||
|
result.UnknownTokens = append(result.UnknownTokens, tok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// score computes a heuristic confidence in [0,1] for a parsed release based
|
||||||
|
// on how many core fields (group, year, format, source, artist) were resolved
|
||||||
|
// and whether any warnings or unknown tokens are present.
|
||||||
func score(result *releaseparser.ParsedRelease) {
|
func score(result *releaseparser.ParsedRelease) {
|
||||||
score := 0.0
|
score := 0.0
|
||||||
if result.Group != "" {
|
if result.Group != "" {
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
func TestParseMoonblood(t *testing.T) {
|
func TestParseMoonblood(t *testing.T) {
|
||||||
file, err := os.Open("../../resources/moonblood.txt")
|
file, err := os.Open("../../resources/moonblood.txt")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
t.Skipf("resource file not found (DB mode), skipping: %v", err)
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
defer file.Close()
|
defer file.Close()
|
||||||
@@ -70,16 +71,17 @@ func TestParseEditionAndDemo(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestParseArtistWithHyphen(t *testing.T) {
|
func TestParseArtistWithHyphen(t *testing.T) {
|
||||||
// Artist itself contains a hyphen; no part of it may leak into UnknownTokens.
|
// [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
|
||||||
|
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
|
||||||
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
|
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
|
||||||
if r.Artist != "Ultra" || r.Title != "Violence-All_My_Life" {
|
if r.Artist != "Ultra" || r.Title != "Violence" {
|
||||||
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence-All_My_Life", r.Artist, r.Title)
|
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
|
||||||
}
|
}
|
||||||
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
|
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
|
||||||
t.Errorf("tail fields wrong: %+v", r)
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
}
|
}
|
||||||
if len(r.UnknownTokens) != 0 {
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
|
||||||
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -144,7 +146,9 @@ func TestParseVinylIsSource(t *testing.T) {
|
|||||||
|
|
||||||
func TestParseCaseInsensitiveDictionaries(t *testing.T) {
|
func TestParseCaseInsensitiveDictionaries(t *testing.T) {
|
||||||
r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood")
|
r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood")
|
||||||
if r.BitDepth != "16BIT" || r.Source != "WEB" || r.Format != "FLAC" {
|
// Значення зберігаються в оригінальному регістрі (як у рядку),
|
||||||
|
// а звірка зі словниками йде регістронезалежно.
|
||||||
|
if r.BitDepth != "16bit" || r.Source != "web" || r.Format != "flac" {
|
||||||
t.Errorf("case-insensitive lookup failed: %+v", r)
|
t.Errorf("case-insensitive lookup failed: %+v", r)
|
||||||
}
|
}
|
||||||
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
||||||
@@ -199,3 +203,199 @@ func TestParseAnyGroupNoWarning(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseCommentExtracted(t *testing.T) {
|
||||||
|
r := Parse("Rain_City_Drive-Rain_City_Drive-DELUXE_EDITION-REPACK-16BIT-WEB-FLAC-2023-FLACCiD")
|
||||||
|
if r.Comment != "REPACK" {
|
||||||
|
t.Errorf("Comment = %q, want REPACK", r.Comment)
|
||||||
|
}
|
||||||
|
if r.Edition != "DELUXE_EDITION" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2023 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Rain_City_Drive" || r.Title != "Rain_City_Drive" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNumericArtistNotAYear(t *testing.T) {
|
||||||
|
// [0]=Artist — навіть якщо це число-рік (1914, 1928), а не "рік" токен.
|
||||||
|
for _, line := range []string{
|
||||||
|
"1914-Eschatology_of_War-24BIT-WEB-FLAC-2015-MOONBLOOD",
|
||||||
|
"1928-Lappel_du_Spectre-FR-CD-FLAC-2025-MOONBLOOD",
|
||||||
|
} {
|
||||||
|
r := Parse(line)
|
||||||
|
if r.Artist != "1914" && r.Artist != "1928" {
|
||||||
|
t.Errorf("%s: Artist = %q, want numeric artist", line, r.Artist)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCatalogNumber(t *testing.T) {
|
||||||
|
r := Parse("Antidot-Beyond_Trance_And_Time-(GLOBAL006)-REPACK-EP-FLAC-2026-AFTERHOUR")
|
||||||
|
if r.CatalogNo != "GLOBAL006" {
|
||||||
|
t.Errorf("CatalogNo = %q, want GLOBAL006", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Comment != "REPACK" || r.Edition != "EP" || r.Format != "FLAC" || r.Year != 2026 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Antidot" || r.Title != "Beyond_Trance_And_Time" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCatalogNumberWithDashesInside(t *testing.T) {
|
||||||
|
r := Parse("Murder_Intentions-Excessive_Display_Of_Human_Nature-(BRC30-001-CD-RRR108)-CD-FLAC-2018-86D")
|
||||||
|
if r.CatalogNo != "BRC30-001-CD-RRR108" {
|
||||||
|
t.Errorf("CatalogNo = %q, want BRC30-001-CD-RRR108", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2018 || r.Group != "86D" {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Murder_Intentions" || r.Title != "Excessive_Display_Of_Human_Nature" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
|
||||||
|
// Катало у дужках у середній зоні — незалежно від вмісту
|
||||||
|
// (крапка, римські, букви, дефіси): все валідне.
|
||||||
|
cases := []struct{ line, catno string }{
|
||||||
|
{"Nervosa-Perpetual_Chaos-(DYN.3140-2)-LIMITED_EDITION-CD-FLAC-2021-MOONBLOOD", "DYN.3140-2"},
|
||||||
|
{"Black_Ceremonial_Kult-Crowned_in_Chaos-(GOWP_DCCCLXXXIX)-CDEP-FLAC-2021-MOONBLOOD", "GOWP_DCCCLXXXIX"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
r := Parse(c.line)
|
||||||
|
if r.CatalogNo != c.catno {
|
||||||
|
t.Errorf("CatalogNo = %q, want %q (release: %s)", r.CatalogNo, c.catno, c.line)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty (release: %s)", r.UnknownTokens, c.line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
|
||||||
|
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
|
||||||
|
// потрапляє в UnknownTokens. (SMG409) — катало.
|
||||||
|
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
|
||||||
|
if r.CatalogNo != "SMG409" {
|
||||||
|
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Artist != "Ferosity" || r.Title != "De" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
|
||||||
|
t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageRequiresKnownNeighbor(t *testing.T) {
|
||||||
|
// Мовний маркер валідний, коли праворуч — відомий хвостовий маркер.
|
||||||
|
r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
|
||||||
|
if r.Language != "DE" || r.Source != "VINYL" {
|
||||||
|
t.Errorf("Language/Source = %q/%q, want DE/VINYL", r.Language, r.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageBetweenTitleAndFormat(t *testing.T) {
|
||||||
|
// A: мовний маркер у зоні між title і format — валідний,
|
||||||
|
// все ліворуч від нього — artist+title, нічого в unknown.
|
||||||
|
r := Parse("Kollektiver_Brechreiz-Heilig_Scheint_Scheinheilig-(NG207)-DE-CD-FLAC-2009-SHELTER")
|
||||||
|
if r.CatalogNo != "NG207" {
|
||||||
|
t.Errorf("CatalogNo = %q, want NG207", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Language != "DE" || r.Source != "CD" || r.Format != "FLAC" || r.Year != 2009 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Kollektiver_Brechreiz" || r.Title != "Heilig_Scheint_Scheinheilig" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageNeverBeforeFormat(t *testing.T) {
|
||||||
|
// DE — словниковий токен у середині (language.txt), позиція неважлива:
|
||||||
|
// Language=DE.
|
||||||
|
r := Parse("Some_Artist-Some_Song-DE-FLAC-2020-SOMEGROUP")
|
||||||
|
if r.Language != "DE" {
|
||||||
|
t.Errorf("Language = %q, want DE", r.Language)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Title != "Some_Song" {
|
||||||
|
t.Errorf("Title = %q, want Some_Song", r.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCompoundEditionRecognized(t *testing.T) {
|
||||||
|
// REMASTERED_REISSUE — складений маркер у dicts/edition.txt, розпізнається
|
||||||
|
// як edition і не йде в UnknownTokens.
|
||||||
|
r := Parse("Ominous_Scriptures-Incarnation_of_the_Unheavenly-(LSR07)-REMASTERED_REISSUE-CD-FLAC-2021-86D")
|
||||||
|
if r.CatalogNo != "LSR07" {
|
||||||
|
t.Errorf("CatalogNo = %q, want LSR07", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Edition != "REMASTERED_REISSUE" {
|
||||||
|
t.Errorf("Edition = %q, want REMASTERED_REISSUE", r.Edition)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Artist != "Ominous_Scriptures" || r.Title != "Incarnation_of_the_Unheavenly" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2021 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
|
||||||
|
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
|
||||||
|
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
|
||||||
|
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
|
||||||
|
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(list []string, s string) bool {
|
||||||
|
for _, v := range list {
|
||||||
|
if v == s {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
|
||||||
|
// Дужки в тайтлі при WEB/BitDepth — не каталономер, лишається в тайтлі.
|
||||||
|
r := Parse("The_Amazons-21st_Century_Fiction_(Epilogue)-16BIT-WEB-FLAC-2026-ENViED")
|
||||||
|
if r.CatalogNo != "" {
|
||||||
|
t.Errorf("CatalogNo = %q, want empty", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Title != "21st_Century_Fiction_(Epilogue)" {
|
||||||
|
t.Errorf("Title = %q, want 21st_Century_Fiction_(Epilogue)", r.Title)
|
||||||
|
}
|
||||||
|
if r.Source != "WEB" || r.BitDepth != "16BIT" || r.Year != 2026 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -38,4 +38,5 @@ var Profile = releaseparser.GroupProfile{
|
|||||||
ResolutionTokens: loadDict("resolution"),
|
ResolutionTokens: loadDict("resolution"),
|
||||||
LanguageMarkers: loadDict("language"),
|
LanguageMarkers: loadDict("language"),
|
||||||
EditionTokens: loadDict("edition"),
|
EditionTokens: loadDict("edition"),
|
||||||
|
CommentTokens: loadDict("comment"),
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ type GroupProfile struct {
|
|||||||
ResolutionTokens map[string]bool
|
ResolutionTokens map[string]bool
|
||||||
LanguageMarkers map[string]bool
|
LanguageMarkers map[string]bool
|
||||||
EditionTokens map[string]bool
|
EditionTokens map[string]bool
|
||||||
|
CommentTokens map[string]bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type ParsedRelease struct {
|
type ParsedRelease struct {
|
||||||
@@ -18,6 +19,7 @@ type ParsedRelease struct {
|
|||||||
BitDepth string
|
BitDepth string
|
||||||
Resolution string
|
Resolution string
|
||||||
Comment string
|
Comment string
|
||||||
|
CatalogNo string
|
||||||
Source string
|
Source string
|
||||||
Format string
|
Format string
|
||||||
Language string
|
Language string
|
||||||
@@ -44,6 +46,8 @@ type GroupStats struct {
|
|||||||
ResolutionDistribution map[string]int
|
ResolutionDistribution map[string]int
|
||||||
LanguageDistribution map[string]int
|
LanguageDistribution map[string]int
|
||||||
EditionDistribution map[string]int
|
EditionDistribution map[string]int
|
||||||
|
CommentDistribution map[string]int
|
||||||
UnknownTokens []UnknownTokenStats
|
UnknownTokens []UnknownTokenStats
|
||||||
|
WarningStats []UnknownTokenStats
|
||||||
AvgConfidence float64
|
AvgConfidence float64
|
||||||
}
|
}
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user