releaseparser: adopt fixed positional model, auto-detect catalogues, add GB doc-blocks
Parse releases with an explicit fixed positional model and document the whole logic in British English doc-blocks. Model: - [0]=Artist, [1]=Title are fixed and never matched against dictionaries, so titles like "Split"/"Demo"/"De" are never eaten. - [len-1]=Group is always read from the line, not from the query key. - [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is resolved as a middle-zone token (e.g. FLAC still picked up as format). - The middle zone is resolved via case-insensitive dictionaries in the order format -> source -> bit-depth -> resolution -> edition -> language -> catalogue number. Position within the zone is irrelevant. Catalogue numbers: - Any "(...)" token in the middle zone is a catalogue number regardless of content (digits, roman numerals, dots, underscores, mixed case). Removed the hasDigit requirement and character whitelist that rejected real catalogues like (DYN.3140-2) and (GOWP_DCCCLXXXIX). - Splitting never breaks tokens inside parentheses. Case handling: lookups are case-insensitive; stored values keep their original case from the line. Documentation: replaced Ukrainian comments in parser.go and analyzer.go with British-English doc-blocks describing the model, tail resolution order, catalogue rule and confidence scoring. Tests: - Add TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2)/(GOWP_DCCCLXXXIX). - Update case-insensitive, split-title, no-year and language expectations. - go build, go vet and go test all pass.
This commit is contained in:
Yevhen Odynets
@@ -10,6 +10,35 @@ import (
|
||||
|
||||
var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
|
||||
|
||||
// Parse splits a release line into tokens and maps them onto a fixed
|
||||
// positional model while the interior is resolved solely through the
|
||||
// group profile's dictionaries.
|
||||
//
|
||||
// The release line is split on "-" (but never inside curved parentheses, so
|
||||
// that a catalogue number such as (BRC30-001-CD-RRR108) stays one token).
|
||||
// The positional layout is:
|
||||
//
|
||||
// [0] Artist - fixed; never matched against dictionaries.
|
||||
// [1] Title - fixed; never matched against dictionaries.
|
||||
// [len-2] Year - if the token is a 4-digit year (19xx | 20xx).
|
||||
// [len-1] Group - always read from the line, never from the query key.
|
||||
//
|
||||
// Everything between [1] and the reserved trailing slots ([2 .. len-3], or
|
||||
// [2 .. len-2] when there is no year) is the middle zone. Each token there is
|
||||
// looked up in the dictionaries (format, source, bit-depth, resolution,
|
||||
// language, edition) and, if it is a parenthesized token, as a catalogue
|
||||
// number. Position within the middle zone is irrelevant. Tokens that match
|
||||
// nothing are NOT silently consumed — they are reported in UnknownTokens so
|
||||
// that they can be reviewed and added to a dictionary.
|
||||
//
|
||||
// Why [0]/[1] are fixed and never matched against the dictionaries: titles
|
||||
// such as "Split" (Alone_In_The_Morgue-Split-...) or "Demo"
|
||||
// (Mystifier-Demo-Nized-...) are legitimate words that collide with edition
|
||||
// markers. Since index 1 is always the title, regardless of case or
|
||||
// dictionary membership, such words are never eaten.
|
||||
//
|
||||
// Values are preserved in their original case (as they appear in the line);
|
||||
// the dictionary lookups themselves are case-insensitive.
|
||||
func Parse(raw string) releaseparser.ParsedRelease {
|
||||
result := releaseparser.ParsedRelease{
|
||||
Raw: strings.TrimSpace(raw),
|
||||
@@ -17,123 +46,140 @@ func Parse(raw string) releaseparser.ParsedRelease {
|
||||
UnknownTokens: []string{},
|
||||
}
|
||||
|
||||
tokens := split(raw)
|
||||
tokens, comments := extractComments(split(raw))
|
||||
if len(comments) > 0 {
|
||||
result.Comment = strings.Join(comments, ",")
|
||||
}
|
||||
if len(tokens) < 5 {
|
||||
result.Warnings = append(result.Warnings, "too few tokens")
|
||||
result.Confidence = 0.2
|
||||
return result
|
||||
}
|
||||
|
||||
// Останній токен — група. Вона ж — ключ до профілю групи (довідників).
|
||||
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
|
||||
// [len-2]=Year (when present).
|
||||
result.Artist = tokens[0]
|
||||
if len(tokens) > 1 {
|
||||
result.Title = tokens[1]
|
||||
}
|
||||
result.Group = tokens[len(tokens)-1]
|
||||
|
||||
// Передостанній — рік
|
||||
last := len(tokens) - 2
|
||||
if yearRegexp.MatchString(tokens[last]) {
|
||||
result.Year, _ = strconv.Atoi(tokens[last])
|
||||
last--
|
||||
}
|
||||
|
||||
// Рухаємось вліво по хвосту: format, source, bit_depth, resolution, language, edition.
|
||||
// Кожен слот бере токен лише зі свого словника (регістронезалежно).
|
||||
// Невідомий токен: репортимо лише якщо ліворуч ще є відомий хвостовий маркер
|
||||
// (токен між двома маркерами = сміття в хвості). Інакше це початок core.
|
||||
tailLoop:
|
||||
for last >= 0 {
|
||||
token := strings.ToUpper(tokens[last])
|
||||
|
||||
switch {
|
||||
case Profile.FormatTokens[token]:
|
||||
if result.Format == "" {
|
||||
result.Format = token
|
||||
}
|
||||
last--
|
||||
|
||||
case Profile.SourceTokens[token]:
|
||||
if result.Source == "" {
|
||||
result.Source = token
|
||||
}
|
||||
last--
|
||||
|
||||
case Profile.BitDepthTokens[token]:
|
||||
if result.BitDepth == "" {
|
||||
result.BitDepth = token
|
||||
}
|
||||
last--
|
||||
|
||||
case Profile.ResolutionTokens[token]:
|
||||
if result.Resolution == "" {
|
||||
result.Resolution = token
|
||||
}
|
||||
last--
|
||||
|
||||
case Profile.LanguageMarkers[token]:
|
||||
if result.Language == "" {
|
||||
result.Language = token
|
||||
}
|
||||
last--
|
||||
|
||||
case Profile.EditionTokens[token]:
|
||||
if result.Edition == "" {
|
||||
result.Edition = token
|
||||
}
|
||||
last--
|
||||
|
||||
default:
|
||||
if hasKnownTailToken(tokens[:last]) {
|
||||
result.UnknownTokens = append(result.UnknownTokens, tokens[last])
|
||||
last--
|
||||
continue
|
||||
}
|
||||
break tailLoop
|
||||
}
|
||||
}
|
||||
|
||||
// Core = artist-title (все, що залишилось ліворуч)
|
||||
coreTokens := tokens[:last+1]
|
||||
result.Core = strings.Join(coreTokens, "-")
|
||||
|
||||
if len(coreTokens) >= 2 {
|
||||
result.Artist = coreTokens[0]
|
||||
result.Title = strings.Join(coreTokens[1:], "-")
|
||||
} else if len(coreTokens) == 1 {
|
||||
result.Artist = coreTokens[0]
|
||||
result.Title = ""
|
||||
result.Warnings = append(result.Warnings, "title not found")
|
||||
end := len(tokens) - 1
|
||||
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
|
||||
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
|
||||
end = len(tokens) - 2
|
||||
}
|
||||
// Resolve the middle zone tokens via the dictionaries.
|
||||
parseTailZone(tokens[2:end], &result)
|
||||
|
||||
score(&result)
|
||||
return result
|
||||
}
|
||||
|
||||
// split breaks a raw release line into tokens on "-", but never inside
|
||||
// curved parentheses, so that a catalogue number such as
|
||||
// (BRC30-001-CD-RRR108) remains a single token.
|
||||
func split(raw string) []string {
|
||||
parts := strings.Split(strings.TrimSpace(raw), "-")
|
||||
out := make([]string, 0, len(parts))
|
||||
for _, part := range parts {
|
||||
if part = strings.TrimSpace(part); part != "" {
|
||||
out = append(out, part)
|
||||
s := strings.TrimSpace(raw)
|
||||
out := make([]string, 0, 8)
|
||||
var cur strings.Builder
|
||||
inParen := false
|
||||
for _, r := range s {
|
||||
switch {
|
||||
case r == '(':
|
||||
inParen = true
|
||||
cur.WriteRune(r)
|
||||
case r == ')':
|
||||
inParen = false
|
||||
cur.WriteRune(r)
|
||||
case r == '-' && !inParen:
|
||||
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||
out = append(out, t)
|
||||
}
|
||||
cur.Reset()
|
||||
default:
|
||||
cur.WriteRune(r)
|
||||
}
|
||||
}
|
||||
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||
out = append(out, t)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// hasKnownTailToken reports whether any token is a known tail marker
|
||||
// (format, source, bit_depth, resolution, edition, language or a year).
|
||||
func hasKnownTailToken(tokens []string) bool {
|
||||
// extractComments removes service comments (CommentTokens) from the token
|
||||
// list and returns the cleaned list together with the removed comments.
|
||||
func extractComments(tokens []string) ([]string, []string) {
|
||||
out := make([]string, 0, len(tokens))
|
||||
comments := []string{}
|
||||
for _, tok := range tokens {
|
||||
up := strings.ToUpper(tok)
|
||||
if Profile.FormatTokens[up] || Profile.SourceTokens[up] ||
|
||||
Profile.BitDepthTokens[up] || Profile.ResolutionTokens[up] ||
|
||||
Profile.EditionTokens[up] || Profile.LanguageMarkers[up] {
|
||||
return true
|
||||
}
|
||||
if yearRegexp.MatchString(up) {
|
||||
return true
|
||||
if Profile.CommentTokens[strings.ToUpper(tok)] {
|
||||
comments = append(comments, tok)
|
||||
continue
|
||||
}
|
||||
out = append(out, tok)
|
||||
}
|
||||
return false
|
||||
return out, comments
|
||||
}
|
||||
|
||||
// catalogNumber reports whether the token is a parenthesized catalogue
|
||||
// number. In the positional model any token wrapped in curved parentheses
|
||||
// that sits in the middle zone ([2..]) is a catalogue number regardless of
|
||||
// its content (digits, letters, roman numerals, dots, underscores, mixed
|
||||
// case). Parenthesized tokens at [0]/[1] are handled as artist/title, not
|
||||
// here. It returns the parenthesized content, or "" when the token is not
|
||||
// a "(...)" group.
|
||||
func catalogNumber(token string) string {
|
||||
if len(token) < 3 || token[0] != '(' || token[len(token)-1] != ')' {
|
||||
return ""
|
||||
}
|
||||
return token[1 : len(token)-1]
|
||||
}
|
||||
|
||||
// parseTailZone resolves middle-zone tokens against the dictionaries.
|
||||
// The first match wins for each field; catalogue numbers are recognised
|
||||
// from parenthesized tokens, and anything unrecognised is appended to
|
||||
// UnknownTokens so that it can be reviewed and curated into a dictionary.
|
||||
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||
for _, tok := range zone {
|
||||
up := strings.ToUpper(tok)
|
||||
switch {
|
||||
case Profile.FormatTokens[up]:
|
||||
if result.Format == "" {
|
||||
result.Format = tok
|
||||
}
|
||||
case Profile.SourceTokens[up]:
|
||||
if result.Source == "" {
|
||||
result.Source = tok
|
||||
}
|
||||
case Profile.BitDepthTokens[up]:
|
||||
if result.BitDepth == "" {
|
||||
result.BitDepth = tok
|
||||
}
|
||||
case Profile.ResolutionTokens[up]:
|
||||
if result.Resolution == "" {
|
||||
result.Resolution = tok
|
||||
}
|
||||
case Profile.EditionTokens[up]:
|
||||
if result.Edition == "" {
|
||||
result.Edition = tok
|
||||
}
|
||||
case Profile.LanguageMarkers[up]:
|
||||
if result.Language == "" {
|
||||
result.Language = tok
|
||||
}
|
||||
default:
|
||||
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
|
||||
result.CatalogNo = cat
|
||||
} else {
|
||||
result.UnknownTokens = append(result.UnknownTokens, tok)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// score computes a heuristic confidence in [0,1] for a parsed release based
|
||||
// on how many core fields (group, year, format, source, artist) were resolved
|
||||
// and whether any warnings or unknown tokens are present.
|
||||
func score(result *releaseparser.ParsedRelease) {
|
||||
score := 0.0
|
||||
if result.Group != "" {
|
||||
|
||||
Reference in New Issue
Block a user