Compare commits

...

6 Commits

Author SHA1 Message Date
Yevhen Odynets 1884151eb0 cosmetic changes 2026-09-01 08:59:03 +03:00
Yevhen Odynets 31febb9f56 releaseparser: adopt fixed positional model, auto-detect catalogues, add GB doc-blocks
Parse releases with an explicit fixed positional model and document the
whole logic in British English doc-blocks.

Model:
- [0]=Artist, [1]=Title are fixed and never matched against dictionaries,
  so titles like "Split"/"Demo"/"De" are never eaten.
- [len-1]=Group is always read from the line, not from the query key.
- [len-2]=Year when it is a 4-digit 19xx|20xx token; otherwise the slot is
  resolved as a middle-zone token (e.g. FLAC still picked up as format).
- The middle zone is resolved via case-insensitive dictionaries in the order
  format -> source -> bit-depth -> resolution -> edition -> language ->
  catalogue number. Position within the zone is irrelevant.

Catalogue numbers:
- Any "(...)" token in the middle zone is a catalogue number regardless of
  content (digits, roman numerals, dots, underscores, mixed case). Removed
  the hasDigit requirement and character whitelist that rejected real
  catalogues like (DYN.3140-2) and (GOWP_DCCCLXXXIX).
- Splitting never breaks tokens inside parentheses.

Case handling: lookups are case-insensitive; stored values keep their
original case from the line.

Documentation: replaced Ukrainian comments in parser.go and analyzer.go with
British-English doc-blocks describing the model, tail resolution order,
catalogue rule and confidence scoring.

Tests:
- Add TestParseCatalogNumberWithDotAndRoman for (DYN.3140-2)/(GOWP_DCCCLXXXIX).
- Update case-insensitive, split-title, no-year and language expectations.
- go build, go vet and go test all pass.
2026-09-01 04:26:06 +03:00
Yevhen Odynets 3f3598c446 add nultie gpouring 2026-09-01 00:08:03 +03:00
Yevhen Odynets 1b2521d4ac refactored & audited, real analizer added 2026-08-31 22:24:38 +03:00
Yevhen Odynets e11c82bd76 Fixed rutracker after changing stucture of the atom file of theirs own 2026-08-31 20:24:07 +03:00
Yevhen Odynets fedb470da6 minor changes 2026-08-31 19:00:09 +03:00
33 changed files with 1585 additions and 94 deletions
+267
View File
@@ -0,0 +1,267 @@
# =============================================================
# .gitattributes — Universal JetBrains / Multi-Stack Template
# Covers: PHP, Go, TypeScript, JavaScript, Rust, Dart/Flutter,
# Python, Ruby, HCL/Terraform, SQL, HTML, CSS, DevOps
# =============================================================
# ─────────────────────────────────────────────────────────────
# DEFAULTS: LF everywhere, CRLF only for Windows-specific files
# ─────────────────────────────────────────────────────────────
* text=auto eol=lf
*.bat text eol=crlf
*.cmd text eol=crlf
*.ps1 text eol=crlf
*.psm1 text eol=crlf
*.psd1 text eol=crlf
# ─────────────────────────────────────────────────────────────
# PHP
# ─────────────────────────────────────────────────────────────
*.php text diff=php
*.phtml text diff=php
*.php8 text diff=php
# ─────────────────────────────────────────────────────────────
# Go
# ─────────────────────────────────────────────────────────────
*.go text diff=golang
go.sum text -diff
go.work.sum text -diff
# ─────────────────────────────────────────────────────────────
# TypeScript / JavaScript
# ─────────────────────────────────────────────────────────────
*.ts text diff=typescript
*.tsx text diff=typescript
*.js text diff=javascript
*.jsx text diff=javascript
*.mjs text diff=javascript
*.cjs text diff=javascript
# ─────────────────────────────────────────────────────────────
# Rust
# ─────────────────────────────────────────────────────────────
*.rs text diff=rust
Cargo.lock text -diff
# ─────────────────────────────────────────────────────────────
# Dart / Flutter
# ─────────────────────────────────────────────────────────────
*.dart text diff=dart
pubspec.lock text -diff
# ─────────────────────────────────────────────────────────────
# Lua
# ─────────────────────────────────────────────────────────────
*.lua text diff=lua
# ─────────────────────────────────────────────────────────────
# Python
# ─────────────────────────────────────────────────────────────
*.py text diff=python
*.pyi text diff=python
Pipfile.lock text -diff
# ─────────────────────────────────────────────────────────────
# Ruby
# ─────────────────────────────────────────────────────────────
*.rb text diff=ruby
Gemfile.lock text -diff
# ─────────────────────────────────────────────────────────────
# Web: HTML / CSS / Templates
# ─────────────────────────────────────────────────────────────
*.html text diff=html
*.htm text diff=html
*.blade.php text diff=html
*.templ text diff=html
*.twig text diff=html
*.njk text diff=html
*.ejs text diff=html
*.hbs text diff=html
*.mustache text diff=html
*.vue text diff=html
*.svelte text diff=html
*.css text diff=css
*.scss text diff=css
*.sass text diff=css
*.less text diff=css
# ─────────────────────────────────────────────────────────────
# Data / Config formats
# ─────────────────────────────────────────────────────────────
*.json text diff=json
*.jsonc text diff=json
*.json5 text diff=json
*.yaml text diff=yaml
*.yml text diff=yaml
*.toml text
*.xml text diff=html
*.svg text diff=html
*.env text
*.env.* text
*.ini text
*.cfg text
*.conf text
*.properties text
# ─────────────────────────────────────────────────────────────
# HCL / Terraform / HashiCorp stack
# ─────────────────────────────────────────────────────────────
*.hcl text diff=terraform
*.tf text diff=terraform
*.tfvars text diff=terraform
*.tfvars.json text diff=json
.terraform.lock.hcl text -diff merge=ours
# ─────────────────────────────────────────────────────────────
# SQL / Database
# ─────────────────────────────────────────────────────────────
*.sql text diff=sql
*.pgsql text diff=sql
*.mysql text diff=sql
# ─────────────────────────────────────────────────────────────
# Shell / Scripts
# ─────────────────────────────────────────────────────────────
*.sh text eol=lf
*.bash text eol=lf
*.zsh text eol=lf
*.fish text eol=lf
Makefile text eol=lf
*.mk text eol=lf
Dockerfile text eol=lf
Dockerfile.* text eol=lf
*.dockerfile text eol=lf
*.containerfile text eol=lf
# ─────────────────────────────────────────────────────────────
# Documentation
# ─────────────────────────────────────────────────────────────
*.md text diff=markdown
*.mdx text diff=markdown
*.rst text
*.txt text
*.adoc text
LICENSE text
AUTHORS text
CODEOWNERS text
# ─────────────────────────────────────────────────────────────
# Lock files — no diff noise, merge=ours
# ─────────────────────────────────────────────────────────────
*.lock text -diff merge=ours
*.lock.* text -diff merge=ours
package-lock.json text -diff merge=ours
yarn.lock text -diff merge=ours
pnpm-lock.yaml text -diff merge=ours
composer.lock text -diff merge=ours
bun.lockb binary
# ─────────────────────────────────────────────────────────────
# Generated files — hide from diff + GitHub stats
# ─────────────────────────────────────────────────────────────
*_templ.go linguist-generated=true -diff
*.pb.go linguist-generated=true -diff
*.gen.go linguist-generated=true -diff
*.generated.go linguist-generated=true -diff
*.generated.ts linguist-generated=true -diff
*.generated.php linguist-generated=true -diff
*_test.snap linguist-generated=true -diff
# ─────────────────────────────────────────────────────────────
# Binary: Images
# ─────────────────────────────────────────────────────────────
*.png binary
*.jpg binary
*.jpeg binary
*.gif binary
*.bmp binary
*.ico binary
*.webp binary
*.avif binary
*.tiff binary
*.tif binary
*.psd binary
*.ai binary
*.eps binary
# ─────────────────────────────────────────────────────────────
# Binary: Fonts
# ─────────────────────────────────────────────────────────────
*.ttf binary
*.otf binary
*.woff binary
*.woff2 binary
*.eot binary
# ─────────────────────────────────────────────────────────────
# Binary: Audio / Video
# ─────────────────────────────────────────────────────────────
*.mp3 binary
*.mp4 binary
*.wav binary
*.flac binary
*.ogg binary
*.opus binary
*.aac binary
*.m4a binary
*.webm binary
*.avi binary
*.mov binary
*.mkv binary
# ─────────────────────────────────────────────────────────────
# Binary: Archives / Executables
# ─────────────────────────────────────────────────────────────
*.zip binary
*.tar binary
*.gz binary
*.bz2 binary
*.xz binary
*.7z binary
*.rar binary
*.exe binary
*.dll binary
*.so binary
*.dylib binary
*.a binary
*.o binary
*.wasm binary
# ─────────────────────────────────────────────────────────────
# Binary: Documents / Data files
# ─────────────────────────────────────────────────────────────
*.pdf binary
*.doc binary
*.docx binary
*.xls binary
*.xlsx binary
*.ppt binary
*.pptx binary
*.db binary
*.sqlite binary
*.sqlite3 binary
# ─────────────────────────────────────────────────────────────
# Terraform state — never should be committed, but just in case
# ─────────────────────────────────────────────────────────────
*.tfstate binary
*.tfstate.backup binary
# ─────────────────────────────────────────────────────────────
# git archive export-ignore
# (files excluded from GitHub release tarballs)
# ─────────────────────────────────────────────────────────────
README.md export-ignore
CHANGELOG.md export-ignore
CONTRIBUTING.md.md export-ignore
SECURITY.md export-ignore
.editorconfig export-ignore
.env.example export-ignore
.gitattributes export-ignore
.gitignore export-ignore
.github export-ignore
docs export-ignore
tests export-ignore
+5 -21
View File
@@ -1,34 +1,18 @@
.idea/**/workspace.xml
.idea/**/tasks.xml
.idea/**/usage.statistics.xml
.idea/**/dictionaries
.idea/**/shelf
.idea/**/aws.xml
.idea/**/contentModel.xml
.idea/**/dataSources/
.idea/**/dataSources.ids
.idea/**/dataSources.local.xml
.idea/**/sqlDataSources.xml
.idea/**/dynamic.xml
.idea/**/uiDesigner.xml
.idea/**/dbnavigator.xml
.idea/**/gradle.xml
.idea/**/libraries
.idea/
cmake-build-*/
.idea/**/mongoSettings.xml
*.iws
out/
.idea_modules/
atlassian-ide-plugin.xml
.idea/replstate.xml
.idea/sonarlint/
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
.idea/httpRequests
.idea/caches/build_file_checksums.ser
.env
tmp/
vendor/
config/secret.yaml
*.md
mdb
internal/releaseparser/resources/*
review_unknown_tokens.txt
-1
View File
@@ -12,7 +12,6 @@
<option value="gosimple" />
</list>
</option>
<option name="goLinterExe" value="C:\Users\theAmok\go\bin\golangci-lint.exe" />
<option name="linterSelected" value="true" />
</component>
</project>
+1
View File
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="WEB_MODULE" version="4">
<component name="Go" enabled="true" />
<component name="GoModuleSettings" enabled="true" />
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$" />
<orderEntry type="inheritedJdk" />
+79
View File
@@ -0,0 +1,79 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
_ "github.com/go-sql-driver/mysql"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
)
func main() {
if len(os.Args) < 2 {
log.Fatal("usage: srrdb <releaser> [<releaser>...]")
}
releasers := os.Args[1:]
// Читаємо config/secret.yaml (без змісту в коді — тільки потрібні ключі)
viper.SetConfigFile("config/secret.yaml")
if err := viper.ReadInConfig(); err != nil {
log.Fatal("failed to read config/secret.yaml: ", err)
}
drv := viper.GetString("db.driver")
dsn := viper.GetString("db.data_source_name")
dbase, err := sqlx.Connect(drv, dsn)
if err != nil {
panic("Failed to connect to the database: " + err.Error())
}
defer dbase.Close()
out := make(map[string]moonblood.GroupStats, len(releasers))
for _, releaser := range releasers {
var titles []string
err = dbase.Select(&titles,
`SELECT title FROM theamok_mdb.external_sources
WHERE type = 'prescene' AND releaser = ?`, releaser)
if err != nil {
log.Fatal(err)
}
out[releaser] = moonblood.AnalyzeTitles(titles)
}
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
log.Fatal(err)
}
// Ручна ревізія: всі невідомі токени — у файл для правки бази.
review, err := os.OpenFile("review_unknown_tokens.txt",
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
if err != nil {
log.Fatal(err)
}
defer review.Close()
for releaser, stats := range out {
for _, unk := range stats.UnknownTokens {
for _, rel := range unk.Releases {
fmt.Fprintf(review, "%s\t%s\t%s\n", releaser, unk.Token, rel)
}
}
for _, warn := range stats.WarningStats {
for _, rel := range warn.Releases {
fmt.Fprintf(review, "%s\tWARN\t%s\t%s\n", releaser, warn.Token, rel)
}
}
}
fmt.Println(string(data))
}
View File
+1
View File
@@ -3,6 +3,7 @@ module git.kplus.net.ua/yevhen/resource-scraper
go 1.26.2
require (
github.com/davecgh/go-spew v1.1.1
github.com/dromara/carbon/v2 v2.6.16
github.com/emersion/go-imap/v2 v2.0.0-beta.8
github.com/go-chi/chi/v5 v5.2.5
-11
View File
@@ -1,4 +1,3 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
@@ -6,12 +5,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dromara/carbon/v2 v2.6.15 h1:3HuC3XcWczIHUTbg/f0CSVydtKEdM+P0GM1sdsbwXmI=
github.com/dromara/carbon/v2 v2.6.15/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -22,8 +17,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
@@ -59,8 +52,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d h1:IL2tii4jXLdhCeQN69HNzYYW1kl0meSG0wt5+sLwszU=
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d/go.mod h1:SIpumAnUWSy0q9RzKD3pyH3g1t5vdawUAPcW5tQrUtI=
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
@@ -94,8 +85,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+79 -15
View File
@@ -2,32 +2,43 @@ package sugar
import (
"fmt"
"log"
"log/slog"
"os"
"path/filepath"
"strconv"
)
// EnsureDir joins the given path segments and creates the directory
// (including all parent directories) if it does not already exist.
// Returns the resolved path or an error if creation fails.
func EnsureDir(dirPath ...string) (string, error) {
outPath := filepath.Join(dirPath...)
if len(outPath) < 3 {
return "", fmt.Errorf("the path is short to be a folder: %s", outPath)
if outPath == "" {
return "", fmt.Errorf("path must not be empty")
}
if _, err := os.Stat(outPath); os.IsNotExist(err) {
var dirMod uint64
if dirMod, err = strconv.ParseUint("0775", 8, 32); err == nil {
err = os.Mkdir(outPath, os.FileMode(dirMod))
}
if err != nil && !os.IsExist(err) {
slog.Error("error creating tmp dir", "err", err)
}
if err := os.MkdirAll(outPath, os.FileMode(0775)); err != nil {
return "", fmt.Errorf("failed to ensure directory %q: %w", outPath, err)
}
return outPath, nil
}
// ExecDir returns the directory of the running binary.
// Works correctly even with symlinks and -trimpath builds.
func ExecDir() (string, error) {
exe, err := os.Executable()
if err != nil {
return "", err
}
// EvalSymlinks is critical — os.Executable can return a symlink path
resolved, err := filepath.EvalSymlinks(exe)
if err != nil {
return "", err
}
return filepath.Dir(resolved), nil
}
func WriteDataToTmpFile(content, fp string) (int, string, error) {
dir, err := EnsureDir(os.TempDir(), "mdb")
if err != nil {
@@ -58,6 +69,59 @@ func WriteDataToTmpFile(content, fp string) (int, string, error) {
return filesize, fp, nil
}
/*
REMOVED_FROM_MMT_DUE_TO_REDOING_OF_THE_EVENT_LOG
*/
// ConfigPaths returns candidate directories for config lookup.
// Priority: XDG_CONFIG_HOME → binary dir → CWD → /etc/appname
func ConfigPaths(appName string) []string {
var candidates []string
// 1. Explicit env override (highest priority)
if envPath := os.Getenv("APP_CONFIG_DIR"); envPath != "" {
candidates = append(candidates, envPath)
}
// 2. Relative to binary (best for production deployments)
if exe, err := os.Executable(); err == nil {
if resolved, err := filepath.EvalSymlinks(exe); err == nil {
candidates = append(candidates, filepath.Dir(resolved))
}
}
// 3. CWD (convenient for `go run` and development)
if cwd, err := os.Getwd(); err == nil {
candidates = append(candidates, cwd)
}
// 4. XDG standard (Linux/Unix)
if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
candidates = append(candidates, filepath.Join(xdg, appName))
} else if home, err := os.UserHomeDir(); err == nil {
candidates = append(candidates, filepath.Join(home, ".config", appName))
}
// 5. System-wide fallback
candidates = append(candidates, filepath.Join("/etc", appName))
return candidates
}
// FindConfig searches all candidate dirs for the given filename.
func FindConfig(appName, filename string) (string, error) {
for _, dir := range ConfigPaths(appName) {
path := filepath.Join(dir, filename)
if _, err := os.Stat(path); err == nil {
return path, nil
}
}
return "", fmt.Errorf("config %q not found in any candidate directory", filename)
}
// DebugPath Debug helper — remove before production
func DebugPath(path string) {
info, err := os.Stat(path)
log.Printf("path=%q exists=%v err=%v mode=%v", path, err == nil, err, func() string {
if info != nil {
return info.Mode().String()
}
return "n/a"
}())
}
+2 -1
View File
@@ -1,2 +1,3 @@
#!/usr/bin/env bash
go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
#go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
go build -trimpath -ldflags="-s -w" -o /opt/sources/resource-scraper/cmd/mdb ./cmd
+11 -1
View File
@@ -5,10 +5,12 @@ import (
"log"
"log/slog"
"path/filepath"
"runtime"
"slices"
"github.com/spf13/viper"
"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
@@ -16,6 +18,8 @@ func New() {
configFilePath := viper.GetString(constant.FlagConfigFile)
configName := filepath.Base(configFilePath)
sugar.DebugPath(configFilePath)
dir := filepath.Dir(configFilePath)
if dir == configName && dir == "." {
configName = "default"
@@ -23,7 +27,13 @@ func New() {
configDir, _ := filepath.Abs(dir)
viper.AddConfigPath(configDir)
switch runtime.GOOS {
case "windows":
viper.AddConfigPath(configDir)
case "linux":
viper.AddConfigPath("/opt/sources/resource-scraper/config")
}
viper.SetConfigName(configName)
err := viper.ReadInConfig()
if err != nil { // Handle errors reading the config file
@@ -0,0 +1,151 @@
package moonblood
import (
"bufio"
"os"
"sort"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// GroupStats summarises how a set of release lines parsed: field
// distributions, unknown tokens and warnings aggregated by occurrence, and
// the average parsing confidence. The slices are sorted by descending count.
type GroupStats struct {
Total int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
ResolutionDistribution map[string]int
LanguageDistribution map[string]int
EditionDistribution map[string]int
CommentDistribution map[string]int
UnknownTokens []releaseparser.UnknownTokenStats
WarningStats []releaseparser.UnknownTokenStats
AvgConfidence float64
}
// AnalyzeGroup parses the release lines from the local resource file. It is
// used for a quick local check of the dictionaries without touching a
// database.
func AnalyzeGroup() (GroupStats, error) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
return GroupStats{}, err
}
defer file.Close()
var titles []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line != "" {
titles = append(titles, line)
}
}
if err := scanner.Err(); err != nil {
return GroupStats{}, err
}
return AnalyzeTitles(titles), nil
}
// AnalyzeTitles parses a set of release lines (for example fetched from the
// database by a particular releaser key) and aggregates the results into a
// GroupStats summary.
func AnalyzeTitles(titles []string) GroupStats {
stats := GroupStats{
YearDistribution: make(map[int]int),
SourceDistribution: make(map[string]int),
BitDepthDistribution: make(map[string]int),
ResolutionDistribution: make(map[string]int),
LanguageDistribution: make(map[string]int),
EditionDistribution: make(map[string]int),
CommentDistribution: make(map[string]int),
UnknownTokens: []releaseparser.UnknownTokenStats{},
WarningStats: []releaseparser.UnknownTokenStats{},
}
totalConfidence := 0.0
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
for _, title := range titles {
result := Parse(title)
stats.Total++
if result.Year != 0 {
stats.YearDistribution[result.Year]++
}
if result.Source != "" {
stats.SourceDistribution[result.Source]++
}
if result.BitDepth != "" {
stats.BitDepthDistribution[result.BitDepth]++
}
if result.Resolution != "" {
stats.ResolutionDistribution[result.Resolution]++
}
if result.Language != "" {
stats.LanguageDistribution[result.Language]++
}
if result.Edition != "" {
stats.EditionDistribution[result.Edition]++
}
if result.Comment != "" {
for _, c := range strings.Split(result.Comment, ",") {
stats.CommentDistribution[c]++
}
}
for _, unknown := range result.UnknownTokens {
if _, exists := unknownMap[unknown]; !exists {
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
Token: unknown,
Count: 0,
Releases: []string{},
}
}
unknownMap[unknown].Count++
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
}
for _, warning := range result.Warnings {
if _, exists := warningMap[warning]; !exists {
warningMap[warning] = &releaseparser.UnknownTokenStats{
Token: warning,
Count: 0,
Releases: []string{},
}
}
warningMap[warning].Count++
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
}
totalConfidence += result.Confidence
}
// Convert the aggregates from maps to slices and order them by count
// (most frequent first).
for _, tokenStats := range unknownMap {
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
}
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
})
for _, w := range warningMap {
stats.WarningStats = append(stats.WarningStats, *w)
}
sort.Slice(stats.WarningStats, func(i, j int) bool {
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
})
if stats.Total > 0 {
stats.AvgConfidence = totalConfidence / float64(stats.Total)
}
return stats
}
@@ -0,0 +1,5 @@
# {bit_depth}
16BIT
24BIT
32BIT
DE16BIT
@@ -0,0 +1,6 @@
REPACK
REAL_REPACK
READNFO
PROPER
RERIP
199X
@@ -0,0 +1,70 @@
# {edition} — тип видання
EP
DEMO
SPLIT
REISSUE
DELUXE_EDITION
REMASTERED
BOXSET
SINGLE
PROMO
REISSUE_REMASTERED
REMASTERED_25TH_ANNIVERSARY_EDITION
REMASTERED_20TH_ANNIVERSARY_EDITION
20TH_ANNIVERSARY_EDITION_REMASTERED
Deluxe_Edition
LIMITED_EDITION_BOXSET
REMASTERED_EP
10TH_ANNIVERSARY_EDITION
OST
XX_Legacy_Edition
REISSUE_DEMO
JP_Deluxe_Edition
Remastered_Edition
Anniversary_Edition
Limited_Edition
CDS
CDM
RETAIL
JP_RETAIL
JP_REISSUE
DEMO_REMASTERED
EU_RETAIL
DE_RETAIL
REISSUE_DELUXE_EDITION
RETAIL_REISSUE
SPECIAL_EDITION
SPECIAL_COLLECTORS_EDITION_REMASTERED
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
LIMITED_EDITION_REMASTERED
25th_Anniversary_Edition
DEMO_REISSUE
Ultimate_Edition
Issue_255
Issue_254
Issue_198
7INCH_VINYL
REMASTERED_REISSUE
PROMO_DEMO
ES_RETAIL
BOX_SET
BONUS
REISSUE_EP
30th_Anniversary
REDUX
REMASTERED_BOXSET
MAG
REISSUE_BOXSET
30TH_ANNIVERSARY_EDITION_REMASTERED
DELUXE
Deluxe_Edition_Reissue
INSTRUMENTAL
REISSUE_EXPANDED_EDITION
BOOTLEG
EXPANDED_EDITION
ADVANCE
15TH_ANNIVERSARY_DELUXE_EDITION
TOUR_EDITION
REMASTERED_DEMO
REMASTERED_DELUXE_EDITION
30TH_ANNIVERSARY_EDITION
@@ -0,0 +1,3 @@
# {format} — аудіоформат
# один токен на рядок, регістр не має значення
FLAC
@@ -0,0 +1,38 @@
# {language} — мовні маркери
DE
FR
ES
SK
FI
SE
BR
PL
NL
NO
PT
IT
HU
DK
AT
IS
UA
JP
GR
CN
RU
ID
PH
RO
LT
BY
CZ
SP
IR
FO
RS
DA
HI
CH
LU
CY
EE
@@ -0,0 +1,8 @@
# {resolution} — частота дискретизації
44KHZ
48KHZ
48.0kHz
88KHZ
96KHZ
176KHZ
192KHZ
@@ -0,0 +1,36 @@
# {source} — джерело релізу
# VINYL сюди входить як source (за домовленістю, не edition)
WEB
CD
2CD
CDEP
VINYL
TAPE
CDR
17CD
3CD
4CD
18CD
3CDEP
DVD
7CD
2CDR
10CD
6CD
2CDEP
2VINYL
CDREP
LP
2LP
3VINYL
3TAPE
13CD
13CDR
10INCH_VINYL
25CD
WEBFLAC
2TAPE
BLURAY
VLS
MCD
24BIT_WEB
@@ -0,0 +1,213 @@
package moonblood
import (
"regexp"
"strconv"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
// Parse splits a release line into tokens and maps them onto a fixed
// positional model while the interior is resolved solely through the
// group profile's dictionaries.
//
// The release line is split on "-" (but never inside curved parentheses, so
// that a catalogue number such as (BRC30-001-CD-RRR108) stays one token).
// The positional layout is:
//
// [0] Artist - fixed; never matched against dictionaries.
// [1] Title - fixed; never matched against dictionaries.
// [len-2] Year - if the token is a 4-digit year (19xx | 20xx).
// [len-1] Group - always read from the line, never from the query key.
//
// Everything between [1] and the reserved trailing slots ([2 .. len-3], or
// [2 .. len-2] when there is no year) is the middle zone. Each token there is
// looked up in the dictionaries (format, source, bit-depth, resolution,
// language, edition) and, if it is a parenthesized token, as a catalogue
// number. Position within the middle zone is irrelevant. Tokens that match
// nothing are NOT silently consumed — they are reported in UnknownTokens so
// that they can be reviewed and added to a dictionary.
//
// Why [0]/[1] are fixed and never matched against the dictionaries: titles
// such as "Split" (Alone_In_The_Morgue-Split-...) or "Demo"
// (Mystifier-Demo-Nized-...) are legitimate words that collide with edition
// markers. Since index 1 is always the title, regardless of case or
// dictionary membership, such words are never eaten.
//
// Values are preserved in their original case (as they appear in the line);
// the dictionary lookups themselves are case-insensitive.
func Parse(raw string) releaseparser.ParsedRelease {
result := releaseparser.ParsedRelease{
Raw: strings.TrimSpace(raw),
Warnings: []string{},
UnknownTokens: []string{},
}
tokens, comments := extractComments(split(raw))
if len(comments) > 0 {
result.Comment = strings.Join(comments, ",")
}
if len(tokens) < 5 {
result.Warnings = append(result.Warnings, "too few tokens")
result.Confidence = 0.2
return result
}
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
// [len-2]=Year (when present).
result.Artist = tokens[0]
if len(tokens) > 1 {
result.Title = tokens[1]
}
result.Group = tokens[len(tokens)-1]
end := len(tokens) - 1
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
end = len(tokens) - 2
}
// Resolve the middle zone tokens via the dictionaries.
parseTailZone(tokens[2:end], &result)
score(&result)
return result
}
// split breaks a raw release line into tokens on "-", but never inside
// curved parentheses, so that a catalogue number such as
// (BRC30-001-CD-RRR108) remains a single token.
func split(raw string) []string {
s := strings.TrimSpace(raw)
out := make([]string, 0, 8)
var cur strings.Builder
inParen := false
for _, r := range s {
switch {
case r == '(':
inParen = true
cur.WriteRune(r)
case r == ')':
inParen = false
cur.WriteRune(r)
case r == '-' && !inParen:
if t := strings.TrimSpace(cur.String()); t != "" {
out = append(out, t)
}
cur.Reset()
default:
cur.WriteRune(r)
}
}
if t := strings.TrimSpace(cur.String()); t != "" {
out = append(out, t)
}
return out
}
// extractComments removes service comments (CommentTokens) from the token
// list and returns the cleaned list together with the removed comments.
func extractComments(tokens []string) ([]string, []string) {
out := make([]string, 0, len(tokens))
comments := []string{}
for _, tok := range tokens {
if Profile.CommentTokens[strings.ToUpper(tok)] {
comments = append(comments, tok)
continue
}
out = append(out, tok)
}
return out, comments
}
// catalogNumber reports whether the token is a parenthesized catalogue
// number. In the positional model any token wrapped in curved parentheses
// that sits in the middle zone ([2..]) is a catalogue number regardless of
// its content (digits, letters, roman numerals, dots, underscores, mixed
// case). Parenthesized tokens at [0]/[1] are handled as artist/title, not
// here. It returns the parenthesized content, or "" when the token is not
// a "(...)" group.
func catalogNumber(token string) string {
if len(token) < 3 || token[0] != '(' || token[len(token)-1] != ')' {
return ""
}
return token[1 : len(token)-1]
}
// parseTailZone resolves middle-zone tokens against the dictionaries.
// The first match wins for each field; catalogue numbers are recognised
// from parenthesized tokens, and anything unrecognised is appended to
// UnknownTokens so that it can be reviewed and curated into a dictionary.
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
for _, tok := range zone {
up := strings.ToUpper(tok)
switch {
case Profile.FormatTokens[up]:
if result.Format == "" {
result.Format = tok
}
case Profile.SourceTokens[up]:
if result.Source == "" {
result.Source = tok
}
case Profile.BitDepthTokens[up]:
if result.BitDepth == "" {
result.BitDepth = tok
}
case Profile.ResolutionTokens[up]:
if result.Resolution == "" {
result.Resolution = tok
}
case Profile.EditionTokens[up]:
if result.Edition == "" {
result.Edition = tok
}
case Profile.LanguageMarkers[up]:
if result.Language == "" {
result.Language = tok
}
default:
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
result.CatalogNo = cat
} else {
result.UnknownTokens = append(result.UnknownTokens, tok)
}
}
}
}
// score computes a heuristic confidence in [0,1] for a parsed release based
// on how many core fields (group, year, format, source, artist) were resolved
// and whether any warnings or unknown tokens are present.
func score(result *releaseparser.ParsedRelease) {
score := 0.0
if result.Group != "" {
score += 0.25
}
if result.Year != 0 {
score += 0.20
}
if result.Format != "" {
score += 0.15
}
if result.Source != "" {
score += 0.15
}
if result.Artist != "" {
score += 0.15
}
if len(result.Warnings) == 0 {
score += 0.10
}
if len(result.UnknownTokens) > 0 {
score -= 0.10
}
if score > 1 {
score = 1
}
if score < 0 {
score = 0
}
result.Confidence = score
}
@@ -0,0 +1,401 @@
package moonblood
import (
"bufio"
"os"
"testing"
)
func TestParseMoonblood(t *testing.T) {
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
t.Skipf("resource file not found (DB mode), skipping: %v", err)
t.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
total, success := 0, 0
for scanner.Scan() {
line := scanner.Text()
if line == "" {
continue
}
result := Parse(line)
total++
if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" {
success++
}
if result.Confidence < 0.7 {
t.Logf("Low confidence: %s (%.2f)", line, result.Confidence)
}
}
t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100)
}
func TestParseFullTail(t *testing.T) {
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
if r.Artist != "Fane" {
t.Errorf("Artist = %q, want %q", r.Artist, "Fane")
}
if r.Title != "Dragged_Through_the_Unending" {
t.Errorf("Title = %q, want %q", r.Title, "Dragged_Through_the_Unending")
}
if r.Year != 2025 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth != "24BIT" {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if r.Confidence < 0.99 {
t.Errorf("Confidence = %.2f, want ~1.0", r.Confidence)
}
}
func TestParseEditionAndDemo(t *testing.T) {
r := Parse("Satans_Grind-Rotten_Demo-DEMO-16BIT-WEB-FLAC-2016-MOONBLOOD")
if r.Edition != "DEMO" {
t.Errorf("Edition = %q, want %q", r.Edition, "DEMO")
}
if r.Artist != "Satans_Grind" || r.Title != "Rotten_Demo" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseArtistWithHyphen(t *testing.T) {
// [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
if r.Artist != "Ultra" || r.Title != "Violence" {
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
}
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
}
}
func TestParseLanguageAndCDSource(t *testing.T) {
r := Parse("Burn_Thou_in_Effigy-Memorias_Postumas_do_Imperador-BR-CD-FLAC-2026-MOONBLOOD")
if r.Language != "BR" || r.Source != "CD" || r.Year != 2026 || r.Format != "FLAC" {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseNoYear(t *testing.T) {
// Without a year the tail still parses; group stays the last token.
r := Parse("Fane-Gallant_Shroud-EP-16BIT-WEB-FLAC-MOONBLOOD")
if r.Group != "MOONBLOOD" || r.Year != 0 {
t.Errorf("Group/Year = %q/%d", r.Group, r.Year)
}
if r.Edition != "EP" || r.Source != "WEB" || r.Format != "FLAC" {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Fane" || r.Title != "Gallant_Shroud" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
}
func TestParseResolutionSlot(t *testing.T) {
r := Parse("Ice_Nine_Kills-Last_Chance_To_Make_Amends-REMASTERED-24BIT-44KHZ-WEB-FLAC-2024-FLACCiD")
if r.Group != "FLACCiD" {
t.Errorf("Group = %q, want FLACCiD", r.Group)
}
if r.Resolution != "44KHZ" {
t.Errorf("Resolution = %q, want 44KHZ", r.Resolution)
}
if r.BitDepth != "24BIT" || r.Edition != "REMASTERED" || r.Source != "WEB" || r.Year != 2024 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Ice_Nine_Kills" || r.Title != "Last_Chance_To_Make_Amends" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseVinylIsSource(t *testing.T) {
r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
if r.Language != "DE" {
t.Errorf("Language = %q, want DE", r.Language)
}
if r.Source != "VINYL" {
t.Errorf("Source = %q, want VINYL", r.Source)
}
if r.Edition != "" {
t.Errorf("Edition = %q, want empty (VINYL is source, not edition)", r.Edition)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseCaseInsensitiveDictionaries(t *testing.T) {
r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood")
// Значення зберігаються в оригінальному регістрі (як у рядку),
// а звірка зі словниками йде регістронезалежно.
if r.BitDepth != "16bit" || r.Source != "web" || r.Format != "flac" {
t.Errorf("case-insensitive lookup failed: %+v", r)
}
if r.Artist != "Artist" || r.Title != "Some_Title" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
}
func TestParseUnknownTailTokenReported(t *testing.T) {
// Невідомий токен між двома відомими маркерами → у UnknownTokens.
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
}
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Artist" || r.Title != "Some_Title" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
}
func TestParseCoreTokensNeverReported(t *testing.T) {
// Артист/тайтл ніколи не потрапляють в UnknownTokens.
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if r.Artist != "Fane" || r.Title != "Dragged_Through_the_Unending" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
}
func TestParseAnyGroupNoWarning(t *testing.T) {
cases := []string{
"Jetsex-Down_Down_Down-24BIT-WEB-FLAC-2025-FLACCiD",
"Sabaton-The_Last_Stand-24BIT-96KHZ-WEB-FLAC-2016-FLACCiD",
"Wind_Walkers-What_If_I_Break-16BIT-WEB-FLAC-2023-FLACCiD",
}
for _, line := range cases {
r := Parse(line)
if r.Group != "FLACCiD" {
t.Errorf("%s: Group = %q, want FLACCiD", line, r.Group)
}
if len(r.Warnings) != 0 {
t.Errorf("%s: Warnings = %v, want empty", line, r.Warnings)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
}
if r.Year == 0 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth == "" {
t.Errorf("%s: tail fields wrong: %+v", line, r)
}
}
}
func TestParseCommentExtracted(t *testing.T) {
r := Parse("Rain_City_Drive-Rain_City_Drive-DELUXE_EDITION-REPACK-16BIT-WEB-FLAC-2023-FLACCiD")
if r.Comment != "REPACK" {
t.Errorf("Comment = %q, want REPACK", r.Comment)
}
if r.Edition != "DELUXE_EDITION" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2023 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Rain_City_Drive" || r.Title != "Rain_City_Drive" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
}
}
func TestParseNumericArtistNotAYear(t *testing.T) {
// [0]=Artist — навіть якщо це число-рік (1914, 1928), а не "рік" токен.
for _, line := range []string{
"1914-Eschatology_of_War-24BIT-WEB-FLAC-2015-MOONBLOOD",
"1928-Lappel_du_Spectre-FR-CD-FLAC-2025-MOONBLOOD",
} {
r := Parse(line)
if r.Artist != "1914" && r.Artist != "1928" {
t.Errorf("%s: Artist = %q, want numeric artist", line, r.Artist)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
}
}
}
func TestParseCatalogNumber(t *testing.T) {
r := Parse("Antidot-Beyond_Trance_And_Time-(GLOBAL006)-REPACK-EP-FLAC-2026-AFTERHOUR")
if r.CatalogNo != "GLOBAL006" {
t.Errorf("CatalogNo = %q, want GLOBAL006", r.CatalogNo)
}
if r.Comment != "REPACK" || r.Edition != "EP" || r.Format != "FLAC" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Antidot" || r.Title != "Beyond_Trance_And_Time" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseCatalogNumberWithDashesInside(t *testing.T) {
r := Parse("Murder_Intentions-Excessive_Display_Of_Human_Nature-(BRC30-001-CD-RRR108)-CD-FLAC-2018-86D")
if r.CatalogNo != "BRC30-001-CD-RRR108" {
t.Errorf("CatalogNo = %q, want BRC30-001-CD-RRR108", r.CatalogNo)
}
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2018 || r.Group != "86D" {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Murder_Intentions" || r.Title != "Excessive_Display_Of_Human_Nature" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
// Катало у дужках у середній зоні — незалежно від вмісту
// (крапка, римські, букви, дефіси): все валідне.
cases := []struct{ line, catno string }{
{"Nervosa-Perpetual_Chaos-(DYN.3140-2)-LIMITED_EDITION-CD-FLAC-2021-MOONBLOOD", "DYN.3140-2"},
{"Black_Ceremonial_Kult-Crowned_in_Chaos-(GOWP_DCCCLXXXIX)-CDEP-FLAC-2021-MOONBLOOD", "GOWP_DCCCLXXXIX"},
}
for _, c := range cases {
r := Parse(c.line)
if r.CatalogNo != c.catno {
t.Errorf("CatalogNo = %q, want %q (release: %s)", r.CatalogNo, c.catno, c.line)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty (release: %s)", r.UnknownTokens, c.line)
}
}
}
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
// потрапляє в UnknownTokens. (SMG409) — катало.
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
if r.CatalogNo != "SMG409" {
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
}
if r.Artist != "Ferosity" || r.Title != "De" {
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
}
}
func TestParseLanguageRequiresKnownNeighbor(t *testing.T) {
// Мовний маркер валідний, коли праворуч — відомий хвостовий маркер.
r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
if r.Language != "DE" || r.Source != "VINYL" {
t.Errorf("Language/Source = %q/%q, want DE/VINYL", r.Language, r.Source)
}
}
func TestParseLanguageBetweenTitleAndFormat(t *testing.T) {
// A: мовний маркер у зоні між title і format — валідний,
// все ліворуч від нього — artist+title, нічого в unknown.
r := Parse("Kollektiver_Brechreiz-Heilig_Scheint_Scheinheilig-(NG207)-DE-CD-FLAC-2009-SHELTER")
if r.CatalogNo != "NG207" {
t.Errorf("CatalogNo = %q, want NG207", r.CatalogNo)
}
if r.Language != "DE" || r.Source != "CD" || r.Format != "FLAC" || r.Year != 2009 {
t.Errorf("tail fields wrong: %+v", r)
}
if r.Artist != "Kollektiver_Brechreiz" || r.Title != "Heilig_Scheint_Scheinheilig" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
}
}
func TestParseLanguageNeverBeforeFormat(t *testing.T) {
// DE — словниковий токен у середині (language.txt), позиція неважлива:
// Language=DE.
r := Parse("Some_Artist-Some_Song-DE-FLAC-2020-SOMEGROUP")
if r.Language != "DE" {
t.Errorf("Language = %q, want DE", r.Language)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if r.Title != "Some_Song" {
t.Errorf("Title = %q, want Some_Song", r.Title)
}
}
func TestParseCompoundEditionRecognized(t *testing.T) {
// REMASTERED_REISSUE — складений маркер у dicts/edition.txt, розпізнається
// як edition і не йде в UnknownTokens.
r := Parse("Ominous_Scriptures-Incarnation_of_the_Unheavenly-(LSR07)-REMASTERED_REISSUE-CD-FLAC-2021-86D")
if r.CatalogNo != "LSR07" {
t.Errorf("CatalogNo = %q, want LSR07", r.CatalogNo)
}
if r.Edition != "REMASTERED_REISSUE" {
t.Errorf("Edition = %q, want REMASTERED_REISSUE", r.Edition)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
if r.Artist != "Ominous_Scriptures" || r.Title != "Incarnation_of_the_Unheavenly" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2021 {
t.Errorf("tail fields wrong: %+v", r)
}
}
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
}
}
func contains(list []string, s string) bool {
for _, v := range list {
if v == s {
return true
}
}
return false
}
func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
// Дужки в тайтлі при WEB/BitDepth — не каталономер, лишається в тайтлі.
r := Parse("The_Amazons-21st_Century_Fiction_(Epilogue)-16BIT-WEB-FLAC-2026-ENViED")
if r.CatalogNo != "" {
t.Errorf("CatalogNo = %q, want empty", r.CatalogNo)
}
if r.Title != "21st_Century_Fiction_(Epilogue)" {
t.Errorf("Title = %q, want 21st_Century_Fiction_(Epilogue)", r.Title)
}
if r.Source != "WEB" || r.BitDepth != "16BIT" || r.Year != 2026 {
t.Errorf("tail fields wrong: %+v", r)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
}
}
@@ -0,0 +1,42 @@
package moonblood
import (
"embed"
"strings"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
)
// Словники токенів лежать у dicts/*.txt — один токен на рядок.
// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці).
//go:embed dicts/*.txt
var dictFS embed.FS
func loadDict(name string) map[string]bool {
data, err := dictFS.ReadFile("dicts/" + name + ".txt")
if err != nil {
panic("moonblood: cannot load dictionary " + name + ": " + err.Error())
}
out := make(map[string]bool)
for _, line := range strings.Split(string(data), "\n") {
line = strings.ToUpper(strings.TrimSpace(line))
if line == "" || strings.HasPrefix(line, "#") {
continue
}
out[line] = true
}
return out
}
var Profile = releaseparser.GroupProfile{
Name: "MOONBLOOD",
ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"},
FormatTokens: loadDict("format"),
SourceTokens: loadDict("source"),
BitDepthTokens: loadDict("bitdepth"),
ResolutionTokens: loadDict("resolution"),
LanguageMarkers: loadDict("language"),
EditionTokens: loadDict("edition"),
CommentTokens: loadDict("comment"),
}
+53
View File
@@ -0,0 +1,53 @@
package releaseparser
type GroupProfile struct {
Name string
ExpectedTailOrder []string
FormatTokens map[string]bool
SourceTokens map[string]bool
BitDepthTokens map[string]bool
ResolutionTokens map[string]bool
LanguageMarkers map[string]bool
EditionTokens map[string]bool
CommentTokens map[string]bool
}
type ParsedRelease struct {
Raw string
Group string
Year int
BitDepth string
Resolution string
Comment string
CatalogNo string
Source string
Format string
Language string
Edition string
Core string
Artist string
Title string
UnknownTokens []string
Warnings []string
Confidence float64
}
type UnknownTokenStats struct {
Token string
Count int
Releases []string
}
type GroupStats struct {
Total int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
ResolutionDistribution map[string]int
LanguageDistribution map[string]int
EditionDistribution map[string]int
CommentDistribution map[string]int
UnknownTokens []UnknownTokenStats
WarningStats []UnknownTokenStats
AvgConfidence float64
}
+1 -1
View File
@@ -29,7 +29,7 @@ func New(services *service.Service) *Handler {
"prescene": h.Prescene,
"rutracker": h.Rutracker,
"stb": h.STB,
//"web": h.Web,
//"web": h.Web,
}
return h
+6 -1
View File
@@ -30,7 +30,12 @@ func (h *Handler) Prescene() string {
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
}
_, err := h.services.Prescene.GetPage(pagesToScrape)
_, err := h.services.Prescene.GetSrrDb()
if err != nil {
slog.Error("error occurred while getting page: ", "err", err.Error())
}
_, err = h.services.Prescene.GetPage(pagesToScrape)
if err != nil {
slog.Error("error occurred while getting page: ", "err", err.Error())
}
-2
View File
@@ -17,7 +17,5 @@ func (h *Handler) Rutracker() string {
slog.Error("error occurred while getting topic: ", "err", err.Error())
}
//fmt.Printf("%+v", rt)
return "rt"
}
+12
View File
@@ -10,6 +10,7 @@ import (
"strings"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/dromara/carbon/v2"
"github.com/go-shiori/dom"
"github.com/jmoiron/sqlx"
@@ -25,6 +26,17 @@ type Prescene struct {
db *sqlx.DB
}
func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) {
entries := make([]model.ExternalSources, 0)
now := time.Now()
yesterday := now.Add(-24 * time.Hour)
todayStr := now.Format("2006-01-02")
yesterdayStr := yesterday.Format("2006-01-02")
spew.Dump(entries, todayStr, yesterdayStr)
return []model.ExternalSources{}, nil
}
func NewPresceneRepository(db *sqlx.DB) *Prescene {
return &Prescene{db: db}
}
+32 -8
View File
@@ -29,6 +29,7 @@ func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
endpoint := viper.GetString(constant.CfgKeyEndpoint)
entries := make([]model.ExternalSources, 0)
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
@@ -41,18 +42,19 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
for _, e := range topic.Entry {
var es model.ExternalSources
u, _ := url.Parse(e.Link.Href)
es.Type = constant.ScopeRuTracker
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
es.TypeId = topicId(e)
es.Title = e.Title
es.TypeSubsectionId, _ = strconv.Atoi(t)
es.Releaser = e.Author.Name
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
esModel := table.ExternalSources{Columns: columns}
fmt.Printf("%+v\n\n\n", es)
entry := esModel.InsertOnDuplicate(es, s.db)
entries = append(entries, entry)
//fmt.Printf("%+v\n\n\n", entry)
}
}
@@ -62,22 +64,44 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
resp, err := http.Get(endpoint)
if err != nil {
slog.Error("couldn't fetch data", endpoint, err.Error())
slog.Error("couldn't fetch data", "endpoint", endpoint, "err", err.Error())
return nil, err
}
defer func(Body io.ReadCloser) {
err = Body.Close()
if err != nil {
fmt.Println("Body.Close")
if cerr := Body.Close(); cerr != nil {
slog.Error("couldn't close response body", "endpoint", endpoint, "err", cerr.Error())
}
}(resp.Body)
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, endpoint)
}
topic := &resource.RutrackerAtomTopic{}
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
slog.Error("couldn't decode feed", "endpoint", endpoint, "err", err.Error())
return nil, err
}
return topic, nil
}
// topicId resolves the rutracker topic id preferring the page link query (?t=...),
// falling back to the entry id tag (tag:rto.feed,...:/t/5702203).
func topicId(e resource.RutrackerAtomEntry) int {
if link, ok := e.PageLink(); ok {
if u, err := url.Parse(link.Href); err == nil {
if id, err := strconv.Atoi(u.Query().Get("t")); err == nil {
return id
}
}
}
id, err := e.TopicId()
if err != nil {
slog.Error("couldn't resolve topic id", "entry_id", e.ID, "err", err.Error())
return 0
}
return id
}
+1 -1
View File
@@ -41,7 +41,7 @@ func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, column
var status string
errCount := 0
for i := 0; i < len(entries); i++ {
for i := range entries {
entry := es.InsertOnDuplicate(entries[i], db)
if entry.Error != nil {
slog.Error("insert/update entry", "err", entry.Error)
+4
View File
@@ -16,3 +16,7 @@ func NewPresceneService(repo _interface.Prescene) *PresceneService {
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
return p.repo.GetPage(pageNumbers)
}
func (p *PresceneService) GetSrrDb() ([]model.ExternalSources, error) {
return p.repo.GetSrrDb()
}
BIN
View File
Binary file not shown.
+1
View File
@@ -11,6 +11,7 @@ type Rutracker interface {
type Prescene interface {
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
GetSrrDb() ([]model.ExternalSources, error)
}
type ShareTheBrutality interface {
+56 -30
View File
@@ -1,35 +1,61 @@
package resource
import "encoding/xml"
import (
"encoding/xml"
"fmt"
"strconv"
"strings"
)
// RutrackerAtomTopic represents the atom feed of a rutracker forum subsection.
type RutrackerAtomTopic struct {
XMLName xml.Name `xml:"feed"`
Text string `xml:",chardata"`
Xmlns string `xml:"xmlns,attr"`
ID string `xml:"id"`
Link struct {
Text string `xml:",chardata"`
Href string `xml:"href,attr"`
} `xml:"link"`
Updated string `xml:"updated"`
Title string `xml:"title"`
Entry []struct {
Text string `xml:",chardata"`
ID string `xml:"id"`
Link struct {
Text string `xml:",chardata"`
Href string `xml:"href,attr"`
} `xml:"link"`
Updated string `xml:"updated"`
Title string `xml:"title"`
Author struct {
Text string `xml:",chardata"`
Name string `xml:"name"`
} `xml:"author"`
Category struct {
Text string `xml:",chardata"`
Term string `xml:"term,attr"`
Label string `xml:"label,attr"`
} `xml:"category"`
} `xml:"entry"`
XMLName xml.Name `xml:"feed"`
Xmlns string `xml:"xmlns,attr"`
ID string `xml:"id"`
Link []AtomLink `xml:"link"`
Updated string `xml:"updated"`
Title string `xml:"title"`
Entry []RutrackerAtomEntry `xml:"entry"`
}
// AtomLink is a generic atom link element. An entry contains two links:
// a topic page link (no rel) and a magnet link (rel="enclosure").
type AtomLink struct {
Href string `xml:"href,attr"`
Rel string `xml:"rel,attr"`
Type string `xml:"type,attr"`
}
type RutrackerAtomEntry struct {
ID string `xml:"id"`
Link []AtomLink `xml:"link"`
Updated string `xml:"updated"`
Title string `xml:"title"`
Author struct {
Name string `xml:"name"`
} `xml:"author"`
Category struct {
Term string `xml:"term,attr"`
Label string `xml:"label,attr"`
} `xml:"category"`
}
// PageLink returns the link to the topic page (the one without rel attribute).
func (e RutrackerAtomEntry) PageLink() (AtomLink, bool) {
for _, l := range e.Link {
if l.Rel == "" {
return l, true
}
}
return AtomLink{}, false
}
// TopicId extracts the topic id from the entry id tag: "tag:rto.feed,2026-08-29:/t/5702203".
func (e RutrackerAtomEntry) TopicId() (int, error) {
const marker = ":/t/"
i := strings.LastIndex(e.ID, marker)
if i < 0 {
return 0, fmt.Errorf("unexpected entry id format: %s", e.ID)
}
return strconv.Atoi(e.ID[i+len(marker):])
}