Compare commits
6 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1884151eb0 | |||
| 31febb9f56 | |||
| 3f3598c446 | |||
| 1b2521d4ac | |||
| e11c82bd76 | |||
| fedb470da6 |
+267
@@ -0,0 +1,267 @@
|
|||||||
|
# =============================================================
|
||||||
|
# .gitattributes — Universal JetBrains / Multi-Stack Template
|
||||||
|
# Covers: PHP, Go, TypeScript, JavaScript, Rust, Dart/Flutter,
|
||||||
|
# Python, Ruby, HCL/Terraform, SQL, HTML, CSS, DevOps
|
||||||
|
# =============================================================
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# DEFAULTS: LF everywhere, CRLF only for Windows-specific files
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
* text=auto eol=lf
|
||||||
|
|
||||||
|
*.bat text eol=crlf
|
||||||
|
*.cmd text eol=crlf
|
||||||
|
*.ps1 text eol=crlf
|
||||||
|
*.psm1 text eol=crlf
|
||||||
|
*.psd1 text eol=crlf
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# PHP
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.php text diff=php
|
||||||
|
*.phtml text diff=php
|
||||||
|
*.php8 text diff=php
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Go
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.go text diff=golang
|
||||||
|
go.sum text -diff
|
||||||
|
go.work.sum text -diff
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# TypeScript / JavaScript
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.ts text diff=typescript
|
||||||
|
*.tsx text diff=typescript
|
||||||
|
*.js text diff=javascript
|
||||||
|
*.jsx text diff=javascript
|
||||||
|
*.mjs text diff=javascript
|
||||||
|
*.cjs text diff=javascript
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Rust
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.rs text diff=rust
|
||||||
|
Cargo.lock text -diff
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Dart / Flutter
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.dart text diff=dart
|
||||||
|
pubspec.lock text -diff
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Lua
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.lua text diff=lua
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Python
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.py text diff=python
|
||||||
|
*.pyi text diff=python
|
||||||
|
Pipfile.lock text -diff
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Ruby
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.rb text diff=ruby
|
||||||
|
Gemfile.lock text -diff
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Web: HTML / CSS / Templates
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.html text diff=html
|
||||||
|
*.htm text diff=html
|
||||||
|
*.blade.php text diff=html
|
||||||
|
*.templ text diff=html
|
||||||
|
*.twig text diff=html
|
||||||
|
*.njk text diff=html
|
||||||
|
*.ejs text diff=html
|
||||||
|
*.hbs text diff=html
|
||||||
|
*.mustache text diff=html
|
||||||
|
*.vue text diff=html
|
||||||
|
*.svelte text diff=html
|
||||||
|
*.css text diff=css
|
||||||
|
*.scss text diff=css
|
||||||
|
*.sass text diff=css
|
||||||
|
*.less text diff=css
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Data / Config formats
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.json text diff=json
|
||||||
|
*.jsonc text diff=json
|
||||||
|
*.json5 text diff=json
|
||||||
|
*.yaml text diff=yaml
|
||||||
|
*.yml text diff=yaml
|
||||||
|
*.toml text
|
||||||
|
*.xml text diff=html
|
||||||
|
*.svg text diff=html
|
||||||
|
*.env text
|
||||||
|
*.env.* text
|
||||||
|
*.ini text
|
||||||
|
*.cfg text
|
||||||
|
*.conf text
|
||||||
|
*.properties text
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# HCL / Terraform / HashiCorp stack
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.hcl text diff=terraform
|
||||||
|
*.tf text diff=terraform
|
||||||
|
*.tfvars text diff=terraform
|
||||||
|
*.tfvars.json text diff=json
|
||||||
|
.terraform.lock.hcl text -diff merge=ours
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# SQL / Database
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.sql text diff=sql
|
||||||
|
*.pgsql text diff=sql
|
||||||
|
*.mysql text diff=sql
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Shell / Scripts
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.sh text eol=lf
|
||||||
|
*.bash text eol=lf
|
||||||
|
*.zsh text eol=lf
|
||||||
|
*.fish text eol=lf
|
||||||
|
Makefile text eol=lf
|
||||||
|
*.mk text eol=lf
|
||||||
|
Dockerfile text eol=lf
|
||||||
|
Dockerfile.* text eol=lf
|
||||||
|
*.dockerfile text eol=lf
|
||||||
|
*.containerfile text eol=lf
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Documentation
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.md text diff=markdown
|
||||||
|
*.mdx text diff=markdown
|
||||||
|
*.rst text
|
||||||
|
*.txt text
|
||||||
|
*.adoc text
|
||||||
|
LICENSE text
|
||||||
|
AUTHORS text
|
||||||
|
CODEOWNERS text
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Lock files — no diff noise, merge=ours
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.lock text -diff merge=ours
|
||||||
|
*.lock.* text -diff merge=ours
|
||||||
|
package-lock.json text -diff merge=ours
|
||||||
|
yarn.lock text -diff merge=ours
|
||||||
|
pnpm-lock.yaml text -diff merge=ours
|
||||||
|
composer.lock text -diff merge=ours
|
||||||
|
bun.lockb binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Generated files — hide from diff + GitHub stats
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*_templ.go linguist-generated=true -diff
|
||||||
|
*.pb.go linguist-generated=true -diff
|
||||||
|
*.gen.go linguist-generated=true -diff
|
||||||
|
*.generated.go linguist-generated=true -diff
|
||||||
|
*.generated.ts linguist-generated=true -diff
|
||||||
|
*.generated.php linguist-generated=true -diff
|
||||||
|
*_test.snap linguist-generated=true -diff
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Binary: Images
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.png binary
|
||||||
|
*.jpg binary
|
||||||
|
*.jpeg binary
|
||||||
|
*.gif binary
|
||||||
|
*.bmp binary
|
||||||
|
*.ico binary
|
||||||
|
*.webp binary
|
||||||
|
*.avif binary
|
||||||
|
*.tiff binary
|
||||||
|
*.tif binary
|
||||||
|
*.psd binary
|
||||||
|
*.ai binary
|
||||||
|
*.eps binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Binary: Fonts
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.ttf binary
|
||||||
|
*.otf binary
|
||||||
|
*.woff binary
|
||||||
|
*.woff2 binary
|
||||||
|
*.eot binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Binary: Audio / Video
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.mp3 binary
|
||||||
|
*.mp4 binary
|
||||||
|
*.wav binary
|
||||||
|
*.flac binary
|
||||||
|
*.ogg binary
|
||||||
|
*.opus binary
|
||||||
|
*.aac binary
|
||||||
|
*.m4a binary
|
||||||
|
*.webm binary
|
||||||
|
*.avi binary
|
||||||
|
*.mov binary
|
||||||
|
*.mkv binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Binary: Archives / Executables
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.zip binary
|
||||||
|
*.tar binary
|
||||||
|
*.gz binary
|
||||||
|
*.bz2 binary
|
||||||
|
*.xz binary
|
||||||
|
*.7z binary
|
||||||
|
*.rar binary
|
||||||
|
*.exe binary
|
||||||
|
*.dll binary
|
||||||
|
*.so binary
|
||||||
|
*.dylib binary
|
||||||
|
*.a binary
|
||||||
|
*.o binary
|
||||||
|
*.wasm binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Binary: Documents / Data files
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.pdf binary
|
||||||
|
*.doc binary
|
||||||
|
*.docx binary
|
||||||
|
*.xls binary
|
||||||
|
*.xlsx binary
|
||||||
|
*.ppt binary
|
||||||
|
*.pptx binary
|
||||||
|
*.db binary
|
||||||
|
*.sqlite binary
|
||||||
|
*.sqlite3 binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# Terraform state — never should be committed, but just in case
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
*.tfstate binary
|
||||||
|
*.tfstate.backup binary
|
||||||
|
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
# git archive export-ignore
|
||||||
|
# (files excluded from GitHub release tarballs)
|
||||||
|
# ─────────────────────────────────────────────────────────────
|
||||||
|
README.md export-ignore
|
||||||
|
CHANGELOG.md export-ignore
|
||||||
|
CONTRIBUTING.md.md export-ignore
|
||||||
|
SECURITY.md export-ignore
|
||||||
|
.editorconfig export-ignore
|
||||||
|
.env.example export-ignore
|
||||||
|
.gitattributes export-ignore
|
||||||
|
.gitignore export-ignore
|
||||||
|
.github export-ignore
|
||||||
|
docs export-ignore
|
||||||
|
tests export-ignore
|
||||||
+5
-21
@@ -1,34 +1,18 @@
|
|||||||
.idea/**/workspace.xml
|
.idea/
|
||||||
.idea/**/tasks.xml
|
|
||||||
.idea/**/usage.statistics.xml
|
|
||||||
.idea/**/dictionaries
|
|
||||||
.idea/**/shelf
|
|
||||||
.idea/**/aws.xml
|
|
||||||
.idea/**/contentModel.xml
|
|
||||||
.idea/**/dataSources/
|
|
||||||
.idea/**/dataSources.ids
|
|
||||||
.idea/**/dataSources.local.xml
|
|
||||||
.idea/**/sqlDataSources.xml
|
|
||||||
.idea/**/dynamic.xml
|
|
||||||
.idea/**/uiDesigner.xml
|
|
||||||
.idea/**/dbnavigator.xml
|
|
||||||
.idea/**/gradle.xml
|
|
||||||
.idea/**/libraries
|
|
||||||
cmake-build-*/
|
cmake-build-*/
|
||||||
.idea/**/mongoSettings.xml
|
|
||||||
*.iws
|
*.iws
|
||||||
out/
|
out/
|
||||||
.idea_modules/
|
.idea_modules/
|
||||||
atlassian-ide-plugin.xml
|
atlassian-ide-plugin.xml
|
||||||
.idea/replstate.xml
|
|
||||||
.idea/sonarlint/
|
|
||||||
com_crashlytics_export_strings.xml
|
com_crashlytics_export_strings.xml
|
||||||
crashlytics.properties
|
crashlytics.properties
|
||||||
crashlytics-build.properties
|
crashlytics-build.properties
|
||||||
fabric.properties
|
fabric.properties
|
||||||
.idea/httpRequests
|
|
||||||
.idea/caches/build_file_checksums.ser
|
|
||||||
.env
|
.env
|
||||||
tmp/
|
tmp/
|
||||||
vendor/
|
vendor/
|
||||||
config/secret.yaml
|
config/secret.yaml
|
||||||
|
*.md
|
||||||
|
mdb
|
||||||
|
internal/releaseparser/resources/*
|
||||||
|
review_unknown_tokens.txt
|
||||||
Generated
-1
@@ -12,7 +12,6 @@
|
|||||||
<option value="gosimple" />
|
<option value="gosimple" />
|
||||||
</list>
|
</list>
|
||||||
</option>
|
</option>
|
||||||
<option name="goLinterExe" value="C:\Users\theAmok\go\bin\golangci-lint.exe" />
|
|
||||||
<option name="linterSelected" value="true" />
|
<option name="linterSelected" value="true" />
|
||||||
</component>
|
</component>
|
||||||
</project>
|
</project>
|
||||||
Generated
+1
@@ -1,6 +1,7 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<module type="WEB_MODULE" version="4">
|
<module type="WEB_MODULE" version="4">
|
||||||
<component name="Go" enabled="true" />
|
<component name="Go" enabled="true" />
|
||||||
|
<component name="GoModuleSettings" enabled="true" />
|
||||||
<component name="NewModuleRootManager">
|
<component name="NewModuleRootManager">
|
||||||
<content url="file://$MODULE_DIR$" />
|
<content url="file://$MODULE_DIR$" />
|
||||||
<orderEntry type="inheritedJdk" />
|
<orderEntry type="inheritedJdk" />
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/jmoiron/sqlx"
|
||||||
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
_ "github.com/go-sql-driver/mysql"
|
||||||
|
|
||||||
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
|
||||||
|
)
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) < 2 {
|
||||||
|
log.Fatal("usage: srrdb <releaser> [<releaser>...]")
|
||||||
|
}
|
||||||
|
releasers := os.Args[1:]
|
||||||
|
|
||||||
|
// Читаємо config/secret.yaml (без змісту в коді — тільки потрібні ключі)
|
||||||
|
viper.SetConfigFile("config/secret.yaml")
|
||||||
|
if err := viper.ReadInConfig(); err != nil {
|
||||||
|
log.Fatal("failed to read config/secret.yaml: ", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
drv := viper.GetString("db.driver")
|
||||||
|
dsn := viper.GetString("db.data_source_name")
|
||||||
|
|
||||||
|
dbase, err := sqlx.Connect(drv, dsn)
|
||||||
|
if err != nil {
|
||||||
|
panic("Failed to connect to the database: " + err.Error())
|
||||||
|
}
|
||||||
|
defer dbase.Close()
|
||||||
|
|
||||||
|
out := make(map[string]moonblood.GroupStats, len(releasers))
|
||||||
|
|
||||||
|
for _, releaser := range releasers {
|
||||||
|
var titles []string
|
||||||
|
|
||||||
|
err = dbase.Select(&titles,
|
||||||
|
`SELECT title FROM theamok_mdb.external_sources
|
||||||
|
WHERE type = 'prescene' AND releaser = ?`, releaser)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
out[releaser] = moonblood.AnalyzeTitles(titles)
|
||||||
|
}
|
||||||
|
|
||||||
|
data, err := json.MarshalIndent(out, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ручна ревізія: всі невідомі токени — у файл для правки бази.
|
||||||
|
review, err := os.OpenFile("review_unknown_tokens.txt",
|
||||||
|
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer review.Close()
|
||||||
|
for releaser, stats := range out {
|
||||||
|
for _, unk := range stats.UnknownTokens {
|
||||||
|
for _, rel := range unk.Releases {
|
||||||
|
fmt.Fprintf(review, "%s\t%s\t%s\n", releaser, unk.Token, rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, warn := range stats.WarningStats {
|
||||||
|
for _, rel := range warn.Releases {
|
||||||
|
fmt.Fprintf(review, "%s\tWARN\t%s\t%s\n", releaser, warn.Token, rel)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println(string(data))
|
||||||
|
}
|
||||||
@@ -3,6 +3,7 @@ module git.kplus.net.ua/yevhen/resource-scraper
|
|||||||
go 1.26.2
|
go 1.26.2
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/davecgh/go-spew v1.1.1
|
||||||
github.com/dromara/carbon/v2 v2.6.16
|
github.com/dromara/carbon/v2 v2.6.16
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8
|
github.com/emersion/go-imap/v2 v2.0.0-beta.8
|
||||||
github.com/go-chi/chi/v5 v5.2.5
|
github.com/go-chi/chi/v5 v5.2.5
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
|
||||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||||
@@ -6,12 +5,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
|
|||||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dromara/carbon/v2 v2.6.15 h1:3HuC3XcWczIHUTbg/f0CSVydtKEdM+P0GM1sdsbwXmI=
|
|
||||||
github.com/dromara/carbon/v2 v2.6.15/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
|
||||||
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
|
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
|
||||||
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
|
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
|
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
|
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
|
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
|
||||||
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
||||||
@@ -22,8 +17,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
|
|
||||||
github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
|
||||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||||
@@ -59,8 +52,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
|
|||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
|
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
|
||||||
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
|
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
|
||||||
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d h1:IL2tii4jXLdhCeQN69HNzYYW1kl0meSG0wt5+sLwszU=
|
|
||||||
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d/go.mod h1:SIpumAnUWSy0q9RzKD3pyH3g1t5vdawUAPcW5tQrUtI=
|
|
||||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
|
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
|
||||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
|
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
|
||||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
|
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
|
||||||
@@ -94,8 +85,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
|||||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
|
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
|
|
||||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
|||||||
+79
-15
@@ -2,32 +2,43 @@ package sugar
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// EnsureDir joins the given path segments and creates the directory
|
||||||
|
// (including all parent directories) if it does not already exist.
|
||||||
|
// Returns the resolved path or an error if creation fails.
|
||||||
func EnsureDir(dirPath ...string) (string, error) {
|
func EnsureDir(dirPath ...string) (string, error) {
|
||||||
outPath := filepath.Join(dirPath...)
|
outPath := filepath.Join(dirPath...)
|
||||||
if len(outPath) < 3 {
|
if outPath == "" {
|
||||||
return "", fmt.Errorf("the path is short to be a folder: %s", outPath)
|
return "", fmt.Errorf("path must not be empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := os.Stat(outPath); os.IsNotExist(err) {
|
if err := os.MkdirAll(outPath, os.FileMode(0775)); err != nil {
|
||||||
var dirMod uint64
|
return "", fmt.Errorf("failed to ensure directory %q: %w", outPath, err)
|
||||||
if dirMod, err = strconv.ParseUint("0775", 8, 32); err == nil {
|
|
||||||
err = os.Mkdir(outPath, os.FileMode(dirMod))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil && !os.IsExist(err) {
|
|
||||||
slog.Error("error creating tmp dir", "err", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return outPath, nil
|
return outPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExecDir returns the directory of the running binary.
|
||||||
|
// Works correctly even with symlinks and -trimpath builds.
|
||||||
|
func ExecDir() (string, error) {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// EvalSymlinks is critical — os.Executable can return a symlink path
|
||||||
|
resolved, err := filepath.EvalSymlinks(exe)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Dir(resolved), nil
|
||||||
|
}
|
||||||
|
|
||||||
func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
||||||
dir, err := EnsureDir(os.TempDir(), "mdb")
|
dir, err := EnsureDir(os.TempDir(), "mdb")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -58,6 +69,59 @@ func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
|||||||
return filesize, fp, nil
|
return filesize, fp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
// ConfigPaths returns candidate directories for config lookup.
|
||||||
REMOVED_FROM_MMT_DUE_TO_REDOING_OF_THE_EVENT_LOG
|
// Priority: XDG_CONFIG_HOME → binary dir → CWD → /etc/appname
|
||||||
*/
|
func ConfigPaths(appName string) []string {
|
||||||
|
var candidates []string
|
||||||
|
|
||||||
|
// 1. Explicit env override (highest priority)
|
||||||
|
if envPath := os.Getenv("APP_CONFIG_DIR"); envPath != "" {
|
||||||
|
candidates = append(candidates, envPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Relative to binary (best for production deployments)
|
||||||
|
if exe, err := os.Executable(); err == nil {
|
||||||
|
if resolved, err := filepath.EvalSymlinks(exe); err == nil {
|
||||||
|
candidates = append(candidates, filepath.Dir(resolved))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. CWD (convenient for `go run` and development)
|
||||||
|
if cwd, err := os.Getwd(); err == nil {
|
||||||
|
candidates = append(candidates, cwd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. XDG standard (Linux/Unix)
|
||||||
|
if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
|
||||||
|
candidates = append(candidates, filepath.Join(xdg, appName))
|
||||||
|
} else if home, err := os.UserHomeDir(); err == nil {
|
||||||
|
candidates = append(candidates, filepath.Join(home, ".config", appName))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. System-wide fallback
|
||||||
|
candidates = append(candidates, filepath.Join("/etc", appName))
|
||||||
|
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindConfig searches all candidate dirs for the given filename.
|
||||||
|
func FindConfig(appName, filename string) (string, error) {
|
||||||
|
for _, dir := range ConfigPaths(appName) {
|
||||||
|
path := filepath.Join(dir, filename)
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("config %q not found in any candidate directory", filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugPath Debug helper — remove before production
|
||||||
|
func DebugPath(path string) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
log.Printf("path=%q exists=%v err=%v mode=%v", path, err == nil, err, func() string {
|
||||||
|
if info != nil {
|
||||||
|
return info.Mode().String()
|
||||||
|
}
|
||||||
|
return "n/a"
|
||||||
|
}())
|
||||||
|
}
|
||||||
|
|||||||
+2
-1
@@ -1,2 +1,3 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
|
#go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
|
||||||
|
go build -trimpath -ldflags="-s -w" -o /opt/sources/resource-scraper/cmd/mdb ./cmd
|
||||||
@@ -5,10 +5,12 @@ import (
|
|||||||
"log"
|
"log"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"runtime"
|
||||||
"slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
|
||||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,6 +18,8 @@ func New() {
|
|||||||
configFilePath := viper.GetString(constant.FlagConfigFile)
|
configFilePath := viper.GetString(constant.FlagConfigFile)
|
||||||
configName := filepath.Base(configFilePath)
|
configName := filepath.Base(configFilePath)
|
||||||
|
|
||||||
|
sugar.DebugPath(configFilePath)
|
||||||
|
|
||||||
dir := filepath.Dir(configFilePath)
|
dir := filepath.Dir(configFilePath)
|
||||||
if dir == configName && dir == "." {
|
if dir == configName && dir == "." {
|
||||||
configName = "default"
|
configName = "default"
|
||||||
@@ -23,7 +27,13 @@ func New() {
|
|||||||
|
|
||||||
configDir, _ := filepath.Abs(dir)
|
configDir, _ := filepath.Abs(dir)
|
||||||
|
|
||||||
|
switch runtime.GOOS {
|
||||||
|
case "windows":
|
||||||
viper.AddConfigPath(configDir)
|
viper.AddConfigPath(configDir)
|
||||||
|
case "linux":
|
||||||
|
viper.AddConfigPath("/opt/sources/resource-scraper/config")
|
||||||
|
}
|
||||||
|
|
||||||
viper.SetConfigName(configName)
|
viper.SetConfigName(configName)
|
||||||
err := viper.ReadInConfig()
|
err := viper.ReadInConfig()
|
||||||
if err != nil { // Handle errors reading the config file
|
if err != nil { // Handle errors reading the config file
|
||||||
|
|||||||
@@ -0,0 +1,151 @@
|
|||||||
|
package moonblood
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"os"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||||
|
)
|
||||||
|
|
||||||
|
// GroupStats summarises how a set of release lines parsed: field
|
||||||
|
// distributions, unknown tokens and warnings aggregated by occurrence, and
|
||||||
|
// the average parsing confidence. The slices are sorted by descending count.
|
||||||
|
type GroupStats struct {
|
||||||
|
Total int
|
||||||
|
YearDistribution map[int]int
|
||||||
|
SourceDistribution map[string]int
|
||||||
|
BitDepthDistribution map[string]int
|
||||||
|
ResolutionDistribution map[string]int
|
||||||
|
LanguageDistribution map[string]int
|
||||||
|
EditionDistribution map[string]int
|
||||||
|
CommentDistribution map[string]int
|
||||||
|
UnknownTokens []releaseparser.UnknownTokenStats
|
||||||
|
WarningStats []releaseparser.UnknownTokenStats
|
||||||
|
AvgConfidence float64
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnalyzeGroup parses the release lines from the local resource file. It is
|
||||||
|
// used for a quick local check of the dictionaries without touching a
|
||||||
|
// database.
|
||||||
|
func AnalyzeGroup() (GroupStats, error) {
|
||||||
|
file, err := os.Open("../../resources/moonblood.txt")
|
||||||
|
if err != nil {
|
||||||
|
return GroupStats{}, err
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
var titles []string
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := strings.TrimSpace(scanner.Text())
|
||||||
|
if line != "" {
|
||||||
|
titles = append(titles, line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
return GroupStats{}, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return AnalyzeTitles(titles), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AnalyzeTitles parses a set of release lines (for example fetched from the
|
||||||
|
// database by a particular releaser key) and aggregates the results into a
|
||||||
|
// GroupStats summary.
|
||||||
|
func AnalyzeTitles(titles []string) GroupStats {
|
||||||
|
stats := GroupStats{
|
||||||
|
YearDistribution: make(map[int]int),
|
||||||
|
SourceDistribution: make(map[string]int),
|
||||||
|
BitDepthDistribution: make(map[string]int),
|
||||||
|
ResolutionDistribution: make(map[string]int),
|
||||||
|
LanguageDistribution: make(map[string]int),
|
||||||
|
EditionDistribution: make(map[string]int),
|
||||||
|
CommentDistribution: make(map[string]int),
|
||||||
|
UnknownTokens: []releaseparser.UnknownTokenStats{},
|
||||||
|
WarningStats: []releaseparser.UnknownTokenStats{},
|
||||||
|
}
|
||||||
|
|
||||||
|
totalConfidence := 0.0
|
||||||
|
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||||
|
warningMap := make(map[string]*releaseparser.UnknownTokenStats)
|
||||||
|
|
||||||
|
for _, title := range titles {
|
||||||
|
result := Parse(title)
|
||||||
|
stats.Total++
|
||||||
|
|
||||||
|
if result.Year != 0 {
|
||||||
|
stats.YearDistribution[result.Year]++
|
||||||
|
}
|
||||||
|
if result.Source != "" {
|
||||||
|
stats.SourceDistribution[result.Source]++
|
||||||
|
}
|
||||||
|
if result.BitDepth != "" {
|
||||||
|
stats.BitDepthDistribution[result.BitDepth]++
|
||||||
|
}
|
||||||
|
if result.Resolution != "" {
|
||||||
|
stats.ResolutionDistribution[result.Resolution]++
|
||||||
|
}
|
||||||
|
if result.Language != "" {
|
||||||
|
stats.LanguageDistribution[result.Language]++
|
||||||
|
}
|
||||||
|
if result.Edition != "" {
|
||||||
|
stats.EditionDistribution[result.Edition]++
|
||||||
|
}
|
||||||
|
if result.Comment != "" {
|
||||||
|
for _, c := range strings.Split(result.Comment, ",") {
|
||||||
|
stats.CommentDistribution[c]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, unknown := range result.UnknownTokens {
|
||||||
|
if _, exists := unknownMap[unknown]; !exists {
|
||||||
|
unknownMap[unknown] = &releaseparser.UnknownTokenStats{
|
||||||
|
Token: unknown,
|
||||||
|
Count: 0,
|
||||||
|
Releases: []string{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
unknownMap[unknown].Count++
|
||||||
|
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, warning := range result.Warnings {
|
||||||
|
if _, exists := warningMap[warning]; !exists {
|
||||||
|
warningMap[warning] = &releaseparser.UnknownTokenStats{
|
||||||
|
Token: warning,
|
||||||
|
Count: 0,
|
||||||
|
Releases: []string{},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
warningMap[warning].Count++
|
||||||
|
warningMap[warning].Releases = append(warningMap[warning].Releases, title)
|
||||||
|
}
|
||||||
|
|
||||||
|
totalConfidence += result.Confidence
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert the aggregates from maps to slices and order them by count
|
||||||
|
// (most frequent first).
|
||||||
|
for _, tokenStats := range unknownMap {
|
||||||
|
stats.UnknownTokens = append(stats.UnknownTokens, *tokenStats)
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(stats.UnknownTokens, func(i, j int) bool {
|
||||||
|
return stats.UnknownTokens[i].Count > stats.UnknownTokens[j].Count
|
||||||
|
})
|
||||||
|
|
||||||
|
for _, w := range warningMap {
|
||||||
|
stats.WarningStats = append(stats.WarningStats, *w)
|
||||||
|
}
|
||||||
|
sort.Slice(stats.WarningStats, func(i, j int) bool {
|
||||||
|
return stats.WarningStats[i].Count > stats.WarningStats[j].Count
|
||||||
|
})
|
||||||
|
|
||||||
|
if stats.Total > 0 {
|
||||||
|
stats.AvgConfidence = totalConfidence / float64(stats.Total)
|
||||||
|
}
|
||||||
|
|
||||||
|
return stats
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
# {bit_depth}
|
||||||
|
16BIT
|
||||||
|
24BIT
|
||||||
|
32BIT
|
||||||
|
DE16BIT
|
||||||
@@ -0,0 +1,6 @@
|
|||||||
|
REPACK
|
||||||
|
REAL_REPACK
|
||||||
|
READNFO
|
||||||
|
PROPER
|
||||||
|
RERIP
|
||||||
|
199X
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
# {edition} — тип видання
|
||||||
|
EP
|
||||||
|
DEMO
|
||||||
|
SPLIT
|
||||||
|
REISSUE
|
||||||
|
DELUXE_EDITION
|
||||||
|
REMASTERED
|
||||||
|
BOXSET
|
||||||
|
SINGLE
|
||||||
|
PROMO
|
||||||
|
REISSUE_REMASTERED
|
||||||
|
REMASTERED_25TH_ANNIVERSARY_EDITION
|
||||||
|
REMASTERED_20TH_ANNIVERSARY_EDITION
|
||||||
|
20TH_ANNIVERSARY_EDITION_REMASTERED
|
||||||
|
Deluxe_Edition
|
||||||
|
LIMITED_EDITION_BOXSET
|
||||||
|
REMASTERED_EP
|
||||||
|
10TH_ANNIVERSARY_EDITION
|
||||||
|
OST
|
||||||
|
XX_Legacy_Edition
|
||||||
|
REISSUE_DEMO
|
||||||
|
JP_Deluxe_Edition
|
||||||
|
Remastered_Edition
|
||||||
|
Anniversary_Edition
|
||||||
|
Limited_Edition
|
||||||
|
CDS
|
||||||
|
CDM
|
||||||
|
RETAIL
|
||||||
|
JP_RETAIL
|
||||||
|
JP_REISSUE
|
||||||
|
DEMO_REMASTERED
|
||||||
|
EU_RETAIL
|
||||||
|
DE_RETAIL
|
||||||
|
REISSUE_DELUXE_EDITION
|
||||||
|
RETAIL_REISSUE
|
||||||
|
SPECIAL_EDITION
|
||||||
|
SPECIAL_COLLECTORS_EDITION_REMASTERED
|
||||||
|
30_YEARS_SPECIAL_COLLECTORS_EDITION_REMASTERED
|
||||||
|
LIMITED_EDITION_REMASTERED
|
||||||
|
25th_Anniversary_Edition
|
||||||
|
DEMO_REISSUE
|
||||||
|
Ultimate_Edition
|
||||||
|
Issue_255
|
||||||
|
Issue_254
|
||||||
|
Issue_198
|
||||||
|
7INCH_VINYL
|
||||||
|
REMASTERED_REISSUE
|
||||||
|
PROMO_DEMO
|
||||||
|
ES_RETAIL
|
||||||
|
BOX_SET
|
||||||
|
BONUS
|
||||||
|
REISSUE_EP
|
||||||
|
30th_Anniversary
|
||||||
|
REDUX
|
||||||
|
REMASTERED_BOXSET
|
||||||
|
MAG
|
||||||
|
REISSUE_BOXSET
|
||||||
|
30TH_ANNIVERSARY_EDITION_REMASTERED
|
||||||
|
DELUXE
|
||||||
|
Deluxe_Edition_Reissue
|
||||||
|
INSTRUMENTAL
|
||||||
|
REISSUE_EXPANDED_EDITION
|
||||||
|
BOOTLEG
|
||||||
|
EXPANDED_EDITION
|
||||||
|
ADVANCE
|
||||||
|
15TH_ANNIVERSARY_DELUXE_EDITION
|
||||||
|
TOUR_EDITION
|
||||||
|
REMASTERED_DEMO
|
||||||
|
REMASTERED_DELUXE_EDITION
|
||||||
|
30TH_ANNIVERSARY_EDITION
|
||||||
@@ -0,0 +1,3 @@
|
|||||||
|
# {format} — аудіоформат
|
||||||
|
# один токен на рядок, регістр не має значення
|
||||||
|
FLAC
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
# {language} — мовні маркери
|
||||||
|
DE
|
||||||
|
FR
|
||||||
|
ES
|
||||||
|
SK
|
||||||
|
FI
|
||||||
|
SE
|
||||||
|
BR
|
||||||
|
PL
|
||||||
|
NL
|
||||||
|
NO
|
||||||
|
PT
|
||||||
|
IT
|
||||||
|
HU
|
||||||
|
DK
|
||||||
|
AT
|
||||||
|
IS
|
||||||
|
UA
|
||||||
|
JP
|
||||||
|
GR
|
||||||
|
CN
|
||||||
|
RU
|
||||||
|
ID
|
||||||
|
PH
|
||||||
|
RO
|
||||||
|
LT
|
||||||
|
BY
|
||||||
|
CZ
|
||||||
|
SP
|
||||||
|
IR
|
||||||
|
FO
|
||||||
|
RS
|
||||||
|
DA
|
||||||
|
HI
|
||||||
|
CH
|
||||||
|
LU
|
||||||
|
CY
|
||||||
|
EE
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
# {resolution} — частота дискретизації
|
||||||
|
44KHZ
|
||||||
|
48KHZ
|
||||||
|
48.0kHz
|
||||||
|
88KHZ
|
||||||
|
96KHZ
|
||||||
|
176KHZ
|
||||||
|
192KHZ
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# {source} — джерело релізу
|
||||||
|
# VINYL сюди входить як source (за домовленістю, не edition)
|
||||||
|
WEB
|
||||||
|
CD
|
||||||
|
2CD
|
||||||
|
CDEP
|
||||||
|
VINYL
|
||||||
|
TAPE
|
||||||
|
CDR
|
||||||
|
17CD
|
||||||
|
3CD
|
||||||
|
4CD
|
||||||
|
18CD
|
||||||
|
3CDEP
|
||||||
|
DVD
|
||||||
|
7CD
|
||||||
|
2CDR
|
||||||
|
10CD
|
||||||
|
6CD
|
||||||
|
2CDEP
|
||||||
|
2VINYL
|
||||||
|
CDREP
|
||||||
|
LP
|
||||||
|
2LP
|
||||||
|
3VINYL
|
||||||
|
3TAPE
|
||||||
|
13CD
|
||||||
|
13CDR
|
||||||
|
10INCH_VINYL
|
||||||
|
25CD
|
||||||
|
WEBFLAC
|
||||||
|
2TAPE
|
||||||
|
BLURAY
|
||||||
|
VLS
|
||||||
|
MCD
|
||||||
|
24BIT_WEB
|
||||||
@@ -0,0 +1,213 @@
|
|||||||
|
package moonblood
|
||||||
|
|
||||||
|
import (
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||||
|
)
|
||||||
|
|
||||||
|
var yearRegexp = regexp.MustCompile(`^(19|20)\d{2}$`)
|
||||||
|
|
||||||
|
// Parse splits a release line into tokens and maps them onto a fixed
|
||||||
|
// positional model while the interior is resolved solely through the
|
||||||
|
// group profile's dictionaries.
|
||||||
|
//
|
||||||
|
// The release line is split on "-" (but never inside curved parentheses, so
|
||||||
|
// that a catalogue number such as (BRC30-001-CD-RRR108) stays one token).
|
||||||
|
// The positional layout is:
|
||||||
|
//
|
||||||
|
// [0] Artist - fixed; never matched against dictionaries.
|
||||||
|
// [1] Title - fixed; never matched against dictionaries.
|
||||||
|
// [len-2] Year - if the token is a 4-digit year (19xx | 20xx).
|
||||||
|
// [len-1] Group - always read from the line, never from the query key.
|
||||||
|
//
|
||||||
|
// Everything between [1] and the reserved trailing slots ([2 .. len-3], or
|
||||||
|
// [2 .. len-2] when there is no year) is the middle zone. Each token there is
|
||||||
|
// looked up in the dictionaries (format, source, bit-depth, resolution,
|
||||||
|
// language, edition) and, if it is a parenthesized token, as a catalogue
|
||||||
|
// number. Position within the middle zone is irrelevant. Tokens that match
|
||||||
|
// nothing are NOT silently consumed — they are reported in UnknownTokens so
|
||||||
|
// that they can be reviewed and added to a dictionary.
|
||||||
|
//
|
||||||
|
// Why [0]/[1] are fixed and never matched against the dictionaries: titles
|
||||||
|
// such as "Split" (Alone_In_The_Morgue-Split-...) or "Demo"
|
||||||
|
// (Mystifier-Demo-Nized-...) are legitimate words that collide with edition
|
||||||
|
// markers. Since index 1 is always the title, regardless of case or
|
||||||
|
// dictionary membership, such words are never eaten.
|
||||||
|
//
|
||||||
|
// Values are preserved in their original case (as they appear in the line);
|
||||||
|
// the dictionary lookups themselves are case-insensitive.
|
||||||
|
func Parse(raw string) releaseparser.ParsedRelease {
|
||||||
|
result := releaseparser.ParsedRelease{
|
||||||
|
Raw: strings.TrimSpace(raw),
|
||||||
|
Warnings: []string{},
|
||||||
|
UnknownTokens: []string{},
|
||||||
|
}
|
||||||
|
|
||||||
|
tokens, comments := extractComments(split(raw))
|
||||||
|
if len(comments) > 0 {
|
||||||
|
result.Comment = strings.Join(comments, ",")
|
||||||
|
}
|
||||||
|
if len(tokens) < 5 {
|
||||||
|
result.Warnings = append(result.Warnings, "too few tokens")
|
||||||
|
result.Confidence = 0.2
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fixed positional edges: [0]=Artist, [1]=Title, [len-1]=Group,
|
||||||
|
// [len-2]=Year (when present).
|
||||||
|
result.Artist = tokens[0]
|
||||||
|
if len(tokens) > 1 {
|
||||||
|
result.Title = tokens[1]
|
||||||
|
}
|
||||||
|
result.Group = tokens[len(tokens)-1]
|
||||||
|
end := len(tokens) - 1
|
||||||
|
if len(tokens) >= 3 && yearRegexp.MatchString(tokens[len(tokens)-2]) {
|
||||||
|
result.Year, _ = strconv.Atoi(tokens[len(tokens)-2])
|
||||||
|
end = len(tokens) - 2
|
||||||
|
}
|
||||||
|
// Resolve the middle zone tokens via the dictionaries.
|
||||||
|
parseTailZone(tokens[2:end], &result)
|
||||||
|
|
||||||
|
score(&result)
|
||||||
|
return result
|
||||||
|
}
|
||||||
|
|
||||||
|
// split breaks a raw release line into tokens on "-", but never inside
|
||||||
|
// curved parentheses, so that a catalogue number such as
|
||||||
|
// (BRC30-001-CD-RRR108) remains a single token.
|
||||||
|
func split(raw string) []string {
|
||||||
|
s := strings.TrimSpace(raw)
|
||||||
|
out := make([]string, 0, 8)
|
||||||
|
var cur strings.Builder
|
||||||
|
inParen := false
|
||||||
|
for _, r := range s {
|
||||||
|
switch {
|
||||||
|
case r == '(':
|
||||||
|
inParen = true
|
||||||
|
cur.WriteRune(r)
|
||||||
|
case r == ')':
|
||||||
|
inParen = false
|
||||||
|
cur.WriteRune(r)
|
||||||
|
case r == '-' && !inParen:
|
||||||
|
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
|
cur.Reset()
|
||||||
|
default:
|
||||||
|
cur.WriteRune(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if t := strings.TrimSpace(cur.String()); t != "" {
|
||||||
|
out = append(out, t)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractComments removes service comments (CommentTokens) from the token
|
||||||
|
// list and returns the cleaned list together with the removed comments.
|
||||||
|
func extractComments(tokens []string) ([]string, []string) {
|
||||||
|
out := make([]string, 0, len(tokens))
|
||||||
|
comments := []string{}
|
||||||
|
for _, tok := range tokens {
|
||||||
|
if Profile.CommentTokens[strings.ToUpper(tok)] {
|
||||||
|
comments = append(comments, tok)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, tok)
|
||||||
|
}
|
||||||
|
return out, comments
|
||||||
|
}
|
||||||
|
|
||||||
|
// catalogNumber reports whether the token is a parenthesized catalogue
|
||||||
|
// number. In the positional model any token wrapped in curved parentheses
|
||||||
|
// that sits in the middle zone ([2..]) is a catalogue number regardless of
|
||||||
|
// its content (digits, letters, roman numerals, dots, underscores, mixed
|
||||||
|
// case). Parenthesized tokens at [0]/[1] are handled as artist/title, not
|
||||||
|
// here. It returns the parenthesized content, or "" when the token is not
|
||||||
|
// a "(...)" group.
|
||||||
|
func catalogNumber(token string) string {
|
||||||
|
if len(token) < 3 || token[0] != '(' || token[len(token)-1] != ')' {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return token[1 : len(token)-1]
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseTailZone resolves middle-zone tokens against the dictionaries.
|
||||||
|
// The first match wins for each field; catalogue numbers are recognised
|
||||||
|
// from parenthesized tokens, and anything unrecognised is appended to
|
||||||
|
// UnknownTokens so that it can be reviewed and curated into a dictionary.
|
||||||
|
func parseTailZone(zone []string, result *releaseparser.ParsedRelease) {
|
||||||
|
for _, tok := range zone {
|
||||||
|
up := strings.ToUpper(tok)
|
||||||
|
switch {
|
||||||
|
case Profile.FormatTokens[up]:
|
||||||
|
if result.Format == "" {
|
||||||
|
result.Format = tok
|
||||||
|
}
|
||||||
|
case Profile.SourceTokens[up]:
|
||||||
|
if result.Source == "" {
|
||||||
|
result.Source = tok
|
||||||
|
}
|
||||||
|
case Profile.BitDepthTokens[up]:
|
||||||
|
if result.BitDepth == "" {
|
||||||
|
result.BitDepth = tok
|
||||||
|
}
|
||||||
|
case Profile.ResolutionTokens[up]:
|
||||||
|
if result.Resolution == "" {
|
||||||
|
result.Resolution = tok
|
||||||
|
}
|
||||||
|
case Profile.EditionTokens[up]:
|
||||||
|
if result.Edition == "" {
|
||||||
|
result.Edition = tok
|
||||||
|
}
|
||||||
|
case Profile.LanguageMarkers[up]:
|
||||||
|
if result.Language == "" {
|
||||||
|
result.Language = tok
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
if cat := catalogNumber(tok); cat != "" && result.CatalogNo == "" {
|
||||||
|
result.CatalogNo = cat
|
||||||
|
} else {
|
||||||
|
result.UnknownTokens = append(result.UnknownTokens, tok)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// score computes a heuristic confidence in [0,1] for a parsed release based
|
||||||
|
// on how many core fields (group, year, format, source, artist) were resolved
|
||||||
|
// and whether any warnings or unknown tokens are present.
|
||||||
|
func score(result *releaseparser.ParsedRelease) {
|
||||||
|
score := 0.0
|
||||||
|
if result.Group != "" {
|
||||||
|
score += 0.25
|
||||||
|
}
|
||||||
|
if result.Year != 0 {
|
||||||
|
score += 0.20
|
||||||
|
}
|
||||||
|
if result.Format != "" {
|
||||||
|
score += 0.15
|
||||||
|
}
|
||||||
|
if result.Source != "" {
|
||||||
|
score += 0.15
|
||||||
|
}
|
||||||
|
if result.Artist != "" {
|
||||||
|
score += 0.15
|
||||||
|
}
|
||||||
|
if len(result.Warnings) == 0 {
|
||||||
|
score += 0.10
|
||||||
|
}
|
||||||
|
if len(result.UnknownTokens) > 0 {
|
||||||
|
score -= 0.10
|
||||||
|
}
|
||||||
|
if score > 1 {
|
||||||
|
score = 1
|
||||||
|
}
|
||||||
|
if score < 0 {
|
||||||
|
score = 0
|
||||||
|
}
|
||||||
|
result.Confidence = score
|
||||||
|
}
|
||||||
@@ -0,0 +1,401 @@
|
|||||||
|
package moonblood
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseMoonblood(t *testing.T) {
|
||||||
|
file, err := os.Open("../../resources/moonblood.txt")
|
||||||
|
if err != nil {
|
||||||
|
t.Skipf("resource file not found (DB mode), skipping: %v", err)
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
total, success := 0, 0
|
||||||
|
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
if line == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
result := Parse(line)
|
||||||
|
total++
|
||||||
|
|
||||||
|
if result.Group == "MOONBLOOD" && result.Year != 0 && result.Format == "FLAC" {
|
||||||
|
success++
|
||||||
|
}
|
||||||
|
|
||||||
|
if result.Confidence < 0.7 {
|
||||||
|
t.Logf("Low confidence: %s (%.2f)", line, result.Confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
t.Logf("Parsed %d releases, %d successful (%.1f%%)", total, success, float64(success)/float64(total)*100)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseFullTail(t *testing.T) {
|
||||||
|
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||||
|
if r.Artist != "Fane" {
|
||||||
|
t.Errorf("Artist = %q, want %q", r.Artist, "Fane")
|
||||||
|
}
|
||||||
|
if r.Title != "Dragged_Through_the_Unending" {
|
||||||
|
t.Errorf("Title = %q, want %q", r.Title, "Dragged_Through_the_Unending")
|
||||||
|
}
|
||||||
|
if r.Year != 2025 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth != "24BIT" {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Confidence < 0.99 {
|
||||||
|
t.Errorf("Confidence = %.2f, want ~1.0", r.Confidence)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseEditionAndDemo(t *testing.T) {
|
||||||
|
r := Parse("Satans_Grind-Rotten_Demo-DEMO-16BIT-WEB-FLAC-2016-MOONBLOOD")
|
||||||
|
if r.Edition != "DEMO" {
|
||||||
|
t.Errorf("Edition = %q, want %q", r.Edition, "DEMO")
|
||||||
|
}
|
||||||
|
if r.Artist != "Satans_Grind" || r.Title != "Rotten_Demo" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseArtistWithHyphen(t *testing.T) {
|
||||||
|
// [0]=Artist, [1]=Title. Artist сам може містити дефіс — тоді тайтл
|
||||||
|
// стає токеном [1], а {"Violence", "All_My_Life"} щось лишається unknown.
|
||||||
|
r := Parse("Ultra-Violence-All_My_Life-EP-16BIT-WEB-FLAC-2026-MOONBLOOD")
|
||||||
|
if r.Artist != "Ultra" || r.Title != "Violence" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q, want Ultra / Violence", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if r.Edition != "EP" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2026 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "All_My_Life" {
|
||||||
|
t.Errorf("UnknownTokens = %v, want [All_My_Life]", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageAndCDSource(t *testing.T) {
|
||||||
|
r := Parse("Burn_Thou_in_Effigy-Memorias_Postumas_do_Imperador-BR-CD-FLAC-2026-MOONBLOOD")
|
||||||
|
if r.Language != "BR" || r.Source != "CD" || r.Year != 2026 || r.Format != "FLAC" {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNoYear(t *testing.T) {
|
||||||
|
// Without a year the tail still parses; group stays the last token.
|
||||||
|
r := Parse("Fane-Gallant_Shroud-EP-16BIT-WEB-FLAC-MOONBLOOD")
|
||||||
|
if r.Group != "MOONBLOOD" || r.Year != 0 {
|
||||||
|
t.Errorf("Group/Year = %q/%d", r.Group, r.Year)
|
||||||
|
}
|
||||||
|
if r.Edition != "EP" || r.Source != "WEB" || r.Format != "FLAC" {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Fane" || r.Title != "Gallant_Shroud" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseResolutionSlot(t *testing.T) {
|
||||||
|
r := Parse("Ice_Nine_Kills-Last_Chance_To_Make_Amends-REMASTERED-24BIT-44KHZ-WEB-FLAC-2024-FLACCiD")
|
||||||
|
if r.Group != "FLACCiD" {
|
||||||
|
t.Errorf("Group = %q, want FLACCiD", r.Group)
|
||||||
|
}
|
||||||
|
if r.Resolution != "44KHZ" {
|
||||||
|
t.Errorf("Resolution = %q, want 44KHZ", r.Resolution)
|
||||||
|
}
|
||||||
|
if r.BitDepth != "24BIT" || r.Edition != "REMASTERED" || r.Source != "WEB" || r.Year != 2024 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Ice_Nine_Kills" || r.Title != "Last_Chance_To_Make_Amends" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseVinylIsSource(t *testing.T) {
|
||||||
|
r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
|
||||||
|
if r.Language != "DE" {
|
||||||
|
t.Errorf("Language = %q, want DE", r.Language)
|
||||||
|
}
|
||||||
|
if r.Source != "VINYL" {
|
||||||
|
t.Errorf("Source = %q, want VINYL", r.Source)
|
||||||
|
}
|
||||||
|
if r.Edition != "" {
|
||||||
|
t.Errorf("Edition = %q, want empty (VINYL is source, not edition)", r.Edition)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCaseInsensitiveDictionaries(t *testing.T) {
|
||||||
|
r := Parse("Artist-Some_Title-16bit-web-flac-2020-moonblood")
|
||||||
|
// Значення зберігаються в оригінальному регістрі (як у рядку),
|
||||||
|
// а звірка зі словниками йде регістронезалежно.
|
||||||
|
if r.BitDepth != "16bit" || r.Source != "web" || r.Format != "flac" {
|
||||||
|
t.Errorf("case-insensitive lookup failed: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseUnknownTailTokenReported(t *testing.T) {
|
||||||
|
// Невідомий токен між двома відомими маркерами → у UnknownTokens.
|
||||||
|
r := Parse("Artist-Some_Title-SPLIT-FOO-16BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||||
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "FOO" {
|
||||||
|
t.Errorf("UnknownTokens = %v, want [FOO]", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Edition != "SPLIT" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2025 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Artist" || r.Title != "Some_Title" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCoreTokensNeverReported(t *testing.T) {
|
||||||
|
// Артист/тайтл ніколи не потрапляють в UnknownTokens.
|
||||||
|
r := Parse("Fane-Dragged_Through_the_Unending-24BIT-WEB-FLAC-2025-MOONBLOOD")
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Artist != "Fane" || r.Title != "Dragged_Through_the_Unending" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseAnyGroupNoWarning(t *testing.T) {
|
||||||
|
cases := []string{
|
||||||
|
"Jetsex-Down_Down_Down-24BIT-WEB-FLAC-2025-FLACCiD",
|
||||||
|
"Sabaton-The_Last_Stand-24BIT-96KHZ-WEB-FLAC-2016-FLACCiD",
|
||||||
|
"Wind_Walkers-What_If_I_Break-16BIT-WEB-FLAC-2023-FLACCiD",
|
||||||
|
}
|
||||||
|
for _, line := range cases {
|
||||||
|
r := Parse(line)
|
||||||
|
if r.Group != "FLACCiD" {
|
||||||
|
t.Errorf("%s: Group = %q, want FLACCiD", line, r.Group)
|
||||||
|
}
|
||||||
|
if len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("%s: Warnings = %v, want empty", line, r.Warnings)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Year == 0 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth == "" {
|
||||||
|
t.Errorf("%s: tail fields wrong: %+v", line, r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCommentExtracted(t *testing.T) {
|
||||||
|
r := Parse("Rain_City_Drive-Rain_City_Drive-DELUXE_EDITION-REPACK-16BIT-WEB-FLAC-2023-FLACCiD")
|
||||||
|
if r.Comment != "REPACK" {
|
||||||
|
t.Errorf("Comment = %q, want REPACK", r.Comment)
|
||||||
|
}
|
||||||
|
if r.Edition != "DELUXE_EDITION" || r.BitDepth != "16BIT" || r.Source != "WEB" || r.Year != 2023 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Rain_City_Drive" || r.Title != "Rain_City_Drive" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseNumericArtistNotAYear(t *testing.T) {
|
||||||
|
// [0]=Artist — навіть якщо це число-рік (1914, 1928), а не "рік" токен.
|
||||||
|
for _, line := range []string{
|
||||||
|
"1914-Eschatology_of_War-24BIT-WEB-FLAC-2015-MOONBLOOD",
|
||||||
|
"1928-Lappel_du_Spectre-FR-CD-FLAC-2025-MOONBLOOD",
|
||||||
|
} {
|
||||||
|
r := Parse(line)
|
||||||
|
if r.Artist != "1914" && r.Artist != "1928" {
|
||||||
|
t.Errorf("%s: Artist = %q, want numeric artist", line, r.Artist)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCatalogNumber(t *testing.T) {
|
||||||
|
r := Parse("Antidot-Beyond_Trance_And_Time-(GLOBAL006)-REPACK-EP-FLAC-2026-AFTERHOUR")
|
||||||
|
if r.CatalogNo != "GLOBAL006" {
|
||||||
|
t.Errorf("CatalogNo = %q, want GLOBAL006", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Comment != "REPACK" || r.Edition != "EP" || r.Format != "FLAC" || r.Year != 2026 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Antidot" || r.Title != "Beyond_Trance_And_Time" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCatalogNumberWithDashesInside(t *testing.T) {
|
||||||
|
r := Parse("Murder_Intentions-Excessive_Display_Of_Human_Nature-(BRC30-001-CD-RRR108)-CD-FLAC-2018-86D")
|
||||||
|
if r.CatalogNo != "BRC30-001-CD-RRR108" {
|
||||||
|
t.Errorf("CatalogNo = %q, want BRC30-001-CD-RRR108", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2018 || r.Group != "86D" {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Murder_Intentions" || r.Title != "Excessive_Display_Of_Human_Nature" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCatalogNumberWithDotAndRoman(t *testing.T) {
|
||||||
|
// Катало у дужках у середній зоні — незалежно від вмісту
|
||||||
|
// (крапка, римські, букви, дефіси): все валідне.
|
||||||
|
cases := []struct{ line, catno string }{
|
||||||
|
{"Nervosa-Perpetual_Chaos-(DYN.3140-2)-LIMITED_EDITION-CD-FLAC-2021-MOONBLOOD", "DYN.3140-2"},
|
||||||
|
{"Black_Ceremonial_Kult-Crowned_in_Chaos-(GOWP_DCCCLXXXIX)-CDEP-FLAC-2021-MOONBLOOD", "GOWP_DCCCLXXXIX"},
|
||||||
|
}
|
||||||
|
for _, c := range cases {
|
||||||
|
r := Parse(c.line)
|
||||||
|
if r.CatalogNo != c.catno {
|
||||||
|
t.Errorf("CatalogNo = %q, want %q (release: %s)", r.CatalogNo, c.catno, c.line)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty (release: %s)", r.UnknownTokens, c.line)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseSplitKeepsParensAndSplitsTitles(t *testing.T) {
|
||||||
|
// [1]=Title (фіксовано), а Evolution — не словниковий токен у середині:
|
||||||
|
// потрапляє в UnknownTokens. (SMG409) — катало.
|
||||||
|
r := Parse("Ferosity-De-Evolution-(SMG409)-CD-FLAC-2025-86D")
|
||||||
|
if r.CatalogNo != "SMG409" {
|
||||||
|
t.Errorf("CatalogNo = %q, want SMG409", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Artist != "Ferosity" || r.Title != "De" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q, want Ferosity / De", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "Evolution" {
|
||||||
|
t.Errorf("UnknownTokens = %v, want [Evolution]", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageRequiresKnownNeighbor(t *testing.T) {
|
||||||
|
// Мовний маркер валідний, коли праворуч — відомий хвостовий маркер.
|
||||||
|
r := Parse("ROTH-Alles_Endet-DE-VINYL-FLAC-2026-TOTENKVLT")
|
||||||
|
if r.Language != "DE" || r.Source != "VINYL" {
|
||||||
|
t.Errorf("Language/Source = %q/%q, want DE/VINYL", r.Language, r.Source)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageBetweenTitleAndFormat(t *testing.T) {
|
||||||
|
// A: мовний маркер у зоні між title і format — валідний,
|
||||||
|
// все ліворуч від нього — artist+title, нічого в unknown.
|
||||||
|
r := Parse("Kollektiver_Brechreiz-Heilig_Scheint_Scheinheilig-(NG207)-DE-CD-FLAC-2009-SHELTER")
|
||||||
|
if r.CatalogNo != "NG207" {
|
||||||
|
t.Errorf("CatalogNo = %q, want NG207", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Language != "DE" || r.Source != "CD" || r.Format != "FLAC" || r.Year != 2009 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if r.Artist != "Kollektiver_Brechreiz" || r.Title != "Heilig_Scheint_Scheinheilig" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 || len(r.Warnings) != 0 {
|
||||||
|
t.Errorf("UnknownTokens=%v Warnings=%v, want empty", r.UnknownTokens, r.Warnings)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLanguageNeverBeforeFormat(t *testing.T) {
|
||||||
|
// DE — словниковий токен у середині (language.txt), позиція неважлива:
|
||||||
|
// Language=DE.
|
||||||
|
r := Parse("Some_Artist-Some_Song-DE-FLAC-2020-SOMEGROUP")
|
||||||
|
if r.Language != "DE" {
|
||||||
|
t.Errorf("Language = %q, want DE", r.Language)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Title != "Some_Song" {
|
||||||
|
t.Errorf("Title = %q, want Some_Song", r.Title)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseCompoundEditionRecognized(t *testing.T) {
|
||||||
|
// REMASTERED_REISSUE — складений маркер у dicts/edition.txt, розпізнається
|
||||||
|
// як edition і не йде в UnknownTokens.
|
||||||
|
r := Parse("Ominous_Scriptures-Incarnation_of_the_Unheavenly-(LSR07)-REMASTERED_REISSUE-CD-FLAC-2021-86D")
|
||||||
|
if r.CatalogNo != "LSR07" {
|
||||||
|
t.Errorf("CatalogNo = %q, want LSR07", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Edition != "REMASTERED_REISSUE" {
|
||||||
|
t.Errorf("Edition = %q, want REMASTERED_REISSUE", r.Edition)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
if r.Artist != "Ominous_Scriptures" || r.Title != "Incarnation_of_the_Unheavenly" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if r.Source != "CD" || r.Format != "FLAC" || r.Year != 2021 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseMixedCaseTitleNeverReported(t *testing.T) {
|
||||||
|
// [1]=Title (один токен), решта, що не словник — UnknownTokens.
|
||||||
|
r := Parse("Enthrallment-Against_the_Will_to_Live_-_Sky_Burial-(RTM103)-CD-FLAC-2021-86D")
|
||||||
|
if r.Artist != "Enthrallment" || r.Title != "Against_the_Will_to_Live_" {
|
||||||
|
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 1 || r.UnknownTokens[0] != "_Sky_Burial" {
|
||||||
|
t.Errorf("UnknownTokens = %v, want [_Sky_Burial]", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func contains(list []string, s string) bool {
|
||||||
|
for _, v := range list {
|
||||||
|
if v == s {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseParenthesizedTitleNotCatNo(t *testing.T) {
|
||||||
|
// Дужки в тайтлі при WEB/BitDepth — не каталономер, лишається в тайтлі.
|
||||||
|
r := Parse("The_Amazons-21st_Century_Fiction_(Epilogue)-16BIT-WEB-FLAC-2026-ENViED")
|
||||||
|
if r.CatalogNo != "" {
|
||||||
|
t.Errorf("CatalogNo = %q, want empty", r.CatalogNo)
|
||||||
|
}
|
||||||
|
if r.Title != "21st_Century_Fiction_(Epilogue)" {
|
||||||
|
t.Errorf("Title = %q, want 21st_Century_Fiction_(Epilogue)", r.Title)
|
||||||
|
}
|
||||||
|
if r.Source != "WEB" || r.BitDepth != "16BIT" || r.Year != 2026 {
|
||||||
|
t.Errorf("tail fields wrong: %+v", r)
|
||||||
|
}
|
||||||
|
if len(r.UnknownTokens) != 0 {
|
||||||
|
t.Errorf("UnknownTokens = %v, want empty", r.UnknownTokens)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package moonblood
|
||||||
|
|
||||||
|
import (
|
||||||
|
"embed"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Словники токенів лежать у dicts/*.txt — один токен на рядок.
|
||||||
|
// Їх можна редагувати без перекомпіляції логіки (embed тягне їх у бінарник на збірці).
|
||||||
|
|
||||||
|
//go:embed dicts/*.txt
|
||||||
|
var dictFS embed.FS
|
||||||
|
|
||||||
|
func loadDict(name string) map[string]bool {
|
||||||
|
data, err := dictFS.ReadFile("dicts/" + name + ".txt")
|
||||||
|
if err != nil {
|
||||||
|
panic("moonblood: cannot load dictionary " + name + ": " + err.Error())
|
||||||
|
}
|
||||||
|
out := make(map[string]bool)
|
||||||
|
for _, line := range strings.Split(string(data), "\n") {
|
||||||
|
line = strings.ToUpper(strings.TrimSpace(line))
|
||||||
|
if line == "" || strings.HasPrefix(line, "#") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out[line] = true
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
var Profile = releaseparser.GroupProfile{
|
||||||
|
Name: "MOONBLOOD",
|
||||||
|
ExpectedTailOrder: []string{"bit_depth", "resolution", "source", "format", "year", "group"},
|
||||||
|
FormatTokens: loadDict("format"),
|
||||||
|
SourceTokens: loadDict("source"),
|
||||||
|
BitDepthTokens: loadDict("bitdepth"),
|
||||||
|
ResolutionTokens: loadDict("resolution"),
|
||||||
|
LanguageMarkers: loadDict("language"),
|
||||||
|
EditionTokens: loadDict("edition"),
|
||||||
|
CommentTokens: loadDict("comment"),
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
package releaseparser
|
||||||
|
|
||||||
|
type GroupProfile struct {
|
||||||
|
Name string
|
||||||
|
ExpectedTailOrder []string
|
||||||
|
FormatTokens map[string]bool
|
||||||
|
SourceTokens map[string]bool
|
||||||
|
BitDepthTokens map[string]bool
|
||||||
|
ResolutionTokens map[string]bool
|
||||||
|
LanguageMarkers map[string]bool
|
||||||
|
EditionTokens map[string]bool
|
||||||
|
CommentTokens map[string]bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type ParsedRelease struct {
|
||||||
|
Raw string
|
||||||
|
Group string
|
||||||
|
Year int
|
||||||
|
BitDepth string
|
||||||
|
Resolution string
|
||||||
|
Comment string
|
||||||
|
CatalogNo string
|
||||||
|
Source string
|
||||||
|
Format string
|
||||||
|
Language string
|
||||||
|
Edition string
|
||||||
|
Core string
|
||||||
|
Artist string
|
||||||
|
Title string
|
||||||
|
UnknownTokens []string
|
||||||
|
Warnings []string
|
||||||
|
Confidence float64
|
||||||
|
}
|
||||||
|
|
||||||
|
type UnknownTokenStats struct {
|
||||||
|
Token string
|
||||||
|
Count int
|
||||||
|
Releases []string
|
||||||
|
}
|
||||||
|
|
||||||
|
type GroupStats struct {
|
||||||
|
Total int
|
||||||
|
YearDistribution map[int]int
|
||||||
|
SourceDistribution map[string]int
|
||||||
|
BitDepthDistribution map[string]int
|
||||||
|
ResolutionDistribution map[string]int
|
||||||
|
LanguageDistribution map[string]int
|
||||||
|
EditionDistribution map[string]int
|
||||||
|
CommentDistribution map[string]int
|
||||||
|
UnknownTokens []UnknownTokenStats
|
||||||
|
WarningStats []UnknownTokenStats
|
||||||
|
AvgConfidence float64
|
||||||
|
}
|
||||||
@@ -30,7 +30,12 @@ func (h *Handler) Prescene() string {
|
|||||||
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := h.services.Prescene.GetPage(pagesToScrape)
|
_, err := h.services.Prescene.GetSrrDb()
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = h.services.Prescene.GetPage(pagesToScrape)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,5 @@ func (h *Handler) Rutracker() string {
|
|||||||
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
//fmt.Printf("%+v", rt)
|
|
||||||
|
|
||||||
return "rt"
|
return "rt"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/davecgh/go-spew/spew"
|
||||||
"github.com/dromara/carbon/v2"
|
"github.com/dromara/carbon/v2"
|
||||||
"github.com/go-shiori/dom"
|
"github.com/go-shiori/dom"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
@@ -25,6 +26,17 @@ type Prescene struct {
|
|||||||
db *sqlx.DB
|
db *sqlx.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) {
|
||||||
|
entries := make([]model.ExternalSources, 0)
|
||||||
|
now := time.Now()
|
||||||
|
yesterday := now.Add(-24 * time.Hour)
|
||||||
|
todayStr := now.Format("2006-01-02")
|
||||||
|
yesterdayStr := yesterday.Format("2006-01-02")
|
||||||
|
spew.Dump(entries, todayStr, yesterdayStr)
|
||||||
|
|
||||||
|
return []model.ExternalSources{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
||||||
return &Prescene{db: db}
|
return &Prescene{db: db}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
|
|||||||
|
|
||||||
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
||||||
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
||||||
|
|
||||||
entries := make([]model.ExternalSources, 0)
|
entries := make([]model.ExternalSources, 0)
|
||||||
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
|
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
|
||||||
|
|
||||||
@@ -41,18 +42,19 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
|||||||
|
|
||||||
for _, e := range topic.Entry {
|
for _, e := range topic.Entry {
|
||||||
var es model.ExternalSources
|
var es model.ExternalSources
|
||||||
u, _ := url.Parse(e.Link.Href)
|
|
||||||
|
|
||||||
es.Type = constant.ScopeRuTracker
|
es.Type = constant.ScopeRuTracker
|
||||||
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
|
es.TypeId = topicId(e)
|
||||||
es.Title = e.Title
|
es.Title = e.Title
|
||||||
|
|
||||||
es.TypeSubsectionId, _ = strconv.Atoi(t)
|
es.TypeSubsectionId, _ = strconv.Atoi(t)
|
||||||
es.Releaser = e.Author.Name
|
es.Releaser = e.Author.Name
|
||||||
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
|
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
|
||||||
esModel := table.ExternalSources{Columns: columns}
|
esModel := table.ExternalSources{Columns: columns}
|
||||||
|
|
||||||
|
fmt.Printf("%+v\n\n\n", es)
|
||||||
|
|
||||||
entry := esModel.InsertOnDuplicate(es, s.db)
|
entry := esModel.InsertOnDuplicate(es, s.db)
|
||||||
entries = append(entries, entry)
|
entries = append(entries, entry)
|
||||||
//fmt.Printf("%+v\n\n\n", entry)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -62,22 +64,44 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
|||||||
func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
|
func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
|
||||||
resp, err := http.Get(endpoint)
|
resp, err := http.Get(endpoint)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("couldn't fetch data", endpoint, err.Error())
|
slog.Error("couldn't fetch data", "endpoint", endpoint, "err", err.Error())
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
defer func(Body io.ReadCloser) {
|
defer func(Body io.ReadCloser) {
|
||||||
err = Body.Close()
|
if cerr := Body.Close(); cerr != nil {
|
||||||
if err != nil {
|
slog.Error("couldn't close response body", "endpoint", endpoint, "err", cerr.Error())
|
||||||
fmt.Println("Body.Close")
|
|
||||||
}
|
}
|
||||||
}(resp.Body)
|
}(resp.Body)
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("unexpected status code %d from %s", resp.StatusCode, endpoint)
|
||||||
|
}
|
||||||
|
|
||||||
topic := &resource.RutrackerAtomTopic{}
|
topic := &resource.RutrackerAtomTopic{}
|
||||||
|
|
||||||
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
|
if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
|
||||||
|
slog.Error("couldn't decode feed", "endpoint", endpoint, "err", err.Error())
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return topic, nil
|
return topic, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// topicId resolves the rutracker topic id preferring the page link query (?t=...),
|
||||||
|
// falling back to the entry id tag (tag:rto.feed,...:/t/5702203).
|
||||||
|
func topicId(e resource.RutrackerAtomEntry) int {
|
||||||
|
if link, ok := e.PageLink(); ok {
|
||||||
|
if u, err := url.Parse(link.Href); err == nil {
|
||||||
|
if id, err := strconv.Atoi(u.Query().Get("t")); err == nil {
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
id, err := e.TopicId()
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("couldn't resolve topic id", "entry_id", e.ID, "err", err.Error())
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return id
|
||||||
|
}
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, column
|
|||||||
var status string
|
var status string
|
||||||
errCount := 0
|
errCount := 0
|
||||||
|
|
||||||
for i := 0; i < len(entries); i++ {
|
for i := range entries {
|
||||||
entry := es.InsertOnDuplicate(entries[i], db)
|
entry := es.InsertOnDuplicate(entries[i], db)
|
||||||
if entry.Error != nil {
|
if entry.Error != nil {
|
||||||
slog.Error("insert/update entry", "err", entry.Error)
|
slog.Error("insert/update entry", "err", entry.Error)
|
||||||
|
|||||||
@@ -16,3 +16,7 @@ func NewPresceneService(repo _interface.Prescene) *PresceneService {
|
|||||||
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
||||||
return p.repo.GetPage(pageNumbers)
|
return p.repo.GetPage(pageNumbers)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *PresceneService) GetSrrDb() ([]model.ExternalSources, error) {
|
||||||
|
return p.repo.GetSrrDb()
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ type Rutracker interface {
|
|||||||
|
|
||||||
type Prescene interface {
|
type Prescene interface {
|
||||||
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
|
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
|
||||||
|
GetSrrDb() ([]model.ExternalSources, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type ShareTheBrutality interface {
|
type ShareTheBrutality interface {
|
||||||
|
|||||||
+41
-15
@@ -1,35 +1,61 @@
|
|||||||
package resource
|
package resource
|
||||||
|
|
||||||
import "encoding/xml"
|
import (
|
||||||
|
"encoding/xml"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// RutrackerAtomTopic represents the atom feed of a rutracker forum subsection.
|
||||||
type RutrackerAtomTopic struct {
|
type RutrackerAtomTopic struct {
|
||||||
XMLName xml.Name `xml:"feed"`
|
XMLName xml.Name `xml:"feed"`
|
||||||
Text string `xml:",chardata"`
|
|
||||||
Xmlns string `xml:"xmlns,attr"`
|
Xmlns string `xml:"xmlns,attr"`
|
||||||
ID string `xml:"id"`
|
ID string `xml:"id"`
|
||||||
Link struct {
|
Link []AtomLink `xml:"link"`
|
||||||
Text string `xml:",chardata"`
|
|
||||||
Href string `xml:"href,attr"`
|
|
||||||
} `xml:"link"`
|
|
||||||
Updated string `xml:"updated"`
|
Updated string `xml:"updated"`
|
||||||
Title string `xml:"title"`
|
Title string `xml:"title"`
|
||||||
Entry []struct {
|
Entry []RutrackerAtomEntry `xml:"entry"`
|
||||||
Text string `xml:",chardata"`
|
}
|
||||||
ID string `xml:"id"`
|
|
||||||
Link struct {
|
// AtomLink is a generic atom link element. An entry contains two links:
|
||||||
Text string `xml:",chardata"`
|
// a topic page link (no rel) and a magnet link (rel="enclosure").
|
||||||
|
type AtomLink struct {
|
||||||
Href string `xml:"href,attr"`
|
Href string `xml:"href,attr"`
|
||||||
} `xml:"link"`
|
Rel string `xml:"rel,attr"`
|
||||||
|
Type string `xml:"type,attr"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type RutrackerAtomEntry struct {
|
||||||
|
ID string `xml:"id"`
|
||||||
|
Link []AtomLink `xml:"link"`
|
||||||
Updated string `xml:"updated"`
|
Updated string `xml:"updated"`
|
||||||
Title string `xml:"title"`
|
Title string `xml:"title"`
|
||||||
Author struct {
|
Author struct {
|
||||||
Text string `xml:",chardata"`
|
|
||||||
Name string `xml:"name"`
|
Name string `xml:"name"`
|
||||||
} `xml:"author"`
|
} `xml:"author"`
|
||||||
Category struct {
|
Category struct {
|
||||||
Text string `xml:",chardata"`
|
|
||||||
Term string `xml:"term,attr"`
|
Term string `xml:"term,attr"`
|
||||||
Label string `xml:"label,attr"`
|
Label string `xml:"label,attr"`
|
||||||
} `xml:"category"`
|
} `xml:"category"`
|
||||||
} `xml:"entry"`
|
}
|
||||||
|
|
||||||
|
// PageLink returns the link to the topic page (the one without rel attribute).
|
||||||
|
func (e RutrackerAtomEntry) PageLink() (AtomLink, bool) {
|
||||||
|
for _, l := range e.Link {
|
||||||
|
if l.Rel == "" {
|
||||||
|
return l, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return AtomLink{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
// TopicId extracts the topic id from the entry id tag: "tag:rto.feed,2026-08-29:/t/5702203".
|
||||||
|
func (e RutrackerAtomEntry) TopicId() (int, error) {
|
||||||
|
const marker = ":/t/"
|
||||||
|
i := strings.LastIndex(e.ID, marker)
|
||||||
|
if i < 0 {
|
||||||
|
return 0, fmt.Errorf("unexpected entry id format: %s", e.ID)
|
||||||
|
}
|
||||||
|
return strconv.Atoi(e.ID[i+len(marker):])
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user