yevhen/resource-scraper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add nultie gpouring

Browse Source
This commit is contained in:
Yevhen Odynets
2026-09-01 00:08:03 +03:00
parent 1b2521d4ac
commit 3f3598c446
9 changed files with 126 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/srrdb/main.go
+40 -4
View File
@@ -4,17 +4,53 @@ import (
"encoding/json"
"fmt"
"log"
"os"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
_ "github.com/go-sql-driver/mysql"
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood"
)
func main() {
stats, err := moonblood.AnalyzeGroup()
if err != nil {
log.Fatal(err)
if len(os.Args) < 2 {
log.Fatal("usage: srrdb <releaser> [<releaser>...]")
}
releasers := os.Args[1:]
// Читаємо config/secret.yaml (без змісту в коді — тільки потрібні ключі)
viper.SetConfigFile("config/secret.yaml")
if err := viper.ReadInConfig(); err != nil {
log.Fatal("failed to read config/secret.yaml: ", err)
}
data, err := json.MarshalIndent(stats, "", " ")
drv := viper.GetString("db.driver")
dsn := viper.GetString("db.data_source_name")
dbase, err := sqlx.Connect(drv, dsn)
if err != nil {
panic("Failed to connect to the database: " + err.Error())
}
defer dbase.Close()
out := make(map[string]moonblood.GroupStats, len(releasers))
for _, releaser := range releasers {
var titles []string
err = dbase.Select(&titles,
`SELECT title FROM theamok_mdb.external_sources
WHERE type = 'prescene' AND releaser = ?`, releaser)
if err != nil {
log.Fatal(err)
}
out[releaser] = moonblood.AnalyzeTitles(titles)
}
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
log.Fatal(err)
}
internal/releaseparser/groups/moonblood/analyzer.go
+40 -29
View File
@@ -10,46 +10,57 @@ import (
)
type GroupStats struct {
Total int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
Total int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
ResolutionDistribution map[string]int
LanguageDistribution map[string]int
EditionDistribution map[string]int
UnknownTokens []releaseparser.UnknownTokenStats
AvgConfidence float64
LanguageDistribution map[string]int
EditionDistribution map[string]int
UnknownTokens []releaseparser.UnknownTokenStats
AvgConfidence float64
}
// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників).
func AnalyzeGroup() (GroupStats, error) {
stats := GroupStats{
YearDistribution: make(map[int]int),
SourceDistribution: make(map[string]int),
BitDepthDistribution: make(map[string]int),
ResolutionDistribution: make(map[string]int),
LanguageDistribution: make(map[string]int),
EditionDistribution: make(map[string]int),
UnknownTokens: []releaseparser.UnknownTokenStats{},
}
file, err := os.Open("internal/releaseparser/resources/moonblood.txt")
file, err := os.Open("../../resources/moonblood.txt")
if err != nil {
return stats, err
return GroupStats{}, err
}
defer file.Close()
var titles []string
scanner := bufio.NewScanner(file)
totalConfidence := 0.0
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
if line != "" {
titles = append(titles, line)
}
}
if err := scanner.Err(); err != nil {
return GroupStats{}, err
}
result := Parse(line)
return AnalyzeTitles(titles), nil
}
// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі).
func AnalyzeTitles(titles []string) GroupStats {
stats := GroupStats{
YearDistribution: make(map[int]int),
SourceDistribution: make(map[string]int),
BitDepthDistribution: make(map[string]int),
ResolutionDistribution: make(map[string]int),
LanguageDistribution: make(map[string]int),
EditionDistribution: make(map[string]int),
UnknownTokens: []releaseparser.UnknownTokenStats{},
}
totalConfidence := 0.0
unknownMap := make(map[string]*releaseparser.UnknownTokenStats)
for _, title := range titles {
result := Parse(title)
stats.Total++
if result.Year != 0 {
@@ -80,7 +91,7 @@ func AnalyzeGroup() (GroupStats, error) {
}
}
unknownMap[unknown].Count++
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, line)
unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title)
}
totalConfidence += result.Confidence
@@ -99,5 +110,5 @@ func AnalyzeGroup() (GroupStats, error) {
stats.AvgConfidence = totalConfidence / float64(stats.Total)
}
return stats, nil
return stats
}
internal/releaseparser/groups/moonblood/dicts/language.txt
+1
View File
@@ -16,3 +16,4 @@ DK
AT
IS
UA
JP
internal/releaseparser/groups/moonblood/dicts/resolution.txt
+3 -2
View File
@@ -1,6 +1,7 @@
# {resolution} — частота дискретизації
44KHZ
44.1KHZ
48KHZ
88.2KHZ
88KHZ
96KHZ
176KHZ
192KHZ
internal/releaseparser/groups/moonblood/parser.go
+2 -5
View File
@@ -24,11 +24,8 @@ func Parse(raw string) releaseparser.ParsedRelease {
return result
}
// Останній токен — група
// Останній токен — група. Вона ж — ключ до профілю групи (довідників).
result.Group = tokens[len(tokens)-1]
if result.Group != "MOONBLOOD" {
result.Warnings = append(result.Warnings, "unexpected group")
}
// Передостанній — рік
last := len(tokens) - 2
@@ -139,7 +136,7 @@ func hasKnownTailToken(tokens []string) bool {
func score(result *releaseparser.ParsedRelease) {
score := 0.0
if result.Group == "MOONBLOOD" {
if result.Group != "" {
score += 0.25
}
if result.Year != 0 {
internal/releaseparser/groups/moonblood/parser_test.go
+23
View File
@@ -176,3 +176,26 @@ func TestParseCoreTokensNeverReported(t *testing.T) {
t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title)
}
}
func TestParseAnyGroupNoWarning(t *testing.T) {
cases := []string{
"Jetsex-Down_Down_Down-24BIT-WEB-FLAC-2025-FLACCiD",
"Sabaton-The_Last_Stand-24BIT-96KHZ-WEB-FLAC-2016-FLACCiD",
"Wind_Walkers-What_If_I_Break-16BIT-WEB-FLAC-2023-FLACCiD",
}
for _, line := range cases {
r := Parse(line)
if r.Group != "FLACCiD" {
t.Errorf("%s: Group = %q, want FLACCiD", line, r.Group)
}
if len(r.Warnings) != 0 {
t.Errorf("%s: Warnings = %v, want empty", line, r.Warnings)
}
if len(r.UnknownTokens) != 0 {
t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens)
}
if r.Year == 0 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth == "" {
t.Errorf("%s: tail fields wrong: %+v", line, r)
}
}
}
internal/releaseparser/types.go
+16 -15
View File
@@ -1,14 +1,14 @@
package releaseparser
type GroupProfile struct {
Name string
Name string
ExpectedTailOrder []string
FormatTokens map[string]bool
SourceTokens map[string]bool
BitDepthTokens map[string]bool
ResolutionTokens map[string]bool
LanguageMarkers map[string]bool
EditionTokens map[string]bool
FormatTokens map[string]bool
SourceTokens map[string]bool
BitDepthTokens map[string]bool
ResolutionTokens map[string]bool
LanguageMarkers map[string]bool
EditionTokens map[string]bool
}
type ParsedRelease struct {
@@ -17,6 +17,7 @@ type ParsedRelease struct {
Year int
BitDepth string
Resolution string
Comment string
Source string
Format string
Language string
@@ -36,13 +37,13 @@ type UnknownTokenStats struct {
}
type GroupStats struct {
Total int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
Total int
YearDistribution map[int]int
SourceDistribution map[string]int
BitDepthDistribution map[string]int
ResolutionDistribution map[string]int
LanguageDistribution map[string]int
EditionDistribution map[string]int
UnknownTokens []UnknownTokenStats
AvgConfidence float64
LanguageDistribution map[string]int
EditionDistribution map[string]int
UnknownTokens []UnknownTokenStats
AvgConfidence float64
}
srrdb.exe
BIN
View File
Binary file not shown.