diff --git a/cmd/srrdb/main.go b/cmd/srrdb/main.go index 3c00451..fc825fd 100644 --- a/cmd/srrdb/main.go +++ b/cmd/srrdb/main.go @@ -4,17 +4,53 @@ import ( "encoding/json" "fmt" "log" + "os" + + "github.com/jmoiron/sqlx" + "github.com/spf13/viper" + + _ "github.com/go-sql-driver/mysql" "git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups/moonblood" ) func main() { - stats, err := moonblood.AnalyzeGroup() - if err != nil { - log.Fatal(err) + if len(os.Args) < 2 { + log.Fatal("usage: srrdb [...]") + } + releasers := os.Args[1:] + + // Читаємо config/secret.yaml (без змісту в коді — тільки потрібні ключі) + viper.SetConfigFile("config/secret.yaml") + if err := viper.ReadInConfig(); err != nil { + log.Fatal("failed to read config/secret.yaml: ", err) } - data, err := json.MarshalIndent(stats, "", " ") + drv := viper.GetString("db.driver") + dsn := viper.GetString("db.data_source_name") + + dbase, err := sqlx.Connect(drv, dsn) + if err != nil { + panic("Failed to connect to the database: " + err.Error()) + } + defer dbase.Close() + + out := make(map[string]moonblood.GroupStats, len(releasers)) + + for _, releaser := range releasers { + var titles []string + + err = dbase.Select(&titles, + `SELECT title FROM theamok_mdb.external_sources + WHERE type = 'prescene' AND releaser = ?`, releaser) + if err != nil { + log.Fatal(err) + } + + out[releaser] = moonblood.AnalyzeTitles(titles) + } + + data, err := json.MarshalIndent(out, "", " ") if err != nil { log.Fatal(err) } diff --git a/internal/releaseparser/groups/moonblood/analyzer.go b/internal/releaseparser/groups/moonblood/analyzer.go index e2670b4..9396e35 100644 --- a/internal/releaseparser/groups/moonblood/analyzer.go +++ b/internal/releaseparser/groups/moonblood/analyzer.go @@ -10,46 +10,57 @@ import ( ) type GroupStats struct { - Total int - YearDistribution map[int]int - SourceDistribution map[string]int - BitDepthDistribution map[string]int + Total int + YearDistribution map[int]int + SourceDistribution map[string]int + BitDepthDistribution map[string]int ResolutionDistribution map[string]int - LanguageDistribution map[string]int - EditionDistribution map[string]int - UnknownTokens []releaseparser.UnknownTokenStats - AvgConfidence float64 + LanguageDistribution map[string]int + EditionDistribution map[string]int + UnknownTokens []releaseparser.UnknownTokenStats + AvgConfidence float64 } +// AnalyzeGroup аналізує релізи з файлового ресурсу (для локальної перевірки довідників). func AnalyzeGroup() (GroupStats, error) { - stats := GroupStats{ - YearDistribution: make(map[int]int), - SourceDistribution: make(map[string]int), - BitDepthDistribution: make(map[string]int), - ResolutionDistribution: make(map[string]int), - LanguageDistribution: make(map[string]int), - EditionDistribution: make(map[string]int), - UnknownTokens: []releaseparser.UnknownTokenStats{}, - } - - file, err := os.Open("internal/releaseparser/resources/moonblood.txt") + file, err := os.Open("../../resources/moonblood.txt") if err != nil { - return stats, err + return GroupStats{}, err } defer file.Close() + var titles []string scanner := bufio.NewScanner(file) - totalConfidence := 0.0 - - unknownMap := make(map[string]*releaseparser.UnknownTokenStats) - for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) - if line == "" { - continue + if line != "" { + titles = append(titles, line) } + } + if err := scanner.Err(); err != nil { + return GroupStats{}, err + } - result := Parse(line) + return AnalyzeTitles(titles), nil +} + +// AnalyzeTitles аналізує набір назв релізів (наприклад, з бази по ключу-групі). +func AnalyzeTitles(titles []string) GroupStats { + stats := GroupStats{ + YearDistribution: make(map[int]int), + SourceDistribution: make(map[string]int), + BitDepthDistribution: make(map[string]int), + ResolutionDistribution: make(map[string]int), + LanguageDistribution: make(map[string]int), + EditionDistribution: make(map[string]int), + UnknownTokens: []releaseparser.UnknownTokenStats{}, + } + + totalConfidence := 0.0 + unknownMap := make(map[string]*releaseparser.UnknownTokenStats) + + for _, title := range titles { + result := Parse(title) stats.Total++ if result.Year != 0 { @@ -80,7 +91,7 @@ func AnalyzeGroup() (GroupStats, error) { } } unknownMap[unknown].Count++ - unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, line) + unknownMap[unknown].Releases = append(unknownMap[unknown].Releases, title) } totalConfidence += result.Confidence @@ -99,5 +110,5 @@ func AnalyzeGroup() (GroupStats, error) { stats.AvgConfidence = totalConfidence / float64(stats.Total) } - return stats, nil + return stats } diff --git a/internal/releaseparser/groups/moonblood/dicts/edition.txt b/internal/releaseparser/groups/moonblood/dicts/edition.txt index d13607c..e80a83d 100644 --- a/internal/releaseparser/groups/moonblood/dicts/edition.txt +++ b/internal/releaseparser/groups/moonblood/dicts/edition.txt @@ -17,4 +17,4 @@ LIMITED_EDITION_BOXSET REMASTERED_EP 10TH_ANNIVERSARY_EDITION OST -REISSUE_DEMO \ No newline at end of file +REISSUE_DEMO diff --git a/internal/releaseparser/groups/moonblood/dicts/language.txt b/internal/releaseparser/groups/moonblood/dicts/language.txt index 5d43261..ae4b187 100644 --- a/internal/releaseparser/groups/moonblood/dicts/language.txt +++ b/internal/releaseparser/groups/moonblood/dicts/language.txt @@ -16,3 +16,4 @@ DK AT IS UA +JP diff --git a/internal/releaseparser/groups/moonblood/dicts/resolution.txt b/internal/releaseparser/groups/moonblood/dicts/resolution.txt index bb8e616..598f71d 100644 --- a/internal/releaseparser/groups/moonblood/dicts/resolution.txt +++ b/internal/releaseparser/groups/moonblood/dicts/resolution.txt @@ -1,6 +1,7 @@ # {resolution} — частота дискретизації 44KHZ -44.1KHZ 48KHZ -88.2KHZ +88KHZ 96KHZ +176KHZ +192KHZ \ No newline at end of file diff --git a/internal/releaseparser/groups/moonblood/parser.go b/internal/releaseparser/groups/moonblood/parser.go index 42bdf24..33d9738 100644 --- a/internal/releaseparser/groups/moonblood/parser.go +++ b/internal/releaseparser/groups/moonblood/parser.go @@ -24,11 +24,8 @@ func Parse(raw string) releaseparser.ParsedRelease { return result } - // Останній токен — група + // Останній токен — група. Вона ж — ключ до профілю групи (довідників). result.Group = tokens[len(tokens)-1] - if result.Group != "MOONBLOOD" { - result.Warnings = append(result.Warnings, "unexpected group") - } // Передостанній — рік last := len(tokens) - 2 @@ -139,7 +136,7 @@ func hasKnownTailToken(tokens []string) bool { func score(result *releaseparser.ParsedRelease) { score := 0.0 - if result.Group == "MOONBLOOD" { + if result.Group != "" { score += 0.25 } if result.Year != 0 { diff --git a/internal/releaseparser/groups/moonblood/parser_test.go b/internal/releaseparser/groups/moonblood/parser_test.go index 4ab5f2c..1425a00 100644 --- a/internal/releaseparser/groups/moonblood/parser_test.go +++ b/internal/releaseparser/groups/moonblood/parser_test.go @@ -176,3 +176,26 @@ func TestParseCoreTokensNeverReported(t *testing.T) { t.Errorf("Artist/Title = %q/%q", r.Artist, r.Title) } } + +func TestParseAnyGroupNoWarning(t *testing.T) { + cases := []string{ + "Jetsex-Down_Down_Down-24BIT-WEB-FLAC-2025-FLACCiD", + "Sabaton-The_Last_Stand-24BIT-96KHZ-WEB-FLAC-2016-FLACCiD", + "Wind_Walkers-What_If_I_Break-16BIT-WEB-FLAC-2023-FLACCiD", + } + for _, line := range cases { + r := Parse(line) + if r.Group != "FLACCiD" { + t.Errorf("%s: Group = %q, want FLACCiD", line, r.Group) + } + if len(r.Warnings) != 0 { + t.Errorf("%s: Warnings = %v, want empty", line, r.Warnings) + } + if len(r.UnknownTokens) != 0 { + t.Errorf("%s: UnknownTokens = %v, want empty", line, r.UnknownTokens) + } + if r.Year == 0 || r.Format != "FLAC" || r.Source != "WEB" || r.BitDepth == "" { + t.Errorf("%s: tail fields wrong: %+v", line, r) + } + } +} diff --git a/internal/releaseparser/types.go b/internal/releaseparser/types.go index c0d7b98..c96b4a7 100644 --- a/internal/releaseparser/types.go +++ b/internal/releaseparser/types.go @@ -1,14 +1,14 @@ package releaseparser type GroupProfile struct { - Name string + Name string ExpectedTailOrder []string - FormatTokens map[string]bool - SourceTokens map[string]bool - BitDepthTokens map[string]bool - ResolutionTokens map[string]bool - LanguageMarkers map[string]bool - EditionTokens map[string]bool + FormatTokens map[string]bool + SourceTokens map[string]bool + BitDepthTokens map[string]bool + ResolutionTokens map[string]bool + LanguageMarkers map[string]bool + EditionTokens map[string]bool } type ParsedRelease struct { @@ -17,6 +17,7 @@ type ParsedRelease struct { Year int BitDepth string Resolution string + Comment string Source string Format string Language string @@ -36,13 +37,13 @@ type UnknownTokenStats struct { } type GroupStats struct { - Total int - YearDistribution map[int]int - SourceDistribution map[string]int - BitDepthDistribution map[string]int + Total int + YearDistribution map[int]int + SourceDistribution map[string]int + BitDepthDistribution map[string]int ResolutionDistribution map[string]int - LanguageDistribution map[string]int - EditionDistribution map[string]int - UnknownTokens []UnknownTokenStats - AvgConfidence float64 + LanguageDistribution map[string]int + EditionDistribution map[string]int + UnknownTokens []UnknownTokenStats + AvgConfidence float64 } diff --git a/srrdb.exe b/srrdb.exe new file mode 100644 index 0000000..6779ede Binary files /dev/null and b/srrdb.exe differ