Yevhen Odynets
126 lines
3.1 KiB
Go
126 lines
3.1 KiB
Go
package main
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"log"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
"github.com/jmoiron/sqlx"
|
|
"github.com/spf13/viper"
|
|
|
|
_ "github.com/go-sql-driver/mysql"
|
|
|
|
"git.kplus.net.ua/yevhen/resource-scraper/internal/releaseparser/groups"
|
|
)
|
|
|
|
func main() {
|
|
if len(os.Args) < 2 {
|
|
log.Fatal("usage: srrdb <releaser> [<releaser>...]")
|
|
}
|
|
releasers := os.Args[1:]
|
|
|
|
// Читаємо config/secret.yaml (без змісту в коді — тільки потрібні ключі)
|
|
viper.SetConfigFile("config/secret.yaml")
|
|
if err := viper.ReadInConfig(); err != nil {
|
|
log.Fatal("failed to read config/secret.yaml: ", err)
|
|
}
|
|
|
|
drv := viper.GetString("db.driver")
|
|
dsn := viper.GetString("db.data_source_name")
|
|
|
|
dbase, err := sqlx.Connect(drv, dsn)
|
|
if err != nil {
|
|
panic("Failed to connect to the database: " + err.Error())
|
|
}
|
|
defer dbase.Close()
|
|
|
|
const outDir = "tmp/releaseparser"
|
|
if err := os.MkdirAll(outDir, 0o755); err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
out := make(map[string]groups.GroupStats, len(releasers))
|
|
|
|
for _, releaser := range releasers {
|
|
var titles []string
|
|
|
|
err = dbase.Select(&titles,
|
|
`SELECT title FROM theamok_mdb.external_sources
|
|
WHERE type = 'prescene' AND releaser = ?`, releaser)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
out[releaser] = groups.AnalyzeTitles(titles)
|
|
|
|
// Per-group parsed records keyed by the actual group read from each
|
|
// line: every release with all resolved fields for manual validation.
|
|
for group, parsed := range groups.ParseByGroup(titles) {
|
|
writeJSON(filepath.Join(outDir, safeName(group)+".json"), parsed)
|
|
}
|
|
|
|
// Per-request report identical to what is printed to the console.
|
|
writeJSON(filepath.Join(outDir, safeName(releaser)+".report.json"), out[releaser])
|
|
}
|
|
|
|
data, err := json.MarshalIndent(out, "", " ")
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
|
|
// Ручна ревізія: всі невідомі токени — у файл для правки бази.
|
|
review, err := os.OpenFile("review_unknown_tokens.txt",
|
|
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o644)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
defer review.Close()
|
|
for releaser, stats := range out {
|
|
for _, unk := range stats.UnknownTokens {
|
|
for _, rel := range unk.Releases {
|
|
fmt.Fprintf(review, "%s\t%s\t%s\n", releaser, unk.Token, rel)
|
|
}
|
|
}
|
|
for _, warn := range stats.WarningStats {
|
|
for _, rel := range warn.Releases {
|
|
fmt.Fprintf(review, "%s\tWARN\t%s\t%s\n", releaser, warn.Token, rel)
|
|
}
|
|
}
|
|
}
|
|
|
|
fmt.Println(string(data))
|
|
}
|
|
|
|
// writeJSON writes value as an indented JSON document to the given path.
|
|
func writeJSON(path string, value any) {
|
|
data, err := json.MarshalIndent(value, "", " ")
|
|
if err != nil {
|
|
log.Fatalf("marshal %s: %v", path, err)
|
|
}
|
|
if err := os.WriteFile(path, data, 0o644); err != nil {
|
|
log.Fatalf("write %s: %v", path, err)
|
|
}
|
|
}
|
|
|
|
// safeName strips characters that are unsafe in file names.
|
|
func safeName(s string) string {
|
|
repl := func(r rune) rune {
|
|
switch {
|
|
case r >= 'a' && r <= 'z':
|
|
return r
|
|
case r >= 'A' && r <= 'Z':
|
|
return r
|
|
case r >= '0' && r <= '9':
|
|
return r
|
|
case r == '-' || r == '_' || r == '.':
|
|
return r
|
|
default:
|
|
return '_'
|
|
}
|
|
}
|
|
return strings.Map(repl, s)
|
|
}
|