From ebb86d094a07684b720246ab9227bf0b0e67c68f Mon Sep 17 00:00:00 2001 From: Yevhen Odynets <6lo5jp@proton.me> Date: Thu, 1 Oct 2026 07:27:26 +0300 Subject: [PATCH] feat(prescene): parse single scnlog release for --single-uri page::1m Handle the page::1m marker before the generic single-uri branch (previously unreachable): - select the latest prescene source from external_sources with meta IS NULL (sql.ErrNoRows -> no-op) - parse article.single: type_id from post-(\d+), size_mb, genre, unique artists, tracks count and total duration (h:M:s) - download the release NFO (anchor "Download NFO" -> onclick href) and save it to //.nfo - update external_sources.meta with compact JSON by id via db.Exec, writing {} when every field is empty Add model.ScnlogMeta / model.ScnlogSource structures. --- config/scope/prescene.yaml | 1 + pkg/repository/prescene.go | 216 ++++++++++++++++++++++++++++++++++++- types/model/prescene.go | 21 ++++ 3 files changed, 237 insertions(+), 1 deletion(-) create mode 100644 types/model/prescene.go diff --git a/config/scope/prescene.yaml b/config/scope/prescene.yaml index 394a34c..48c5478 100644 --- a/config/scope/prescene.yaml +++ b/config/scope/prescene.yaml @@ -3,6 +3,7 @@ endpoint: https://scnlog.me/music/flac/ endpoint-next: page/%s/ sleep-before-next-iteration: 5s levels-to-scrape: 6 +nfo_dir: k:\data\mdb\nfo groups: tags: neformat: diff --git a/pkg/repository/prescene.go b/pkg/repository/prescene.go index afdc981..0f0abef 100644 --- a/pkg/repository/prescene.go +++ b/pkg/repository/prescene.go @@ -1,9 +1,15 @@ package repository import ( + "database/sql" + "encoding/json/v2" "errors" "fmt" + "io" "log/slog" + "net/http" + "os" + "path/filepath" "regexp" "slices" "strconv" @@ -46,7 +52,14 @@ func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error endpoint := viper.GetString(constant.CfgKeyEndpoint) uri := viper.GetString(constant.FlagSingleUri) - if uri != "" { + + if uri == "page::1m" { + if result, err := parseSingleScnlog(s.db); err != nil { + slog.Error("parsing single scnlog page", "err", err) + } else { + entries = append(entries, result...) + } + } else if uri != "" { url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/")) result, _ := parseUrl(url, s.db) entries = append(entries, result...) @@ -151,3 +164,204 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) { return entries, nil } + +func parseSingleScnlog(db *sqlx.DB) ([]model.ExternalSources, error) { + entries := make([]model.ExternalSources, 0) + + const query = "SELECT id, type_id, title, meta, eXsource " + + "FROM theamok_mdb.external_sources " + + "WHERE type = 'prescene' AND created >= DATE_SUB(NOW(), INTERVAL 1 MONTH) AND meta IS NULL " + + "ORDER BY created DESC LIMIT 1" + + var src model.ScnlogSource + if err := db.Get(&src, query); err != nil { + if errors.Is(err, sql.ErrNoRows) { + slog.Info("no prescene rows with meta IS NULL - nothing to do") + return entries, nil + } + return nil, err + } + + doc, err := parser.HTMLSourceFromURL(src.ExSource) + if err != nil { + return nil, err + } + if doc == nil { + return nil, errors.New("document is nil") + } + + article := dom.QuerySelector(doc, "article.single") + if article == nil { + return nil, fmt.Errorf("article.single not found: %s", src.ExSource) + } + + // type_id з class="single post-5227778 ..." + typeId := src.TypeId + if m := regexp.MustCompile(`post-(\d+)`).FindStringSubmatch(dom.GetAttribute(article, "class")); len(m) > 1 { + if v, err := strconv.ParseInt(m[1], 10, 64); err == nil { + typeId = v + } + } + + content := dom.QuerySelector(article, ".entry-content") + contentText := "" + if content != nil { + contentText = dom.TextContent(content) + } + + meta := model.ScnlogMeta{} + + // size_mb — тільки число з "Size: 217.26 MB" + if m := regexp.MustCompile(`Size:\s*([\d.]+)\s*MB`).FindStringSubmatch(contentText); len(m) > 1 { + if v, err := strconv.ParseFloat(m[1], 64); err == nil { + meta.SizeMb = &v + } + } + + // genre — один рядок + if m := regexp.MustCompile(`Genre:\s*([^\n]+)`).FindStringSubmatch(contentText); len(m) > 1 { + if g := strings.TrimSpace(m[1]); g != "" { + meta.Genre = &g + } + } + + // artist / tracks / duration з .mp3tbl + if tbl := dom.QuerySelector(article, ".mp3tbl"); tbl != nil { + all := make([]string, 0) + for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-artist") { + all = append(all, strings.TrimSpace(dom.TextContent(td))) + } + if unique := uniqueStrings(all); len(unique) > 0 { + meta.Artist = &unique + } + + tracks := len(dom.QuerySelectorAll(tbl, "tbody tr")) + meta.Tracks = &tracks + + total := 0 + for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-time") { + total += durationToSeconds(strings.TrimSpace(dom.TextContent(td))) + } + duration := secondsToHMS(total) + meta.Duration = &duration + } + + // NFO: анкор із текстом "Download NFO", шлях з onclick + if content == nil { + slog.Warn("entry-content not found", "url", src.ExSource) + } else { + for _, a := range dom.QuerySelectorAll(content, "a") { + if !strings.Contains(dom.TextContent(a), "Download NFO") { + continue + } + onclick := dom.GetAttribute(a, "onclick") + spew.Dump("NFO-ANCHOR", dom.TextContent(a), onclick) + if m := regexp.MustCompile(`'([^']+)'`).FindStringSubmatch(onclick); len(m) > 1 { + nfoURL := "https://scnlog.me" + m[1] + spew.Dump("NFO-URL", nfoURL, "NFO_DIR", viper.GetString("nfo_dir")) + if err := saveNfo(src.Title, nfoURL); err != nil { + return nil, err + } + } + break + } + } + + payload, err := json.Marshal(meta) // compact = "uglify" + if err != nil { + return nil, err + } + // якщо всі поля порожні — пишемо порожній об'єкт, а не набір null + if meta.SizeMb == nil && meta.Genre == nil && meta.Artist == nil && meta.Tracks == nil && meta.Duration == nil { + payload = []byte("{}") + } + + if _, err := db.Exec( + "UPDATE `theamok_mdb`.`external_sources` SET `meta` = ? WHERE `id` = ?", + string(payload), src.Id, + ); err != nil { + return nil, err + } + + slog.Info("scnlog meta updated", + "id", src.Id, "type_id", typeId, "title", src.Title, "meta", string(payload)) + + return entries, nil +} + +// uniqueStrings прибирає дублікати, зберігаючи порядок першої появи. +func uniqueStrings(in []string) []string { + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} + +// durationToSeconds парсить "m:ss" / "h:mm:ss" у секунди. +func durationToSeconds(v string) int { + parts := strings.Split(strings.TrimSpace(v), ":") + if len(parts) == 0 { + return 0 + } + total := 0 + for _, p := range parts { + n, _ := strconv.Atoi(strings.TrimSpace(p)) + total = total*60 + n + } + return total +} + +// secondsToHMS форматує секунди у "h:M:s" (напр. 1802 -> "0:30:02"). +func secondsToHMS(total int) string { + return fmt.Sprintf("%d:%02d:%02d", total/3600, (total%3600)/60, total%60) +} + +// saveNfo пише NFO у <nfo_dir>/<перша літера title в lower>/<title>.nfo +func saveNfo(title, nfoURL string) error { + dir := viper.GetString("nfo_dir") + if dir == "" { + return errors.New("nfo_dir is empty") + } + + letter := strings.ToLower(string([]rune(title)[0])) + sub := filepath.Join(dir, letter) + if err := os.MkdirAll(sub, 0o755); err != nil { + return err + } + + data, err := fetchNfoBytes(nfoURL) + if err != nil { + return err + } + + return os.WriteFile(filepath.Join(sub, title+".nfo"), data, 0o644) +} + +// fetchNfoBytes завантажує сирі байти NFO (не HTML) у []byte. +func fetchNfoBytes(nfoURL string) ([]byte, error) { + client := &http.Client{Timeout: 30 * time.Second} + + req, err := http.NewRequest(http.MethodGet, nfoURL, nil) + if err != nil { + return nil, err + } + req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; resource-scraper)") + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, nfoURL) + } + + return io.ReadAll(resp.Body) +} diff --git a/types/model/prescene.go b/types/model/prescene.go new file mode 100644 index 0000000..81d038e --- /dev/null +++ b/types/model/prescene.go @@ -0,0 +1,21 @@ +package model + +import "database/sql" + +// ScnlogMeta — структура, що кладеться в external_sources.meta (compact JSON). +type ScnlogMeta struct { + SizeMb *float64 `json:"size_mb"` + Genre *string `json:"genre"` + Artist *[]string `json:"artist"` + Tracks *int `json:"tracks"` + Duration *string `json:"duration"` +} + +// ScnlogSource — рядок вибірки останнього необробленого prescene-джерела. +type ScnlogSource struct { + Id int64 `db:"id"` + TypeId int64 `db:"type_id"` + Title string `db:"title"` + Meta sql.NullString `db:"meta"` + ExSource string `db:"eXsource"` +}