feat(prescene): parse single scnlog release for --single-uri page::1m

Handle the page::1m marker before the generic single-uri branch (previously unreachable): - select the latest prescene source from external_sources with meta IS NULL (sql.ErrNoRows -> no-op) - parse article.single: type_id from post-(\d+), size_mb, genre, unique artists, tracks count and total duration (h:M:s) - download the release NFO (anchor "Download NFO" -> onclick href) and save it to <nfo_dir>/<first-letter-of-title>/<title>.nfo - update external_sources.meta with compact JSON by id via db.Exec, writing {} when every field is empty Add model.ScnlogMeta / model.ScnlogSource structures.
2026-10-01 07:27:26 +03:00
parent a2bfa8ea71
commit ebb86d094a
3 changed files with 237 additions and 1 deletions
@@ -3,6 +3,7 @@ endpoint: https://scnlog.me/music/flac/
 endpoint-next: page/%s/
 sleep-before-next-iteration: 5s
 levels-to-scrape: 6
+nfo_dir: k:\data\mdb\nfo
 groups:
  tags:
    neformat:
@@ -1,9 +1,15 @@
 package repository

 import (
+	"database/sql"
+	"encoding/json/v2"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
+	"net/http"
+	"os"
+	"path/filepath"
 	"regexp"
 	"slices"
 	"strconv"
@@ -46,7 +52,14 @@ func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error
 	endpoint := viper.GetString(constant.CfgKeyEndpoint)

 	uri := viper.GetString(constant.FlagSingleUri)
-	if uri != "" {
+
+	if uri == "page::1m" {
+		if result, err := parseSingleScnlog(s.db); err != nil {
+			slog.Error("parsing single scnlog page", "err", err)
+		} else {
+			entries = append(entries, result...)
+		}
+	} else if uri != "" {
 		url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/"))
 		result, _ := parseUrl(url, s.db)
 		entries = append(entries, result...)
@@ -151,3 +164,204 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {

 	return entries, nil
 }
+
+func parseSingleScnlog(db *sqlx.DB) ([]model.ExternalSources, error) {
+	entries := make([]model.ExternalSources, 0)
+
+	const query = "SELECT id, type_id, title, meta, eXsource " +
+		"FROM theamok_mdb.external_sources " +
+		"WHERE type = 'prescene' AND created >= DATE_SUB(NOW(), INTERVAL 1 MONTH) AND meta IS NULL " +
+		"ORDER BY created DESC LIMIT 1"
+
+	var src model.ScnlogSource
+	if err := db.Get(&src, query); err != nil {
+		if errors.Is(err, sql.ErrNoRows) {
+			slog.Info("no prescene rows with meta IS NULL - nothing to do")
+			return entries, nil
+		}
+		return nil, err
+	}
+
+	doc, err := parser.HTMLSourceFromURL(src.ExSource)
+	if err != nil {
+		return nil, err
+	}
+	if doc == nil {
+		return nil, errors.New("document is nil")
+	}
+
+	article := dom.QuerySelector(doc, "article.single")
+	if article == nil {
+		return nil, fmt.Errorf("article.single not found: %s", src.ExSource)
+	}
+
+	// type_id з class="single post-5227778 ..."
+	typeId := src.TypeId
+	if m := regexp.MustCompile(`post-(\d+)`).FindStringSubmatch(dom.GetAttribute(article, "class")); len(m) > 1 {
+		if v, err := strconv.ParseInt(m[1], 10, 64); err == nil {
+			typeId = v
+		}
+	}
+
+	content := dom.QuerySelector(article, ".entry-content")
+	contentText := ""
+	if content != nil {
+		contentText = dom.TextContent(content)
+	}
+
+	meta := model.ScnlogMeta{}
+
+	// size_mb — тільки число з "Size: 217.26 MB"
+	if m := regexp.MustCompile(`Size:\s*([\d.]+)\s*MB`).FindStringSubmatch(contentText); len(m) > 1 {
+		if v, err := strconv.ParseFloat(m[1], 64); err == nil {
+			meta.SizeMb = &v
+		}
+	}
+
+	// genre — один рядок
+	if m := regexp.MustCompile(`Genre:\s*([^\n]+)`).FindStringSubmatch(contentText); len(m) > 1 {
+		if g := strings.TrimSpace(m[1]); g != "" {
+			meta.Genre = &g
+		}
+	}
+
+	// artist / tracks / duration з .mp3tbl
+	if tbl := dom.QuerySelector(article, ".mp3tbl"); tbl != nil {
+		all := make([]string, 0)
+		for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-artist") {
+			all = append(all, strings.TrimSpace(dom.TextContent(td)))
+		}
+		if unique := uniqueStrings(all); len(unique) > 0 {
+			meta.Artist = &unique
+		}
+
+		tracks := len(dom.QuerySelectorAll(tbl, "tbody tr"))
+		meta.Tracks = &tracks
+
+		total := 0
+		for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-time") {
+			total += durationToSeconds(strings.TrimSpace(dom.TextContent(td)))
+		}
+		duration := secondsToHMS(total)
+		meta.Duration = &duration
+	}
+
+	// NFO: анкор із текстом "Download NFO", шлях з onclick
+	if content == nil {
+		slog.Warn("entry-content not found", "url", src.ExSource)
+	} else {
+		for _, a := range dom.QuerySelectorAll(content, "a") {
+			if !strings.Contains(dom.TextContent(a), "Download NFO") {
+				continue
+			}
+			onclick := dom.GetAttribute(a, "onclick")
+			spew.Dump("NFO-ANCHOR", dom.TextContent(a), onclick)
+			if m := regexp.MustCompile(`'([^']+)'`).FindStringSubmatch(onclick); len(m) > 1 {
+				nfoURL := "https://scnlog.me" + m[1]
+				spew.Dump("NFO-URL", nfoURL, "NFO_DIR", viper.GetString("nfo_dir"))
+				if err := saveNfo(src.Title, nfoURL); err != nil {
+					return nil, err
+				}
+			}
+			break
+		}
+	}
+
+	payload, err := json.Marshal(meta) // compact = "uglify"
+	if err != nil {
+		return nil, err
+	}
+	// якщо всі поля порожні — пишемо порожній об'єкт, а не набір null
+	if meta.SizeMb == nil && meta.Genre == nil && meta.Artist == nil && meta.Tracks == nil && meta.Duration == nil {
+		payload = []byte("{}")
+	}
+
+	if _, err := db.Exec(
+		"UPDATE `theamok_mdb`.`external_sources` SET `meta` = ? WHERE `id` = ?",
+		string(payload), src.Id,
+	); err != nil {
+		return nil, err
+	}
+
+	slog.Info("scnlog meta updated",
+		"id", src.Id, "type_id", typeId, "title", src.Title, "meta", string(payload))
+
+	return entries, nil
+}
+
+// uniqueStrings прибирає дублікати, зберігаючи порядок першої появи.
+func uniqueStrings(in []string) []string {
+	seen := make(map[string]struct{}, len(in))
+	out := make([]string, 0, len(in))
+	for _, s := range in {
+		if _, ok := seen[s]; ok {
+			continue
+		}
+		seen[s] = struct{}{}
+		out = append(out, s)
+	}
+	return out
+}
+
+// durationToSeconds парсить "m:ss" / "h:mm:ss" у секунди.
+func durationToSeconds(v string) int {
+	parts := strings.Split(strings.TrimSpace(v), ":")
+	if len(parts) == 0 {
+		return 0
+	}
+	total := 0
+	for _, p := range parts {
+		n, _ := strconv.Atoi(strings.TrimSpace(p))
+		total = total*60 + n
+	}
+	return total
+}
+
+// secondsToHMS форматує секунди у "h:M:s" (напр. 1802 -> "0:30:02").
+func secondsToHMS(total int) string {
+	return fmt.Sprintf("%d:%02d:%02d", total/3600, (total%3600)/60, total%60)
+}
+
+// saveNfo пише NFO у <nfo_dir>/<перша літера title в lower>/<title>.nfo
+func saveNfo(title, nfoURL string) error {
+	dir := viper.GetString("nfo_dir")
+	if dir == "" {
+		return errors.New("nfo_dir is empty")
+	}
+
+	letter := strings.ToLower(string([]rune(title)[0]))
+	sub := filepath.Join(dir, letter)
+	if err := os.MkdirAll(sub, 0o755); err != nil {
+		return err
+	}
+
+	data, err := fetchNfoBytes(nfoURL)
+	if err != nil {
+		return err
+	}
+
+	return os.WriteFile(filepath.Join(sub, title+".nfo"), data, 0o644)
+}
+
+// fetchNfoBytes завантажує сирі байти NFO (не HTML) у []byte.
+func fetchNfoBytes(nfoURL string) ([]byte, error) {
+	client := &http.Client{Timeout: 30 * time.Second}
+
+	req, err := http.NewRequest(http.MethodGet, nfoURL, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; resource-scraper)")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, nfoURL)
+	}
+
+	return io.ReadAll(resp.Body)
+}
@@ -0,0 +1,21 @@
+package model
+
+import "database/sql"
+
+// ScnlogMeta — структура, що кладеться в external_sources.meta (compact JSON).
+type ScnlogMeta struct {
+	SizeMb   *float64  `json:"size_mb"`
+	Genre    *string   `json:"genre"`
+	Artist   *[]string `json:"artist"`
+	Tracks   *int      `json:"tracks"`
+	Duration *string   `json:"duration"`
+}
+
+// ScnlogSource — рядок вибірки останнього необробленого prescene-джерела.
+type ScnlogSource struct {
+	Id       int64          `db:"id"`
+	TypeId   int64          `db:"type_id"`
+	Title    string         `db:"title"`
+	Meta     sql.NullString `db:"meta"`
+	ExSource string         `db:"eXsource"`
+}