Yevhen Odynets
ebb86d094a
Handle the page::1m marker before the generic single-uri branch
(previously unreachable):
- select the latest prescene source from external_sources with meta IS NULL
(sql.ErrNoRows -> no-op)
- parse article.single: type_id from post-(\d+), size_mb, genre,
unique artists, tracks count and total duration (h:M:s)
- download the release NFO (anchor "Download NFO" -> onclick href)
and save it to <nfo_dir>/<first-letter-of-title>/<title>.nfo
- update external_sources.meta with compact JSON by id via db.Exec,
writing {} when every field is empty
Add model.ScnlogMeta / model.ScnlogSource structures.
368 lines
10 KiB
Go
368 lines
10 KiB
Go
package repository
|
||
|
||
import (
|
||
"database/sql"
|
||
"encoding/json/v2"
|
||
"errors"
|
||
"fmt"
|
||
"io"
|
||
"log/slog"
|
||
"net/http"
|
||
"os"
|
||
"path/filepath"
|
||
"regexp"
|
||
"slices"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
|
||
"github.com/davecgh/go-spew/spew"
|
||
"github.com/dromara/carbon/v2"
|
||
"github.com/go-shiori/dom"
|
||
"github.com/jmoiron/sqlx"
|
||
"github.com/spf13/viper"
|
||
|
||
"git.kplus.net.ua/yevhen/resource-scraper/helper/parser"
|
||
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
|
||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
||
)
|
||
|
||
type Prescene struct {
|
||
db *sqlx.DB
|
||
}
|
||
|
||
func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) {
|
||
entries := make([]model.ExternalSources, 0)
|
||
now := time.Now()
|
||
yesterday := now.Add(-24 * time.Hour)
|
||
todayStr := now.Format("2006-01-02")
|
||
yesterdayStr := yesterday.Format("2006-01-02")
|
||
spew.Dump(entries, todayStr, yesterdayStr)
|
||
|
||
return []model.ExternalSources{}, nil
|
||
}
|
||
|
||
func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
||
return &Prescene{db: db}
|
||
}
|
||
|
||
func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
||
entries := make([]model.ExternalSources, 0)
|
||
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
||
|
||
uri := viper.GetString(constant.FlagSingleUri)
|
||
|
||
if uri == "page::1m" {
|
||
if result, err := parseSingleScnlog(s.db); err != nil {
|
||
slog.Error("parsing single scnlog page", "err", err)
|
||
} else {
|
||
entries = append(entries, result...)
|
||
}
|
||
} else if uri != "" {
|
||
url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/"))
|
||
result, _ := parseUrl(url, s.db)
|
||
entries = append(entries, result...)
|
||
} else {
|
||
for _, t := range pageNumbers {
|
||
if t != "1" {
|
||
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
|
||
}
|
||
|
||
//doc, err := parser.HTMLSourceFromURL("https://mdb.kplus.net.ua /$/scnlog.html")
|
||
if result, err := parseUrl(endpoint, s.db); err == nil {
|
||
entries = append(entries, result...)
|
||
} else {
|
||
slog.Error("parsing url", "err", err)
|
||
}
|
||
|
||
//fmt.Println("Sleeping...", j)
|
||
time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
|
||
}
|
||
}
|
||
|
||
return entries, nil
|
||
}
|
||
|
||
func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
|
||
entries := make([]model.ExternalSources, 0)
|
||
tags := viper.GetStringMapStringSlice("groups.tags")
|
||
slog.Info("singleton", "url", endpoint)
|
||
doc, err := parser.HTMLSourceFromURL(endpoint)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
if doc == nil {
|
||
return nil, errors.New("document is nil")
|
||
}
|
||
var validID = regexp.MustCompile(`-\d+\/$`)
|
||
|
||
for i, item := range dom.QuerySelectorAll(doc, "li.row") {
|
||
var es model.ExternalSources
|
||
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
|
||
|
||
// TypeId тепер напряму з data-id на <li>, без regex по class
|
||
idAttr := dom.GetAttribute(item, "data-id")
|
||
es.TypeId, _ = strconv.Atoi(idAttr)
|
||
|
||
title := dom.QuerySelector(item, ".title")
|
||
if title != nil {
|
||
anchor := dom.QuerySelector(title, "a")
|
||
if anchor != nil {
|
||
es.Type = constant.ScopePrescene
|
||
|
||
titleSpan := dom.QuerySelector(anchor, "span.title-start")
|
||
if titleSpan != nil {
|
||
es.Title = dom.TextContent(titleSpan)
|
||
}
|
||
|
||
if es.Title == "Auto Draft" {
|
||
slog.Info("Skipped", "title", es.Title)
|
||
continue
|
||
}
|
||
|
||
es.ExSource = dom.GetAttribute(anchor, "href")
|
||
if validID.MatchString(es.ExSource) {
|
||
continue
|
||
}
|
||
|
||
pattern := regexp.MustCompile(`(?is)-(\w+)$`)
|
||
es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
|
||
|
||
for flag, groups := range tags {
|
||
if slices.Contains(groups, es.Releaser) {
|
||
es.A = flag
|
||
es.H = flag
|
||
columns = append(columns, "a", "h")
|
||
break
|
||
}
|
||
}
|
||
}
|
||
|
||
if es.A == constant.TagIgnore {
|
||
slog.Info("Skipped", "releaser", es.Releaser)
|
||
continue
|
||
}
|
||
|
||
localtime := dom.QuerySelector(item, "span.localtime")
|
||
if localtime != nil {
|
||
lc := dom.GetAttribute(localtime, "data-lttime")
|
||
es.Created = carbon.Parse(lc, "Europe/Kyiv")
|
||
}
|
||
}
|
||
|
||
esModel := table.ExternalSources{Columns: columns}
|
||
entry := esModel.InsertOnDuplicate(es, db)
|
||
entries = append(entries, entry)
|
||
|
||
fmt.Println("====================== ", i, " ==============================")
|
||
//fmt.Printf("%+v\n", es)
|
||
fmt.Printf("%s %d %d %s\n", entry.Created.ToString(), entry.Id, entry.TypeId, entry.Title)
|
||
//spew.Dump(entry)
|
||
}
|
||
|
||
return entries, nil
|
||
}
|
||
|
||
func parseSingleScnlog(db *sqlx.DB) ([]model.ExternalSources, error) {
|
||
entries := make([]model.ExternalSources, 0)
|
||
|
||
const query = "SELECT id, type_id, title, meta, eXsource " +
|
||
"FROM theamok_mdb.external_sources " +
|
||
"WHERE type = 'prescene' AND created >= DATE_SUB(NOW(), INTERVAL 1 MONTH) AND meta IS NULL " +
|
||
"ORDER BY created DESC LIMIT 1"
|
||
|
||
var src model.ScnlogSource
|
||
if err := db.Get(&src, query); err != nil {
|
||
if errors.Is(err, sql.ErrNoRows) {
|
||
slog.Info("no prescene rows with meta IS NULL - nothing to do")
|
||
return entries, nil
|
||
}
|
||
return nil, err
|
||
}
|
||
|
||
doc, err := parser.HTMLSourceFromURL(src.ExSource)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if doc == nil {
|
||
return nil, errors.New("document is nil")
|
||
}
|
||
|
||
article := dom.QuerySelector(doc, "article.single")
|
||
if article == nil {
|
||
return nil, fmt.Errorf("article.single not found: %s", src.ExSource)
|
||
}
|
||
|
||
// type_id з class="single post-5227778 ..."
|
||
typeId := src.TypeId
|
||
if m := regexp.MustCompile(`post-(\d+)`).FindStringSubmatch(dom.GetAttribute(article, "class")); len(m) > 1 {
|
||
if v, err := strconv.ParseInt(m[1], 10, 64); err == nil {
|
||
typeId = v
|
||
}
|
||
}
|
||
|
||
content := dom.QuerySelector(article, ".entry-content")
|
||
contentText := ""
|
||
if content != nil {
|
||
contentText = dom.TextContent(content)
|
||
}
|
||
|
||
meta := model.ScnlogMeta{}
|
||
|
||
// size_mb — тільки число з "Size: 217.26 MB"
|
||
if m := regexp.MustCompile(`Size:\s*([\d.]+)\s*MB`).FindStringSubmatch(contentText); len(m) > 1 {
|
||
if v, err := strconv.ParseFloat(m[1], 64); err == nil {
|
||
meta.SizeMb = &v
|
||
}
|
||
}
|
||
|
||
// genre — один рядок
|
||
if m := regexp.MustCompile(`Genre:\s*([^\n]+)`).FindStringSubmatch(contentText); len(m) > 1 {
|
||
if g := strings.TrimSpace(m[1]); g != "" {
|
||
meta.Genre = &g
|
||
}
|
||
}
|
||
|
||
// artist / tracks / duration з .mp3tbl
|
||
if tbl := dom.QuerySelector(article, ".mp3tbl"); tbl != nil {
|
||
all := make([]string, 0)
|
||
for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-artist") {
|
||
all = append(all, strings.TrimSpace(dom.TextContent(td)))
|
||
}
|
||
if unique := uniqueStrings(all); len(unique) > 0 {
|
||
meta.Artist = &unique
|
||
}
|
||
|
||
tracks := len(dom.QuerySelectorAll(tbl, "tbody tr"))
|
||
meta.Tracks = &tracks
|
||
|
||
total := 0
|
||
for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-time") {
|
||
total += durationToSeconds(strings.TrimSpace(dom.TextContent(td)))
|
||
}
|
||
duration := secondsToHMS(total)
|
||
meta.Duration = &duration
|
||
}
|
||
|
||
// NFO: анкор із текстом "Download NFO", шлях з onclick
|
||
if content == nil {
|
||
slog.Warn("entry-content not found", "url", src.ExSource)
|
||
} else {
|
||
for _, a := range dom.QuerySelectorAll(content, "a") {
|
||
if !strings.Contains(dom.TextContent(a), "Download NFO") {
|
||
continue
|
||
}
|
||
onclick := dom.GetAttribute(a, "onclick")
|
||
spew.Dump("NFO-ANCHOR", dom.TextContent(a), onclick)
|
||
if m := regexp.MustCompile(`'([^']+)'`).FindStringSubmatch(onclick); len(m) > 1 {
|
||
nfoURL := "https://scnlog.me" + m[1]
|
||
spew.Dump("NFO-URL", nfoURL, "NFO_DIR", viper.GetString("nfo_dir"))
|
||
if err := saveNfo(src.Title, nfoURL); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
break
|
||
}
|
||
}
|
||
|
||
payload, err := json.Marshal(meta) // compact = "uglify"
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
// якщо всі поля порожні — пишемо порожній об'єкт, а не набір null
|
||
if meta.SizeMb == nil && meta.Genre == nil && meta.Artist == nil && meta.Tracks == nil && meta.Duration == nil {
|
||
payload = []byte("{}")
|
||
}
|
||
|
||
if _, err := db.Exec(
|
||
"UPDATE `theamok_mdb`.`external_sources` SET `meta` = ? WHERE `id` = ?",
|
||
string(payload), src.Id,
|
||
); err != nil {
|
||
return nil, err
|
||
}
|
||
|
||
slog.Info("scnlog meta updated",
|
||
"id", src.Id, "type_id", typeId, "title", src.Title, "meta", string(payload))
|
||
|
||
return entries, nil
|
||
}
|
||
|
||
// uniqueStrings прибирає дублікати, зберігаючи порядок першої появи.
|
||
func uniqueStrings(in []string) []string {
|
||
seen := make(map[string]struct{}, len(in))
|
||
out := make([]string, 0, len(in))
|
||
for _, s := range in {
|
||
if _, ok := seen[s]; ok {
|
||
continue
|
||
}
|
||
seen[s] = struct{}{}
|
||
out = append(out, s)
|
||
}
|
||
return out
|
||
}
|
||
|
||
// durationToSeconds парсить "m:ss" / "h:mm:ss" у секунди.
|
||
func durationToSeconds(v string) int {
|
||
parts := strings.Split(strings.TrimSpace(v), ":")
|
||
if len(parts) == 0 {
|
||
return 0
|
||
}
|
||
total := 0
|
||
for _, p := range parts {
|
||
n, _ := strconv.Atoi(strings.TrimSpace(p))
|
||
total = total*60 + n
|
||
}
|
||
return total
|
||
}
|
||
|
||
// secondsToHMS форматує секунди у "h:M:s" (напр. 1802 -> "0:30:02").
|
||
func secondsToHMS(total int) string {
|
||
return fmt.Sprintf("%d:%02d:%02d", total/3600, (total%3600)/60, total%60)
|
||
}
|
||
|
||
// saveNfo пише NFO у <nfo_dir>/<перша літера title в lower>/<title>.nfo
|
||
func saveNfo(title, nfoURL string) error {
|
||
dir := viper.GetString("nfo_dir")
|
||
if dir == "" {
|
||
return errors.New("nfo_dir is empty")
|
||
}
|
||
|
||
letter := strings.ToLower(string([]rune(title)[0]))
|
||
sub := filepath.Join(dir, letter)
|
||
if err := os.MkdirAll(sub, 0o755); err != nil {
|
||
return err
|
||
}
|
||
|
||
data, err := fetchNfoBytes(nfoURL)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
|
||
return os.WriteFile(filepath.Join(sub, title+".nfo"), data, 0o644)
|
||
}
|
||
|
||
// fetchNfoBytes завантажує сирі байти NFO (не HTML) у []byte.
|
||
func fetchNfoBytes(nfoURL string) ([]byte, error) {
|
||
client := &http.Client{Timeout: 30 * time.Second}
|
||
|
||
req, err := http.NewRequest(http.MethodGet, nfoURL, nil)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; resource-scraper)")
|
||
|
||
resp, err := client.Do(req)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
defer resp.Body.Close()
|
||
|
||
if resp.StatusCode != http.StatusOK {
|
||
return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, nfoURL)
|
||
}
|
||
|
||
return io.ReadAll(resp.Body)
|
||
}
|