package repository
import (
"database/sql"
"encoding/json/v2"
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"os"
"path/filepath"
"regexp"
"slices"
"strconv"
"strings"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/dromara/carbon/v2"
"github.com/go-shiori/dom"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"git.kplus.net.ua/yevhen/resource-scraper/helper/parser"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type Prescene struct {
db *sqlx.DB
}
func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) {
entries := make([]model.ExternalSources, 0)
now := time.Now()
yesterday := now.Add(-24 * time.Hour)
todayStr := now.Format("2006-01-02")
yesterdayStr := yesterday.Format("2006-01-02")
spew.Dump(entries, todayStr, yesterdayStr)
return []model.ExternalSources{}, nil
}
func NewPresceneRepository(db *sqlx.DB) *Prescene {
return &Prescene{db: db}
}
func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
entries := make([]model.ExternalSources, 0)
endpoint := viper.GetString(constant.CfgKeyEndpoint)
uri := viper.GetString(constant.FlagSingleUri)
if uri == "page::1m" {
if result, err := parseSingleScnlog(s.db); err != nil {
slog.Error("parsing single scnlog page", "err", err)
} else {
entries = append(entries, result...)
}
} else if uri != "" {
url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/"))
result, _ := parseUrl(url, s.db)
entries = append(entries, result...)
} else {
for _, t := range pageNumbers {
if t != "1" {
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
}
//doc, err := parser.HTMLSourceFromURL("https://mdb.kplus.net.ua /$/scnlog.html")
if result, err := parseUrl(endpoint, s.db); err == nil {
entries = append(entries, result...)
} else {
slog.Error("parsing url", "err", err)
}
//fmt.Println("Sleeping...", j)
time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
}
}
return entries, nil
}
func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
entries := make([]model.ExternalSources, 0)
tags := viper.GetStringMapStringSlice("groups.tags")
slog.Info("singleton", "url", endpoint)
doc, err := parser.HTMLSourceFromURL(endpoint)
if err != nil {
return nil, err
}
if doc == nil {
return nil, errors.New("document is nil")
}
var validID = regexp.MustCompile(`-\d+\/$`)
for i, item := range dom.QuerySelectorAll(doc, "li.row") {
var es model.ExternalSources
columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
// TypeId тепер напряму з data-id на
, без regex по class
idAttr := dom.GetAttribute(item, "data-id")
es.TypeId, _ = strconv.Atoi(idAttr)
title := dom.QuerySelector(item, ".title")
if title != nil {
anchor := dom.QuerySelector(title, "a")
if anchor != nil {
es.Type = constant.ScopePrescene
titleSpan := dom.QuerySelector(anchor, "span.title-start")
if titleSpan != nil {
es.Title = dom.TextContent(titleSpan)
}
if es.Title == "Auto Draft" {
slog.Info("Skipped", "title", es.Title)
continue
}
es.ExSource = dom.GetAttribute(anchor, "href")
if validID.MatchString(es.ExSource) {
continue
}
pattern := regexp.MustCompile(`(?is)-(\w+)$`)
es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
for flag, groups := range tags {
if slices.Contains(groups, es.Releaser) {
es.A = flag
es.H = flag
columns = append(columns, "a", "h")
break
}
}
}
if es.A == constant.TagIgnore {
slog.Info("Skipped", "releaser", es.Releaser)
continue
}
localtime := dom.QuerySelector(item, "span.localtime")
if localtime != nil {
lc := dom.GetAttribute(localtime, "data-lttime")
es.Created = carbon.Parse(lc, "Europe/Kyiv")
}
}
esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, db)
entries = append(entries, entry)
fmt.Println("====================== ", i, " ==============================")
//fmt.Printf("%+v\n", es)
fmt.Printf("%s %d %d %s\n", entry.Created.ToString(), entry.Id, entry.TypeId, entry.Title)
//spew.Dump(entry)
}
return entries, nil
}
func parseSingleScnlog(db *sqlx.DB) ([]model.ExternalSources, error) {
entries := make([]model.ExternalSources, 0)
const query = "SELECT id, type_id, title, meta, eXsource " +
"FROM theamok_mdb.external_sources " +
"WHERE type = 'prescene' AND created >= DATE_SUB(NOW(), INTERVAL 1 MONTH) AND meta IS NULL " +
"ORDER BY created DESC LIMIT 1"
var src model.ScnlogSource
if err := db.Get(&src, query); err != nil {
if errors.Is(err, sql.ErrNoRows) {
slog.Info("no prescene rows with meta IS NULL - nothing to do")
return entries, nil
}
return nil, err
}
doc, err := parser.HTMLSourceFromURL(src.ExSource)
if err != nil {
return nil, err
}
if doc == nil {
return nil, errors.New("document is nil")
}
article := dom.QuerySelector(doc, "article.single")
if article == nil {
return nil, fmt.Errorf("article.single not found: %s", src.ExSource)
}
// type_id з class="single post-5227778 ..."
typeId := src.TypeId
if m := regexp.MustCompile(`post-(\d+)`).FindStringSubmatch(dom.GetAttribute(article, "class")); len(m) > 1 {
if v, err := strconv.ParseInt(m[1], 10, 64); err == nil {
typeId = v
}
}
content := dom.QuerySelector(article, ".entry-content")
contentText := ""
if content != nil {
contentText = dom.TextContent(content)
}
meta := model.ScnlogMeta{}
// size_mb — тільки число з "Size: 217.26 MB"
if m := regexp.MustCompile(`Size:\s*([\d.]+)\s*MB`).FindStringSubmatch(contentText); len(m) > 1 {
if v, err := strconv.ParseFloat(m[1], 64); err == nil {
meta.SizeMb = &v
}
}
// genre — один рядок
if m := regexp.MustCompile(`Genre:\s*([^\n]+)`).FindStringSubmatch(contentText); len(m) > 1 {
if g := strings.TrimSpace(m[1]); g != "" {
meta.Genre = &g
}
}
// artist / tracks / duration з .mp3tbl
if tbl := dom.QuerySelector(article, ".mp3tbl"); tbl != nil {
all := make([]string, 0)
for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-artist") {
all = append(all, strings.TrimSpace(dom.TextContent(td)))
}
if unique := uniqueStrings(all); len(unique) > 0 {
meta.Artist = &unique
}
tracks := len(dom.QuerySelectorAll(tbl, "tbody tr"))
meta.Tracks = &tracks
total := 0
for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-time") {
total += durationToSeconds(strings.TrimSpace(dom.TextContent(td)))
}
duration := secondsToHMS(total)
meta.Duration = &duration
}
// NFO: анкор із текстом "Download NFO", шлях з onclick
if content == nil {
slog.Warn("entry-content not found", "url", src.ExSource)
} else {
for _, a := range dom.QuerySelectorAll(content, "a") {
if !strings.Contains(dom.TextContent(a), "Download NFO") {
continue
}
onclick := dom.GetAttribute(a, "onclick")
spew.Dump("NFO-ANCHOR", dom.TextContent(a), onclick)
if m := regexp.MustCompile(`'([^']+)'`).FindStringSubmatch(onclick); len(m) > 1 {
nfoURL := "https://scnlog.me" + m[1]
spew.Dump("NFO-URL", nfoURL, "NFO_DIR", viper.GetString("nfo_dir"))
if err := saveNfo(src.Title, nfoURL); err != nil {
return nil, err
}
}
break
}
}
payload, err := json.Marshal(meta) // compact = "uglify"
if err != nil {
return nil, err
}
// якщо всі поля порожні — пишемо порожній об'єкт, а не набір null
if meta.SizeMb == nil && meta.Genre == nil && meta.Artist == nil && meta.Tracks == nil && meta.Duration == nil {
payload = []byte("{}")
}
if _, err := db.Exec(
"UPDATE `theamok_mdb`.`external_sources` SET `meta` = ? WHERE `id` = ?",
string(payload), src.Id,
); err != nil {
return nil, err
}
slog.Info("scnlog meta updated",
"id", src.Id, "type_id", typeId, "title", src.Title, "meta", string(payload))
return entries, nil
}
// uniqueStrings прибирає дублікати, зберігаючи порядок першої появи.
func uniqueStrings(in []string) []string {
seen := make(map[string]struct{}, len(in))
out := make([]string, 0, len(in))
for _, s := range in {
if _, ok := seen[s]; ok {
continue
}
seen[s] = struct{}{}
out = append(out, s)
}
return out
}
// durationToSeconds парсить "m:ss" / "h:mm:ss" у секунди.
func durationToSeconds(v string) int {
parts := strings.Split(strings.TrimSpace(v), ":")
if len(parts) == 0 {
return 0
}
total := 0
for _, p := range parts {
n, _ := strconv.Atoi(strings.TrimSpace(p))
total = total*60 + n
}
return total
}
// secondsToHMS форматує секунди у "h:M:s" (напр. 1802 -> "0:30:02").
func secondsToHMS(total int) string {
return fmt.Sprintf("%d:%02d:%02d", total/3600, (total%3600)/60, total%60)
}
// saveNfo пише NFO у /<перша літера title в lower>/.nfo
func saveNfo(title, nfoURL string) error {
dir := viper.GetString("nfo_dir")
if dir == "" {
return errors.New("nfo_dir is empty")
}
letter := strings.ToLower(string([]rune(title)[0]))
sub := filepath.Join(dir, letter)
if err := os.MkdirAll(sub, 0o755); err != nil {
return err
}
data, err := fetchNfoBytes(nfoURL)
if err != nil {
return err
}
return os.WriteFile(filepath.Join(sub, title+".nfo"), data, 0o644)
}
// fetchNfoBytes завантажує сирі байти NFO (не HTML) у []byte.
func fetchNfoBytes(nfoURL string) ([]byte, error) {
client := &http.Client{Timeout: 30 * time.Second}
req, err := http.NewRequest(http.MethodGet, nfoURL, nil)
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; resource-scraper)")
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, nfoURL)
}
return io.ReadAll(resp.Body)
}