package repository import ( "database/sql" "encoding/json/v2" "errors" "fmt" "io" "log/slog" "net/http" "os" "path/filepath" "regexp" "slices" "strconv" "strings" "time" "github.com/davecgh/go-spew/spew" "github.com/dromara/carbon/v2" "github.com/go-shiori/dom" "github.com/jmoiron/sqlx" "github.com/spf13/viper" "git.kplus.net.ua/yevhen/resource-scraper/helper/parser" "git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table" "git.kplus.net.ua/yevhen/resource-scraper/types/constant" "git.kplus.net.ua/yevhen/resource-scraper/types/model" ) type Prescene struct { db *sqlx.DB } func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) { entries := make([]model.ExternalSources, 0) now := time.Now() yesterday := now.Add(-24 * time.Hour) todayStr := now.Format("2006-01-02") yesterdayStr := yesterday.Format("2006-01-02") spew.Dump(entries, todayStr, yesterdayStr) return []model.ExternalSources{}, nil } func NewPresceneRepository(db *sqlx.DB) *Prescene { return &Prescene{db: db} } func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) { entries := make([]model.ExternalSources, 0) endpoint := viper.GetString(constant.CfgKeyEndpoint) uri := viper.GetString(constant.FlagSingleUri) if uri == "page::1m" { if result, err := parseSingleScnlog(s.db); err != nil { slog.Error("parsing single scnlog page", "err", err) } else { entries = append(entries, result...) } } else if uri != "" { url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/")) result, _ := parseUrl(url, s.db) entries = append(entries, result...) } else { for _, t := range pageNumbers { if t != "1" { endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t) } //doc, err := parser.HTMLSourceFromURL("https://mdb.kplus.net.ua /$/scnlog.html") if result, err := parseUrl(endpoint, s.db); err == nil { entries = append(entries, result...) } else { slog.Error("parsing url", "err", err) } //fmt.Println("Sleeping...", j) time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration)) } } return entries, nil } func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) { entries := make([]model.ExternalSources, 0) tags := viper.GetStringMapStringSlice("groups.tags") slog.Info("singleton", "url", endpoint) doc, err := parser.HTMLSourceFromURL(endpoint) if err != nil { return nil, err } if doc == nil { return nil, errors.New("document is nil") } var validID = regexp.MustCompile(`-\d+\/$`) for i, item := range dom.QuerySelectorAll(doc, "li.row") { var es model.ExternalSources columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"} // TypeId тепер напряму з data-id на
  • , без regex по class idAttr := dom.GetAttribute(item, "data-id") es.TypeId, _ = strconv.Atoi(idAttr) title := dom.QuerySelector(item, ".title") if title != nil { anchor := dom.QuerySelector(title, "a") if anchor != nil { es.Type = constant.ScopePrescene titleSpan := dom.QuerySelector(anchor, "span.title-start") if titleSpan != nil { es.Title = dom.TextContent(titleSpan) } if es.Title == "Auto Draft" { slog.Info("Skipped", "title", es.Title) continue } es.ExSource = dom.GetAttribute(anchor, "href") if validID.MatchString(es.ExSource) { continue } pattern := regexp.MustCompile(`(?is)-(\w+)$`) es.Releaser = pattern.FindStringSubmatch(es.Title)[1] for flag, groups := range tags { if slices.Contains(groups, es.Releaser) { es.A = flag es.H = flag columns = append(columns, "a", "h") break } } } if es.A == constant.TagIgnore { slog.Info("Skipped", "releaser", es.Releaser) continue } localtime := dom.QuerySelector(item, "span.localtime") if localtime != nil { lc := dom.GetAttribute(localtime, "data-lttime") es.Created = carbon.Parse(lc, "Europe/Kyiv") } } esModel := table.ExternalSources{Columns: columns} entry := esModel.InsertOnDuplicate(es, db) entries = append(entries, entry) fmt.Println("====================== ", i, " ==============================") //fmt.Printf("%+v\n", es) fmt.Printf("%s %d %d %s\n", entry.Created.ToString(), entry.Id, entry.TypeId, entry.Title) //spew.Dump(entry) } return entries, nil } func parseSingleScnlog(db *sqlx.DB) ([]model.ExternalSources, error) { entries := make([]model.ExternalSources, 0) const query = "SELECT id, type_id, title, meta, eXsource " + "FROM theamok_mdb.external_sources " + "WHERE type = 'prescene' AND created >= DATE_SUB(NOW(), INTERVAL 1 MONTH) AND meta IS NULL " + "ORDER BY created DESC LIMIT 1" var src model.ScnlogSource if err := db.Get(&src, query); err != nil { if errors.Is(err, sql.ErrNoRows) { slog.Info("no prescene rows with meta IS NULL - nothing to do") return entries, nil } return nil, err } doc, err := parser.HTMLSourceFromURL(src.ExSource) if err != nil { return nil, err } if doc == nil { return nil, errors.New("document is nil") } article := dom.QuerySelector(doc, "article.single") if article == nil { return nil, fmt.Errorf("article.single not found: %s", src.ExSource) } // type_id з class="single post-5227778 ..." typeId := src.TypeId if m := regexp.MustCompile(`post-(\d+)`).FindStringSubmatch(dom.GetAttribute(article, "class")); len(m) > 1 { if v, err := strconv.ParseInt(m[1], 10, 64); err == nil { typeId = v } } content := dom.QuerySelector(article, ".entry-content") contentText := "" if content != nil { contentText = dom.TextContent(content) } meta := model.ScnlogMeta{} // size_mb — тільки число з "Size: 217.26 MB" if m := regexp.MustCompile(`Size:\s*([\d.]+)\s*MB`).FindStringSubmatch(contentText); len(m) > 1 { if v, err := strconv.ParseFloat(m[1], 64); err == nil { meta.SizeMb = &v } } // genre — один рядок if m := regexp.MustCompile(`Genre:\s*([^\n]+)`).FindStringSubmatch(contentText); len(m) > 1 { if g := strings.TrimSpace(m[1]); g != "" { meta.Genre = &g } } // artist / tracks / duration з .mp3tbl if tbl := dom.QuerySelector(article, ".mp3tbl"); tbl != nil { all := make([]string, 0) for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-artist") { all = append(all, strings.TrimSpace(dom.TextContent(td))) } if unique := uniqueStrings(all); len(unique) > 0 { meta.Artist = &unique } tracks := len(dom.QuerySelectorAll(tbl, "tbody tr")) meta.Tracks = &tracks total := 0 for _, td := range dom.QuerySelectorAll(tbl, "td.mp3tbl-time") { total += durationToSeconds(strings.TrimSpace(dom.TextContent(td))) } duration := secondsToHMS(total) meta.Duration = &duration } // NFO: анкор із текстом "Download NFO", шлях з onclick if content == nil { slog.Warn("entry-content not found", "url", src.ExSource) } else { for _, a := range dom.QuerySelectorAll(content, "a") { if !strings.Contains(dom.TextContent(a), "Download NFO") { continue } onclick := dom.GetAttribute(a, "onclick") spew.Dump("NFO-ANCHOR", dom.TextContent(a), onclick) if m := regexp.MustCompile(`'([^']+)'`).FindStringSubmatch(onclick); len(m) > 1 { nfoURL := "https://scnlog.me" + m[1] spew.Dump("NFO-URL", nfoURL, "NFO_DIR", viper.GetString("nfo_dir")) if err := saveNfo(src.Title, nfoURL); err != nil { return nil, err } } break } } payload, err := json.Marshal(meta) // compact = "uglify" if err != nil { return nil, err } // якщо всі поля порожні — пишемо порожній об'єкт, а не набір null if meta.SizeMb == nil && meta.Genre == nil && meta.Artist == nil && meta.Tracks == nil && meta.Duration == nil { payload = []byte("{}") } if _, err := db.Exec( "UPDATE `theamok_mdb`.`external_sources` SET `meta` = ? WHERE `id` = ?", string(payload), src.Id, ); err != nil { return nil, err } slog.Info("scnlog meta updated", "id", src.Id, "type_id", typeId, "title", src.Title, "meta", string(payload)) return entries, nil } // uniqueStrings прибирає дублікати, зберігаючи порядок першої появи. func uniqueStrings(in []string) []string { seen := make(map[string]struct{}, len(in)) out := make([]string, 0, len(in)) for _, s := range in { if _, ok := seen[s]; ok { continue } seen[s] = struct{}{} out = append(out, s) } return out } // durationToSeconds парсить "m:ss" / "h:mm:ss" у секунди. func durationToSeconds(v string) int { parts := strings.Split(strings.TrimSpace(v), ":") if len(parts) == 0 { return 0 } total := 0 for _, p := range parts { n, _ := strconv.Atoi(strings.TrimSpace(p)) total = total*60 + n } return total } // secondsToHMS форматує секунди у "h:M:s" (напр. 1802 -> "0:30:02"). func secondsToHMS(total int) string { return fmt.Sprintf("%d:%02d:%02d", total/3600, (total%3600)/60, total%60) } // saveNfo пише NFO у /<перша літера title в lower>/.nfo func saveNfo(title, nfoURL string) error { dir := viper.GetString("nfo_dir") if dir == "" { return errors.New("nfo_dir is empty") } letter := strings.ToLower(string([]rune(title)[0])) sub := filepath.Join(dir, letter) if err := os.MkdirAll(sub, 0o755); err != nil { return err } data, err := fetchNfoBytes(nfoURL) if err != nil { return err } return os.WriteFile(filepath.Join(sub, title+".nfo"), data, 0o644) } // fetchNfoBytes завантажує сирі байти NFO (не HTML) у []byte. func fetchNfoBytes(nfoURL string) ([]byte, error) { client := &http.Client{Timeout: 30 * time.Second} req, err := http.NewRequest(http.MethodGet, nfoURL, nil) if err != nil { return nil, err } req.Header.Set("User-Agent", "Mozilla/5.0 (compatible; resource-scraper)") resp, err := client.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("unexpected status %d for %s", resp.StatusCode, nfoURL) } return io.ReadAll(resp.Body) }