add scraping the torrent topics rss files

pkg/handler/handler.go

@@ -0,0 +1,28 @@
+package handler
+
+import (
+	"git.amok.space/yevhen/resource-scraper/pkg/service"
+)
+
+type Handler struct {
+	services *service.Service
+}
+
+func New(services *service.Service) *Handler {
+	return &Handler{services: services}
+}
+
+func (h *Handler) InitConsole() string {
+	return h.rutracker()
+}
+
+//func (h *Handler) Base(services *service.Service) *Handler {
+//	return &Handler{services: services}
+//}
+
+//func (h *Handler) InitApi() *chi.Mux {
+//	api := chi.NewRouter()
+//	api.Get("/", web.ApiFallbackHandler)
+//
+//	return api
+//}

pkg/handler/rutracker.go

@@ -0,0 +1,21 @@
+package handler
+
+import (
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+func (h *Handler) rutracker() string {
+	key := fmt.Sprintf("topic.%v", time.Now().Hour())
+	topics := viper.GetStringSlice(key)
+
+	err := h.services.Rutracker.GetTopic(topics)
+	if err != nil {
+		slog.Error("error occurred while getting topic: ", err.Error())
+	}
+
+	return "rt"
+}

pkg/repository/repository.go

@@ -0,0 +1,23 @@
+package repository
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/types"
+)
+
+type Repository struct {
+	types.Rutracker
+}
+
+func New(db *sqlx.DB) *Repository {
+	switch viper.GetString("scope") {
+	case types.RuTracker:
+		return &Repository{
+			Rutracker: NewRutracker(db),
+		}
+	}
+
+	return &Repository{}
+}

pkg/repository/rutracker.go

@@ -0,0 +1,85 @@
+package repository
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strconv"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	iface "git.amok.space/yevhen/resource-scraper/types"
+	"git.amok.space/yevhen/resource-scraper/types/table"
+)
+
+type Rutracker struct {
+	db *sqlx.DB
+}
+
+func NewRutracker(db *sqlx.DB) *Rutracker {
+	return &Rutracker{db: db}
+}
+
+func (s *Rutracker) GetTopic(topics []string) error {
+	endpoint := viper.GetString("endpoint")
+
+	for _, t := range topics {
+		topic, err := fetch(fmt.Sprintf(endpoint, t))
+		if err != nil {
+			slog.Error("couldn't parse topic data", "err", err.Error())
+		}
+
+		for i, e := range topic.Entry {
+			var id int
+			var es table.ExternalSources
+
+			u, _ := url.Parse(e.Link.Href)
+			es.Type = "rutracker"
+			es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
+			es.Title = e.Title
+			es.TypeSubsectionId, _ = strconv.Atoi(t)
+			es.Releaser = e.Author.Name
+			es.Created, _ = time.Parse(time.RFC3339, e.Updated)
+			created := es.Created.Format(iface.DateTimeFormat)
+
+			query := fmt.Sprintf("INSERT INTO %s (`type`, type_id, title, type_subsection_id, releaser, created) VALUES (?, ?, ?, ?, ?, ?) ON DUPLICATE KEY UPDATE title=?, created=? RETURNING id", iface.ExternalSourcesTable)
+
+			row := s.db.QueryRow(query, es.Type, es.TypeId, es.Title, es.TypeSubsectionId, es.Releaser, created, es.Title, created)
+			if err = row.Scan(&id); err != nil {
+				return err
+			}
+
+			fmt.Println("<< ----------------- ", i+1, id, " ----------------- >>")
+		}
+	}
+
+	return nil
+}
+
+func fetch(endpoint string) (*iface.RutrackerAtomTopic, error) {
+	resp, err := http.Get(endpoint)
+	if err != nil {
+		slog.Error("couldn't fetch data", endpoint, err.Error())
+		return nil, err
+	}
+
+	defer func(Body io.ReadCloser) {
+		err = Body.Close()
+		if err != nil {
+			fmt.Println("Body.Close")
+		}
+	}(resp.Body)
+
+	topic := &iface.RutrackerAtomTopic{}
+
+	if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
+		return nil, err
+	}
+
+	return topic, nil
+}

pkg/service/rutracker.go

@@ -0,0 +1,17 @@
+package service
+
+import (
+	iface "git.amok.space/yevhen/resource-scraper/types"
+)
+
+type RutrackerService struct {
+	repo iface.Rutracker
+}
+
+func NewRutrackerService(repo iface.Rutracker) *RutrackerService {
+	return &RutrackerService{repo: repo}
+}
+
+func (s *RutrackerService) GetTopic(topic []string) error {
+	return s.repo.GetTopic(topic)
+}

pkg/service/service.go

@@ -0,0 +1,24 @@
+package service
+
+import (
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/pkg/repository"
+	"git.amok.space/yevhen/resource-scraper/types"
+)
+
+type Service struct {
+	types.Rutracker
+}
+
+func New(repos *repository.Repository) *Service {
+
+	switch viper.GetString("scope") {
+	case types.RuTracker:
+		return &Service{
+			Rutracker: NewRutrackerService(repos.Rutracker),
+		}
+	}
+
+	return &Service{}
+}