preset scraper added

types/constant/config.go

@@ -0,0 +1,10 @@
+package constant
+
+const (
+	CfgKeyScopeEnable              = FlagScopeEnable
+	CfgKeyConfigDir                = "config-dir"
+	CfgKeySleepBeforeNextIteration = "sleep-before-next-iteration"
+	CfgKeyEndpoint                 = "endpoint"
+	CfgKeyEndpointNext             = "endpoint-next"
+	CfgKeyLevelsToScrape           = "levels-to-scrape"
+)

types/constant/constants.go

@@ -0,0 +1,18 @@
+package constant
+
+const (
+	ScopeRuTracker string = "rutracker"
+	ScopePrescene  string = "prescene"
+	ScopeWeb       string = "web"
+	ScopeInfo      string = "info"
+)
+
+const (
+	RoleConsole string = "console"
+	RoleWeb     string = "web"
+)
+
+const (
+	TagIgnore            string = "ignore"
+	ExternalSourcesTable string = "external_sources"
+)

types/constant/flag.go

@@ -0,0 +1,11 @@
+package constant
+
+const (
+	FlagHelp         = "help"
+	FlagHelpShort    = "h"
+	FlagDebug        = "debug"
+	FlagVersion      = "version"
+	FlagVersionShort = "v"
+	FlagConfigFile   = "config-file"
+	FlagScopeEnable  = "scope-enable"
+)

types/constants.go

@@ -1,14 +0,0 @@
-package types
-
-const (
-	RuTracker string = "rutracker"
-)
-
-const (
-	RoleConsole string = "console"
-)
-
-const (
-	DateTimeFormat       string = "2006-01-02 15:04:05"
-	ExternalSourcesTable string = "external_sources"
-)

types/interface/interfaces.go

@@ -0,0 +1,11 @@
+package _interface
+
+import "git.amok.space/yevhen/resource-scraper/types/model"
+
+type Rutracker interface {
+	GetTopic(topics []string) ([]model.ExternalSources, error)
+}
+
+type Prescene interface {
+	GetPage(pageNumbers []string) ([]model.ExternalSources, error)
+}

types/interfaces.go

@@ -1,5 +0,0 @@
-package types
-
-type Rutracker interface {
-	GetTopic(topics []string) error
-}

types/model/external_sources.go

@@ -0,0 +1,42 @@
+package model
+
+import (
+	"github.com/golang-module/carbon/v2"
+)
+
+type Type string
+
+/*const (
+	mmt Type = iota
+	prescene
+	rutracker
+	locate
+	darkabyss
+	bfm
+	trash
+	nnmc
+	stb
+	bandcamp
+	deathgrind
+)*/
+
+type ExternalSources struct {
+	Id                 int           `json:"id" db:"id"`
+	Type               string        `json:"type" db:"type"`
+	TypeId             int           `json:"type_id" db:"type_id"`
+	Title              string        `json:"title" db:"title"`
+	TypeSubsectionId   int           `json:"type_subsection_id" db:"type_subsection_id"`
+	Releaser           string        `json:"releaser" db:"releaser"`
+	ExSource           string        `json:"ex_source" db:"eXsource"`
+	Created            carbon.Carbon `json:"created_at" db:"created"`
+	Modified           carbon.Carbon `json:"modified_at" db:"modified"`
+	Contents           string        `json:"contents" db:"contents"`
+	A                  string        `json:"a" db:"a"`
+	H                  string        `json:"h" db:"h"`
+	Fingerprint        string        `json:"fingerprint" db:"fingerprint"`
+	FsFingerprint      string        `json:"fs_fingerprint" db:"fs_fingerprint"`
+	Vid                int           `json:"vid" db:"vid"`
+	G                  string        `json:"g" db:"g"`
+	ExternalSourcesCol string        `json:"external_sources_column" db:"external_sourcescol"`
+	Error              error
+}

types/resource/rutracker.go

@@ -1,4 +1,4 @@
-package types
+package resource
 
 import "encoding/xml"
 

types/resource/scnlog.go

@@ -0,0 +1,46 @@
+package resource
+
+import "encoding/xml"
+
+type ScnLogRss struct {
+	XMLName xml.Name `xml:"rss"`
+	Text    string   `xml:",chardata"`
+	Version string   `xml:"version,attr"`
+	Content string   `xml:"content,attr"`
+	Wfw     string   `xml:"wfw,attr"`
+	Dc      string   `xml:"dc,attr"`
+	Atom    string   `xml:"atom,attr"`
+	Sy      string   `xml:"sy,attr"`
+	Slash   string   `xml:"slash,attr"`
+	Channel struct {
+		Text  string `xml:",chardata"`
+		Title string `xml:"title"`
+		Link  struct {
+			Text string `xml:",chardata"`
+			Href string `xml:"href,attr"`
+			Rel  string `xml:"rel,attr"`
+			Type string `xml:"type,attr"`
+		} `xml:"link"`
+		Description     string `xml:"description"`
+		LastBuildDate   string `xml:"lastBuildDate"`
+		Language        string `xml:"language"`
+		UpdatePeriod    string `xml:"updatePeriod"`
+		UpdateFrequency string `xml:"updateFrequency"`
+		Generator       string `xml:"generator"`
+		Item            []struct {
+			Text     string   `xml:",chardata"`
+			Title    string   `xml:"title"`
+			Link     string   `xml:"link"`
+			Comments string   `xml:"comments"`
+			Creator  string   `xml:"creator"`
+			PubDate  string   `xml:"pubDate"`
+			Category []string `xml:"category"`
+			Guid     struct {
+				Text        string `xml:",chardata"`
+				IsPermaLink string `xml:"isPermaLink,attr"`
+			} `xml:"guid"`
+			Description string `xml:"description"`
+			CommentRss  string `xml:"commentRss"`
+		} `xml:"item"`
+	} `xml:"channel"`
+}

types/table/external_sources.go

@@ -1,39 +0,0 @@
-package table
-
-import "time"
-
-type Type string
-
-/*const (
-	mmt Type = iota
-	prescene
-	rutracker
-	locate
-	darkabyss
-	bfm
-	trash
-	nnmc
-	stb
-	bandcamp
-	deathgrind
-)*/
-
-type ExternalSources struct {
-	Id                 int       `json:"id" db:"id"`
-	Type               string    `json:"type" db:"type"`
-	TypeId             int       `json:"type_id" db:"type_id"`
-	Title              string    `json:"title" db:"title"`
-	TypeSubsectionId   int       `json:"type_subsection_id" db:"type_subsection_id"`
-	Releaser           string    `json:"releaser" db:"releaser"`
-	ExSource           string    `json:"ex_source" db:"eXsource"`
-	Created            time.Time `json:"created_at" db:"created"`
-	Modified           time.Time `json:"modified_at" db:"modified"`
-	Contents           string    `json:"contents" db:"contents"`
-	A                  string    `json:"a" db:"a"`
-	H                  string    `json:"h" db:"h"`
-	Fingerprint        string    `json:"fingerprint" db:"fingerprint"`
-	FsFingerprint      string    `json:"fs_fingerprint" db:"fs_fingerprint"`
-	Vid                int       `json:"vid" db:"vid"`
-	G                  string    `json:"g" db:"g"`
-	ExternalSourcesCol string    `json:"external_sources_column" db:"external_sourcescol"`
-}