changed handlers summoning

logging feature finished

.air.toml

@@ -4,7 +4,7 @@ tmp_dir = "tmp"
 
 [build]
   args_bin = []
-  bin = "tmp\\main.exe"
+  bin = "tmp\\main.exe --config-file config/default --scope rutracker"
   cmd = "go build -o ./tmp/main.exe ./cmd"
   delay = 1000
   exclude_dir = ["assets", "tmp", "vendor", "testdata"]

.gitignore

@@ -29,4 +29,6 @@ fabric.properties
 .idea/httpRequests
 .idea/caches/build_file_checksums.ser
 .env
-tmp/
+tmp/
+vendor/
+config/secret.yaml

.idea/dataSources.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourceManagerImpl" format="xml" multifile-model="true">
+    <data-source source="LOCAL" name="@arm" uuid="e3965414-f0c4-4a05-af48-11e028c626da">
+      <driver-ref>mariadb</driver-ref>
+      <synchronize>true</synchronize>
+      <jdbc-driver>org.mariadb.jdbc.Driver</jdbc-driver>
+      <jdbc-url>jdbc:mariadb://10.14.88.1:6033/go_tut_tokill</jdbc-url>
+      <jdbc-additional-properties>
+        <property name="com.intellij.clouds.kubernetes.db.host.port" />
+        <property name="com.intellij.clouds.kubernetes.db.enabled" value="false" />
+        <property name="com.intellij.clouds.kubernetes.db.resource.type" value="Deployment" />
+        <property name="com.intellij.clouds.kubernetes.db.container.port" />
+      </jdbc-additional-properties>
+      <working-dir>$ProjectFileDir$</working-dir>
+    </data-source>
+  </component>
+</project>

.idea/data_source_mapping.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DataSourcePerFileMappings">
+    <file url="file://$PROJECT_DIR$/pkg/repository/table/external_sources.go" value="e3965414-f0c4-4a05-af48-11e028c626da" />
+  </component>
+</project>

.idea/golinter.xml

@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <project version="4">
   <component name="GoLinterSettings">
+    <option name="concurrency" value="5" />
     <option name="enabledLinters">
       <list>
         <option value="ineffassign" />

cmd/main.go

@@ -2,27 +2,45 @@ package main
 
 import (
 	"fmt"
-	"log"
+	"time"
-	"os"
 
-	"github.com/joho/godotenv"
+	_ "github.com/go-sql-driver/mysql"
+
+	scraper "git.kplus.net.ua/yevhen/resource-scraper/internal"
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/config"
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/logging"
 )
 
-// init is invoked before main()
+var (
+	st time.Time
+)
+
+//var tokenAuth *jwtauth.JWTAuth
+
 func init() {
-	// loads values from .env into the system
+	st = time.Now()
-	if err := godotenv.Load(); err != nil {
+	logging.New()
-		log.Print("No .env file found")
+	scraper.ParseFlags()
-	}
+	config.New()
+
+	/*verifyKey := []byte(viper.GetString("jwt.sign_key"))
+	tokenAuth = jwtauth.New("HS256", verifyKey, nil)
+
+	// For debugging/example purposes, we generate and print
+	// a sample jwt token with claims `user_id:123` here:
+	_, tokenString, _ := tokenAuth.Encode(map[string]interface{}{"user_id": 123})
+	slog.Info("jwt", "token", tokenString)*/
 }
 
 func main() {
-	// Get the GREETING environment variable
+	defer func() {
-	greeting, exists := os.LookupEnv("GREETING")
+		fmt.Printf("\nExecution took %v", time.Since(st))
+	}()
 
-	if exists {
+	//a, b := runtime.GOOS
-		fmt.Println(greeting)
+	//fmt.Printf("%v  - - -  \n", runtime.GOOS)
-	}
 
-	fmt.Println("fin")
+	scraper.Bootstrap()
+	//fmt.Printf("%v\n", viper.Get("23").([]interface{}))
+	// Кінґстонська стіна
 }

config/default.yaml

@@ -0,0 +1,6 @@
+scope:
+  allow: [info, web, rutracker, prescene, stb, metal-archives, console]
+  default: console
+mail:
+  kplus.net.ua:
+    dial-tls: s21.thehost.com.ua:993

config/scope/console.yaml

@@ -0,0 +1,4 @@
+role: console
+console:
+  cmd:
+    create: [scope]

config/scope/info.yaml

@@ -0,0 +1 @@
+role: console

config/scope/metal-archives.yaml

@@ -0,0 +1,2 @@
+role: console
+metal-archives:

config/scope/prescene.yaml

@@ -0,0 +1,78 @@
+role: console
+endpoint: https://scnlog.me/music/flac/
+endpoint-next: page/%s/
+sleep-before-next-iteration: 5s
+levels-to-scrape: 5
+groups:
+  tags:
+    neformat:
+      - SHGZ
+    remove:
+      - BIGLOVE
+      - CT
+      - ENViED
+      - GARLICKNOTS
+      - Ouzo
+      - PTCx
+      - SHGZ
+      - XiVERO
+    ignore:
+      - "401"
+      - 6DM
+      - AFO
+      - AOVF
+      - AOVF_Classics
+      - AUDiOFiLE
+      - BABAS
+      - BBD
+      - BEATOCUL
+      - CALiFLAC
+      - CEBAD
+      - D2H
+      - DALiAS
+      - dh
+      - DWM
+      - EMP
+      - ENRiCH
+      - ERP
+      - ESGFLAC
+      - FATHEAD
+      - FiXIE
+      - FLACME
+      - FrB
+      - HOUND
+      - JRO
+      - KINDA
+      - MAHOU
+      - MARiBOR
+      - MEOWZiK
+      - MFDOS
+      - MLS
+      - Mrflac
+      - MUNDANE
+      - MyDad
+      - NRS
+      #      - OBZEN
+      - OTT
+      - PERFECT
+      - PLAYLiST
+      - PTC
+      - PWT
+      - RAGEFLAC
+      - RAWBEATS
+      - RECTiFY
+      - ROSiN
+      - STAX
+      - THEVOiD
+      #      - TiMES
+      - TM
+      - TVRf
+      #      - VEXED
+      - WAVED
+      - YARD
+      - HiTE
+      - RPO
+      - MEOWZiK
+      - STONERD
+      - TR
+      - W4GN3R

config/scope/rutracker.yaml

@@ -0,0 +1,27 @@
+role: console
+endpoint: https://feed.rutracker.cc/atom/f/%s.atom
+topic:
+  0: [ 1719, 1779 ]
+  1: [ 1796, 1730 ]
+  2: [ 1720, 1728 ]
+  3: [ 1724, 1815 ]
+  4: [ 1726, 1719 ]
+  5: [ 2230, 1779 ]
+  6: [ 1868, 1796 ]
+  7: [ 1877, 1730 ]
+  8: [ 1866, 1720 ]
+  9: [ 1864, 1728 ]
+  10: [ 1871, 1724 ]
+  11: [ 1869, 1815 ]
+  12: [ 1788, 1719 ]
+  13: [ 739, 1779 ]
+  14: [ 951, 1702 ]
+  15: [ 1740, 1730 ]
+  16: [ 1736, 1720 ]
+  17: [ 1746, 1728 ]
+  18: [ 1744, 1724 ]
+  19: [ 1748, 1815 ]
+  20: [ 1706, 1719 ]
+  21: [ 1757, 1779 ]
+  22: [ 1133, 739 ]
+  23: [ 1755, 1756 ]

config/scope/stb.yaml

@@ -0,0 +1,29 @@
+role: console
+stb:
+  endpoint: stb@kplus.net.ua
+  mailboxes: ["INBOX"]
+  #mailboxes: ["INBOX", "Junk Mail", "Done"]
+  #mailboxes: ["Junk Mail"]
+  #mailboxes: ["Done"]
+  search-criteria: ["Your Share The Brutality digest", "noreply@sharethebrutality.info"]
+  move-processed-to-mailbox:
+    succeed: Succeed
+    #succeed: Done
+    failed: Failed
+    suspicious: Suspicious
+  sender:
+    mailbox: noreply
+    host: sharethebrutality.info
+    subject: Your Share The Brutality digest
+  db-type: nnmc
+  regex:
+    type-id: (?is)^\/forum\/topic\/(\d+)-
+    who-genre: (?is)^(.*) created a topic in (Discographies|Death Metal|Old School Death|Other Genres|Your Own Rips)
+  topics:
+    discographies: 9
+    death metal: 11
+    old school death: 33
+    other genres: 40
+    your own rips: 14
+  storage:
+    filepath: c:\tmp\mdb\stb\

config/scope/web.yaml

@@ -0,0 +1,3 @@
+role: web
+http:
+  port: 19576

config/secret.yaml-sample

@@ -0,0 +1,6 @@
+db:
+  driver: mysql
+  data_source_name: <user:pass@(localhost:3306)/bd_name>
+jwt:
+  ttl: 6h12m24s
+  sign_key: <your key>

go.mod

@@ -1,5 +1,42 @@
-module git.amok.space/yevhen/resource-scraper
+module git.kplus.net.ua/yevhen/resource-scraper
 
-go 1.23.1
+go 1.25
 
-require github.com/joho/godotenv v1.5.1 // indirect
+require (
+	github.com/dromara/carbon/v2 v2.6.13
+	github.com/emersion/go-imap/v2 v2.0.0-beta.7
+	github.com/go-chi/chi/v5 v5.2.3
+	github.com/go-chi/cors v1.2.2
+	github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
+	github.com/go-sql-driver/mysql v1.9.3
+	github.com/iancoleman/strcase v0.3.0
+	github.com/jmoiron/sqlx v1.4.0
+	github.com/logrusorgru/aurora/v4 v4.0.0
+	github.com/mewkiz/flac v1.0.13
+	github.com/spf13/pflag v1.0.10
+	github.com/spf13/viper v1.21.0
+	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
+	golang.org/x/net v0.46.0
+)
+
+require (
+	filippo.io/edwards25519 v1.1.0 // indirect
+	github.com/andybalholm/cascadia v1.3.3 // indirect
+	github.com/emersion/go-message v0.18.2 // indirect
+	github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
+	github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
+	github.com/icza/bitio v1.1.0 // indirect
+	github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d // indirect
+	github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.4 // indirect
+	github.com/sagikazarmark/locafero v0.12.0 // indirect
+	github.com/spf13/afero v1.15.0 // indirect
+	github.com/spf13/cast v1.10.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	go.yaml.in/yaml/v3 v3.0.4 // indirect
+	golang.org/x/sys v0.37.0 // indirect
+	golang.org/x/text v0.30.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+)

go.sum

@@ -1,2 +1,156 @@
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
+github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
+github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dromara/carbon/v2 v2.6.13 h1:lSXQpikgbBN/Gj3tfqjTXwBWnpidN0bnI6qnlcQXExs=
+github.com/dromara/carbon/v2 v2.6.13/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
+github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
+github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
+github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
+github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
+github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
+github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
+github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
+github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
+github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
+github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
+github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
+github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
+github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
+github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
+github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
+github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
+github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
+github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
+github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0=
+github.com/icza/bitio v1.1.0/go.mod h1:0jGnlLAx8MKMr9VGnn/4YrvZiprkvBelsVIbA9Jjr9A=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lToqnXgA8Mz1DP11X4zSJ159C3k=
+github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
+github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
+github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
+github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
+github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
+github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d h1:IL2tii4jXLdhCeQN69HNzYYW1kl0meSG0wt5+sLwszU=
+github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d/go.mod h1:SIpumAnUWSy0q9RzKD3pyH3g1t5vdawUAPcW5tQrUtI=
+github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
+github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
+github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
+github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
+github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
+github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
+github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
+github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
+github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
+github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
+github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
+github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
+golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
+golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
+golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
+golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
+golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
+golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
+golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
+golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU=
+golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
+golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
+golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

helper/parser/html.go

@@ -0,0 +1,65 @@
+package parser
+
+import (
+	"errors"
+	"io"
+	"log"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+func setClient(url string) (*http.Response, error) {
+	client := &http.Client{}
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36")
+
+	return client.Do(req)
+}
+
+func HTMLSourceFromURL(url string) (*html.Node, error) {
+	resp, err := setClient(url)
+	if resp == nil {
+		slog.Error("client return nil response", "err", err)
+		return nil, err
+	}
+
+	defer func(Body io.ReadCloser) {
+		err = Body.Close()
+		if err != nil {
+			slog.Error("closing response body", "err", err)
+		}
+	}(resp.Body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode >= 400 {
+		return nil, errors.New("http status code: " + strconv.Itoa(resp.StatusCode))
+	}
+
+	doc, err := html.Parse(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	return doc, nil
+}
+
+func HTMLSource(htmlSource string) (*html.Node, error) {
+	doc, err := html.Parse(strings.NewReader(htmlSource))
+	if err != nil {
+		return nil, err
+	}
+	return doc, nil
+}
+
+//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang

helper/sugar/console.go

@@ -0,0 +1,19 @@
+package sugar
+
+import (
+	"fmt"
+
+	"github.com/logrusorgru/aurora/v4"
+)
+
+func LogSuccess(msg string) {
+	fmt.Printf("%s: %s\n", aurora.Green("SUCCESS"), aurora.Yellow(msg))
+}
+
+func LogError(msg string) {
+	fmt.Printf("%s: %s\n", aurora.Red("ERROR"), aurora.White(msg))
+}
+
+func LogWarning(msg string) {
+	fmt.Printf("%s: %s\n", aurora.Magenta("WARNING"), aurora.White(msg))
+}

helper/sugar/env.go

@@ -0,0 +1,11 @@
+package sugar
+
+import (
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+func IsDev() bool {
+	return viper.GetString(constant.FlagEnv) == constant.DefaultEnvDev
+}

helper/sugar/fs.go

@@ -0,0 +1,63 @@
+package sugar
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"strconv"
+)
+
+func EnsureDir(dirPath ...string) (string, error) {
+	outPath := filepath.Join(dirPath...)
+	if len(outPath) < 3 {
+		return "", fmt.Errorf("the path is short to be a folder: %s", outPath)
+	}
+
+	if _, err := os.Stat(outPath); os.IsNotExist(err) {
+		var dirMod uint64
+		if dirMod, err = strconv.ParseUint("0775", 8, 32); err == nil {
+			err = os.Mkdir(outPath, os.FileMode(dirMod))
+		}
+
+		if err != nil && !os.IsExist(err) {
+			slog.Error("error creating tmp dir", "err", err)
+		}
+	}
+
+	return outPath, nil
+}
+
+func WriteDataToTmpFile(content, fp string) (int, string, error) {
+	dir, err := EnsureDir(os.TempDir(), "mdb")
+	if err != nil {
+		slog.Error("EnsureDir", "err", err.Error())
+		return 0, dir, err
+	}
+
+	fp = filepath.Join(dir, fp)
+
+	f, err := os.Create(fp)
+	if err != nil {
+		slog.Error("CreateFile", "e", err)
+		return 0, fp, err
+	}
+
+	filesize, err := f.WriteString(content)
+	if err != nil {
+		slog.Error("WriteString", "e", err)
+		return 0, fp, err
+	}
+
+	err = f.Close()
+	if err != nil {
+		slog.Error("CloseFile", "e", err)
+		return 0, fp, err
+	}
+
+	return filesize, fp, nil
+}
+
+/*
+	REMOVED_FROM_MMT_DUE_TO_REDOING_OF_THE_EVENT_LOG
+*/

helper/sugar/string.go

@@ -0,0 +1,8 @@
+package sugar
+
+import "regexp"
+
+// SqueezeLine remove extra spaces and line breaks from a string
+func SqueezeLine(line string) string {
+	return regexp.MustCompile(`(?m)[\n\r\t\s]+`).ReplaceAllString(line, " ")
+}

helper/thither/thither.go

@@ -0,0 +1,28 @@
+package thither
+
+import (
+	"reflect"
+	"strconv"
+)
+
+func FieldValueToIntSlice[T interface{}](object []T, key string) []int {
+	fields := make([]int, len(object))
+
+	for i, el := range object {
+		immutable := reflect.ValueOf(el)
+		fields[i] = immutable.FieldByName(key).Interface().(int)
+	}
+
+	return fields
+}
+
+func FieldValueToStrSlice[T interface{}](object []T, key string) []string {
+	fields := make([]string, len(object))
+
+	for i, el := range object {
+		immutable := reflect.ValueOf(el)
+		fields[i] = strconv.Itoa(immutable.FieldByName(key).Interface().(int))
+	}
+
+	return fields
+}

helper/web/handlerReadiness.go

@@ -0,0 +1,7 @@
+package web
+
+import "net/http"
+
+func FallbackHandler(w http.ResponseWriter, r *http.Request) {
+	respondWithJSON(w, http.StatusOK, struct{}{})
+}

helper/web/json.go

@@ -0,0 +1,35 @@
+package web
+
+import (
+	"encoding/json"
+	"log"
+	"net/http"
+)
+
+type errorResponse struct {
+	Error string `json:"error"`
+}
+
+func respondWithJSON(w http.ResponseWriter, code int, payload interface{}) {
+	data, err := json.Marshal(payload)
+	if err != nil {
+		log.Printf("failed to marshal JSON response: %v\n", err)
+		w.WriteHeader(http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Add("Content-Type", "application/json")
+	w.WriteHeader(code)
+	_, err = w.Write(data)
+	if err != nil {
+		log.Printf("failed to write data to output buffer: %v\n", err)
+	}
+}
+
+func respondWithError(w http.ResponseWriter, code int, message string) {
+	if code >= http.StatusInternalServerError {
+		log.Println("responding with 5xx error:", message)
+	}
+
+	respondWithJSON(w, code, errorResponse{Error: message})
+}

install.sh

@@ -0,0 +1,2 @@
+#!/usr/bin/env bash
+go build -o /usr/local/bin/mdb ./cmd

internal/bootstrap.go

@@ -0,0 +1,47 @@
+package scraper
+
+import (
+	"fmt"
+
+	"github.com/iancoleman/strcase"
+	"github.com/logrusorgru/aurora/v4"
+	"github.com/spf13/viper"
+	"golang.org/x/exp/slices"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/db"
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/handler"
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository"
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/service"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+func isAllowScope() bool {
+	scopesAllow := viper.GetStringSlice("scope.allow")
+	scopeEnabled := viper.GetString(constant.FlagScopeEnable)
+
+	return slices.Contains(scopesAllow, scopeEnabled)
+}
+
+func init() {
+	strcase.ConfigureAcronym("stb", "STB")
+}
+
+func Bootstrap() {
+	if !isAllowScope() {
+		fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml"))
+		return
+	}
+
+	dbase := db.New()
+	repos := repository.New(dbase)
+	services := service.New(repos)
+	handlers := handler.New(services)
+
+	switch viper.GetString("role") {
+	case constant.RoleConsole:
+		fmt.Printf("init console console: %s\n", handlers.InitConsole())
+	case constant.RoleWeb:
+		fmt.Printf("who: %s\n", handlers.InitRoutes())
+		///http.Run()
+	}
+}

internal/config/config.go

@@ -0,0 +1,64 @@
+package config
+
+import (
+	"fmt"
+	"log"
+	"log/slog"
+	"path/filepath"
+	"slices"
+
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+func New() {
+	configFilePath := viper.GetString(constant.FlagConfigFile)
+	configName := filepath.Base(configFilePath)
+
+	dir := filepath.Dir(configFilePath)
+	if dir == configName && dir == "." {
+		configName = "default"
+	}
+
+	configDir, _ := filepath.Abs(dir)
+
+	viper.AddConfigPath(configDir)
+	viper.SetConfigName(configName)
+	err := viper.ReadInConfig()
+	if err != nil { // Handle errors reading the config file
+		panic(fmt.Errorf("fatal error config file: %w", err))
+	}
+
+	viper.SetConfigName("secret")
+	err = viper.MergeInConfig()
+	if err != nil { // Handle errors reading the config file
+		panic(fmt.Errorf("fatal error secret file: %w", err))
+	}
+
+	slog.Info("using config", "path", filepath.Join(configDir, configName))
+	viper.SetDefault(constant.CfgKeyConfigDir, configDir)
+
+	// Scopes validating
+	scope := viper.GetString(constant.CfgKeyScopeEnable)
+	scopesAllowed := viper.GetStringSlice("scope.allow")
+
+	if !slices.Contains(scopesAllowed, scope) {
+		scope = viper.GetString("scope.default")
+
+		if scope == "" {
+			scope = constant.ScopeInfo
+		}
+	}
+
+	fmt.Println("scope", scope)
+
+	viper.SetConfigName(scope)
+	viper.AddConfigPath(filepath.Join(configDir, "scope"))
+	err = viper.MergeInConfig()
+	if err == nil {
+		viper.SetDefault(constant.CfgKeyScopeEnable, scope)
+	} else {
+		log.Fatalf("fatal error config file: %v", err)
+	}
+}

internal/db/connector.go

@@ -0,0 +1,24 @@
+package db
+
+import (
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+)
+
+func New() *sqlx.DB {
+	drv := viper.GetString("db.driver")
+	dsn := viper.GetString("db.data_source_name")
+
+	db, err := sqlx.Connect(drv, dsn)
+	if err != nil {
+		panic("Failed to connect to the database: " + err.Error())
+	}
+
+	// Verify the connection to the database is still alive
+	err = db.Ping()
+	if err != nil {
+		panic("Failed to ping the database: " + err.Error())
+	}
+
+	return db
+}

internal/flag.go

@@ -0,0 +1,60 @@
+package scraper
+
+import (
+	"flag"
+	"fmt"
+	"log"
+	"runtime"
+
+	"github.com/spf13/pflag"
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+const usage = `Music Database (MDB) server/cli craftware'
+
+Usage:
+	mdb [command] -c [config-file-path]
+
+Commands:
+	-help,		-h  Print this message
+	-version	-v  Print version.
+	-config		-c  Set configuration file without extension. It is %s by default.
+	-section	-s  Select section e.g.: rutracker, bandcamp etc.
+
+The Go runtime version: %s
+Report bugs to https://git.kplus.net.ua/yevhen/resource-scraper/issues`
+
+const (
+	version           = "0.1"
+	defaultConfigPath = constant.DefaultConfigPath
+	defaultConfigEnv  = constant.DefaultEnvProd
+)
+
+func ParseFlags() {
+	flag.Bool(constant.FlagHelpShort, false, "")
+	flag.Bool(constant.FlagHelp, false, "")
+	flag.Bool(constant.FlagVersionShort, false, "")
+	flag.Bool(constant.FlagVersion, false, "")
+	flag.Bool(constant.FlagDebug, false, "")
+	flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location")
+	flag.String(constant.FlagScopeEnable, "", "")
+	flag.String(constant.FlagSingleUri, "", "")
+	flag.String(constant.FlagEnv, defaultConfigEnv, "")
+	flag.String("create", defaultConfigEnv, "used to create e.g. scope etc.")
+
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	pflag.Parse()
+	err := viper.BindPFlags(pflag.CommandLine)
+	if err != nil {
+		log.Fatalf("[ERR] Failed to bind flags to config: %s", err)
+	}
+
+	if viper.GetBool(constant.FlagHelpShort) || viper.GetBool(constant.FlagHelp) {
+		msg := fmt.Sprintf(usage, defaultConfigPath, runtime.Version())
+		fmt.Println(msg)
+	} else if viper.GetBool(constant.FlagVersionShort) || viper.GetBool(constant.FlagVersion) {
+		fmt.Println("MDB version", version)
+	}
+}

internal/http/http.go

@@ -0,0 +1,54 @@
+package http
+
+import (
+	"log"
+	"net/http"
+
+	"github.com/go-chi/chi/v5"
+	"github.com/go-chi/cors"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/helper/web"
+)
+
+/*type Server struct {
+	router *chi.Mux
+}*/
+
+/*type Routes struct {
+	Api *chi.Mux
+}*/
+
+/*type Handler func(w http.ResponseWriter, r *http.Request) error
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if err := h(w, r); err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+
+		_, err := w.Write([]byte("bad"))
+		if err != nil {
+			log.Fatalf("Error starting HTTP server: %v", err)
+		}
+	}
+}*/
+
+func Run() {
+	port := "19576"
+
+	r := chi.NewRouter()
+	r.Use(cors.Handler(cors.Options{
+		AllowedOrigins:   []string{"https://*", "http://*"},
+		AllowedMethods:   []string{"GET", "POST"},
+		AllowedHeaders:   []string{"*"},
+		ExposedHeaders:   []string{"Link"},
+		AllowCredentials: false,
+		MaxAge:           300,
+	}))
+
+	r.Get("/", web.FallbackHandler)
+
+	log.Printf("Server starting on port %v", port)
+	err := http.ListenAndServe("localhost:"+port, r)
+	if err != nil {
+		log.Fatalf("Error starting HTTP server: %v", err)
+	}
+}

internal/logging/logging.go

@@ -0,0 +1,23 @@
+package logging
+
+import (
+	"log/slog"
+	"os"
+)
+
+func custom() {
+	//logger := log.New(os.Stderr, "INFO\t", log.LstdFlags|log.Lmicroseconds|log.Lshortfile)
+}
+
+// New check this later  https://github.com/hedzr/logg
+func New() {
+	handlerOpts := &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+		//AddSource: true,
+	}
+
+	logger := slog.New(slog.NewJSONHandler(os.Stderr, handlerOpts))
+	//.WithAttrs([]slog.Attr{slog.String("app_version", "v0.0.1")})
+
+	slog.SetDefault(logger)
+}

internal/mail/mail.go

@@ -0,0 +1,193 @@
+package mail
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/emersion/go-imap/v2"
+	client "github.com/emersion/go-imap/v2/imapclient"
+	"github.com/logrusorgru/aurora/v4"
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
+)
+
+type EmailService struct {
+	User      string
+	pass      string
+	Err       error
+	client    *client.Client
+	Mailboxes []*imap.SelectData
+	Messages  []*client.FetchMessageBuffer
+}
+
+func (s *EmailService) CheckErr(msg string, err error) bool {
+	if err != nil {
+		if sugar.IsDev() {
+			sugar.LogError(err.Error())
+		}
+		s.Err = err
+		return true
+	}
+
+	return false
+}
+
+func (s *EmailService) success(msg string) {
+	if sugar.IsDev() {
+		sugar.LogSuccess(msg)
+	}
+}
+
+func (s *EmailService) Warn(msg string) {
+	if sugar.IsDev() {
+		sugar.LogWarning(msg)
+	}
+}
+
+func (s *EmailService) Connect() {
+	box := strings.Split(s.User, "@")
+	mail := viper.GetStringMapString("mail." + box[1])
+
+	conn, err := client.DialTLS(mail["dial-tls"], nil)
+	if s.CheckErr("DialTLS", err) {
+		return
+	}
+
+	s.client = conn
+	s.pass = mail[s.User]
+
+	s.success("connected to " + mail["dial-tls"])
+
+	//defer s.Logout()
+}
+
+func (s *EmailService) Login() {
+	s.Connect()
+
+	err := s.client.Login(s.User, s.pass).Wait()
+	if s.CheckErr("Login", err) {
+		fmt.Println("========================================")
+		fmt.Println(err.Error())
+		fmt.Println("========================================")
+		return
+	}
+
+	s.success(s.User + " logged")
+}
+
+func (s *EmailService) ListMailboxes(mailboxes []string) {
+	selectOptions := &imap.SelectOptions{ReadOnly: true}
+
+	for _, mailbox := range mailboxes {
+		mbox, err := s.client.Select(mailbox, selectOptions).Wait()
+		if s.CheckErr("Listing mailbox", err) {
+			return
+		}
+		s.Mailboxes = append(s.Mailboxes, mbox)
+	}
+
+	if len(s.Mailboxes) == 0 {
+		s.Warn(sugar.SqueezeLine("mailboxes " + strings.Join(mailboxes, ", ") + " not found"))
+	}
+}
+
+func (s *EmailService) ListMessages(mailboxes []string, criteria *imap.SearchCriteria) {
+	for _, mailbox := range mailboxes {
+		mbox, err := s.client.Select(mailbox, nil).Wait()
+		if s.CheckErr("Listing mailbox", err) {
+			continue
+		}
+
+		seqs, err := s.client.UIDSearch(criteria, nil).Wait()
+		if s.CheckErr("UIDSearch", err) {
+			return
+		}
+
+		seqSet := imap.SeqSet{}
+		seqSet.AddRange(1, mbox.NumMessages)
+		if len(seqs.AllUIDs()) == 0 {
+			s.Warn("no messages found in mailbox: " + mailbox)
+			continue
+		}
+		s.success(fmt.Sprintf("Search complete, found %d messages: %+v", len(seqs.AllUIDs()), mbox.UIDNext))
+
+		fetchOptions := &imap.FetchOptions{
+			Envelope:      true,
+			Flags:         true,
+			BodyStructure: &imap.FetchItemBodyStructure{Extended: true},
+			BodySection:   []*imap.FetchItemBodySection{{Peek: true, Part: []int{2}}},
+		}
+
+		messages, err := s.client.Fetch(seqSet, fetchOptions).Collect()
+
+		if s.CheckErr("Fetch", err) {
+			continue
+		}
+
+		s.Messages = messages
+	}
+}
+
+func (s *EmailService) LogOut() {
+	err := s.client.Logout().Wait()
+	if !s.CheckErr("failed to logout", err) {
+		s.success("logged out successfully")
+	}
+}
+
+func (s *EmailService) CreateMailbox(mailboxName string) {
+	if err := s.client.Create(mailboxName, nil); err != nil {
+		log.Println("Помилка створення папки:", mailboxName, err)
+	} else {
+		log.Println("Створено папку:", mailboxName)
+	}
+}
+
+func (s *EmailService) DeleteMailbox(mailboxName string) {
+	// Вибираємо папку
+	if err := s.client.Delete(mailboxName); err != nil {
+		log.Println("Помилка видалення папки:", mailboxName, err)
+	} else {
+		log.Println("Папка видалена:", mailboxName)
+	}
+}
+
+func (s *EmailService) MailboxesList() {
+	listCmd := s.client.List("", "%", &imap.ListOptions{
+		ReturnStatus: &imap.StatusOptions{
+			NumMessages: true,
+			NumUnseen:   true,
+		},
+	})
+
+	for {
+		mbox := listCmd.Next()
+		if mbox == nil {
+			break
+		}
+
+		fmt.Printf("%+v\n", mbox)
+	}
+
+	if err := listCmd.Close(); err != nil {
+		log.Fatalf("LIST command failed: %v", err)
+	}
+}
+
+func (s *EmailService) MoveMessageToMailbox(msg *client.FetchMessageBuffer, status string) bool {
+	movable := imap.SeqSet{}
+	movable.AddNum(msg.SeqNum)
+	mailbox := viper.GetStringMapString("stb.move-processed-to-mailbox")
+	wait, err := s.client.Move(movable, mailbox[status]).Wait()
+
+	if s.CheckErr("Moving to archive", err) {
+		fmt.Println("Error moving to mail:", mailbox[status], "Error:", err.Error())
+		return false
+	}
+
+	fmt.Printf("Message %s moved, srcUIDs: %s, dstUIDs: %s\n", aurora.White(msg.Envelope.MessageID), aurora.Yellow(wait.SourceUIDs), aurora.Yellow(wait.DestUIDs))
+
+	return true
+}

logpresece

@@ -0,0 +1,59 @@
+scope prescene
+FLOAT: 33.000000, 12.000000, 3
+false
+======================  1  ==============================
+{Id:825647 Type:prescene TypeId:4313240 Title:Phaethon-Wielder_Of_The_Steel-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/phaethon-wielder_of_the_steel-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:09:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  2  ==============================
+{Id:825648 Type:prescene TypeId:4313238 Title:Helvetets_Port-Warlords-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/helvetets_port-warlords-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:08:33 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  4  ==============================
+{Id:825649 Type:prescene TypeId:4313236 Title:Circle_of_Ouroborus-Ajattoman_Uni-FI-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/circle_of_ouroborus-ajattoman_uni-fi-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:07:20 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  5  ==============================
+{Id:825650 Type:prescene TypeId:4313234 Title:Blitzkrieg-Blitzkrieg-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/blitzkrieg-blitzkrieg-24bit-web-flac-2024-moonblood/ Created:2024-09-10 15:06:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  6  ==============================
+{Id:825651 Type:prescene TypeId:4313233 Title:Antioch-Onward_With_Obliteration-EP-16BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/antioch-onward_with_obliteration-ep-16bit-web-flac-2024-moonblood/ Created:2024-09-10 15:05:27 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  8  ==============================
+{Id:825631 Type:prescene TypeId:4313221 Title:Wretchgod-Suffering_Upon_Suffering-EP-24BIT-WEB-FLAC-2024-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-suffering_upon_suffering-ep-24bit-web-flac-2024-moonblood/ Created:2024-09-10 14:07:36 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  9  ==============================
+{Id:825632 Type:prescene TypeId:4313220 Title:Wretchgod-Stygian_Blood_Ritual-EP-16BIT-WEB-FLAC-2022-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/wretchgod-stygian_blood_ritual-ep-16bit-web-flac-2022-moonblood/ Created:2024-09-10 14:06:51 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  11  ==============================
+{Id:825633 Type:prescene TypeId:4313188 Title:1349-The_Infernal_Pathway-REPACK-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-repack-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:34:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  15  ==============================
+{Id:825634 Type:prescene TypeId:4313177 Title:1349-The_Infernal_Pathway-Deluxe_Edition-16BIT-WEB-FLAC-2019-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-the_infernal_pathway-deluxe_edition-16bit-web-flac-2019-moonblood/ Created:2024-09-10 13:17:42 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  16  ==============================
+{Id:825635 Type:prescene TypeId:4313175 Title:1349-Revelations_of_the_Black_Flame-16BIT-WEB-FLAC-2009-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-revelations_of_the_black_flame-16bit-web-flac-2009-moonblood/ Created:2024-09-10 13:16:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  17  ==============================
+{Id:825636 Type:prescene TypeId:4313173 Title:1349-Massive_Cauldron_of_Chaos-16BIT-WEB-FLAC-2014-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-massive_cauldron_of_chaos-16bit-web-flac-2014-moonblood/ Created:2024-09-10 13:15:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  19  ==============================
+{Id:825637 Type:prescene TypeId:4313171 Title:1349-Liberation-16BIT-WEB-FLAC-2003-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-liberation-16bit-web-flac-2003-moonblood/ Created:2024-09-10 13:14:38 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  21  ==============================
+{Id:825638 Type:prescene TypeId:4313168 Title:1349-Hellfire-16BIT-WEB-FLAC-2005-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-hellfire-16bit-web-flac-2005-moonblood/ Created:2024-09-10 13:13:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  22  ==============================
+{Id:825639 Type:prescene TypeId:4313166 Title:1349-Demonoir-REISSUE-16BIT-WEB-FLAC-2011-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-demonoir-reissue-16bit-web-flac-2011-moonblood/ Created:2024-09-10 13:12:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  23  ==============================
+{Id:825640 Type:prescene TypeId:4313164 Title:1349-Beyond_the_Apocalypse-16BIT-WEB-FLAC-2004-MOONBLOOD TypeSubsectionId:0 Releaser:MOONBLOOD ExSource:https://scnlog.me/music/1349-beyond_the_apocalypse-16bit-web-flac-2004-moonblood/ Created:2024-09-10 13:12:00 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+Sleeping... 0
+false
+======================  1  ==============================
+{Id:825652 Type:prescene TypeId:4300530 Title:Zuckerbaby--Zuckerbaby-CD-FLAC-1997-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/zuckerbaby-zuckerbaby-cd-flac-1997-meowzik/ Created:2024-08-31 18:17:03 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  2  ==============================
+{Id:825653 Type:prescene TypeId:4300529 Title:Silmarils--Vegas_76-FR-CD-FLAC-2000-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/silmarils-vegas_76-fr-cd-flac-2000-meowzik/ Created:2024-08-31 18:16:29 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  3  ==============================
+{Id:825654 Type:prescene TypeId:4300527 Title:Michael_Kaeshammer--Days_Like_These-CD-FLAC-2007-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/michael_kaeshammer-days_like_these-cd-flac-2007-meowzik/ Created:2024-08-31 18:14:25 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  4  ==============================
+{Id:825655 Type:prescene TypeId:4300521 Title:Butch_Walker--Letters-CD-FLAC-2004-MEOWZiK TypeSubsectionId:0 Releaser:MEOWZiK ExSource:https://scnlog.me/music/butch_walker-letters-cd-flac-2004-meowzik/ Created:2024-08-31 18:09:22 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  16  ==============================
+{Id:825656 Type:prescene TypeId:4300320 Title:Embodiment_Elimination-Metamorphosis_Incarnate_Through_Genetic_Devastation-(ISR071-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/embodiment_elimination-metamorphosis_incarnate_through_genetic_devastation-isr071-24-cd-flac-2024-86d/ Created:2024-08-31 11:19:45 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  19  ==============================
+{Id:825657 Type:prescene TypeId:4300309 Title:Putrified_Degradation-Manufractured-(ISR072-24)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/putrified_degradation-manufractured-isr072-24-cd-flac-2024-86d/ Created:2024-08-31 10:45:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  20  ==============================
+{Id:825658 Type:prescene TypeId:4300307 Title:Post_Mortal_Possession-The_Dead_Space_Between_the_Stars-(LORD091)-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/post_mortal_possession-the_dead_space_between_the_stars-lord091-cd-flac-2024-86d/ Created:2024-08-31 10:43:09 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  21  ==============================
+{Id:825659 Type:prescene TypeId:4300306 Title:Invirulant-Necrothuggin-(ISR070-24)-CDEP-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/invirulant-necrothuggin-isr070-24-cdep-flac-2024-86d/ Created:2024-08-31 10:41:04 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+======================  22  ==============================
+{Id:825660 Type:prescene TypeId:4300304 Title:Dead_and_Dripping-Disillusioned_by_Excessive_Human_Consumption-(LORD092)-DEMO_REISSUE-CD-FLAC-2024-86D TypeSubsectionId:0 Releaser:86D ExSource:https://scnlog.me/music/dead_and_dripping-disillusioned_by_excessive_human_consumption-lord092-demo_reissue-cd-flac-2024-86d/ Created:2024-08-31 10:38:13 Modified: Contents: A: H: Fingerprint: FsFingerprint: Vid:0 G: ExternalSourcesCol: Error:<nil>}
+Sleeping... 1
+scope: prescene
+[1 45]
+init console console: ps
+
+Execution took 12.6276275s

pkg/handler/console.go

@@ -0,0 +1,26 @@
+package handler
+
+import (
+	"fmt"
+	"slices"
+
+	"github.com/logrusorgru/aurora/v4"
+	"github.com/spf13/viper"
+)
+
+func (h *Handler) Console() string {
+	viper.SetDefault("env", "devel")
+	cmd := viper.GetString("create")
+	cmdList := viper.Sub("console.cmd")
+	allowCreate := cmdList.GetStringSlice("create")
+
+	if !slices.Contains(allowCreate, cmd) {
+		fmt.Printf("%s Not allowed command %s used\n", aurora.BgMagenta("[WARN]"), aurora.Magenta(cmd))
+		return ""
+	}
+
+	fmt.Printf("%v\n", slices.Contains(allowCreate, cmd))
+
+	return viper.GetString("env")
+
+}

pkg/handler/handler.go

@@ -0,0 +1,52 @@
+package handler
+
+import (
+	"log" // Додано імпорт log
+
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/service"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+// consoleCommandHandler визначає тип для функцій-обробників консольних команд.
+type consoleCommandHandler func() string
+
+type Handler struct {
+	services *service.Service
+	// consoleCommands зберігає мапу назв команд до відповідних функцій-обробників.
+	consoleCommands map[string]consoleCommandHandler
+}
+
+func New(services *service.Service) *Handler {
+	h := &Handler{services: services}
+
+	// Ініціалізація мапи команд.
+	h.consoleCommands = map[string]consoleCommandHandler{
+		"console":        h.Console,
+		"info":           h.Info,
+		"metal-archives": h.MetalArchives,
+		"prescene":       h.Prescene,
+		"rutracker":      h.Rutracker,
+		"stb":            h.STB,
+		//"web":            h.Web,
+	}
+
+	return h
+}
+
+func (h *Handler) InitConsole() string {
+	commandName := viper.GetString(constant.FlagScopeEnable)
+
+	if cmdFunc, ok := h.consoleCommands[commandName]; ok {
+		return commandName + " launched, " + cmdFunc() + "\n"
+	} else {
+		log.Printf("Error: unknown console command '%s'", commandName)
+		return "Error: unknown console command"
+	}
+}
+
+func (h *Handler) InitRoutes() string {
+	//TODO:
+	return "i am the web initiator"
+}

pkg/handler/info.go

@@ -0,0 +1,36 @@
+package handler
+
+import (
+	"fmt"
+	"log"
+
+	"github.com/logrusorgru/aurora/v4"
+	"github.com/spf13/viper"
+
+	"github.com/mewkiz/flac"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+func (h *Handler) Info() string {
+	md5()
+
+	fmt.Printf("%s: %s; %s: %s\n",
+		aurora.Cyan("ENV"),
+		viper.GetString(constant.FlagEnv), aurora.Cyan("SCOPE"), viper.GetString(constant.FlagScopeEnable))
+	return "info"
+}
+
+func md5() {
+	stream, err := flac.ParseFile("C:\\arm.amok.space\\.incoming\\Ancient Storm\\Forever and Never (2024)\\06 Old Mountain.flac")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer stream.Close()
+
+	fmt.Printf("unencoded audio md5sum: %032x\n", stream.Info.MD5sum[:])
+	fmt.Printf("Total number of inter-channel samples in the stream: %+v\n", stream.Info.NSamples)
+	for i, block := range stream.Blocks {
+		fmt.Printf("block %d: %v\n", i, block.Type)
+	}
+}

pkg/handler/metal-archives.go

@@ -0,0 +1,5 @@
+package handler
+
+func (h *Handler) MetalArchives() string {
+	return "MetalArchives................."
+}

pkg/handler/prescene.go

@@ -0,0 +1,46 @@
+package handler
+
+import (
+	"fmt"
+	"log/slog"
+	"math"
+	"strconv"
+	"time"
+
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+func (h *Handler) Prescene() string {
+	pagesToScrape := []string{"1"}
+	levels := viper.GetInt(constant.CfgKeyLevelsToScrape)
+	if levels > 1 {
+		var q = (60.0 / 100.0) * (100.0 / viper.GetFloat64(constant.CfgKeyLevelsToScrape))
+		minute := float64(time.Now().Minute())
+		if minute < 1 {
+			minute = 1
+		}
+
+		fmt.Printf("FLOAT: %f, %f, %v\n", minute, q, math.Ceil(minute/q))
+
+		hour := time.Now().Hour()
+		if hour == 1 {
+			hour = 25
+		} else if hour == 0 {
+			hour = 1
+		}
+
+		page := hour * int(math.Ceil(minute/q))
+		pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
+	}
+
+	_, err := h.services.Prescene.GetPage(pagesToScrape)
+	if err != nil {
+		slog.Error("error occurred while getting page: ", "err", err.Error())
+	}
+
+	fmt.Printf("%+v\n", pagesToScrape)
+
+	return "ps"
+}

pkg/handler/rutracker.go

@@ -0,0 +1,23 @@
+package handler
+
+import (
+	"fmt"
+	"log/slog"
+	"time"
+
+	"github.com/spf13/viper"
+)
+
+func (h *Handler) Rutracker() string {
+	key := fmt.Sprintf("topic.%v", time.Now().Hour())
+	topics := viper.GetStringSlice(key)
+
+	_, err := h.services.Rutracker.GetTopic(topics)
+	if err != nil {
+		slog.Error("error occurred while getting topic: ", "err", err.Error())
+	}
+
+	//fmt.Printf("%+v", rt)
+
+	return "rt"
+}

pkg/handler/stb.go

@@ -0,0 +1,36 @@
+package handler
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+)
+
+func (h *Handler) STB() string {
+	endpoint := fmt.Sprintf("%s.%s", constant.ScopeShareTheBrutality, constant.CfgKeyEndpoint)
+	endpoint = viper.GetString(endpoint)
+
+	if endpoint == "" {
+		slog.Error("getting endpoint from config", "err", errors.New("no endpoint provided"))
+		return "stb"
+	}
+
+	es, ms := h.services.ShareTheBrutality.GetMail(endpoint)
+
+	//fmt.Printf("%+v\n", es)
+	ms.LogOut()
+
+	for _, record := range es {
+		fmt.Printf("%s %d: %s [#%s]\n", record.Created, record.Id, record.Title, record.Releaser)
+	}
+
+	/*if err != nil {
+		slog.Error("error occurred while getting topic: ", "err", err)
+	}*/
+
+	return fmt.Sprintf("Added %d records\n", len(es))
+}

pkg/repository/prescene.go

@@ -0,0 +1,134 @@
+package repository
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"regexp"
+	"slices"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/dromara/carbon/v2"
+	"github.com/go-shiori/dom"
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/helper/parser"
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type Prescene struct {
+	db *sqlx.DB
+}
+
+func NewPresceneRepository(db *sqlx.DB) *Prescene {
+	return &Prescene{db: db}
+}
+
+func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
+	entries := make([]model.ExternalSources, 0)
+	endpoint := viper.GetString(constant.CfgKeyEndpoint)
+
+	uri := viper.GetString(constant.FlagSingleUri)
+	if uri != "" {
+		url := fmt.Sprintf("%s/%s", strings.Trim(endpoint, "/"), strings.Trim(uri, "/"))
+		result, _ := parseUrl(url, s.db)
+		entries = append(entries, result...)
+	} else {
+		for _, t := range pageNumbers {
+			if t != "1" {
+				endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
+			}
+
+			//doc, err := parser.HTMLSourceFromURL("https://mdb.kplus.net.ua	/$/scnlog.html")
+			if result, err := parseUrl(endpoint, s.db); err == nil {
+				entries = append(entries, result...)
+			} else {
+				slog.Error("parsing url", "err", err)
+			}
+
+			//fmt.Println("Sleeping...", j)
+			time.Sleep(viper.GetDuration(constant.CfgKeySleepBeforeNextIteration))
+		}
+	}
+
+	return entries, nil
+}
+
+func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
+	entries := make([]model.ExternalSources, 0)
+	tags := viper.GetStringMapStringSlice("groups.tags")
+	slog.Info("singleton", "url", endpoint)
+	doc, err := parser.HTMLSourceFromURL(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	if doc == nil {
+		return nil, errors.New("document is nil")
+	}
+	var validID = regexp.MustCompile(`-\d+\/$`)
+
+	for i, item := range dom.QuerySelectorAll(doc, ".post.type-post.category-flac.category-music") {
+		var es model.ExternalSources
+		columns := []string{"`type`", "type_id", "title", "eXsource", "releaser", "created"}
+
+		title := dom.QuerySelector(item, ".title")
+		if title != nil {
+			anchor := dom.QuerySelector(title, "h1 > a")
+			if anchor != nil {
+				es.Type = constant.ScopePrescene
+				es.Title = dom.GetAttribute(anchor, "title")
+				if es.Title == "Auto Draft" {
+					slog.Info("Skipped", "title", es.Title)
+					continue
+				}
+
+				es.ExSource = dom.GetAttribute(anchor, "href")
+				if validID.MatchString(es.ExSource) {
+					continue
+				}
+
+				pattern := regexp.MustCompile(`(?is)-(\w+)$`)
+				es.Releaser = pattern.FindStringSubmatch(es.Title)[1]
+
+				for flag, groups := range tags {
+					if slices.Contains(groups, es.Releaser) {
+						es.A = flag
+						es.H = flag
+						columns = append(columns, "a", "h")
+						break
+					}
+				}
+			}
+
+			if es.A == constant.TagIgnore {
+				slog.Info("Skipped", "releaser", es.Releaser)
+				continue
+			}
+
+			localtime := dom.QuerySelector(title, "small > span.localtime")
+			if localtime != nil {
+				lc := dom.GetAttribute(localtime, "data-lttime")
+				es.Created = carbon.Parse(lc, "Europe/Kyiv")
+			}
+		}
+
+		cls := dom.GetAttribute(item, "class")
+		pattern := regexp.MustCompile(`(?s)^post-(\d+)\spost`)
+		es.TypeId, _ = strconv.Atoi(pattern.FindStringSubmatch(cls)[1])
+
+		esModel := table.ExternalSources{Columns: columns}
+		entry := esModel.InsertOnDuplicate(es, db)
+		entries = append(entries, entry)
+
+		fmt.Println("====================== ", i, " ==============================")
+		fmt.Printf("%+v\n", entry)
+	}
+
+	return entries, nil
+}

pkg/repository/repository.go

@@ -0,0 +1,21 @@
+package repository
+
+import (
+	"github.com/jmoiron/sqlx"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
+)
+
+type Repository struct {
+	_interface.Rutracker
+	_interface.Prescene
+	_interface.ShareTheBrutality
+}
+
+func New(db *sqlx.DB) *Repository {
+	return &Repository{
+		Rutracker:         NewRutrackerRepository(db),
+		Prescene:          NewPresceneRepository(db),
+		ShareTheBrutality: NewShareTheBrutalityRepository(db),
+	}
+}

pkg/repository/rutracker.go

@@ -0,0 +1,83 @@
+package repository
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log/slog"
+	"net/http"
+	"net/url"
+	"strconv"
+
+	"github.com/dromara/carbon/v2"
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
+)
+
+type Rutracker struct {
+	db *sqlx.DB
+}
+
+func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
+	return &Rutracker{db: db}
+}
+
+func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
+	endpoint := viper.GetString(constant.CfgKeyEndpoint)
+	entries := make([]model.ExternalSources, 0)
+	columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
+
+	for _, t := range topics {
+		topic, err := fetch(fmt.Sprintf(endpoint, t))
+		if err != nil {
+			slog.Error("couldn't parse topic data", "err", err.Error())
+			return entries, err
+		}
+
+		for _, e := range topic.Entry {
+			var es model.ExternalSources
+			u, _ := url.Parse(e.Link.Href)
+
+			es.Type = constant.ScopeRuTracker
+			es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
+			es.Title = e.Title
+			es.TypeSubsectionId, _ = strconv.Atoi(t)
+			es.Releaser = e.Author.Name
+			es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
+			esModel := table.ExternalSources{Columns: columns}
+			entry := esModel.InsertOnDuplicate(es, s.db)
+			entries = append(entries, entry)
+			//fmt.Printf("%+v\n\n\n", entry)
+		}
+	}
+
+	return entries, nil
+}
+
+func fetch(endpoint string) (*resource.RutrackerAtomTopic, error) {
+	resp, err := http.Get(endpoint)
+	if err != nil {
+		slog.Error("couldn't fetch data", endpoint, err.Error())
+		return nil, err
+	}
+
+	defer func(Body io.ReadCloser) {
+		err = Body.Close()
+		if err != nil {
+			fmt.Println("Body.Close")
+		}
+	}(resp.Body)
+
+	topic := &resource.RutrackerAtomTopic{}
+
+	if err = xml.NewDecoder(resp.Body).Decode(topic); err != nil {
+		return nil, err
+	}
+
+	return topic, nil
+}

pkg/repository/stb.go

@@ -0,0 +1,156 @@
+package repository
+
+import (
+	"fmt"
+	"net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"github.com/dromara/carbon/v2"
+	"github.com/emersion/go-imap/v2"
+	"github.com/go-shiori/dom"
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+	"golang.org/x/net/html"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/helper/parser"
+	"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/mail"
+	_table "git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type ShareTheBrutality struct {
+	scope        string
+	EmailService mail.EmailService
+	db           *sqlx.DB
+}
+
+func NewShareTheBrutalityRepository(db *sqlx.DB) *ShareTheBrutality {
+	return &ShareTheBrutality{db: db, scope: constant.ScopeShareTheBrutality}
+}
+
+func (s *ShareTheBrutality) GetMail(email string) ([]model.ExternalSources, *mail.EmailService) {
+	s.EmailService = mail.EmailService{
+		User: email,
+	}
+
+	s.EmailService.Login()
+
+	mailboxes := fmt.Sprintf("%s.mailboxes", s.scope)
+	criteria := fmt.Sprintf("%s.search-criteria", s.scope)
+	searchCriteria := &imap.SearchCriteria{
+		Text: viper.GetStringSlice(criteria),
+	}
+
+	s.EmailService.ListMessages(viper.GetStringSlice(mailboxes), searchCriteria)
+	//s.EmailService.CreateMailbox("Succeed")
+	//s.EmailService.DeleteMailbox("Succeed")
+	//s.EmailService.MailboxesList()
+
+	entries := s.Processing(viper.GetStringMapString(fmt.Sprintf("%s.sender", s.scope)))
+
+	return entries, &s.EmailService
+}
+
+func (s *ShareTheBrutality) Processing(sender map[string]string) []model.ExternalSources {
+	columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created", "fingerprint"}
+	entriesBatched := make([]model.ExternalSources, 0)
+	if len(s.EmailService.Messages) == 0 {
+		return entriesBatched
+	}
+
+	//tmpPath := viper.GetString(fmt.Sprintf("%s.storage.filepath", s.scope))
+	dbType := viper.GetString(fmt.Sprintf("%s.db-type", s.scope))
+	regexPatterns := viper.GetStringMapString(fmt.Sprintf("%s.regex", s.scope))
+	topics := viper.GetStringMap(fmt.Sprintf("%s.topics", s.scope))
+
+	for _, msg := range s.EmailService.Messages {
+		entries := make([]model.ExternalSources, 0)
+		from := msg.Envelope.From[0]
+		subject := msg.Envelope.Subject
+
+		if !(from.Mailbox == sender["mailbox"] && from.Host == sender["host"] && subject == sender["subject"]) {
+			continue
+		}
+
+		for _, section := range msg.BodySection {
+			//sugar.WriteDataToTmpFile(msg.BodySection, tmpPath)
+
+			if section.Bytes != nil {
+				doc, err := parser.HTMLSource(string(section.Bytes))
+				if s.EmailService.CheckErr("parsing message body", err) {
+					continue
+				}
+
+				table := dom.QuerySelector(doc, "body > table:nth-of-type(1n) table:nth-of-type(1n) table:nth-of-type(2n) > tbody")
+				if table == nil {
+					s.EmailService.Warn("dom.QuerySelector had not queried any data, returned nil")
+					continue
+				}
+
+				var es model.ExternalSources
+				for _, td := range dom.QuerySelectorAll(table, "tr > td:nth-child(2)") {
+					anchor := dom.QuerySelector(td, "h2 > a")
+					if anchor == nil {
+						s.EmailService.Warn("dom.QuerySelector couldn't find title")
+						continue
+					}
+					es.Title = sugar.SqueezeLine(dom.InnerHTML(anchor))
+
+					u, err := url.Parse(dom.GetAttribute(anchor, "href"))
+					if s.EmailService.CheckErr("parsing url", err) {
+						continue
+					}
+
+					es.Fingerprint = u.RequestURI()
+					pattern := regexp.MustCompile(regexPatterns["type-id"])
+					typeIdMatch := pattern.FindStringSubmatch(es.Fingerprint)
+					if len(typeIdMatch) != 2 {
+						s.EmailService.Warn("Regexp => typeIdMatch not matched")
+						continue
+					}
+					es.TypeId, _ = strconv.Atoi(typeIdMatch[1])
+
+					sourceData := dom.QuerySelector(td, "p:first-child")
+					if sourceData == nil {
+						s.EmailService.Warn("dom.QuerySelector couldn't find sourceData in paragraph")
+						continue
+					}
+					sourceDataString := html.UnescapeString(sugar.SqueezeLine(dom.InnerHTML(sourceData)))
+					pattern = regexp.MustCompile(regexPatterns["who-genre"])
+					sourceDataMatch := pattern.FindStringSubmatch(sourceDataString)
+
+					if len(sourceDataMatch) != 3 {
+						s.EmailService.Warn("Regexp => sourceData not matched")
+						continue
+					}
+
+					es.Releaser = sourceDataMatch[1]
+					es.Created = carbon.Parse(msg.Envelope.Date.String(), "Europe/Kyiv")
+
+					es.Type = dbType
+
+					genre := strings.ToLower(sourceDataMatch[2])
+					es.TypeSubsectionId = topics[genre].(int)
+
+					entries = append(entries, es)
+				}
+
+				//fmt.Println(entries)
+				//os.Exit(0)
+
+				result, status := _table.BatchInsertOnDuplicate(entries, s.db, columns)
+				if status != constant.StatusFailed {
+					entriesBatched = append(entriesBatched, result...)
+				}
+
+				s.EmailService.MoveMessageToMailbox(msg, status)
+			}
+		}
+	}
+
+	return entriesBatched
+}

pkg/repository/table/external_sources.go

@@ -0,0 +1,84 @@
+package table
+
+import (
+	"fmt"
+	"log/slog"
+	"slices"
+	"strings"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/logrusorgru/aurora/v4"
+
+	"git.kplus.net.ua/yevhen/resource-scraper/helper/thither"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type ExternalSources struct {
+	Columns []string
+}
+
+func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
+	stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
+	placeholders := strings.Join(f.Columns, ", :")
+	placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
+	query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
+
+	if rows, err := db.NamedQuery(query, &es); err == nil {
+		for rows.Next() {
+			es.Error = rows.StructScan(&es)
+		}
+	} else {
+		es.Error = err
+	}
+
+	return es
+}
+
+func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, columns []string) ([]model.ExternalSources, string) {
+	es := &ExternalSources{Columns: columns}
+	typeIds := es.GetTypeIds(entries, db)
+	var status string
+	errCount := 0
+
+	for i := 0; i < len(entries); i++ {
+		entry := es.InsertOnDuplicate(entries[i], db)
+		if entry.Error != nil {
+			slog.Error("insert/update entry", "err", entry.Error)
+			errCount++
+		}
+
+		if !slices.Contains(typeIds, entry.TypeId) {
+			fmt.Printf("%s: %s\n", aurora.Green("ADDED"), aurora.White(entry.Title))
+		}
+
+		entries[i] = es.InsertOnDuplicate(entries[i], db)
+	}
+
+	if errCount == 0 {
+		status = constant.StatusSucceed
+	} else if errCount > 0 && errCount == len(entries) {
+		status = constant.StatusFailed
+	} else {
+		status = constant.StatusSuspicious
+	}
+
+	return entries, status
+}
+
+func (f *ExternalSources) GetTypeIds(entries []model.ExternalSources, db *sqlx.DB) []int {
+	if len(entries) == 0 {
+		fmt.Printf("Entries are empty")
+		return nil // або логування помилки
+	}
+
+	var typeIds []int
+	ids := thither.FieldValueToStrSlice(entries, "TypeId")
+	query := fmt.Sprintf("SELECT type_id FROM %s WHERE `type` = '%s' AND type_id IN (%s) LIMIT %d", constant.ExternalSourcesTable, entries[0].Type, strings.Join(ids, ","), len(ids))
+	fmt.Println(query)
+	err := db.Select(&typeIds, query)
+	if err != nil {
+		slog.Error("getting type ids", "err", err)
+	}
+	return typeIds
+}

pkg/service/info.go

@@ -0,0 +1,7 @@
+package service
+
+type InfoService struct{}
+
+func NewInfoService() *InfoService {
+	return &InfoService{}
+}

pkg/service/metal-archives.go

@@ -0,0 +1 @@
+package service

pkg/service/prescene.go

@@ -0,0 +1,18 @@
+package service
+
+import (
+	"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type PresceneService struct {
+	repo _interface.Prescene
+}
+
+func NewPresceneService(repo _interface.Prescene) *PresceneService {
+	return &PresceneService{repo: repo}
+}
+
+func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
+	return p.repo.GetPage(pageNumbers)
+}

pkg/service/rutracker.go

@@ -0,0 +1,18 @@
+package service
+
+import (
+	"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type RutrackerService struct {
+	repo _interface.Rutracker
+}
+
+func NewRutrackerService(repo _interface.Rutracker) *RutrackerService {
+	return &RutrackerService{repo: repo}
+}
+
+func (s *RutrackerService) GetTopic(topic []string) ([]model.ExternalSources, error) {
+	return s.repo.GetTopic(topic)
+}

pkg/service/service.go

@@ -0,0 +1,22 @@
+package service
+
+import (
+	"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
+)
+
+type Service struct {
+	_interface.Rutracker
+	_interface.Prescene
+	_interface.Info
+	_interface.ShareTheBrutality
+}
+
+func New(repos *repository.Repository) *Service {
+	return &Service{
+		Rutracker:         NewRutrackerService(repos.Rutracker),
+		Prescene:          NewPresceneService(repos.Prescene),
+		Info:              NewInfoService(),
+		ShareTheBrutality: NewShareTheBrutalityService(repos.ShareTheBrutality),
+	}
+}

pkg/service/stb.go

@@ -0,0 +1,19 @@
+package service
+
+import (
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/mail"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type ShareTheBrutalityService struct {
+	repo _interface.ShareTheBrutality
+}
+
+func NewShareTheBrutalityService(repo _interface.ShareTheBrutality) *ShareTheBrutalityService {
+	return &ShareTheBrutalityService{repo: repo}
+}
+
+func (stb *ShareTheBrutalityService) GetMail(email string) ([]model.ExternalSources, *mail.EmailService) {
+	return stb.repo.GetMail(email)
+}

types/constant/config.go

@@ -0,0 +1,10 @@
+package constant
+
+const (
+	CfgKeyScopeEnable              = FlagScopeEnable
+	CfgKeyConfigDir                = "config-dir"
+	CfgKeySleepBeforeNextIteration = "sleep-before-next-iteration"
+	CfgKeyEndpoint                 = "endpoint"
+	CfgKeyEndpointNext             = "endpoint-next"
+	CfgKeyLevelsToScrape           = "levels-to-scrape"
+)

types/constant/constants.go

@@ -0,0 +1,28 @@
+package constant
+
+const (
+	ScopeRuTracker         string = "rutracker"
+	ScopePrescene          string = "prescene"
+	ScopeWeb               string = "web"
+	ScopeInfo              string = "info"
+	ScopeShareTheBrutality string = "stb"
+)
+
+const (
+	RoleConsole string = "console"
+	RoleWeb     string = "web"
+)
+
+const (
+	TagIgnore            string = "ignore"
+	ExternalSourcesTable string = "external_sources"
+)
+
+const (
+	StatusSuccess string = "success"
+	StatusError   string = "error"
+
+	StatusSucceed    string = "succeed"
+	StatusSuspicious string = "suspicious"
+	StatusFailed     string = "failed"
+)

types/constant/flag.go

@@ -0,0 +1,17 @@
+package constant
+
+const (
+	FlagHelp         = "help"
+	FlagHelpShort    = "h"
+	FlagDebug        = "debug"
+	FlagVersion      = "version"
+	FlagVersionShort = "v"
+	FlagConfigFile   = "config-file"
+	FlagScopeEnable  = "scope-enable"
+	FlagSingleUri    = "single-uri"
+	FlagEnv          = "env"
+
+	DefaultEnvProd    = "prod"
+	DefaultEnvDev     = "devel"
+	DefaultConfigPath = "config/default"
+)

types/interface/interfaces.go

@@ -0,0 +1,22 @@
+package _interface
+
+import (
+	"git.kplus.net.ua/yevhen/resource-scraper/internal/mail"
+	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
+)
+
+type Rutracker interface {
+	GetTopic(topics []string) ([]model.ExternalSources, error)
+}
+
+type Prescene interface {
+	GetPage(pageNumbers []string) ([]model.ExternalSources, error)
+}
+
+type ShareTheBrutality interface {
+	GetMail(email string) ([]model.ExternalSources, *mail.EmailService)
+}
+
+type MetalArchiveInterface interface{}
+
+type Info interface{}

types/model/external_sources.go

@@ -0,0 +1,42 @@
+package model
+
+import (
+	"github.com/dromara/carbon/v2"
+)
+
+type Type string
+
+/*const (
+	mmt Type = iota
+	prescene
+	rutracker
+	locate
+	darkabyss
+	bfm
+	trash
+	nnmc
+	stb
+	bandcamp
+	deathgrind
+)*/
+
+type ExternalSources struct {
+	Id                 int64          `json:"id" db:"id"`
+	Type               string         `json:"type" db:"type"`
+	TypeId             int            `json:"type_id" db:"type_id"`
+	Title              string         `json:"title" db:"title"`
+	TypeSubsectionId   int            `json:"type_subsection_id" db:"type_subsection_id"`
+	Releaser           string         `json:"releaser" db:"releaser"`
+	ExSource           string         `json:"ex_source" db:"eXsource"`
+	Created            *carbon.Carbon `json:"created_at" db:"created"`
+	Modified           carbon.Carbon  `json:"modified_at" db:"modified"`
+	Contents           string         `json:"contents" db:"contents"`
+	A                  string         `json:"a" db:"a"`
+	H                  string         `json:"h" db:"h"`
+	Fingerprint        string         `json:"fingerprint" db:"fingerprint"`
+	FsFingerprint      string         `json:"fs_fingerprint" db:"fs_fingerprint"`
+	Vid                int            `json:"vid" db:"vid"`
+	G                  string         `json:"g" db:"g"`
+	ExternalSourcesCol string         `json:"external_sources_column" db:"external_sourcescol"`
+	Error              error
+}

types/resource/metal-archive.go

@@ -0,0 +1,45 @@
+package resource
+
+import "github.com/dromara/carbon/v2"
+
+/*https://www.metal-archives.com/search/ajax-advanced/searching/albums/?bandName=&releaseTitle=&releaseYearFrom=2024&releaseMonthFrom=09&releaseYearTo=&releaseMonthTo=&country=&location=&releaseLabelName=&releaseCatalogNumber=&releaseIdentifiers=&releaseRecordingInfo=&releaseDescription=&releaseNotes=&genre=&sEcho=6&iColumns=4&sColumns=&iDisplayStart=1000&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&_=1726217541027*/
+
+type AutoGenerated struct {
+	Error                string     `json:"error"`
+	ITotalRecords        int        `json:"iTotalRecords"`
+	ITotalDisplayRecords int        `json:"iTotalDisplayRecords"`
+	SEcho                int        `json:"sEcho"`
+	AaData               [][]string `json:"aaData"`
+}
+
+/*ALTER TABLE `go_tut_tokill`.`external_sources`
+CHANGE COLUMN `type` `type` ENUM('mmt', 'prescene', 'rutracker', 'locate', 'darkabyss', 'bfm', 'trash', 'nnmc', 'stb', 'bandcamp', 'deathgrind', 'ma') NULL DEFAULT NULL ;
+
+*/
+
+// AlbumQuery easy to debug https://www.freeformatter.com/url-parser-query-string-splitter.html
+type AlbumQuery struct {
+	BandName             string        `param:"bandName"`
+	ReleaseTitle         string        `param:"releaseTitle"`
+	ReleaseYearFrom      int           `param:"releaseYearFrom"`
+	ReleaseMonthFrom     int           `param:"releaseMonthFrom"`
+	ReleaseYearTo        int           `param:"releaseYearTo"`
+	ReleaseMonthTo       int           `param:"releaseMonthTo"`
+	Country              []string      `param:"country"`
+	Location             string        `param:"location"`
+	ReleaseLabelName     string        `param:"releaseLabelName"`
+	ReleaseCatalogNumber int           `param:"releaseCatalogNumber"`
+	ReleaseIdentifiers   string        `param:"releaseIdentifiers"`
+	ReleaseRecordingInfo string        `param:"releaseRecordingInfo"`
+	ReleaseDescription   string        `param:"releaseDescription"`
+	ReleaseNotes         string        `param:"releaseNotes"`
+	Genre                string        `param:"genre"`
+	ReleaseType          []string      `param:"releaseType"`
+	SEcho                string        `param:"sEcho"`
+	IColumns             uint8         `param:"iColumns" default:"4"`
+	SColumns             string        `param:"sColumns"`
+	IDisplayStart        uint8         `param:"iDisplayStart" default:"0"`
+	IDisplayLength       uint8         `param:"iDisplayLength" default:"200"`
+	MDataProp            uint8         `param:"mDataProp_%d"`
+	Timestamp            carbon.Carbon `param:"_"`
+}

types/resource/rutracker.go

@@ -0,0 +1,35 @@
+package resource
+
+import "encoding/xml"
+
+type RutrackerAtomTopic struct {
+	XMLName xml.Name `xml:"feed"`
+	Text    string   `xml:",chardata"`
+	Xmlns   string   `xml:"xmlns,attr"`
+	ID      string   `xml:"id"`
+	Link    struct {
+		Text string `xml:",chardata"`
+		Href string `xml:"href,attr"`
+	} `xml:"link"`
+	Updated string `xml:"updated"`
+	Title   string `xml:"title"`
+	Entry   []struct {
+		Text string `xml:",chardata"`
+		ID   string `xml:"id"`
+		Link struct {
+			Text string `xml:",chardata"`
+			Href string `xml:"href,attr"`
+		} `xml:"link"`
+		Updated string `xml:"updated"`
+		Title   string `xml:"title"`
+		Author  struct {
+			Text string `xml:",chardata"`
+			Name string `xml:"name"`
+		} `xml:"author"`
+		Category struct {
+			Text  string `xml:",chardata"`
+			Term  string `xml:"term,attr"`
+			Label string `xml:"label,attr"`
+		} `xml:"category"`
+	} `xml:"entry"`
+}

types/resource/scnlog.go

@@ -0,0 +1,46 @@
+package resource
+
+import "encoding/xml"
+
+type ScnLogRss struct {
+	XMLName xml.Name `xml:"rss"`
+	Text    string   `xml:",chardata"`
+	Version string   `xml:"version,attr"`
+	Content string   `xml:"content,attr"`
+	Wfw     string   `xml:"wfw,attr"`
+	Dc      string   `xml:"dc,attr"`
+	Atom    string   `xml:"atom,attr"`
+	Sy      string   `xml:"sy,attr"`
+	Slash   string   `xml:"slash,attr"`
+	Channel struct {
+		Text  string `xml:",chardata"`
+		Title string `xml:"title"`
+		Link  struct {
+			Text string `xml:",chardata"`
+			Href string `xml:"href,attr"`
+			Rel  string `xml:"rel,attr"`
+			Type string `xml:"type,attr"`
+		} `xml:"link"`
+		Description     string `xml:"description"`
+		LastBuildDate   string `xml:"lastBuildDate"`
+		Language        string `xml:"language"`
+		UpdatePeriod    string `xml:"updatePeriod"`
+		UpdateFrequency string `xml:"updateFrequency"`
+		Generator       string `xml:"generator"`
+		Item            []struct {
+			Text     string   `xml:",chardata"`
+			Title    string   `xml:"title"`
+			Link     string   `xml:"link"`
+			Comments string   `xml:"comments"`
+			Creator  string   `xml:"creator"`
+			PubDate  string   `xml:"pubDate"`
+			Category []string `xml:"category"`
+			Guid     struct {
+				Text        string `xml:",chardata"`
+				IsPermaLink string `xml:"isPermaLink,attr"`
+			} `xml:"guid"`
+			Description string `xml:"description"`
+			CommentRss  string `xml:"commentRss"`
+		} `xml:"item"`
+	} `xml:"channel"`
+}