diff --git a/.gitignore b/.gitignore
index 4d32dcb..916cd2e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,34 +1,15 @@
-.idea/**/workspace.xml
-.idea/**/tasks.xml
-.idea/**/usage.statistics.xml
-.idea/**/dictionaries
-.idea/**/shelf
-.idea/**/aws.xml
-.idea/**/contentModel.xml
-.idea/**/dataSources/
-.idea/**/dataSources.ids
-.idea/**/dataSources.local.xml
-.idea/**/sqlDataSources.xml
-.idea/**/dynamic.xml
-.idea/**/uiDesigner.xml
-.idea/**/dbnavigator.xml
-.idea/**/gradle.xml
-.idea/**/libraries
+.idea/
cmake-build-*/
-.idea/**/mongoSettings.xml
*.iws
out/
.idea_modules/
atlassian-ide-plugin.xml
-.idea/replstate.xml
-.idea/sonarlint/
com_crashlytics_export_strings.xml
crashlytics.properties
crashlytics-build.properties
fabric.properties
-.idea/httpRequests
-.idea/caches/build_file_checksums.ser
.env
tmp/
vendor/
-config/secret.yaml
\ No newline at end of file
+config/secret.yaml
+*.md
\ No newline at end of file
diff --git a/.idea/golinter.xml b/.idea/golinter.xml
index 2c04dd9..bb58ec5 100644
--- a/.idea/golinter.xml
+++ b/.idea/golinter.xml
@@ -12,7 +12,6 @@
-
\ No newline at end of file
diff --git a/.idea/harvester.iml b/.idea/harvester.iml
index 5e764c4..9c408cc 100644
--- a/.idea/harvester.iml
+++ b/.idea/harvester.iml
@@ -1,6 +1,7 @@
+
diff --git a/go.mod b/go.mod
index 9592184..66d25d4 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@ module git.kplus.net.ua/yevhen/resource-scraper
go 1.26.2
require (
+ github.com/davecgh/go-spew v1.1.1
github.com/dromara/carbon/v2 v2.6.16
github.com/emersion/go-imap/v2 v2.0.0-beta.8
github.com/go-chi/chi/v5 v5.2.5
diff --git a/go.sum b/go.sum
index 0a14383..a89feb1 100644
--- a/go.sum
+++ b/go.sum
@@ -1,4 +1,3 @@
-filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
@@ -6,12 +5,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dromara/carbon/v2 v2.6.15 h1:3HuC3XcWczIHUTbg/f0CSVydtKEdM+P0GM1sdsbwXmI=
-github.com/dromara/carbon/v2 v2.6.15/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
-github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
-github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
@@ -22,8 +17,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
-github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
-github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
@@ -59,8 +52,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
-github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d h1:IL2tii4jXLdhCeQN69HNzYYW1kl0meSG0wt5+sLwszU=
-github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d/go.mod h1:SIpumAnUWSy0q9RzKD3pyH3g1t5vdawUAPcW5tQrUtI=
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
@@ -94,8 +85,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
-golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
diff --git a/helper/sugar/fs.go b/helper/sugar/fs.go
index 452456f..3083605 100644
--- a/helper/sugar/fs.go
+++ b/helper/sugar/fs.go
@@ -2,32 +2,43 @@ package sugar
import (
"fmt"
+ "log"
"log/slog"
"os"
"path/filepath"
- "strconv"
)
+// EnsureDir joins the given path segments and creates the directory
+// (including all parent directories) if it does not already exist.
+// Returns the resolved path or an error if creation fails.
func EnsureDir(dirPath ...string) (string, error) {
outPath := filepath.Join(dirPath...)
- if len(outPath) < 3 {
- return "", fmt.Errorf("the path is short to be a folder: %s", outPath)
+ if outPath == "" {
+ return "", fmt.Errorf("path must not be empty")
}
- if _, err := os.Stat(outPath); os.IsNotExist(err) {
- var dirMod uint64
- if dirMod, err = strconv.ParseUint("0775", 8, 32); err == nil {
- err = os.Mkdir(outPath, os.FileMode(dirMod))
- }
-
- if err != nil && !os.IsExist(err) {
- slog.Error("error creating tmp dir", "err", err)
- }
+ if err := os.MkdirAll(outPath, os.FileMode(0775)); err != nil {
+ return "", fmt.Errorf("failed to ensure directory %q: %w", outPath, err)
}
return outPath, nil
}
+// ExecDir returns the directory of the running binary.
+// Works correctly even with symlinks and -trimpath builds.
+func ExecDir() (string, error) {
+ exe, err := os.Executable()
+ if err != nil {
+ return "", err
+ }
+ // EvalSymlinks is critical — os.Executable can return a symlink path
+ resolved, err := filepath.EvalSymlinks(exe)
+ if err != nil {
+ return "", err
+ }
+ return filepath.Dir(resolved), nil
+}
+
func WriteDataToTmpFile(content, fp string) (int, string, error) {
dir, err := EnsureDir(os.TempDir(), "mdb")
if err != nil {
@@ -58,6 +69,59 @@ func WriteDataToTmpFile(content, fp string) (int, string, error) {
return filesize, fp, nil
}
-/*
- REMOVED_FROM_MMT_DUE_TO_REDOING_OF_THE_EVENT_LOG
-*/
+// ConfigPaths returns candidate directories for config lookup.
+// Priority: XDG_CONFIG_HOME → binary dir → CWD → /etc/appname
+func ConfigPaths(appName string) []string {
+ var candidates []string
+
+ // 1. Explicit env override (highest priority)
+ if envPath := os.Getenv("APP_CONFIG_DIR"); envPath != "" {
+ candidates = append(candidates, envPath)
+ }
+
+ // 2. Relative to binary (best for production deployments)
+ if exe, err := os.Executable(); err == nil {
+ if resolved, err := filepath.EvalSymlinks(exe); err == nil {
+ candidates = append(candidates, filepath.Dir(resolved))
+ }
+ }
+
+ // 3. CWD (convenient for `go run` and development)
+ if cwd, err := os.Getwd(); err == nil {
+ candidates = append(candidates, cwd)
+ }
+
+ // 4. XDG standard (Linux/Unix)
+ if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
+ candidates = append(candidates, filepath.Join(xdg, appName))
+ } else if home, err := os.UserHomeDir(); err == nil {
+ candidates = append(candidates, filepath.Join(home, ".config", appName))
+ }
+
+ // 5. System-wide fallback
+ candidates = append(candidates, filepath.Join("/etc", appName))
+
+ return candidates
+}
+
+// FindConfig searches all candidate dirs for the given filename.
+func FindConfig(appName, filename string) (string, error) {
+ for _, dir := range ConfigPaths(appName) {
+ path := filepath.Join(dir, filename)
+ if _, err := os.Stat(path); err == nil {
+ return path, nil
+ }
+ }
+ return "", fmt.Errorf("config %q not found in any candidate directory", filename)
+}
+
+// DebugPath Debug helper — remove before production
+func DebugPath(path string) {
+ info, err := os.Stat(path)
+ log.Printf("path=%q exists=%v err=%v mode=%v", path, err == nil, err, func() string {
+ if info != nil {
+ return info.Mode().String()
+ }
+ return "n/a"
+ }())
+}
diff --git a/install.sh b/install.sh
index 4c122e0..5f5735c 100755
--- a/install.sh
+++ b/install.sh
@@ -1,2 +1,3 @@
#!/usr/bin/env bash
-go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
\ No newline at end of file
+#go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
+go build -trimpath -ldflags="-s -w" -o /opt/sources/resource-scraper/cmd/mdb ./cmd
\ No newline at end of file
diff --git a/internal/config/config.go b/internal/config/config.go
index 0567762..dbd97ab 100644
--- a/internal/config/config.go
+++ b/internal/config/config.go
@@ -9,6 +9,7 @@ import (
"github.com/spf13/viper"
+ "git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
@@ -16,6 +17,8 @@ func New() {
configFilePath := viper.GetString(constant.FlagConfigFile)
configName := filepath.Base(configFilePath)
+ sugar.DebugPath(configFilePath)
+
dir := filepath.Dir(configFilePath)
if dir == configName && dir == "." {
configName = "default"
diff --git a/pkg/handler/handler.go b/pkg/handler/handler.go
index fba153f..6d33264 100644
--- a/pkg/handler/handler.go
+++ b/pkg/handler/handler.go
@@ -29,7 +29,7 @@ func New(services *service.Service) *Handler {
"prescene": h.Prescene,
"rutracker": h.Rutracker,
"stb": h.STB,
- //"web": h.Web,
+ //"web": h.Web,
}
return h
diff --git a/pkg/handler/prescene.go b/pkg/handler/prescene.go
index b0f6fbf..63ad139 100644
--- a/pkg/handler/prescene.go
+++ b/pkg/handler/prescene.go
@@ -30,7 +30,12 @@ func (h *Handler) Prescene() string {
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
}
- _, err := h.services.Prescene.GetPage(pagesToScrape)
+ _, err := h.services.Prescene.GetSrrDb()
+ if err != nil {
+ slog.Error("error occurred while getting page: ", "err", err.Error())
+ }
+
+ _, err = h.services.Prescene.GetPage(pagesToScrape)
if err != nil {
slog.Error("error occurred while getting page: ", "err", err.Error())
}
diff --git a/pkg/handler/rutracker.go b/pkg/handler/rutracker.go
index e3677d3..c7d746f 100644
--- a/pkg/handler/rutracker.go
+++ b/pkg/handler/rutracker.go
@@ -17,7 +17,5 @@ func (h *Handler) Rutracker() string {
slog.Error("error occurred while getting topic: ", "err", err.Error())
}
- //fmt.Printf("%+v", rt)
-
return "rt"
}
diff --git a/pkg/repository/prescene.go b/pkg/repository/prescene.go
index 77cdb3b..a728cc7 100644
--- a/pkg/repository/prescene.go
+++ b/pkg/repository/prescene.go
@@ -10,6 +10,7 @@ import (
"strings"
"time"
+ "github.com/davecgh/go-spew/spew"
"github.com/dromara/carbon/v2"
"github.com/go-shiori/dom"
"github.com/jmoiron/sqlx"
@@ -25,6 +26,17 @@ type Prescene struct {
db *sqlx.DB
}
+func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) {
+ entries := make([]model.ExternalSources, 0)
+ now := time.Now()
+ yesterday := now.Add(-24 * time.Hour)
+ todayStr := now.Format("2006-01-02")
+ yesterdayStr := yesterday.Format("2006-01-02")
+ spew.Dump(entries, todayStr, yesterdayStr)
+
+ return []model.ExternalSources{}, nil
+}
+
func NewPresceneRepository(db *sqlx.DB) *Prescene {
return &Prescene{db: db}
}
diff --git a/pkg/repository/rutracker.go b/pkg/repository/rutracker.go
index 2281b5b..fbf280c 100644
--- a/pkg/repository/rutracker.go
+++ b/pkg/repository/rutracker.go
@@ -29,6 +29,7 @@ func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
endpoint := viper.GetString(constant.CfgKeyEndpoint)
+
entries := make([]model.ExternalSources, 0)
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
@@ -46,13 +47,17 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
es.Type = constant.ScopeRuTracker
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
es.Title = e.Title
+
es.TypeSubsectionId, _ = strconv.Atoi(t)
es.Releaser = e.Author.Name
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, s.db)
entries = append(entries, entry)
- //fmt.Printf("%+v\n\n\n", entry)
+
+ ///fmt.Println(entries)
+
+ fmt.Printf("%+v\n\n\n", entry)
}
}
diff --git a/pkg/repository/table/external_sources.go b/pkg/repository/table/external_sources.go
index e26fe36..f86f377 100644
--- a/pkg/repository/table/external_sources.go
+++ b/pkg/repository/table/external_sources.go
@@ -41,7 +41,7 @@ func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, column
var status string
errCount := 0
- for i := 0; i < len(entries); i++ {
+ for i := range entries {
entry := es.InsertOnDuplicate(entries[i], db)
if entry.Error != nil {
slog.Error("insert/update entry", "err", entry.Error)
diff --git a/pkg/service/prescene.go b/pkg/service/prescene.go
index 7599db5..5957b46 100644
--- a/pkg/service/prescene.go
+++ b/pkg/service/prescene.go
@@ -16,3 +16,7 @@ func NewPresceneService(repo _interface.Prescene) *PresceneService {
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
return p.repo.GetPage(pageNumbers)
}
+
+func (p *PresceneService) GetSrrDb() ([]model.ExternalSources, error) {
+ return p.repo.GetSrrDb()
+}
diff --git a/types/interface/interfaces.go b/types/interface/interfaces.go
index dd1f81e..39b4c97 100644
--- a/types/interface/interfaces.go
+++ b/types/interface/interfaces.go
@@ -11,6 +11,7 @@ type Rutracker interface {
type Prescene interface {
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
+ GetSrrDb() ([]model.ExternalSources, error)
}
type ShareTheBrutality interface {