minor changes
This commit is contained in:
+2
-21
@@ -1,34 +1,15 @@
|
|||||||
.idea/**/workspace.xml
|
.idea/
|
||||||
.idea/**/tasks.xml
|
|
||||||
.idea/**/usage.statistics.xml
|
|
||||||
.idea/**/dictionaries
|
|
||||||
.idea/**/shelf
|
|
||||||
.idea/**/aws.xml
|
|
||||||
.idea/**/contentModel.xml
|
|
||||||
.idea/**/dataSources/
|
|
||||||
.idea/**/dataSources.ids
|
|
||||||
.idea/**/dataSources.local.xml
|
|
||||||
.idea/**/sqlDataSources.xml
|
|
||||||
.idea/**/dynamic.xml
|
|
||||||
.idea/**/uiDesigner.xml
|
|
||||||
.idea/**/dbnavigator.xml
|
|
||||||
.idea/**/gradle.xml
|
|
||||||
.idea/**/libraries
|
|
||||||
cmake-build-*/
|
cmake-build-*/
|
||||||
.idea/**/mongoSettings.xml
|
|
||||||
*.iws
|
*.iws
|
||||||
out/
|
out/
|
||||||
.idea_modules/
|
.idea_modules/
|
||||||
atlassian-ide-plugin.xml
|
atlassian-ide-plugin.xml
|
||||||
.idea/replstate.xml
|
|
||||||
.idea/sonarlint/
|
|
||||||
com_crashlytics_export_strings.xml
|
com_crashlytics_export_strings.xml
|
||||||
crashlytics.properties
|
crashlytics.properties
|
||||||
crashlytics-build.properties
|
crashlytics-build.properties
|
||||||
fabric.properties
|
fabric.properties
|
||||||
.idea/httpRequests
|
|
||||||
.idea/caches/build_file_checksums.ser
|
|
||||||
.env
|
.env
|
||||||
tmp/
|
tmp/
|
||||||
vendor/
|
vendor/
|
||||||
config/secret.yaml
|
config/secret.yaml
|
||||||
|
*.md
|
||||||
Generated
-1
@@ -12,7 +12,6 @@
|
|||||||
<option value="gosimple" />
|
<option value="gosimple" />
|
||||||
</list>
|
</list>
|
||||||
</option>
|
</option>
|
||||||
<option name="goLinterExe" value="C:\Users\theAmok\go\bin\golangci-lint.exe" />
|
|
||||||
<option name="linterSelected" value="true" />
|
<option name="linterSelected" value="true" />
|
||||||
</component>
|
</component>
|
||||||
</project>
|
</project>
|
||||||
Generated
+1
@@ -1,6 +1,7 @@
|
|||||||
<?xml version="1.0" encoding="UTF-8"?>
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
<module type="WEB_MODULE" version="4">
|
<module type="WEB_MODULE" version="4">
|
||||||
<component name="Go" enabled="true" />
|
<component name="Go" enabled="true" />
|
||||||
|
<component name="GoModuleSettings" enabled="true" />
|
||||||
<component name="NewModuleRootManager">
|
<component name="NewModuleRootManager">
|
||||||
<content url="file://$MODULE_DIR$" />
|
<content url="file://$MODULE_DIR$" />
|
||||||
<orderEntry type="inheritedJdk" />
|
<orderEntry type="inheritedJdk" />
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ module git.kplus.net.ua/yevhen/resource-scraper
|
|||||||
go 1.26.2
|
go 1.26.2
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/davecgh/go-spew v1.1.1
|
||||||
github.com/dromara/carbon/v2 v2.6.16
|
github.com/dromara/carbon/v2 v2.6.16
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8
|
github.com/emersion/go-imap/v2 v2.0.0-beta.8
|
||||||
github.com/go-chi/chi/v5 v5.2.5
|
github.com/go-chi/chi/v5 v5.2.5
|
||||||
|
|||||||
@@ -1,4 +1,3 @@
|
|||||||
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
|
|
||||||
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
|
||||||
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
|
||||||
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
|
||||||
@@ -6,12 +5,8 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
|
|||||||
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/dromara/carbon/v2 v2.6.15 h1:3HuC3XcWczIHUTbg/f0CSVydtKEdM+P0GM1sdsbwXmI=
|
|
||||||
github.com/dromara/carbon/v2 v2.6.15/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
|
||||||
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
|
github.com/dromara/carbon/v2 v2.6.16 h1:AbxrnW1kJhR3KHdS8G96NFmxDwPFyre+t+xSiJIUD1I=
|
||||||
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
github.com/dromara/carbon/v2 v2.6.16/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
|
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
|
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
|
github.com/emersion/go-imap/v2 v2.0.0-beta.8 h1:5IXZK1E33DyeP526320J3RS7eFlCYGFgtbrfapqDPug=
|
||||||
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
|
github.com/emersion/go-imap/v2 v2.0.0-beta.8/go.mod h1:dhoFe2Q0PwLrMD7oZw8ODuaD0vLYPe5uj2wcOMnvh48=
|
||||||
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
|
||||||
@@ -22,8 +17,6 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
|
|||||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||||
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
|
||||||
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||||
github.com/go-chi/chi/v5 v5.2.4 h1:WtFKPHwlywe8Srng8j2BhOD9312j9cGUxG1SP4V2cR4=
|
|
||||||
github.com/go-chi/chi/v5 v5.2.4/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
|
||||||
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
github.com/go-chi/chi/v5 v5.2.5 h1:Eg4myHZBjyvJmAFjFvWgrqDTXFyOzjj7YIm3L3mu6Ug=
|
||||||
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
github.com/go-chi/chi/v5 v5.2.5/go.mod h1:X7Gx4mteadT3eDOMTsXzmI4/rwUpOwBHLpAfupzFJP0=
|
||||||
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
|
||||||
@@ -59,8 +52,6 @@ github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o
|
|||||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||||
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
|
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
|
||||||
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
|
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
|
||||||
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d h1:IL2tii4jXLdhCeQN69HNzYYW1kl0meSG0wt5+sLwszU=
|
|
||||||
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d/go.mod h1:SIpumAnUWSy0q9RzKD3pyH3g1t5vdawUAPcW5tQrUtI=
|
|
||||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
|
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de h1:tVseKKgTIPOo8L0gFK4qX+kqyINtWncwER/t1n2im1A=
|
||||||
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
|
github.com/mewkiz/pkg v0.0.0-20260331151047-10214ccde7de/go.mod h1:omNJr4dHOKbrBeoY/idmLDFw8OIdDUUZvj3uB1cJxwA=
|
||||||
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
|
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
|
||||||
@@ -94,8 +85,6 @@ golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliY
|
|||||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
||||||
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
|
||||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96 h1:Z/6YuSHTLOHfNFdb8zVZomZr7cqNgTJvA8+Qz75D8gU=
|
|
||||||
golang.org/x/exp v0.0.0-20260112195511-716be5621a96/go.mod h1:nzimsREAkjBCIEFtHiYkrJyT+2uy9YZJB7H1k68CXZU=
|
|
||||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM=
|
||||||
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80=
|
||||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||||
|
|||||||
+79
-15
@@ -2,32 +2,43 @@ package sugar
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strconv"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// EnsureDir joins the given path segments and creates the directory
|
||||||
|
// (including all parent directories) if it does not already exist.
|
||||||
|
// Returns the resolved path or an error if creation fails.
|
||||||
func EnsureDir(dirPath ...string) (string, error) {
|
func EnsureDir(dirPath ...string) (string, error) {
|
||||||
outPath := filepath.Join(dirPath...)
|
outPath := filepath.Join(dirPath...)
|
||||||
if len(outPath) < 3 {
|
if outPath == "" {
|
||||||
return "", fmt.Errorf("the path is short to be a folder: %s", outPath)
|
return "", fmt.Errorf("path must not be empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := os.Stat(outPath); os.IsNotExist(err) {
|
if err := os.MkdirAll(outPath, os.FileMode(0775)); err != nil {
|
||||||
var dirMod uint64
|
return "", fmt.Errorf("failed to ensure directory %q: %w", outPath, err)
|
||||||
if dirMod, err = strconv.ParseUint("0775", 8, 32); err == nil {
|
|
||||||
err = os.Mkdir(outPath, os.FileMode(dirMod))
|
|
||||||
}
|
|
||||||
|
|
||||||
if err != nil && !os.IsExist(err) {
|
|
||||||
slog.Error("error creating tmp dir", "err", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return outPath, nil
|
return outPath, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ExecDir returns the directory of the running binary.
|
||||||
|
// Works correctly even with symlinks and -trimpath builds.
|
||||||
|
func ExecDir() (string, error) {
|
||||||
|
exe, err := os.Executable()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
// EvalSymlinks is critical — os.Executable can return a symlink path
|
||||||
|
resolved, err := filepath.EvalSymlinks(exe)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return filepath.Dir(resolved), nil
|
||||||
|
}
|
||||||
|
|
||||||
func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
||||||
dir, err := EnsureDir(os.TempDir(), "mdb")
|
dir, err := EnsureDir(os.TempDir(), "mdb")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -58,6 +69,59 @@ func WriteDataToTmpFile(content, fp string) (int, string, error) {
|
|||||||
return filesize, fp, nil
|
return filesize, fp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
// ConfigPaths returns candidate directories for config lookup.
|
||||||
REMOVED_FROM_MMT_DUE_TO_REDOING_OF_THE_EVENT_LOG
|
// Priority: XDG_CONFIG_HOME → binary dir → CWD → /etc/appname
|
||||||
*/
|
func ConfigPaths(appName string) []string {
|
||||||
|
var candidates []string
|
||||||
|
|
||||||
|
// 1. Explicit env override (highest priority)
|
||||||
|
if envPath := os.Getenv("APP_CONFIG_DIR"); envPath != "" {
|
||||||
|
candidates = append(candidates, envPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Relative to binary (best for production deployments)
|
||||||
|
if exe, err := os.Executable(); err == nil {
|
||||||
|
if resolved, err := filepath.EvalSymlinks(exe); err == nil {
|
||||||
|
candidates = append(candidates, filepath.Dir(resolved))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. CWD (convenient for `go run` and development)
|
||||||
|
if cwd, err := os.Getwd(); err == nil {
|
||||||
|
candidates = append(candidates, cwd)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. XDG standard (Linux/Unix)
|
||||||
|
if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" {
|
||||||
|
candidates = append(candidates, filepath.Join(xdg, appName))
|
||||||
|
} else if home, err := os.UserHomeDir(); err == nil {
|
||||||
|
candidates = append(candidates, filepath.Join(home, ".config", appName))
|
||||||
|
}
|
||||||
|
|
||||||
|
// 5. System-wide fallback
|
||||||
|
candidates = append(candidates, filepath.Join("/etc", appName))
|
||||||
|
|
||||||
|
return candidates
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindConfig searches all candidate dirs for the given filename.
|
||||||
|
func FindConfig(appName, filename string) (string, error) {
|
||||||
|
for _, dir := range ConfigPaths(appName) {
|
||||||
|
path := filepath.Join(dir, filename)
|
||||||
|
if _, err := os.Stat(path); err == nil {
|
||||||
|
return path, nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return "", fmt.Errorf("config %q not found in any candidate directory", filename)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DebugPath Debug helper — remove before production
|
||||||
|
func DebugPath(path string) {
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
log.Printf("path=%q exists=%v err=%v mode=%v", path, err == nil, err, func() string {
|
||||||
|
if info != nil {
|
||||||
|
return info.Mode().String()
|
||||||
|
}
|
||||||
|
return "n/a"
|
||||||
|
}())
|
||||||
|
}
|
||||||
|
|||||||
+2
-1
@@ -1,2 +1,3 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
|
#go build -trimpath -ldflags="-s -w" -o /usr/local/bin/mdb ./cmd
|
||||||
|
go build -trimpath -ldflags="-s -w" -o /opt/sources/resource-scraper/cmd/mdb ./cmd
|
||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
|
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
|
|
||||||
|
"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
|
||||||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -16,6 +17,8 @@ func New() {
|
|||||||
configFilePath := viper.GetString(constant.FlagConfigFile)
|
configFilePath := viper.GetString(constant.FlagConfigFile)
|
||||||
configName := filepath.Base(configFilePath)
|
configName := filepath.Base(configFilePath)
|
||||||
|
|
||||||
|
sugar.DebugPath(configFilePath)
|
||||||
|
|
||||||
dir := filepath.Dir(configFilePath)
|
dir := filepath.Dir(configFilePath)
|
||||||
if dir == configName && dir == "." {
|
if dir == configName && dir == "." {
|
||||||
configName = "default"
|
configName = "default"
|
||||||
|
|||||||
@@ -29,7 +29,7 @@ func New(services *service.Service) *Handler {
|
|||||||
"prescene": h.Prescene,
|
"prescene": h.Prescene,
|
||||||
"rutracker": h.Rutracker,
|
"rutracker": h.Rutracker,
|
||||||
"stb": h.STB,
|
"stb": h.STB,
|
||||||
//"web": h.Web,
|
//"web": h.Web,
|
||||||
}
|
}
|
||||||
|
|
||||||
return h
|
return h
|
||||||
|
|||||||
@@ -30,7 +30,12 @@ func (h *Handler) Prescene() string {
|
|||||||
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
pagesToScrape = append(pagesToScrape, strconv.Itoa(page))
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err := h.services.Prescene.GetPage(pagesToScrape)
|
_, err := h.services.Prescene.GetSrrDb()
|
||||||
|
if err != nil {
|
||||||
|
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = h.services.Prescene.GetPage(pagesToScrape)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
slog.Error("error occurred while getting page: ", "err", err.Error())
|
slog.Error("error occurred while getting page: ", "err", err.Error())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,7 +17,5 @@ func (h *Handler) Rutracker() string {
|
|||||||
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
slog.Error("error occurred while getting topic: ", "err", err.Error())
|
||||||
}
|
}
|
||||||
|
|
||||||
//fmt.Printf("%+v", rt)
|
|
||||||
|
|
||||||
return "rt"
|
return "rt"
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/davecgh/go-spew/spew"
|
||||||
"github.com/dromara/carbon/v2"
|
"github.com/dromara/carbon/v2"
|
||||||
"github.com/go-shiori/dom"
|
"github.com/go-shiori/dom"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
@@ -25,6 +26,17 @@ type Prescene struct {
|
|||||||
db *sqlx.DB
|
db *sqlx.DB
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *Prescene) GetSrrDb() ([]model.ExternalSources, error) {
|
||||||
|
entries := make([]model.ExternalSources, 0)
|
||||||
|
now := time.Now()
|
||||||
|
yesterday := now.Add(-24 * time.Hour)
|
||||||
|
todayStr := now.Format("2006-01-02")
|
||||||
|
yesterdayStr := yesterday.Format("2006-01-02")
|
||||||
|
spew.Dump(entries, todayStr, yesterdayStr)
|
||||||
|
|
||||||
|
return []model.ExternalSources{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
func NewPresceneRepository(db *sqlx.DB) *Prescene {
|
||||||
return &Prescene{db: db}
|
return &Prescene{db: db}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ func NewRutrackerRepository(db *sqlx.DB) *Rutracker {
|
|||||||
|
|
||||||
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
||||||
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
endpoint := viper.GetString(constant.CfgKeyEndpoint)
|
||||||
|
|
||||||
entries := make([]model.ExternalSources, 0)
|
entries := make([]model.ExternalSources, 0)
|
||||||
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
|
columns := []string{"`type`", "type_id", "title", "type_subsection_id", "releaser", "created"}
|
||||||
|
|
||||||
@@ -46,13 +47,17 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
|
|||||||
es.Type = constant.ScopeRuTracker
|
es.Type = constant.ScopeRuTracker
|
||||||
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
|
es.TypeId, _ = strconv.Atoi(u.Query().Get("t"))
|
||||||
es.Title = e.Title
|
es.Title = e.Title
|
||||||
|
|
||||||
es.TypeSubsectionId, _ = strconv.Atoi(t)
|
es.TypeSubsectionId, _ = strconv.Atoi(t)
|
||||||
es.Releaser = e.Author.Name
|
es.Releaser = e.Author.Name
|
||||||
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
|
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
|
||||||
esModel := table.ExternalSources{Columns: columns}
|
esModel := table.ExternalSources{Columns: columns}
|
||||||
entry := esModel.InsertOnDuplicate(es, s.db)
|
entry := esModel.InsertOnDuplicate(es, s.db)
|
||||||
entries = append(entries, entry)
|
entries = append(entries, entry)
|
||||||
//fmt.Printf("%+v\n\n\n", entry)
|
|
||||||
|
///fmt.Println(entries)
|
||||||
|
|
||||||
|
fmt.Printf("%+v\n\n\n", entry)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -41,7 +41,7 @@ func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, column
|
|||||||
var status string
|
var status string
|
||||||
errCount := 0
|
errCount := 0
|
||||||
|
|
||||||
for i := 0; i < len(entries); i++ {
|
for i := range entries {
|
||||||
entry := es.InsertOnDuplicate(entries[i], db)
|
entry := es.InsertOnDuplicate(entries[i], db)
|
||||||
if entry.Error != nil {
|
if entry.Error != nil {
|
||||||
slog.Error("insert/update entry", "err", entry.Error)
|
slog.Error("insert/update entry", "err", entry.Error)
|
||||||
|
|||||||
@@ -16,3 +16,7 @@ func NewPresceneService(repo _interface.Prescene) *PresceneService {
|
|||||||
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
func (p *PresceneService) GetPage(pageNumbers []string) ([]model.ExternalSources, error) {
|
||||||
return p.repo.GetPage(pageNumbers)
|
return p.repo.GetPage(pageNumbers)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (p *PresceneService) GetSrrDb() ([]model.ExternalSources, error) {
|
||||||
|
return p.repo.GetSrrDb()
|
||||||
|
}
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ type Rutracker interface {
|
|||||||
|
|
||||||
type Prescene interface {
|
type Prescene interface {
|
||||||
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
|
GetPage(pageNumbers []string) ([]model.ExternalSources, error)
|
||||||
|
GetSrrDb() ([]model.ExternalSources, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type ShareTheBrutality interface {
|
type ShareTheBrutality interface {
|
||||||
|
|||||||
Reference in New Issue
Block a user