Compare commits

10 Commits

Author SHA1 Message Date
c6d1a8d455 fixed mails moving 2025-10-28 13:36:53 +02:00
c8745e1f60 some mirror fixed 2025-10-28 12:59:54 +02:00
ac4b0ef47b added to ignore STONERD in prescene 2025-10-28 11:38:00 +02:00
879b33d385 all fixed 2025-10-28 11:36:11 +02:00
54aaa80f23 migrate to 1.25.3 2025-10-28 10:16:56 +02:00
a41ccec4a0 change modelu name to git.kplus.net.ua 2025-10-06 12:30:33 +03:00
b26c43ee4b change git imports 2025-10-06 12:28:15 +03:00
1544b943fc cahnge git server and fixed time parsing 2025-10-06 12:21:22 +03:00
970f6c2bfd change done folder 2025-03-04 11:39:56 +02:00
1c17e73e2d stb config changed 2025-03-04 08:47:42 +02:00
29 changed files with 194 additions and 178 deletions

6
.idea/data_source_mapping.xml generated Normal file
View File

@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="DataSourcePerFileMappings">
<file url="file://$PROJECT_DIR$/pkg/repository/table/external_sources.go" value="e3965414-f0c4-4a05-af48-11e028c626da" />
</component>
</project>

View File

@@ -6,9 +6,9 @@ import (
_ "github.com/go-sql-driver/mysql"
scraper "git.amok.space/yevhen/resource-scraper/internal"
"git.amok.space/yevhen/resource-scraper/internal/config"
"git.amok.space/yevhen/resource-scraper/internal/logging"
scraper "git.kplus.net.ua/yevhen/resource-scraper/internal"
"git.kplus.net.ua/yevhen/resource-scraper/internal/config"
"git.kplus.net.ua/yevhen/resource-scraper/internal/logging"
)
var (

View File

@@ -2,5 +2,5 @@ scope:
allow: [info, web, rutracker, prescene, stb, metal-archives, console]
default: console
mail:
amok.space:
dial-tls: mail.amok.space:993
kplus.net.ua:
dial-tls: s21.thehost.com.ua:993

View File

@@ -1,16 +1,16 @@
role: console
stb:
endpoint: stb@amok.space
#mailboxes: ["INBOX"]
endpoint: stb@kplus.net.ua
mailboxes: ["INBOX"]
#mailboxes: ["INBOX", "Junk Mail", "Done"]
mailboxes: ["Junk Mail"]
#mailboxes: ["Junk Mail"]
#mailboxes: ["Done"]
search-criteria: ["Your Share The Brutality digest", "noreply@sharethebrutality.info"]
move-processed-to-mailbox:
succeed: Processed/Succeed
succeed: Succeed
#succeed: Done
failed: Processed/Failed
suspicious: Processed/Suspicious
failed: Failed
suspicious: Suspicious
sender:
mailbox: noreply
host: sharethebrutality.info

51
go.mod
View File

@@ -1,22 +1,22 @@
module git.amok.space/yevhen/resource-scraper
module git.kplus.net.ua/yevhen/resource-scraper
go 1.24
go 1.25
require (
github.com/emersion/go-imap/v2 v2.0.0-beta.5
github.com/go-chi/chi/v5 v5.2.1
github.com/go-chi/cors v1.2.1
github.com/dromara/carbon/v2 v2.6.13
github.com/emersion/go-imap/v2 v2.0.0-beta.7
github.com/go-chi/chi/v5 v5.2.3
github.com/go-chi/cors v1.2.2
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c
github.com/go-sql-driver/mysql v1.9.0
github.com/golang-module/carbon/v2 v2.4.1
github.com/go-sql-driver/mysql v1.9.3
github.com/iancoleman/strcase v0.3.0
github.com/jmoiron/sqlx v1.4.0
github.com/logrusorgru/aurora/v4 v4.0.0
github.com/mewkiz/flac v1.0.12
github.com/spf13/pflag v1.0.6
github.com/spf13/viper v1.19.0
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa
golang.org/x/net v0.35.0
github.com/mewkiz/flac v1.0.13
github.com/spf13/pflag v1.0.10
github.com/spf13/viper v1.21.0
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546
golang.org/x/net v0.46.0
)
require (
@@ -24,24 +24,19 @@ require (
github.com/andybalholm/cascadia v1.3.3 // indirect
github.com/emersion/go-message v0.18.2 // indirect
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 // indirect
github.com/fsnotify/fsnotify v1.8.0 // indirect
github.com/fsnotify/fsnotify v1.9.0 // indirect
github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f // indirect
github.com/hashicorp/hcl v1.0.0 // indirect
github.com/icza/bitio v1.1.0 // indirect
github.com/magiconair/properties v1.8.9 // indirect
github.com/mewkiz/pkg v0.0.0-20241223220703-7f3c7df797ff // indirect
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d // indirect
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/pelletier/go-toml/v2 v2.2.3 // indirect
github.com/sagikazarmark/locafero v0.7.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.12.0 // indirect
github.com/spf13/cast v1.7.1 // indirect
github.com/pelletier/go-toml/v2 v2.2.4 // indirect
github.com/sagikazarmark/locafero v0.12.0 // indirect
github.com/spf13/afero v1.15.0 // indirect
github.com/spf13/cast v1.10.0 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/sys v0.30.0 // indirect
golang.org/x/text v0.22.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/sys v0.37.0 // indirect
golang.org/x/text v0.30.0 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
)

107
go.sum
View File

@@ -2,37 +2,35 @@ filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kktS1LM=
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/d4l3k/messagediff v1.2.2-0.20190829033028-7e0a312ae40b/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkEQxENCrlLo=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/emersion/go-imap/v2 v2.0.0-beta.5 h1:H3858DNmBuXyMK1++YrQIRdpKE1MwBc+ywBtg3n+0wA=
github.com/emersion/go-imap/v2 v2.0.0-beta.5/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
github.com/dromara/carbon/v2 v2.6.13 h1:lSXQpikgbBN/Gj3tfqjTXwBWnpidN0bnI6qnlcQXExs=
github.com/dromara/carbon/v2 v2.6.13/go.mod h1:NGo3reeV5vhWCYWcSqbJRZm46MEwyfYI5EJRdVFoLJo=
github.com/emersion/go-imap/v2 v2.0.0-beta.7 h1:lNznYWa5uhMrngnSYEklzCeye4DBq9TEJ+pr0K593+8=
github.com/emersion/go-imap/v2 v2.0.0-beta.7/go.mod h1:BZTFHsS1hmgBkFlHqbxGLXk2hnRqTItUgwjSSCsYNAk=
github.com/emersion/go-message v0.18.2 h1:rl55SQdjd9oJcIoQNhubD2Acs1E6IzlZISRTK7x/Lpg=
github.com/emersion/go-message v0.18.2/go.mod h1:XpJyL70LwRvq2a8rVbHXikPgKj8+aI0kGdHlg16ibYA=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6 h1:oP4q0fw+fOSWn3DfFi4EXdT+B+gTtzx8GC9xsc26Znk=
github.com/emersion/go-sasl v0.0.0-20241020182733-b788ff22d5a6/go.mod h1:iL2twTeMvZnrg54ZoPDNfJaJaqy0xIQFuBdrLsmspwQ=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-chi/chi/v5 v5.2.1 h1:KOIHODQj58PmL80G2Eak4WdvUzjSJSm0vG72crDCqb8=
github.com/go-chi/chi/v5 v5.2.1/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-chi/cors v1.2.1 h1:xEC8UT3Rlp2QuWNEr4Fs/c2EAGVKBwy/1vHx3bppil4=
github.com/go-chi/cors v1.2.1/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/fsnotify/fsnotify v1.9.0 h1:2Ml+OJNzbYCTzsxtv8vKSFD9PbJjmhYF14k/jKC7S9k=
github.com/fsnotify/fsnotify v1.9.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
github.com/go-chi/chi/v5 v5.2.3 h1:WQIt9uxdsAbgIYgid+BpYc+liqQZGMHRaUwp0JUcvdE=
github.com/go-chi/chi/v5 v5.2.3/go.mod h1:L2yAIGWB3H+phAw1NxKwWM+7eUH/lU8pOMm5hHcoops=
github.com/go-chi/cors v1.2.2 h1:Jmey33TE+b+rB7fT8MUy1u0I4L+NARQlK6LhzKPSyQE=
github.com/go-chi/cors v1.2.2/go.mod h1:sSbTewc+6wYHBBCW7ytsFSn836hqM7JxpglAy2Vzc58=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c h1:wpkoddUomPfHiOziHZixGO5ZBS73cKqVzZipfrLmO1w=
github.com/go-shiori/dom v0.0.0-20230515143342-73569d674e1c/go.mod h1:oVDCh3qjJMLVUSILBRwrm+Bc6RNXGZYtoh9xdvf1ffM=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/go-sql-driver/mysql v1.9.0 h1:Y0zIbQXhQKmQgTp44Y1dp3wTXcn804QoTptLZT1vtvo=
github.com/go-sql-driver/mysql v1.9.0/go.mod h1:pDetrLJeA3oMujJuvXc8RJoasr589B6A9fwzD3QMrqw=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs=
github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f h1:3BSP1Tbs2djlpprl7wCLuiqMaUh5SJkkzI2gDs+FgLs=
github.com/gogs/chardet v0.0.0-20211120154057-b7413eaefb8f/go.mod h1:Pcatq5tYkCW2Q6yrR2VRHlbHpZ/R4/7qyL1TCF7vl14=
github.com/golang-module/carbon/v2 v2.4.1 h1:cYUD8T+rHeX+qIybGYpnJ8I90F10dvyEF67VNOO+zZM=
github.com/golang-module/carbon/v2 v2.4.1/go.mod h1:1jP9AZ4k2+lmfgY/wZgmtsN52VcHC5YuPM6varKDTkM=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
github.com/icza/bitio v1.1.0 h1:ysX4vtldjdi3Ygai5m1cWy4oLkhWTAi+SyO6HC8L9T0=
@@ -41,7 +39,6 @@ github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6 h1:8UsGZ2rr2ksmEru6lTo
github.com/icza/mighty v0.0.0-20180919140131-cfd07d671de6/go.mod h1:xQig96I1VNBDIWGCdTt54nHt6EeI639SmHycLYL7FkA=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/jszwec/csvutil v1.5.1/go.mod h1:Rpu7Uu9giO9subDyMCIQfHVDuLrcaC36UA4YcJjGBkg=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
@@ -50,56 +47,45 @@ github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/logrusorgru/aurora/v4 v4.0.0 h1:sRjfPpun/63iADiSvGGjgA1cAYegEWMPCJdUpJYn9JA=
github.com/logrusorgru/aurora/v4 v4.0.0/go.mod h1:lP0iIa2nrnT/qoFXcOZSrZQpJ1o6n2CUf/hyHi2Q4ZQ=
github.com/magiconair/properties v1.8.9 h1:nWcCbLq1N2v/cpNsy5WvQ37Fb+YElfq20WJ/a8RkpQM=
github.com/magiconair/properties v1.8.9/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mewkiz/flac v1.0.12 h1:5Y1BRlUebfiVXPmz7hDD7h3ceV2XNrGNMejNVjDpgPY=
github.com/mewkiz/flac v1.0.12/go.mod h1:1UeXlFRJp4ft2mfZnPLRpQTd7cSjb/s17o7JQzzyrCA=
github.com/mewkiz/pkg v0.0.0-20230226050401-4010bf0fec14/go.mod h1:QYCFBiH5q6XTHEbWhR0uhR3M9qNPoD2CSQzr0g75kE4=
github.com/mewkiz/pkg v0.0.0-20241223220703-7f3c7df797ff h1:ts1OPi6zzzHE4nosvaXfkvhVNHeBghtxwwJL/PBsVHk=
github.com/mewkiz/pkg v0.0.0-20241223220703-7f3c7df797ff/go.mod h1:dpOqQ/SkbEbPklAuIQqDGHc6K6kygt0L1XYa/IMlQeI=
github.com/mewkiz/flac v1.0.13 h1:6wF8rRQKBFW159Daqx6Ro7K5ZnlVhHUKfS5aTsC4oXs=
github.com/mewkiz/flac v1.0.13/go.mod h1:HfPYDA+oxjyuqMu2V+cyKcxF51KM6incpw5eZXmfA6k=
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d h1:IL2tii4jXLdhCeQN69HNzYYW1kl0meSG0wt5+sLwszU=
github.com/mewkiz/pkg v0.0.0-20250417130911-3f050ff8c56d/go.mod h1:SIpumAnUWSy0q9RzKD3pyH3g1t5vdawUAPcW5tQrUtI=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985 h1:h8O1byDZ1uk6RUXMhj1QJU3VXFKXHDZxr4TXRPGeBa8=
github.com/mewpkg/term v0.0.0-20241026122259-37a80af23985/go.mod h1:uiPmbdUbdt1NkGApKl7htQjZ8S7XaGUAVulJUJ9v6q4=
github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
github.com/pelletier/go-toml/v2 v2.2.4/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/sagikazarmark/locafero v0.7.0 h1:5MqpDsTGNDhY8sGp0Aowyf0qKsPrhewaLSsFaodPcyo=
github.com/sagikazarmark/locafero v0.7.0/go.mod h1:2za3Cg5rMaTMoG/2Ulr9AwtFaIppKXTRYnozin4aB5k=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
github.com/spf13/cast v1.7.1 h1:cuNEagBQEHWN1FnbGEjCXL2szYEXqfJPbP2HNUaca9Y=
github.com/spf13/cast v1.7.1/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=
github.com/spf13/viper v1.19.0/go.mod h1:GQUN9bilAbhU/jgc1bKs99f/suXKeUMct8Adx5+Ntkg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/sagikazarmark/locafero v0.12.0 h1:/NQhBAkUb4+fH1jivKHWusDYFjMOOKU88eegjfxfHb4=
github.com/sagikazarmark/locafero v0.12.0/go.mod h1:sZh36u/YSZ918v0Io+U9ogLYQJ9tLLBmM4eneO6WwsI=
github.com/spf13/afero v1.15.0 h1:b/YBCLWAJdFWJTN9cLhiXXcD7mzKn9Dm86dNnfyQw1I=
github.com/spf13/afero v1.15.0/go.mod h1:NC2ByUVxtQs4b3sIUphxK0NioZnmxgyCrfzeuq8lxMg=
github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY=
github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk=
github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.21.0 h1:x5S+0EU27Lbphp4UKm1C+1oQO+rKx36vfCoaVebLFSU=
github.com/spf13/viper v1.21.0/go.mod h1:P0lhsswPGWD/1lZJ9ny3fYnVqxiegrlNrEmgLjbTCAY=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa h1:t2QcU6V556bFjYgu4L6C+6VrCPyJZ+eyRsABUPs1mz4=
golang.org/x/exp v0.0.0-20250218142911-aa4b98e5adaa/go.mod h1:BHOTPb3L19zxehTsLoJXVaTktb06DFgmdW6Wb9s8jqk=
golang.org/x/image v0.5.0/go.mod h1:FVC7BI/5Ym8R25iw5OLsgshdUBbT1h5jZTpA+mvAdZ4=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
@@ -109,14 +95,13 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4=
golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -135,8 +120,8 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
@@ -155,8 +140,8 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k=
golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
@@ -167,7 +152,5 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -3,7 +3,7 @@ package sugar
import (
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
func IsDev() bool {

2
install.sh Normal file
View File

@@ -0,0 +1,2 @@
#!/usr/bin/env bash
env GOBIN=/usr/local/bin go install ./cmd

View File

@@ -8,11 +8,11 @@ import (
"github.com/spf13/viper"
"golang.org/x/exp/slices"
"git.amok.space/yevhen/resource-scraper/internal/db"
"git.amok.space/yevhen/resource-scraper/pkg/handler"
"git.amok.space/yevhen/resource-scraper/pkg/repository"
"git.amok.space/yevhen/resource-scraper/pkg/service"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/internal/db"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/handler"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/service"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
func isAllowScope() bool {

View File

@@ -9,7 +9,7 @@ import (
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
func New() {

View File

@@ -9,7 +9,7 @@ import (
"github.com/spf13/pflag"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
const usage = `Music Database (MDB) server/cli craftware'
@@ -24,7 +24,7 @@ Commands:
-section -s Select section e.g.: rutracker, bandcamp etc.
The Go runtime version: %s
Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
Report bugs to https://git.kplus.net.ua/yevhen/resource-scraper/issues`
const (
version = "0.1"

View File

@@ -7,7 +7,7 @@ import (
"github.com/go-chi/chi/v5"
"github.com/go-chi/cors"
"git.amok.space/yevhen/resource-scraper/helper/web"
"git.kplus.net.ua/yevhen/resource-scraper/helper/web"
)
/*type Server struct {

View File

@@ -10,7 +10,7 @@ import (
"github.com/logrusorgru/aurora/v4"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/helper/sugar"
"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
)
type EmailService struct {
@@ -57,6 +57,7 @@ func (s *EmailService) Connect() {
s.client = conn
s.pass = mail[s.User]
s.success("connected to " + mail["dial-tls"])
//defer s.Logout()
@@ -67,8 +68,12 @@ func (s *EmailService) Login() {
err := s.client.Login(s.User, s.pass).Wait()
if s.CheckErr("Login", err) {
fmt.Println("========================================")
fmt.Println(err.Error())
fmt.Println("========================================")
return
}
s.success(s.User + " logged")
}
@@ -116,6 +121,7 @@ func (s *EmailService) ListMessages(mailboxes []string, criteria *imap.SearchCri
}
messages, err := s.client.Fetch(seqSet, fetchOptions).Collect()
if s.CheckErr("Fetch", err) {
continue
}
@@ -132,7 +138,20 @@ func (s *EmailService) LogOut() {
}
func (s *EmailService) CreateMailbox(mailboxName string) {
s.client.Create(mailboxName, nil)
if err := s.client.Create(mailboxName, nil); err != nil {
log.Println("Помилка створення папки:", mailboxName, err)
} else {
log.Println("Створено папку:", mailboxName)
}
}
func (s *EmailService) DeleteMailbox(mailboxName string) {
// Вибираємо папку
if err := s.client.Delete(mailboxName); err != nil {
log.Println("Помилка видалення папки:", mailboxName, err)
} else {
log.Println("Папка видалена:", mailboxName)
}
}
func (s *EmailService) MailboxesList() {
@@ -162,7 +181,9 @@ func (s *EmailService) MoveMessageToMailbox(msg *client.FetchMessageBuffer, stat
movable.AddNum(msg.SeqNum)
mailbox := viper.GetStringMapString("stb.move-processed-to-mailbox")
wait, err := s.client.Move(movable, mailbox[status]).Wait()
if s.CheckErr("Moving to archive", err) {
fmt.Println("Error moving to mail:", mailbox[status], "Error:", err.Error())
return false
}

View File

@@ -6,8 +6,8 @@ import (
"github.com/iancoleman/strcase"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/pkg/service"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/service"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
type Handler struct {

View File

@@ -9,7 +9,7 @@ import (
"github.com/mewkiz/flac"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
func (h *Handler) Info() string {

View File

@@ -9,7 +9,7 @@ import (
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
func (h *Handler) Prescene() string {

View File

@@ -7,7 +7,7 @@ import (
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
)
func (h *Handler) STB() string {

View File

@@ -10,15 +10,15 @@ import (
"strings"
"time"
"github.com/dromara/carbon/v2"
"github.com/go-shiori/dom"
"github.com/golang-module/carbon/v2"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/helper/parser"
"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/helper/parser"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type Prescene struct {
@@ -44,7 +44,7 @@ func (s *Prescene) GetPage(pageNumbers []string) ([]model.ExternalSources, error
endpoint += fmt.Sprintf(viper.GetString(constant.CfgKeyEndpointNext), t)
}
//doc, err := parser.HTMLSourceFromURL("https://mdb.amok.space/$/scnlog.html")
//doc, err := parser.HTMLSourceFromURL("https://mdb.kplus.net.ua /$/scnlog.html")
if result, err := parseUrl(endpoint, s.db); err == nil {
entries = append(entries, result...)
} else {
@@ -114,7 +114,7 @@ func parseUrl(endpoint string, db *sqlx.DB) ([]model.ExternalSources, error) {
localtime := dom.QuerySelector(title, "small > span.localtime")
if localtime != nil {
lc := dom.GetAttribute(localtime, "data-lttime")
es.Created = carbon.Parse(lc)
es.Created = carbon.Parse(lc, "Europe/Kyiv")
}
}

View File

@@ -3,7 +3,7 @@ package repository
import (
"github.com/jmoiron/sqlx"
"git.amok.space/yevhen/resource-scraper/types/interface"
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
)
type Repository struct {

View File

@@ -9,14 +9,14 @@ import (
"net/url"
"strconv"
"github.com/golang-module/carbon/v2"
"github.com/dromara/carbon/v2"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"git.amok.space/yevhen/resource-scraper/pkg/repository/table"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.amok.space/yevhen/resource-scraper/types/resource"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)
type Rutracker struct {
@@ -48,8 +48,7 @@ func (s *Rutracker) GetTopic(topics []string) ([]model.ExternalSources, error) {
es.Title = e.Title
es.TypeSubsectionId, _ = strconv.Atoi(t)
es.Releaser = e.Author.Name
es.Created = carbon.Parse(e.Updated)
es.Created = carbon.Parse(e.Updated, "Europe/Kyiv")
esModel := table.ExternalSources{Columns: columns}
entry := esModel.InsertOnDuplicate(es, s.db)
entries = append(entries, entry)

View File

@@ -7,19 +7,19 @@ import (
"strconv"
"strings"
"github.com/dromara/carbon/v2"
"github.com/emersion/go-imap/v2"
"github.com/go-shiori/dom"
"github.com/golang-module/carbon/v2"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
"golang.org/x/net/html"
"git.amok.space/yevhen/resource-scraper/helper/parser"
"git.amok.space/yevhen/resource-scraper/helper/sugar"
"git.amok.space/yevhen/resource-scraper/internal/mail"
_table "git.amok.space/yevhen/resource-scraper/pkg/repository/table"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/helper/parser"
"git.kplus.net.ua/yevhen/resource-scraper/helper/sugar"
"git.kplus.net.ua/yevhen/resource-scraper/internal/mail"
_table "git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type ShareTheBrutality struct {
@@ -47,10 +47,10 @@ func (s *ShareTheBrutality) GetMail(email string) ([]model.ExternalSources, *mai
//now := carbon.Now()
s.EmailService.ListMessages(viper.GetStringSlice(mailboxes), searchCriteria)
//box.CreateMailbox("INBOX/Processed")
//s.EmailService.CreateMailbox("Processed/Succeed")
//s.EmailService.CreateMailbox("Processed/Failed")
//s.EmailService.CreateMailbox("Processed/Suspicious")
//s.EmailService.MailboxesList()
//s.EmailService.CreateMailbox("Succeed")
//s.EmailService.DeleteMailbox("Succeed")
s.EmailService.MailboxesList()
entries := s.Processing(viper.GetStringMapString(fmt.Sprintf("%s.sender", s.scope)))
@@ -131,7 +131,8 @@ func (s *ShareTheBrutality) Processing(sender map[string]string) []model.Externa
}
es.Releaser = sourceDataMatch[1]
es.Created = carbon.Parse(msg.Envelope.Date.String())
es.Created = carbon.Parse(msg.Envelope.Date.String(), "Europe/Kyiv")
es.Type = dbType
genre := strings.ToLower(sourceDataMatch[2])
@@ -140,6 +141,9 @@ func (s *ShareTheBrutality) Processing(sender map[string]string) []model.Externa
entries = append(entries, es)
}
//fmt.Println(entries)
//os.Exit(0)
result, status := _table.BatchInsertOnDuplicate(entries, s.db, columns)
if status != constant.StatusFailed {
entriesBatched = append(entriesBatched, result...)

View File

@@ -9,9 +9,9 @@ import (
"github.com/jmoiron/sqlx"
"github.com/logrusorgru/aurora/v4"
"git.amok.space/yevhen/resource-scraper/helper/thither"
"git.amok.space/yevhen/resource-scraper/types/constant"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/helper/thither"
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type ExternalSources struct {
@@ -67,9 +67,15 @@ func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, column
}
func (f *ExternalSources) GetTypeIds(entries []model.ExternalSources, db *sqlx.DB) []int {
if len(entries) == 0 {
fmt.Printf("Entries are empty")
return nil // або логування помилки
}
var typeIds []int
ids := thither.FieldValueToStrSlice(entries, "TypeId")
query := fmt.Sprintf("SELECT type_id FROM %s WHERE `type` = '%s' AND type_id IN (%s) LIMIT %d", constant.ExternalSourcesTable, entries[0].Type, strings.Join(ids, ","), len(ids))
fmt.Println(query)
err := db.Select(&typeIds, query)
if err != nil {
slog.Error("getting type ids", "err", err)

View File

@@ -1,8 +1,8 @@
package service
import (
"git.amok.space/yevhen/resource-scraper/types/interface"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type PresceneService struct {

View File

@@ -1,8 +1,8 @@
package service
import (
"git.amok.space/yevhen/resource-scraper/types/interface"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type RutrackerService struct {

View File

@@ -1,8 +1,8 @@
package service
import (
"git.amok.space/yevhen/resource-scraper/pkg/repository"
"git.amok.space/yevhen/resource-scraper/types/interface"
"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository"
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
)
type Service struct {

View File

@@ -1,9 +1,9 @@
package service
import (
"git.amok.space/yevhen/resource-scraper/internal/mail"
"git.amok.space/yevhen/resource-scraper/types/interface"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/internal/mail"
"git.kplus.net.ua/yevhen/resource-scraper/types/interface"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type ShareTheBrutalityService struct {

View File

@@ -1,8 +1,8 @@
package _interface
import (
"git.amok.space/yevhen/resource-scraper/internal/mail"
"git.amok.space/yevhen/resource-scraper/types/model"
"git.kplus.net.ua/yevhen/resource-scraper/internal/mail"
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)
type Rutracker interface {

View File

@@ -1,7 +1,7 @@
package model
import (
"github.com/golang-module/carbon/v2"
"github.com/dromara/carbon/v2"
)
type Type string
@@ -21,22 +21,22 @@ type Type string
)*/
type ExternalSources struct {
Id int64 `json:"id" db:"id"`
Type string `json:"type" db:"type"`
TypeId int `json:"type_id" db:"type_id"`
Title string `json:"title" db:"title"`
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
Releaser string `json:"releaser" db:"releaser"`
ExSource string `json:"ex_source" db:"eXsource"`
Created carbon.Carbon `json:"created_at" db:"created"`
Modified carbon.Carbon `json:"modified_at" db:"modified"`
Contents string `json:"contents" db:"contents"`
A string `json:"a" db:"a"`
H string `json:"h" db:"h"`
Fingerprint string `json:"fingerprint" db:"fingerprint"`
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
Vid int `json:"vid" db:"vid"`
G string `json:"g" db:"g"`
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
Id int64 `json:"id" db:"id"`
Type string `json:"type" db:"type"`
TypeId int `json:"type_id" db:"type_id"`
Title string `json:"title" db:"title"`
TypeSubsectionId int `json:"type_subsection_id" db:"type_subsection_id"`
Releaser string `json:"releaser" db:"releaser"`
ExSource string `json:"ex_source" db:"eXsource"`
Created *carbon.Carbon `json:"created_at" db:"created"`
Modified carbon.Carbon `json:"modified_at" db:"modified"`
Contents string `json:"contents" db:"contents"`
A string `json:"a" db:"a"`
H string `json:"h" db:"h"`
Fingerprint string `json:"fingerprint" db:"fingerprint"`
FsFingerprint string `json:"fs_fingerprint" db:"fs_fingerprint"`
Vid int `json:"vid" db:"vid"`
G string `json:"g" db:"g"`
ExternalSourcesCol string `json:"external_sources_column" db:"external_sourcescol"`
Error error
}

View File

@@ -1,6 +1,6 @@
package resource
import "github.com/golang-module/carbon/v2"
import "github.com/dromara/carbon/v2"
/*https://www.metal-archives.com/search/ajax-advanced/searching/albums/?bandName=&releaseTitle=&releaseYearFrom=2024&releaseMonthFrom=09&releaseYearTo=&releaseMonthTo=&country=&location=&releaseLabelName=&releaseCatalogNumber=&releaseIdentifiers=&releaseRecordingInfo=&releaseDescription=&releaseNotes=&genre=&sEcho=6&iColumns=4&sColumns=&iDisplayStart=1000&iDisplayLength=200&mDataProp_0=0&mDataProp_1=1&mDataProp_2=2&mDataProp_3=3&_=1726217541027*/