upadted to 1.24.0; fixed stb and spiddped doubles in precene

internal/bootstrap.go

@@ -3,7 +3,10 @@ package scraper
 import (
 	"fmt"
 
+	"github.com/iancoleman/strcase"
+	"github.com/logrusorgru/aurora/v4"
 	"github.com/spf13/viper"
+	"golang.org/x/exp/slices"
 
 	"git.amok.space/yevhen/resource-scraper/internal/db"
 	"git.amok.space/yevhen/resource-scraper/pkg/handler"
@@ -12,8 +15,22 @@ import (
 	"git.amok.space/yevhen/resource-scraper/types/constant"
 )
 
+func isAllowScope() bool {
+	scopesAllow := viper.GetStringSlice("scope.allow")
+	scopeEnabled := viper.GetString(constant.FlagScopeEnable)
+
+	return slices.Contains(scopesAllow, scopeEnabled)
+}
+
+func init() {
+	strcase.ConfigureAcronym("stb", "STB")
+}
+
 func Bootstrap() {
-	//https://ahmadrosid.com/blog/how-to-query-html-dom-in-golang
+	if !isAllowScope() {
+		fmt.Printf("%s You are in not allowed scope, check %s config file\n", aurora.BgMagenta("[WARN]"), aurora.Magenta("default.yaml"))
+		return
+	}
 
 	dbase := db.New()
 	repos := repository.New(dbase)
@@ -23,10 +40,8 @@ func Bootstrap() {
 	switch viper.GetString("role") {
 	case constant.RoleConsole:
 		fmt.Printf("init console console: %s\n", handlers.InitConsole())
-		break
 	case constant.RoleWeb:
 		fmt.Printf("who: %s\n", handlers.InitRoutes())
 		///http.Run()
-		break
 	}
 }

internal/flag.go

@@ -28,7 +28,8 @@ Report bugs to https://git.amok.space/yevhen/resource-scraper/issues`
 
 const (
 	version           = "0.1"
-	defaultConfigPath = "config/default"
+	defaultConfigPath = constant.DefaultConfigPath
+	defaultConfigEnv  = constant.DefaultEnvProd
 )
 
 func ParseFlags() {
@@ -37,8 +38,11 @@ func ParseFlags() {
 	flag.Bool(constant.FlagVersionShort, false, "")
 	flag.Bool(constant.FlagVersion, false, "")
 	flag.Bool(constant.FlagDebug, false, "")
-	flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location used for the program")
+	flag.String(constant.FlagConfigFile, defaultConfigPath, "config file location")
 	flag.String(constant.FlagScopeEnable, "", "")
+	flag.String(constant.FlagSingleUri, "", "")
+	flag.String(constant.FlagEnv, defaultConfigEnv, "")
+	flag.String("create", defaultConfigEnv, "used to create e.g. scope etc.")
 
 	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
 	pflag.Parse()

internal/logging/logging.go

@@ -0,0 +1,23 @@
+package logging
+
+import (
+	"log/slog"
+	"os"
+)
+
+func custom() {
+	//logger := log.New(os.Stderr, "INFO\t", log.LstdFlags|log.Lmicroseconds|log.Lshortfile)
+}
+
+// New check this later  https://github.com/hedzr/logg
+func New() {
+	handlerOpts := &slog.HandlerOptions{
+		Level: slog.LevelDebug,
+		//AddSource: true,
+	}
+
+	logger := slog.New(slog.NewJSONHandler(os.Stderr, handlerOpts))
+	//.WithAttrs([]slog.Attr{slog.String("app_version", "v0.0.1")})
+
+	slog.SetDefault(logger)
+}

internal/mail/mail.go

@@ -0,0 +1,172 @@
+package mail
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/emersion/go-imap/v2"
+	client "github.com/emersion/go-imap/v2/imapclient"
+	"github.com/logrusorgru/aurora/v4"
+	"github.com/spf13/viper"
+
+	"git.amok.space/yevhen/resource-scraper/helper/sugar"
+)
+
+type EmailService struct {
+	User      string
+	pass      string
+	Err       error
+	client    *client.Client
+	Mailboxes []*imap.SelectData
+	Messages  []*client.FetchMessageBuffer
+}
+
+func (s *EmailService) CheckErr(msg string, err error) bool {
+	if err != nil {
+		if sugar.IsDev() {
+			sugar.LogError(err.Error())
+		}
+		s.Err = err
+		return true
+	}
+
+	return false
+}
+
+func (s *EmailService) success(msg string) {
+	if sugar.IsDev() {
+		sugar.LogSuccess(msg)
+	}
+}
+
+func (s *EmailService) Warn(msg string) {
+	if sugar.IsDev() {
+		sugar.LogWarning(msg)
+	}
+}
+
+func (s *EmailService) Connect() {
+	box := strings.Split(s.User, "@")
+	mail := viper.GetStringMapString("mail." + box[1])
+
+	conn, err := client.DialTLS(mail["dial-tls"], nil)
+	if s.CheckErr("DialTLS", err) {
+		return
+	}
+
+	s.client = conn
+	s.pass = mail[s.User]
+	s.success("connected to " + mail["dial-tls"])
+
+	//defer s.Logout()
+}
+
+func (s *EmailService) Login() {
+	s.Connect()
+
+	err := s.client.Login(s.User, s.pass).Wait()
+	if s.CheckErr("Login", err) {
+		return
+	}
+	s.success(s.User + " logged")
+}
+
+func (s *EmailService) ListMailboxes(mailboxes []string) {
+	selectOptions := &imap.SelectOptions{ReadOnly: true}
+
+	for _, mailbox := range mailboxes {
+		mbox, err := s.client.Select(mailbox, selectOptions).Wait()
+		if s.CheckErr("Listing mailbox", err) {
+			return
+		}
+		s.Mailboxes = append(s.Mailboxes, mbox)
+	}
+
+	if len(s.Mailboxes) == 0 {
+		s.Warn(sugar.SqueezeLine("mailboxes " + strings.Join(mailboxes, ", ") + " not found"))
+	}
+}
+
+func (s *EmailService) ListMessages(mailboxes []string, criteria *imap.SearchCriteria) {
+	for _, mailbox := range mailboxes {
+		mbox, err := s.client.Select(mailbox, nil).Wait()
+		if s.CheckErr("Listing mailbox", err) {
+			continue
+		}
+
+		seqs, err := s.client.UIDSearch(criteria, nil).Wait()
+		if s.CheckErr("UIDSearch", err) {
+			return
+		}
+
+		seqSet := imap.SeqSet{}
+		seqSet.AddRange(1, mbox.NumMessages)
+		if len(seqs.AllUIDs()) == 0 {
+			s.Warn("no messages found in mailbox: " + mailbox)
+			continue
+		}
+		s.success(fmt.Sprintf("Search complete, found %d messages: %+v", len(seqs.AllUIDs()), mbox.UIDNext))
+
+		fetchOptions := &imap.FetchOptions{
+			Envelope:      true,
+			Flags:         true,
+			BodyStructure: &imap.FetchItemBodyStructure{Extended: true},
+			BodySection:   []*imap.FetchItemBodySection{{Peek: true, Part: []int{2}}},
+		}
+
+		messages, err := s.client.Fetch(seqSet, fetchOptions).Collect()
+		if s.CheckErr("Fetch", err) {
+			continue
+		}
+
+		s.Messages = messages
+	}
+}
+
+func (s *EmailService) LogOut() {
+	err := s.client.Logout().Wait()
+	if !s.CheckErr("failed to logout", err) {
+		s.success("logged out successfully")
+	}
+}
+
+func (s *EmailService) CreateMailbox(mailboxName string) {
+	s.client.Create(mailboxName, nil)
+}
+
+func (s *EmailService) MailboxesList() {
+	listCmd := s.client.List("", "%", &imap.ListOptions{
+		ReturnStatus: &imap.StatusOptions{
+			NumMessages: true,
+			NumUnseen:   true,
+		},
+	})
+
+	for {
+		mbox := listCmd.Next()
+		if mbox == nil {
+			break
+		}
+
+		fmt.Printf("%+v\n", mbox)
+	}
+
+	if err := listCmd.Close(); err != nil {
+		log.Fatalf("LIST command failed: %v", err)
+	}
+}
+
+func (s *EmailService) MoveMessageToMailbox(msg *client.FetchMessageBuffer, status string) bool {
+	movable := imap.SeqSet{}
+	movable.AddNum(msg.SeqNum)
+	mailbox := viper.GetStringMapString("stb.move-processed-to-mailbox")
+	wait, err := s.client.Move(movable, mailbox[status]).Wait()
+	if s.CheckErr("Moving to archive", err) {
+		return false
+	}
+
+	fmt.Printf("Message %s moved, srcUIDs: %s, dstUIDs: %s\n", aurora.White(msg.Envelope.MessageID), aurora.Yellow(wait.SourceUIDs), aurora.Yellow(wait.DestUIDs))
+
+	return true
+}