upadted to 1.24.0; fixed stb and spiddped doubles in precene

pkg/repository/table/external_sources.go

@@ -2,10 +2,14 @@ package table
 
 import (
 	"fmt"
+	"log/slog"
+	"slices"
 	"strings"
 
 	"github.com/jmoiron/sqlx"
+	"github.com/logrusorgru/aurora/v4"
 
+	"git.amok.space/yevhen/resource-scraper/helper/thither"
 	"git.amok.space/yevhen/resource-scraper/types/constant"
 	"git.amok.space/yevhen/resource-scraper/types/model"
 )
@@ -20,8 +24,6 @@ func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.D
 	placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
 	query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
 
-	//fmt.Printf("%s\n", query)
-
 	if rows, err := db.NamedQuery(query, &es); err == nil {
 		for rows.Next() {
 			es.Error = rows.StructScan(&es)
@@ -32,3 +34,45 @@ func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.D
 
 	return es
 }
+
+func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, columns []string) ([]model.ExternalSources, string) {
+	es := &ExternalSources{Columns: columns}
+	typeIds := es.GetTypeIds(entries, db)
+	var status string
+	errCount := 0
+
+	for i := 0; i < len(entries); i++ {
+		entry := es.InsertOnDuplicate(entries[i], db)
+		if entry.Error != nil {
+			slog.Error("insert/update entry", "err", entry.Error)
+			errCount++
+		}
+
+		if !slices.Contains(typeIds, entry.TypeId) {
+			fmt.Printf("%s: %s\n", aurora.Green("ADDED"), aurora.White(entry.Title))
+		}
+
+		entries[i] = es.InsertOnDuplicate(entries[i], db)
+	}
+
+	if errCount == 0 {
+		status = constant.StatusSucceed
+	} else if errCount > 0 && errCount == len(entries) {
+		status = constant.StatusFailed
+	} else {
+		status = constant.StatusSuspicious
+	}
+
+	return entries, status
+}
+
+func (f *ExternalSources) GetTypeIds(entries []model.ExternalSources, db *sqlx.DB) []int {
+	var typeIds []int
+	ids := thither.FieldValueToStrSlice(entries, "TypeId")
+	query := fmt.Sprintf("SELECT type_id FROM %s WHERE `type` = '%s' AND type_id IN (%s) LIMIT %d", constant.ExternalSourcesTable, entries[0].Type, strings.Join(ids, ","), len(ids))
+	err := db.Select(&typeIds, query)
+	if err != nil {
+		slog.Error("getting type ids", "err", err)
+	}
+	return typeIds
+}