85 lines
2.3 KiB
Go
85 lines
2.3 KiB
Go
package table
|
||
|
||
import (
|
||
"fmt"
|
||
"log/slog"
|
||
"slices"
|
||
"strings"
|
||
|
||
"github.com/jmoiron/sqlx"
|
||
"github.com/logrusorgru/aurora/v4"
|
||
|
||
"git.kplus.net.ua/yevhen/resource-scraper/helper/thither"
|
||
"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
|
||
"git.kplus.net.ua/yevhen/resource-scraper/types/model"
|
||
)
|
||
|
||
type ExternalSources struct {
|
||
Columns []string
|
||
}
|
||
|
||
func (f *ExternalSources) InsertOnDuplicate(es model.ExternalSources, db *sqlx.DB) model.ExternalSources {
|
||
stmt := "INSERT INTO %s (%s) VALUES (%s) ON DUPLICATE KEY UPDATE title=:title, created=:created RETURNING id"
|
||
placeholders := strings.Join(f.Columns, ", :")
|
||
placeholders = ":" + strings.Replace(placeholders, "`", "", -1)
|
||
query := fmt.Sprintf(stmt, constant.ExternalSourcesTable, strings.Join(f.Columns, ", "), placeholders)
|
||
|
||
if rows, err := db.NamedQuery(query, &es); err == nil {
|
||
for rows.Next() {
|
||
es.Error = rows.StructScan(&es)
|
||
}
|
||
} else {
|
||
es.Error = err
|
||
}
|
||
|
||
return es
|
||
}
|
||
|
||
func BatchInsertOnDuplicate(entries []model.ExternalSources, db *sqlx.DB, columns []string) ([]model.ExternalSources, string) {
|
||
es := &ExternalSources{Columns: columns}
|
||
typeIds := es.GetTypeIds(entries, db)
|
||
var status string
|
||
errCount := 0
|
||
|
||
for i := 0; i < len(entries); i++ {
|
||
entry := es.InsertOnDuplicate(entries[i], db)
|
||
if entry.Error != nil {
|
||
slog.Error("insert/update entry", "err", entry.Error)
|
||
errCount++
|
||
}
|
||
|
||
if !slices.Contains(typeIds, entry.TypeId) {
|
||
fmt.Printf("%s: %s\n", aurora.Green("ADDED"), aurora.White(entry.Title))
|
||
}
|
||
|
||
entries[i] = es.InsertOnDuplicate(entries[i], db)
|
||
}
|
||
|
||
if errCount == 0 {
|
||
status = constant.StatusSucceed
|
||
} else if errCount > 0 && errCount == len(entries) {
|
||
status = constant.StatusFailed
|
||
} else {
|
||
status = constant.StatusSuspicious
|
||
}
|
||
|
||
return entries, status
|
||
}
|
||
|
||
func (f *ExternalSources) GetTypeIds(entries []model.ExternalSources, db *sqlx.DB) []int {
|
||
if len(entries) == 0 {
|
||
fmt.Printf("Entries are empty")
|
||
return nil // або логування помилки
|
||
}
|
||
|
||
var typeIds []int
|
||
ids := thither.FieldValueToStrSlice(entries, "TypeId")
|
||
query := fmt.Sprintf("SELECT type_id FROM %s WHERE `type` = '%s' AND type_id IN (%s) LIMIT %d", constant.ExternalSourcesTable, entries[0].Type, strings.Join(ids, ","), len(ids))
|
||
fmt.Println(query)
|
||
err := db.Select(&typeIds, query)
|
||
if err != nil {
|
||
slog.Error("getting type ids", "err", err)
|
||
}
|
||
return typeIds
|
||
}
|