package repository import ( "errors" "fmt" "log/slog" "strings" "time" "github.com/jmoiron/sqlx" "git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table" "git.kplus.net.ua/yevhen/resource-scraper/types/model" ) // srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the // API itself uses in its date field, because the column is TEXT. const srrdbTimestamp = "2006-01-02 15:04:05" // Srrdb harvests the srrDB search API into the separate SQLite database. type Srrdb struct { db *sqlx.DB } // NewSrrdbRepository returns a collector writing into the SQLite handle. The // handle is nil for every scope except srrdb, because Bootstrap connects the // SQLite database lazily, so the collector refuses to run without it instead of // failing later on a nil handle. func NewSrrdbRepository(db *sqlx.DB) *Srrdb { return &Srrdb{db: db} } // Search harvests one search path. Nothing is written to the database until the // whole set is collected and its size matches what the API reported, so the // dump and the success log can only ever describe a complete day. func (s *Srrdb) Search(rawURI string) (int, error) { if s.db == nil { err := errors.New("srrdb collector has no database handle") srrdbLogError(rawURI, err) return 0, err } if !strings.Contains(rawURI, "category:") { err := fmt.Errorf("search path %q has no category: segment", rawURI) srrdbLogError(rawURI, err) return 0, err } auto := strings.Contains(rawURI, srrdbAutoSegment) now := time.Now() uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now) switch { case errors.Is(err, errSrrdbFloor): slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI) return 0, nil case err != nil: srrdbLogError(rawURI, err) return 0, err } // A slash typed on either end of the command line would build a double // separator in the URL and an odd dump file name. uri = strings.Trim(uri, "/") // The guard applies to the automatic walk only. A day named on the command // line is fetched again and its dump rewritten every time: a day can still // gain releases later, either from another time zone or from a late upload, // and cron refreshes today and yesterday on purpose. dump := srrdbDumpPath(uri) if auto { exists, err := srrdbExists(dump) if err != nil { srrdbLogError(uri, err) return 0, err } if exists { slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump) return 0, nil } } collected, err := srrdbCollectAll(uri) if err != nil { srrdbLogError(uri, err) return 0, err } // An empty day is a normal outcome: zero results and no warnings. The dump // records it as harvested, which keeps the walk moving. Warnings mean the // path was not understood, and that must never be remembered as done. switch { case len(collected.Warnings) > 0: srrdbLogWarning(uri, collected.Warnings) return 0, fmt.Errorf("srrdb %q reported warnings: %s", uri, strings.Join(collected.Warnings, "; ")) case len(collected.Results) != collected.ResultsCount: err := fmt.Errorf("srrdb %q pulled %d of %d reported releases", uri, len(collected.Results), collected.ResultsCount) srrdbLogError(uri, err) return 0, err } run := time.Now().Format(srrdbTimestamp) category := srrdbCategory(uri) rows := make([]model.SrrdbRelease, 0, len(collected.Results)) for _, result := range collected.Results { rows = append(rows, srrdbRow(result, run, category)) } if err := table.UpsertAllSrrdb(s.db, rows); err != nil { srrdbLogError(uri, err) return 0, err } if err := srrdbWriteDump(dump, collected); err != nil { srrdbLogError(uri, err) return len(rows), err } srrdbLogStatus(uri, len(rows), collected.ResultsCount) return len(rows), nil }