package repository import ( "errors" "fmt" "log/slog" "strings" "time" "github.com/jmoiron/sqlx" "git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table" "git.kplus.net.ua/yevhen/resource-scraper/types/model" ) // srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the // API itself uses in its date field, because the column is TEXT. const srrdbTimestamp = "2006-01-02 15:04:05" // Srrdb harvests the srrDB search API into the separate SQLite database. type Srrdb struct { db *sqlx.DB } // NewSrrdbRepository returns a collector writing into the SQLite handle. The // handle is nil for every scope except srrdb, because Bootstrap connects the // SQLite database lazily, so the collector refuses to run without it instead of // failing later on a nil handle. func NewSrrdbRepository(db *sqlx.DB) *Srrdb { return &Srrdb{db: db} } // Search harvests one search path. Nothing is written to the database until the // whole set is collected and its size matches what the API reported, so the // dump and the success log can only ever describe a complete day. func (s *Srrdb) Search(rawURI string) (int, error) { if s.db == nil { err := errors.New("srrdb collector has no database handle") srrdbLogError(rawURI, err) return 0, err } if !strings.Contains(rawURI, "category:") { err := fmt.Errorf("search path %q has no category: segment", rawURI) srrdbLogError(rawURI, err) return 0, err } rules, err := srrdbLoadGroupRules() if err != nil { srrdbLogError(rawURI, err) return 0, err } auto := strings.Contains(rawURI, srrdbAutoSegment) now := time.Now() uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now) switch { case errors.Is(err, errSrrdbFloor): slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI) return 0, nil case err != nil: srrdbLogError(rawURI, err) return 0, err } // A slash typed on either end of the command line would build a double // separator in the URL and an odd dump file name. uri = strings.Trim(uri, "/") // The guard applies to the automatic walk only. A day named on the command // line is fetched again and its dump rewritten every time: a day can still // gain releases later, either from another time zone or from a late upload, // and cron refreshes today and yesterday on purpose. dump := srrdbDumpPath(uri) if auto { exists, err := srrdbExists(dump) if err != nil { srrdbLogError(uri, err) return 0, err } if exists { slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump) return 0, nil } } collected, err := srrdbCollectAll(uri) if err != nil { srrdbLogError(uri, err) return 0, err } // An empty day is a normal outcome: zero results and no warnings. The dump // records it as harvested, which keeps the walk moving. Warnings mean the // path was not understood, and that must never be remembered as done. switch { case len(collected.Warnings) > 0: srrdbLogWarning(uri, collected.Warnings) return 0, fmt.Errorf("srrdb %q reported warnings: %s", uri, strings.Join(collected.Warnings, "; ")) case len(collected.Results) != collected.ResultsCount: err := fmt.Errorf("srrdb %q pulled %d of %d reported releases", uri, len(collected.Results), collected.ResultsCount) srrdbLogError(uri, err) return 0, err } run := time.Now().Format(srrdbTimestamp) category := srrdbCategory(uri) rows := make([]model.SrrdbRelease, 0, len(collected.Results)) day := &model.SrrdbLog{ Category: category, RunAt: run, } day.Found = len(collected.Results) for _, result := range collected.Results { row := srrdbRow(result, run, category) if rules.ignore[row.GroupName] { slog.Info("srrdb release ignored by group rule", "release", row.Release, "group", row.GroupName) day.Ignored++ continue } if status, exists := rules.status[row.GroupName]; exists { row.Status = &status } rows = append(rows, row) } day.Stored = len(rows) _, dateRest, ok := strings.Cut(uri, "date:") if !ok { err := fmt.Errorf("search path %q has no date: segment", uri) srrdbLogError(uri, err) return 0, err } dayName, _, _ := strings.Cut(dateRest, "/") if _, err := time.Parse(srrdbDayLayout, dayName); err != nil { err := fmt.Errorf("search path %q has an invalid date %q", uri, dayName) srrdbLogError(uri, err) return 0, err } day.Day = dayName if err := table.UpsertAllSrrdbWithDay(s.db, rows, day); err != nil { srrdbLogError(uri, err) return 0, err } if err := srrdbWriteDump(dump, collected); err != nil { srrdbLogError(uri, err) return len(rows), err } srrdbLogStatus(uri, len(rows), collected.ResultsCount) return len(rows), nil }