resource-scraper/pkg/repository/srrdb.go

package repository

import (
	"errors"
	"fmt"
	"log/slog"
	"strings"
	"time"

	"github.com/jmoiron/sqlx"

	"git.kplus.net.ua/yevhen/resource-scraper/pkg/repository/table"
	"git.kplus.net.ua/yevhen/resource-scraper/types/model"
)

// srrdbTimestamp keeps created_at in the same "YYYY-MM-DD HH:MM:SS" layout the
// API itself uses in its date field, because the column is TEXT.
const srrdbTimestamp = "2006-01-02 15:04:05"

// Srrdb harvests the srrDB search API into the separate SQLite database.
type Srrdb struct {
	db *sqlx.DB
}

// NewSrrdbRepository returns a collector writing into the SQLite handle. The
// handle is nil for every scope except srrdb, because Bootstrap connects the
// SQLite database lazily, so the collector refuses to run without it instead of
// failing later on a nil handle.
func NewSrrdbRepository(db *sqlx.DB) *Srrdb {
	return &Srrdb{db: db}
}

// Search harvests one search path. Nothing is written to the database until the
// whole set is collected and its size matches what the API reported, so the
// dump and the success log can only ever describe a complete day.
func (s *Srrdb) Search(rawURI string) (int, error) {
	if s.db == nil {
		err := errors.New("srrdb collector has no database handle")
		srrdbLogError(rawURI, err)

		return 0, err
	}

	if !strings.Contains(rawURI, "category:") {
		err := fmt.Errorf("search path %q has no category: segment", rawURI)
		srrdbLogError(rawURI, err)

		return 0, err
	}

	rules, err := srrdbLoadGroupRules()
	if err != nil {
		srrdbLogError(rawURI, err)

		return 0, err
	}

	auto := strings.Contains(rawURI, srrdbAutoSegment)

	now := time.Now()

	uri, err := srrdbResolveAutoDate(srrdbResolveRelativeDays(rawURI, now), now)

	switch {
	case errors.Is(err, errSrrdbFloor):
		slog.Info("srrdb backfill complete, srrdb.floor reached", "uri", rawURI)

		return 0, nil
	case err != nil:
		srrdbLogError(rawURI, err)

		return 0, err
	}

	// A slash typed on either end of the command line would build a double
	// separator in the URL and an odd dump file name.
	uri = strings.Trim(uri, "/")

	// The guard applies to the automatic walk only. A day named on the command
	// line is fetched again and its dump rewritten every time: a day can still
	// gain releases later, either from another time zone or from a late upload,
	// and cron refreshes today and yesterday on purpose.
	dump := srrdbDumpPath(uri)

	if auto {
		exists, err := srrdbExists(dump)
		if err != nil {
			srrdbLogError(uri, err)

			return 0, err
		}

		if exists {
			slog.Info("srrdb already harvested, skipping", "uri", uri, "dump", dump)

			return 0, nil
		}
	}

	collected, err := srrdbCollectAll(uri)
	if err != nil {
		srrdbLogError(uri, err)

		return 0, err
	}

	// An empty day is a normal outcome: zero results and no warnings. The dump
	// records it as harvested, which keeps the walk moving. Warnings mean the
	// path was not understood, and that must never be remembered as done.
	switch {
	case len(collected.Warnings) > 0:
		srrdbLogWarning(uri, collected.Warnings)

		return 0, fmt.Errorf("srrdb %q reported warnings: %s",
			uri, strings.Join(collected.Warnings, "; "))
	case len(collected.Results) != collected.ResultsCount:
		err := fmt.Errorf("srrdb %q pulled %d of %d reported releases",
			uri, len(collected.Results), collected.ResultsCount)
		srrdbLogError(uri, err)

		return 0, err
	}

	run := time.Now().Format(srrdbTimestamp)
	category := srrdbCategory(uri)
	rows := make([]model.SrrdbRelease, 0, len(collected.Results))
	day := &model.SrrdbLog{
		Category: category,
		RunAt:    run,
	}
	day.Found = len(collected.Results)

	for _, result := range collected.Results {
		row := srrdbRow(result, run, category)

		if rules.ignore[row.GroupName] {
			slog.Info("srrdb release ignored by group rule", "release", row.Release, "group", row.GroupName)
			day.Ignored++

			continue
		}

		if status, exists := rules.status[row.GroupName]; exists {
			row.Status = &status
		}

		rows = append(rows, row)
	}

	day.Stored = len(rows)

	_, dateRest, ok := strings.Cut(uri, "date:")
	if !ok {
		err := fmt.Errorf("search path %q has no date: segment", uri)
		srrdbLogError(uri, err)

		return 0, err
	}

	dayName, _, _ := strings.Cut(dateRest, "/")
	if _, err := time.Parse(srrdbDayLayout, dayName); err != nil {
		err := fmt.Errorf("search path %q has an invalid date %q", uri, dayName)
		srrdbLogError(uri, err)

		return 0, err
	}

	day.Day = dayName

	if err := table.UpsertAllSrrdbWithDay(s.db, rows, day); err != nil {
		srrdbLogError(uri, err)

		return 0, err
	}

	if err := srrdbWriteDump(dump, collected); err != nil {
		srrdbLogError(uri, err)

		return len(rows), err
	}

	srrdbLogStatus(uri, len(rows), collected.ResultsCount)

	return len(rows), nil
}