resource-scraper/pkg/repository/srrdb_date.go

package repository

import (
	"errors"
	"fmt"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/spf13/viper"
)

// srrdbAutoSegment is the placeholder replaced by the date the run has to
// harvest, so cron can repeat the same command every day.
const srrdbAutoSegment = "date:auto"

// srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for
// the neighbouring days, so cron can name a day without hard-coding a date.
const (
	srrdbYesterdaySegment = "date:yesterday"
	srrdbTodaySegment     = "date:today"
	srrdbTomorrowSegment  = "date:tomorrow"
)

// srrdbDayLayout is the layout of a date segment in a search path.
const srrdbDayLayout = "2006-01-02"

// errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is
// the archive being fully covered rather than a failure.
var errSrrdbFloor = errors.New("srrdb.floor reached")

// srrdbResolveAutoDate replaces date:auto with the newest day in
// [srrdb.floor..yesterday] that has not been harvested yet.
//
// Scanning from the top-down means a new day is picked up the moment it appears,
// and a hole left in the middle is closed once the walk reaches it. The previous
// rule (the day before the earliest dump) treated the harvested days as one
// uninterrupted run, so a single dump below the floor made it declare the whole
// archive covered while everything above stayed missing.
//
// The current moment is passed in rather than read here, so the walk can be tested
// without depending on the wall clock.
func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) {
	if !strings.Contains(rawURI, srrdbAutoSegment) {
		return rawURI, nil
	}

	floor, err := srrdbFloorDate()
	if err != nil {
		return "", err
	}

	harvested, err := srrdbDumpedDays(rawURI)
	if err != nil {
		return "", err
	}

	done := make(map[string]bool, len(harvested))
	for _, day := range harvested {
		done[day.Format(srrdbDayLayout)] = true
	}

	// The current day is never taken: it is still being filled, so it would have
	// no dump and would be chosen by every single run, freezing the walk. cron
	// refreshes today explicitly instead.
	for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) {
		if !done[day.Format(srrdbDayLayout)] {
			return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil
		}
	}

	return "", errSrrdbFloor
}

// srrdbDumpedDays lists the days already harvested for the same query, read back
// from the dump file names inside srrdb.dump_dir.
//
// A missing directory is an error rather than "nothing harvested yet": the dumps
// are the cursor of the walk, so a renamed, unmounted or mistyped directory would
// otherwise send the run silently back to the beginning. The directory is created
// once by hand, exactly like the database file.
func srrdbDumpedDays(uri string) ([]time.Time, error) {
	pattern, err := srrdbDumpPattern(uri)
	if err != nil {
		return nil, err
	}

	dir := viper.GetString("srrdb.dump_dir")

	entries, err := os.ReadDir(dir)
	if err != nil {
		return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err)
	}

	days := make([]time.Time, 0, len(entries))

	for _, entry := range entries {
		match := pattern.FindStringSubmatch(entry.Name())
		if match == nil {
			continue
		}

		day, err := time.Parse(srrdbDayLayout, match[1])
		if err != nil {
			continue
		}

		days = append(days, day)
	}

	return days, nil
}

// srrdbDumpPattern builds the regular expression matching the dump names of the
// same query, capturing the day segment so it can be read back. The literal part
// is quoted first, therefore the capture group is the only pattern syntax left in
// the result.
func srrdbDumpPattern(uri string) (*regexp.Regexp, error) {
	const normalizedAuto = "date：auto"

	escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri))

	head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto))
	if !ok {
		return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri)
	}

	return regexp.Compile("^" + head + `date：(\d{4}-\d{2}-\d{2})` + tail + "$")
}

// srrdbFloorDate is the earliest day the automatic walk may ask for, kept in
// configuration because it is a property of the archive, not of the code.
//
// The value is read through viper.Get rather than viper.GetString: YAML decodes
// an unquoted date as a timestamp, and asking for its string form yields
// "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the
// quoted and the unquoted spelling are accepted here.
func srrdbFloorDate() (time.Time, error) {
	switch raw := viper.Get("srrdb.floor").(type) {
	case nil:
		return time.Time{}, errors.New("srrdb.floor is not configured")
	case time.Time:
		return raw, nil
	case string:
		floor, err := time.Parse(srrdbDayLayout, raw)
		if err != nil {
			return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err)
		}

		return floor, nil
	default:
		return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw)
	}
}

// srrdbReplaceDate swaps a date placeholder for a concrete day.
func srrdbReplaceDate(uri, segment string, day time.Time) string {
	return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1)
}

// srrdbRelativeDays holds the date placeholders that stand for a day counted
// from now. date:auto is deliberately absent: it means "the next day the walk
// has to cover", which only srrdbResolveAutoDate can decide.
var srrdbRelativeDays = []struct {
	segment string
	offset  int
}{
	{segment: srrdbYesterdaySegment, offset: -1},
	{segment: srrdbTodaySegment, offset: 0},
	{segment: srrdbTomorrowSegment, offset: 1},
}

// srrdbResolveRelativeDays expands every relative date placeholder present in the
// path. The current moment is passed in for the same reason as in
// srrdbResolveAutoDate: a test has to be able to pin it.
func srrdbResolveRelativeDays(rawURI string, now time.Time) string {
	uri := rawURI

	for _, relative := range srrdbRelativeDays {
		if !strings.Contains(uri, relative.segment) {
			continue
		}

		uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset))
	}

	return uri
}