package repository import ( "errors" "fmt" "os" "regexp" "strings" "time" "github.com/spf13/viper" ) // srrdbAutoSegment is the placeholder replaced by the date the run has to // harvest, so cron can repeat the same command every day. const srrdbAutoSegment = "date:auto" // srrdbYesterdaySegment, srrdbTodaySegment and srrdbTomorrowSegment stand for // the neighbouring days, so cron can name a day without hard-coding a date. const ( srrdbYesterdaySegment = "date:yesterday" srrdbTodaySegment = "date:today" srrdbTomorrowSegment = "date:tomorrow" ) // srrdbDayLayout is the layout of a date segment in a search path. const srrdbDayLayout = "2006-01-02" // errSrrdbFloor reports that the automatic walk reached srrdb.floor, which is // the archive being fully covered rather than a failure. var errSrrdbFloor = errors.New("srrdb.floor reached") // srrdbResolveAutoDate replaces date:auto with the newest day in // [srrdb.floor..yesterday] that has not been harvested yet. // // Scanning from the top-down means a new day is picked up the moment it appears, // and a hole left in the middle is closed once the walk reaches it. The previous // rule (the day before the earliest dump) treated the harvested days as one // uninterrupted run, so a single dump below the floor made it declare the whole // archive covered while everything above stayed missing. // // The current moment is passed in rather than read here, so the walk can be tested // without depending on the wall clock. func srrdbResolveAutoDate(rawURI string, now time.Time) (string, error) { if !strings.Contains(rawURI, srrdbAutoSegment) { return rawURI, nil } floor, err := srrdbFloorDate() if err != nil { return "", err } harvested, err := srrdbDumpedDays(rawURI) if err != nil { return "", err } done := make(map[string]bool, len(harvested)) for _, day := range harvested { done[day.Format(srrdbDayLayout)] = true } // The current day is never taken: it is still being filled, so it would have // no dump and would be chosen by every single run, freezing the walk. cron // refreshes today explicitly instead. for day := now.AddDate(0, 0, -1); !day.Before(floor); day = day.AddDate(0, 0, -1) { if !done[day.Format(srrdbDayLayout)] { return srrdbReplaceDate(rawURI, srrdbAutoSegment, day), nil } } return "", errSrrdbFloor } // srrdbDumpedDays lists the days already harvested for the same query, read back // from the dump file names inside srrdb.dump_dir. // // A missing directory is an error rather than "nothing harvested yet": the dumps // are the cursor of the walk, so a renamed, unmounted or mistyped directory would // otherwise send the run silently back to the beginning. The directory is created // once by hand, exactly like the database file. func srrdbDumpedDays(uri string) ([]time.Time, error) { pattern, err := srrdbDumpPattern(uri) if err != nil { return nil, err } dir := viper.GetString("srrdb.dump_dir") entries, err := os.ReadDir(dir) if err != nil { return nil, fmt.Errorf("read srrdb dump dir %q: %w", dir, err) } days := make([]time.Time, 0, len(entries)) for _, entry := range entries { match := pattern.FindStringSubmatch(entry.Name()) if match == nil { continue } day, err := time.Parse(srrdbDayLayout, match[1]) if err != nil { continue } days = append(days, day) } return days, nil } // srrdbDumpPattern builds the regular expression matching the dump names of the // same query, capturing the day segment so it can be read back. The literal part // is quoted first, therefore the capture group is the only pattern syntax left in // the result. func srrdbDumpPattern(uri string) (*regexp.Regexp, error) { const normalizedAuto = "date:auto" escaped := regexp.QuoteMeta(srrdbNormalizedPath(uri)) head, tail, ok := strings.Cut(escaped, regexp.QuoteMeta(normalizedAuto)) if !ok { return nil, fmt.Errorf("srrdb dump pattern needs a %s segment, got %q", srrdbAutoSegment, uri) } return regexp.Compile("^" + head + `date:(\d{4}-\d{2}-\d{2})` + tail + "$") } // srrdbFloorDate is the earliest day the automatic walk may ask for, kept in // configuration because it is a property of the archive, not of the code. // // The value is read through viper.Get rather than viper.GetString: YAML decodes // an unquoted date as a timestamp, and asking for its string form yields // "2013-03-16 00:00:00 +0000 UTC", which the day layout cannot parse. Both the // quoted and the unquoted spelling are accepted here. func srrdbFloorDate() (time.Time, error) { switch raw := viper.Get("srrdb.floor").(type) { case nil: return time.Time{}, errors.New("srrdb.floor is not configured") case time.Time: return raw, nil case string: floor, err := time.Parse(srrdbDayLayout, raw) if err != nil { return time.Time{}, fmt.Errorf("parse srrdb.floor %q: %w", raw, err) } return floor, nil default: return time.Time{}, fmt.Errorf("srrdb.floor has unexpected type %T", raw) } } // srrdbReplaceDate swaps a date placeholder for a concrete day. func srrdbReplaceDate(uri, segment string, day time.Time) string { return strings.Replace(uri, segment, "date:"+day.Format(srrdbDayLayout), 1) } // srrdbRelativeDays holds the date placeholders that stand for a day counted // from now. date:auto is deliberately absent: it means "the next day the walk // has to cover", which only srrdbResolveAutoDate can decide. var srrdbRelativeDays = []struct { segment string offset int }{ {segment: srrdbYesterdaySegment, offset: -1}, {segment: srrdbTodaySegment, offset: 0}, {segment: srrdbTomorrowSegment, offset: 1}, } // srrdbResolveRelativeDays expands every relative date placeholder present in the // path. The current moment is passed in for the same reason as in // srrdbResolveAutoDate: a test has to be able to pin it. func srrdbResolveRelativeDays(rawURI string, now time.Time) string { uri := rawURI for _, relative := range srrdbRelativeDays { if !strings.Contains(uri, relative.segment) { continue } uri = srrdbReplaceDate(uri, relative.segment, now.AddDate(0, 0, relative.offset)) } return uri }