resource-scraper/pkg/repository/srrdb_api.go

package repository

import (
	"encoding/json"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/spf13/viper"

	"git.kplus.net.ua/yevhen/resource-scraper/types/constant"
	"git.kplus.net.ua/yevhen/resource-scraper/types/resource"
)

// srrdbPageSegment locates the page:N segment so a run can continue from N into
// the following pages.
var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`)

// srrdbCollectAll walks every page before anything is written, so the caller
// only ever receives a complete set or an error.
func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) {
	endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/")

	// The flag binds to its own key and wins over the config value; an unused
	// flag yields zero.
	sleep := viper.GetDuration(constant.FlagSleep)
	if sleep <= 0 {
		sleep = viper.GetDuration("srrdb.sleep")
	}
	if sleep <= 0 {
		sleep = 2 * time.Second
	}

	slog.Info("srrdb pacing", "uri", uri, "sleep", sleep)

	page, paginated := srrdbStartPage(uri)
	collected := &resource.SrrdbSearchResponse{}

	for {
		url := endpoint + "/" + srrdbWithPage(uri, page, paginated)

		response, err := srrdbFetch(url)
		if err != nil {
			return nil, err
		}

		srrdbCollect(collected, response)

		slog.Info("srrdb page fetched",
			"url", url,
			"page", page,
			"fetched", len(response.Results),
			"reported", response.ResultsCount)

		if !paginated || len(response.Results) == 0 {
			break
		}

		// Everything the API reported is already collected: one more page would
		// cost a request and a pause to receive an empty answer.
		if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount {
			break
		}

		page++

		time.Sleep(sleep)
	}

	return collected, nil
}

// srrdbCollect merges one page into the answer kept for the dump: results are
// appended, while resultsCount and query come from the first page.
func srrdbCollect(all, page *resource.SrrdbSearchResponse) {
	if all.ResultsCount == 0 {
		all.ResultsCount = page.ResultsCount
	}

	if len(all.Query) == 0 {
		all.Query = page.Query
	}

	all.Results = append(all.Results, page.Results...)
	all.Warnings = append(all.Warnings, page.Warnings...)
}

// srrdbStartPage reads the page:N segment. Without it the path is a single
// request and no further pages are attempted.
func srrdbStartPage(uri string) (int, bool) {
	match := srrdbPageSegment.FindStringSubmatch(uri)
	if match == nil {
		return 1, false
	}

	page, err := strconv.Atoi(match[1])
	if err != nil || page < 1 {
		return 1, true
	}

	return page, true
}

// srrdbWithPage swaps the page:N segment for the page requested now.
func srrdbWithPage(uri string, page int, paginated bool) string {
	if !paginated {
		return uri
	}

	return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page))
}

// srrdbFetch performs one search request and decodes the JSON answer.
func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) {
	client := &http.Client{Timeout: 30 * time.Second}

	request, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return nil, fmt.Errorf("build srrdb request %q: %w", url, err)
	}

	request.Header.Set("Accept", "application/json")

	response, err := client.Do(request)
	if err != nil {
		return nil, fmt.Errorf("request srrdb %q: %w", url, err)
	}

	defer func() {
		if cerr := response.Body.Close(); cerr != nil {
			slog.Error("closing srrdb response body", "url", url, "err", cerr)
		}
	}()

	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status)
	}

	body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20))
	if err != nil {
		return nil, fmt.Errorf("read srrdb body %q: %w", url, err)
	}

	decoded := &resource.SrrdbSearchResponse{}
	if err := json.Unmarshal(body, decoded); err != nil {
		return nil, fmt.Errorf("decode srrdb body %q: %w", url, err)
	}

	return decoded, nil
}