package repository import ( "encoding/json" "fmt" "io" "log/slog" "net/http" "regexp" "strconv" "strings" "time" "github.com/spf13/viper" "git.kplus.net.ua/yevhen/resource-scraper/types/constant" "git.kplus.net.ua/yevhen/resource-scraper/types/resource" ) // srrdbPageSegment locates the page:N segment so a run can continue from N into // the following pages. var srrdbPageSegment = regexp.MustCompile(`page:(\d+)`) // srrdbCollectAll walks every page before anything is written, so the caller // only ever receives a complete set or an error. func srrdbCollectAll(uri string) (*resource.SrrdbSearchResponse, error) { endpoint := strings.TrimRight(viper.GetString("srrdb.endpoint"), "/") // The flag binds to its own key and wins over the config value; an unused // flag yields zero. sleep := viper.GetDuration(constant.FlagSleep) if sleep <= 0 { sleep = viper.GetDuration("srrdb.sleep") } if sleep <= 0 { sleep = 2 * time.Second } slog.Info("srrdb pacing", "uri", uri, "sleep", sleep) page, paginated := srrdbStartPage(uri) collected := &resource.SrrdbSearchResponse{} for { url := endpoint + "/" + srrdbWithPage(uri, page, paginated) response, err := srrdbFetch(url) if err != nil { return nil, err } srrdbCollect(collected, response) slog.Info("srrdb page fetched", "url", url, "page", page, "fetched", len(response.Results), "reported", response.ResultsCount) if !paginated || len(response.Results) == 0 { break } // Everything the API reported is already collected: one more page would // cost a request and a pause to receive an empty answer. if collected.ResultsCount > 0 && len(collected.Results) >= collected.ResultsCount { break } page++ time.Sleep(sleep) } return collected, nil } // srrdbCollect merges one page into the answer kept for the dump: results are // appended, while resultsCount and query come from the first page. func srrdbCollect(all, page *resource.SrrdbSearchResponse) { if all.ResultsCount == 0 { all.ResultsCount = page.ResultsCount } if len(all.Query) == 0 { all.Query = page.Query } all.Results = append(all.Results, page.Results...) all.Warnings = append(all.Warnings, page.Warnings...) } // srrdbStartPage reads the page:N segment. Without it the path is a single // request and no further pages are attempted. func srrdbStartPage(uri string) (int, bool) { match := srrdbPageSegment.FindStringSubmatch(uri) if match == nil { return 1, false } page, err := strconv.Atoi(match[1]) if err != nil || page < 1 { return 1, true } return page, true } // srrdbWithPage swaps the page:N segment for the page requested now. func srrdbWithPage(uri string, page int, paginated bool) string { if !paginated { return uri } return srrdbPageSegment.ReplaceAllString(uri, "page:"+strconv.Itoa(page)) } // srrdbFetch performs one search request and decodes the JSON answer. func srrdbFetch(url string) (*resource.SrrdbSearchResponse, error) { client := &http.Client{Timeout: 30 * time.Second} request, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("build srrdb request %q: %w", url, err) } request.Header.Set("Accept", "application/json") response, err := client.Do(request) if err != nil { return nil, fmt.Errorf("request srrdb %q: %w", url, err) } defer func() { if cerr := response.Body.Close(); cerr != nil { slog.Error("closing srrdb response body", "url", url, "err", cerr) } }() if response.StatusCode != http.StatusOK { return nil, fmt.Errorf("srrdb %q replied %s", url, response.Status) } body, err := io.ReadAll(io.LimitReader(response.Body, 16<<20)) if err != nil { return nil, fmt.Errorf("read srrdb body %q: %w", url, err) } decoded := &resource.SrrdbSearchResponse{} if err := json.Unmarshal(body, decoded); err != nil { return nil, fmt.Errorf("decode srrdb body %q: %w", url, err) } return decoded, nil }