From 776ca68e3e04140045d83c363f53517151267b8f Mon Sep 17 00:00:00 2001 From: Timmy Welch Date: Sat, 19 Oct 2024 09:39:08 -0700 Subject: [PATCH] Wait for page download failures --- cv/cv.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/cv/cv.go b/cv/cv.go index b7b4dd3..577679f 100644 --- a/cv/cv.go +++ b/cv/cv.go @@ -193,6 +193,7 @@ func (c *CVDownloader) updateIssues() { if failCount > 2 { sleepTime = time.Minute * 10 } + log.Println("This page failed to download, lets wait for", sleepTime, "and hope it works") select { case <-c.Context.Done(): // allows us to return immediately even during a timeout return false @@ -263,6 +264,18 @@ func (c *CVDownloader) updateIssues() { } return } + if resp.StatusCode != 200 { + cancelDownloadCTX() + if retry(URI.String(), nil) { + continue + } + log.Println("Failed to download this page, we'll wait for an hour to see if it clears up") + select { + case <-c.Context.Done(): // allows us to return immediately even during a timeout + return + case <-time.After(1 * time.Hour): + } + } file, err := os.Create(filepath.Join(c.JSONPath, "cv-"+strconv.Itoa(offset)+".json")) if err != nil { log.Fatal(err)