lordwelch/comic-hasher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Wait for page download failures

Browse Source
This commit is contained in:
Timmy Welch 2024-10-19 09:39:08 -07:00
parent 95fa44fb97
commit 776ca68e3e
1 changed files with 13 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
cv/cv.go
View File

@ -193,6 +193,7 @@ func (c *CVDownloader) updateIssues() {
if failCount > 2 { if failCount > 2 {
sleepTime = time.Minute * 10 sleepTime = time.Minute * 10
} }
log.Println("This page failed to download, lets wait for", sleepTime, "and hope it works")
select { select {
case <-c.Context.Done(): // allows us to return immediately even during a timeout case <-c.Context.Done(): // allows us to return immediately even during a timeout
return false return false
@ -263,6 +264,18 @@ func (c *CVDownloader) updateIssues() {
} }
return return
} }
if resp.StatusCode != 200 {
cancelDownloadCTX()
if retry(URI.String(), nil) {
continue
}
log.Println("Failed to download this page, we'll wait for an hour to see if it clears up")
select {
case <-c.Context.Done(): // allows us to return immediately even during a timeout
return
case <-time.After(1 * time.Hour):
}
}
file, err := os.Create(filepath.Join(c.JSONPath, "cv-"+strconv.Itoa(offset)+".json")) file, err := os.Create(filepath.Join(c.JSONPath, "cv-"+strconv.Itoa(offset)+".json"))
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)