lordwelch/comic-hasher
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Handle reading existing invalid json

Browse Source
This commit is contained in:
Timmy Welch 2024-10-19 09:50:23 -07:00
parent 50fcfb9513
commit 840e97cff0
1 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
cv/cv.go
View File

@ -78,7 +78,10 @@ type CVDownloader struct {
notFound chan download notFound chan download
} }
var ErrQuit = errors.New("Quit") var (
ErrQuit = errors.New("Quit")
ErrInvalidPage = errors.New("Invalid ComicVine Page")
)
func (c *CVDownloader) InsertBadURL(url string) { func (c *CVDownloader) InsertBadURL(url string) {
c.bMut.Lock() c.bMut.Lock()
@ -110,11 +113,13 @@ func (c *CVDownloader) readJson() ([]*CVResult, error) {
} }
result, err := c.loadIssues(file_entry) result, err := c.loadIssues(file_entry)
if err != nil { if err != nil {
if err == ErrInvalidPage {
continue
}
return issues, err return issues, err
} }
if result.NumberOfTotalResults > c.totalResults {
c.totalResults = result.NumberOfTotalResults c.totalResults = max(result.NumberOfTotalResults, c.totalResults)
}
issues = append(issues, result) issues = append(issues, result)
} }
return issues, nil return issues, nil
@ -134,6 +139,9 @@ func (c *CVDownloader) loadIssues(file_entry fs.DirEntry) (*CVResult, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
if getOffset(file_entry) != tmp.Offset {
return nil, ErrInvalidPage
}
return tmp, nil return tmp, nil
} }
@ -223,6 +231,8 @@ func (c *CVDownloader) updateIssues() {
continue continue
} else { } else {
log.Println("Failed to read page at offset ", offset, err) log.Println("Failed to read page at offset ", offset, err)
os.Remove(filepath.Join(c.JSONPath, c.fileList[offset/100].Name()))
c.fileList = slices.Delete(c.fileList, offset/100, (offset/100)+1)
} }
} }
log.Printf("Expected Offset %d got Offset %d", offset, getOffset(c.fileList[offset/100])) log.Printf("Expected Offset %d got Offset %d", offset, getOffset(c.fileList[offset/100]))
@ -241,6 +251,10 @@ func (c *CVDownloader) updateIssues() {
case c.downloadQueue <- issue: case c.downloadQueue <- issue:
} }
continue continue
} else {
log.Println("Failed to read page at offset ", offset, err)
os.Remove(filepath.Join(c.JSONPath, c.fileList[offset/100].Name()))
c.fileList = slices.Delete(c.fileList, offset/100, (offset/100)+1)
} }
} }