Limit download buffer pool to 10MB buffers

cmd/comic-hasher/main.go

@@ -804,18 +804,14 @@ func downloadProcessor(chdb ch.CHDB, opts Opts, imagePaths chan cv.Download, ser
 			file = io.NopCloser(path.Image)
 		}
 		i, format, err := image.Decode(bufio.NewReader(file))
-		if err != nil {
-			file.Close()
-			log.Println("Reading image failed", path.Dest, err)
-			if path.Image != nil {
-				bufPool.Put(path.Image)
-			}
-			continue // skip this image
-		}
 		file.Close()
-		if path.Image != nil {
+		if path.Image != nil && path.Image.Cap() < 10*1024*1024 {
 			bufPool.Put(path.Image)
 		}
+		if err != nil {
+			log.Println("Reading image failed", path.Dest, err)
+			continue // skip this image
+		}
 		chdb.AddPath(path.Dest) // Add to sqlite db and remove file if opts.deleteHashedImages is true
 
 		im := ch.Im{

cv/cv.go

@@ -385,9 +385,7 @@ func (c *CVDownloader) start_downloader() {
 						log.Println("Failed when downloading image", err)
 						cleanup()
 						os.Remove(dl.dest)
-						if image != nil {
+						// Something failed let this buffer GC instead of saving it
-							c.bufPool.Put(image)
-						}
 						continue
 					}
 
@@ -576,10 +574,10 @@ func NewCVDownloader(ctx context.Context, bufPool *sync.Pool, chdb ch.CHDB, work
 		JSONPath:              filepath.Join(workPath, "_json"),
 		ImagePath:             filepath.Join(workPath, "_image"),
 		APIKey:                APIKey,
-		downloadQueue:         make(chan *CVResult, 1), // This is just json it shouldn't take up much more than 122 MB
+		downloadQueue:         make(chan *CVResult, 100), // This is just json it shouldn't take up much more than 122 MB
-		imageDownloads:        make(chan download, 1),  // These are just URLs should only take a few MB
+		imageDownloads:        make(chan download, 1),    // These are just URLs should only take a few MB
-		notFound:              make(chan download, 1),  // Same here
+		notFound:              make(chan download, 1),    // Same here
-		bufPool:               bufPool,                 // Only used if keepDownloadedImages is false to save space on byte buffers. The buffers get sent back via finishedDownloadQueue
+		bufPool:               bufPool,                   // Only used if keepDownloadedImages is false to save space on byte buffers. The buffers get sent back via finishedDownloadQueue
 		FinishedDownloadQueue: finishedDownloadQueue,
 		SendExistingImages:    sendExistingImages,
 		KeepDownloadedImages:  keepDownloadedImages,