From 29f58e7fe7160408bcae6fcac23b3216be8df71d Mon Sep 17 00:00:00 2001 From: Timmy Welch Date: Sat, 11 Jan 2025 16:11:55 -0800 Subject: [PATCH] Limit download buffer pool to 10MB buffers --- cmd/comic-hasher/main.go | 14 +++++--------- cv/cv.go | 12 +++++------- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/cmd/comic-hasher/main.go b/cmd/comic-hasher/main.go index be07a38..c0478b1 100644 --- a/cmd/comic-hasher/main.go +++ b/cmd/comic-hasher/main.go @@ -804,18 +804,14 @@ func downloadProcessor(chdb ch.CHDB, opts Opts, imagePaths chan cv.Download, ser file = io.NopCloser(path.Image) } i, format, err := image.Decode(bufio.NewReader(file)) - if err != nil { - file.Close() - log.Println("Reading image failed", path.Dest, err) - if path.Image != nil { - bufPool.Put(path.Image) - } - continue // skip this image - } file.Close() - if path.Image != nil { + if path.Image != nil && path.Image.Cap() < 10*1024*1024 { bufPool.Put(path.Image) } + if err != nil { + log.Println("Reading image failed", path.Dest, err) + continue // skip this image + } chdb.AddPath(path.Dest) // Add to sqlite db and remove file if opts.deleteHashedImages is true im := ch.Im{ diff --git a/cv/cv.go b/cv/cv.go index 981e477..1574a73 100644 --- a/cv/cv.go +++ b/cv/cv.go @@ -385,9 +385,7 @@ func (c *CVDownloader) start_downloader() { log.Println("Failed when downloading image", err) cleanup() os.Remove(dl.dest) - if image != nil { - c.bufPool.Put(image) - } + // Something failed let this buffer GC instead of saving it continue } @@ -576,10 +574,10 @@ func NewCVDownloader(ctx context.Context, bufPool *sync.Pool, chdb ch.CHDB, work JSONPath: filepath.Join(workPath, "_json"), ImagePath: filepath.Join(workPath, "_image"), APIKey: APIKey, - downloadQueue: make(chan *CVResult, 1), // This is just json it shouldn't take up much more than 122 MB - imageDownloads: make(chan download, 1), // These are just URLs should only take a few MB - notFound: make(chan download, 1), // Same here - bufPool: bufPool, // Only used if keepDownloadedImages is false to save space on byte buffers. The buffers get sent back via finishedDownloadQueue + downloadQueue: make(chan *CVResult, 100), // This is just json it shouldn't take up much more than 122 MB + imageDownloads: make(chan download, 1), // These are just URLs should only take a few MB + notFound: make(chan download, 1), // Same here + bufPool: bufPool, // Only used if keepDownloadedImages is false to save space on byte buffers. The buffers get sent back via finishedDownloadQueue FinishedDownloadQueue: finishedDownloadQueue, SendExistingImages: sendExistingImages, KeepDownloadedImages: keepDownloadedImages,