Limit download buffer pool to 10MB buffers

This commit is contained in:
Timmy Welch 2025-01-11 16:11:55 -08:00
parent fe3f045c6e
commit 29f58e7fe7
2 changed files with 10 additions and 16 deletions

View File

@ -804,18 +804,14 @@ func downloadProcessor(chdb ch.CHDB, opts Opts, imagePaths chan cv.Download, ser
file = io.NopCloser(path.Image)
}
i, format, err := image.Decode(bufio.NewReader(file))
file.Close()
if path.Image != nil && path.Image.Cap() < 10*1024*1024 {
bufPool.Put(path.Image)
}
if err != nil {
file.Close()
log.Println("Reading image failed", path.Dest, err)
if path.Image != nil {
bufPool.Put(path.Image)
}
continue // skip this image
}
file.Close()
if path.Image != nil {
bufPool.Put(path.Image)
}
chdb.AddPath(path.Dest) // Add to sqlite db and remove file if opts.deleteHashedImages is true
im := ch.Im{

View File

@ -385,9 +385,7 @@ func (c *CVDownloader) start_downloader() {
log.Println("Failed when downloading image", err)
cleanup()
os.Remove(dl.dest)
if image != nil {
c.bufPool.Put(image)
}
// Something failed let this buffer GC instead of saving it
continue
}
@ -576,7 +574,7 @@ func NewCVDownloader(ctx context.Context, bufPool *sync.Pool, chdb ch.CHDB, work
JSONPath: filepath.Join(workPath, "_json"),
ImagePath: filepath.Join(workPath, "_image"),
APIKey: APIKey,
downloadQueue: make(chan *CVResult, 1), // This is just json it shouldn't take up much more than 122 MB
downloadQueue: make(chan *CVResult, 100), // This is just json it shouldn't take up much more than 122 MB
imageDownloads: make(chan download, 1), // These are just URLs should only take a few MB
notFound: make(chan download, 1), // Same here
bufPool: bufPool, // Only used if keepDownloadedImages is false to save space on byte buffers. The buffers get sent back via finishedDownloadQueue