Limit download buffer pool to 10MB buffers
This commit is contained in:
parent
fe3f045c6e
commit
29f58e7fe7
@ -804,18 +804,14 @@ func downloadProcessor(chdb ch.CHDB, opts Opts, imagePaths chan cv.Download, ser
|
||||
file = io.NopCloser(path.Image)
|
||||
}
|
||||
i, format, err := image.Decode(bufio.NewReader(file))
|
||||
file.Close()
|
||||
if path.Image != nil && path.Image.Cap() < 10*1024*1024 {
|
||||
bufPool.Put(path.Image)
|
||||
}
|
||||
if err != nil {
|
||||
file.Close()
|
||||
log.Println("Reading image failed", path.Dest, err)
|
||||
if path.Image != nil {
|
||||
bufPool.Put(path.Image)
|
||||
}
|
||||
continue // skip this image
|
||||
}
|
||||
file.Close()
|
||||
if path.Image != nil {
|
||||
bufPool.Put(path.Image)
|
||||
}
|
||||
chdb.AddPath(path.Dest) // Add to sqlite db and remove file if opts.deleteHashedImages is true
|
||||
|
||||
im := ch.Im{
|
||||
|
6
cv/cv.go
6
cv/cv.go
@ -385,9 +385,7 @@ func (c *CVDownloader) start_downloader() {
|
||||
log.Println("Failed when downloading image", err)
|
||||
cleanup()
|
||||
os.Remove(dl.dest)
|
||||
if image != nil {
|
||||
c.bufPool.Put(image)
|
||||
}
|
||||
// Something failed let this buffer GC instead of saving it
|
||||
continue
|
||||
}
|
||||
|
||||
@ -576,7 +574,7 @@ func NewCVDownloader(ctx context.Context, bufPool *sync.Pool, chdb ch.CHDB, work
|
||||
JSONPath: filepath.Join(workPath, "_json"),
|
||||
ImagePath: filepath.Join(workPath, "_image"),
|
||||
APIKey: APIKey,
|
||||
downloadQueue: make(chan *CVResult, 1), // This is just json it shouldn't take up much more than 122 MB
|
||||
downloadQueue: make(chan *CVResult, 100), // This is just json it shouldn't take up much more than 122 MB
|
||||
imageDownloads: make(chan download, 1), // These are just URLs should only take a few MB
|
||||
notFound: make(chan download, 1), // Same here
|
||||
bufPool: bufPool, // Only used if keepDownloadedImages is false to save space on byte buffers. The buffers get sent back via finishedDownloadQueue
|
||||
|
Loading…
Reference in New Issue
Block a user