From 260a13688a69746ba5ed6626dbda5d2390ad4d31 Mon Sep 17 00:00:00 2001 From: Timmy Welch Date: Thu, 26 Dec 2024 17:50:25 -0800 Subject: [PATCH] Wait every 200 downloads so CV doesn't get overloaded --- cmd/comic-hasher/main.go | 4 ++-- cv/cv.go | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/cmd/comic-hasher/main.go b/cmd/comic-hasher/main.go index 6f08e46..dba3474 100644 --- a/cmd/comic-hasher/main.go +++ b/cmd/comic-hasher/main.go @@ -843,8 +843,8 @@ func startServer(opts Opts) { cancel: cancel, signalQueue: make(chan os.Signal, 1), readerQueue: make(chan string, 100), - hashingQueue: make(chan ch.Im), - mappingQueue: make(chan ch.ImageHash), + hashingQueue: make(chan ch.Im, 1), + mappingQueue: make(chan ch.ImageHash, 1), mux: mux, httpServer: &http.Server{ Addr: ":8080", diff --git a/cv/cv.go b/cv/cv.go index fd6c42a..863dd55 100644 --- a/cv/cv.go +++ b/cv/cv.go @@ -481,10 +481,10 @@ func (c *CVDownloader) downloadImages() { if added > 200 { // On a clean single image type run each page would have 100 downloads of a single cover type but stuff happens so we only wait once we have sent 200 to the queue log.Println("waiting for", added, "downloads at offset", list.Offset) - added = 0 beforeWait := time.Now() c.imageWG.Wait() waited := time.Since(beforeWait) + added = 0 // If we had to wait for the arbitrarily picked time of 7.4 seconds it means we had a backed up queue (slow hashing can also cause it to wait longer), lets wait to give the CV servers a break if waited > time.Duration(7.4*float64(time.Second)) { t := 10 * time.Second @@ -494,6 +494,10 @@ func (c *CVDownloader) downloadImages() { return case <-time.After(t): } + } else { + // Things are too fast we can't depend CV being slow to manage our download speed + // We sleep for 3 seconds so we don't overload CV + time.Sleep(3 * time.Second) } } }