diff --git a/comictaggerlib/autotagstartwindow.py b/comictaggerlib/autotagstartwindow.py
index 94380e8..1f8f138 100644
--- a/comictaggerlib/autotagstartwindow.py
+++ b/comictaggerlib/autotagstartwindow.py
@@ -48,7 +48,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.cbxAssumeIssueOne.setChecked(self.config.autotag_assume_1_if_no_issue_num)
self.cbxIgnoreLeadingDigitsInFilename.setChecked(self.config.autotag_ignore_leading_numbers_in_filename)
self.cbxRemoveAfterSuccess.setChecked(self.config.autotag_remove_archive_after_successful_match)
- self.cbxWaitForRateLimit.setChecked(self.config.autotag_wait_and_retry_on_rate_limit)
self.cbxAutoImprint.setChecked(self.config.identifier_auto_imprint)
nlmt_tip = """The Name Match Ratio Threshold: Auto-Identify is for eliminating automatic
@@ -73,7 +72,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.assume_issue_one = False
self.ignore_leading_digits_in_filename = False
self.remove_after_success = False
- self.wait_and_retry_on_rate_limit = False
self.search_string = ""
self.name_length_match_tolerance = self.config.identifier_series_match_search_thresh
self.split_words = self.cbxSplitWords.isChecked()
@@ -91,7 +89,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.ignore_leading_digits_in_filename = self.cbxIgnoreLeadingDigitsInFilename.isChecked()
self.remove_after_success = self.cbxRemoveAfterSuccess.isChecked()
self.name_length_match_tolerance = self.sbNameMatchSearchThresh.value()
- self.wait_and_retry_on_rate_limit = self.cbxWaitForRateLimit.isChecked()
self.split_words = self.cbxSplitWords.isChecked()
# persist some settings
@@ -100,7 +97,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.config.autotag_assume_1_if_no_issue_num = self.assume_issue_one
self.config.autotag_ignore_leading_numbers_in_filename = self.ignore_leading_digits_in_filename
self.config.autotag_remove_archive_after_successful_match = self.remove_after_success
- self.config.autotag_wait_and_retry_on_rate_limit = self.wait_and_retry_on_rate_limit
if self.cbxSpecifySearchString.isChecked():
self.search_string = self.leSearchString.text()
diff --git a/comictaggerlib/ctsettings/file.py b/comictaggerlib/ctsettings/file.py
index fb3e0d4..8283a5a 100644
--- a/comictaggerlib/ctsettings/file.py
+++ b/comictaggerlib/ctsettings/file.py
@@ -207,14 +207,6 @@ def autotag(parser: settngs.Manager) -> None:
help="When searching ignore leading numbers in the filename",
)
parser.add_setting("remove_archive_after_successful_match", default=False, cmdline=False)
- parser.add_setting(
- "-w",
- "--wait-on-rate-limit",
- dest="wait_and_retry_on_rate_limit",
- action=argparse.BooleanOptionalAction,
- default=True,
- help="When encountering a Comic Vine rate limit\nerror, wait and retry query.\n\n",
- )
def validate_file_settings(config: settngs.Config[ct_ns]) -> settngs.Config[ct_ns]:
diff --git a/comictaggerlib/ctsettings/settngs_namespace.py b/comictaggerlib/ctsettings/settngs_namespace.py
index 453d56a..eaa2e0e 100644
--- a/comictaggerlib/ctsettings/settngs_namespace.py
+++ b/comictaggerlib/ctsettings/settngs_namespace.py
@@ -102,4 +102,3 @@ class settngs_namespace(settngs.TypedNS):
autotag_assume_1_if_no_issue_num: bool
autotag_ignore_leading_numbers_in_filename: bool
autotag_remove_archive_after_successful_match: bool
- autotag_wait_and_retry_on_rate_limit: bool
diff --git a/comictaggerlib/taggerwindow.py b/comictaggerlib/taggerwindow.py
index 33963b4..1e59531 100644
--- a/comictaggerlib/taggerwindow.py
+++ b/comictaggerlib/taggerwindow.py
@@ -257,6 +257,10 @@ class TaggerWindow(QtWidgets.QMainWindow):
Also, be aware that writing tags to comic archives will change their file hashes,
which has implications with respect to other software packages. It's best to
use ComicTagger on local copies of your comics.
+ COMIC VINE NOTE: Using the default API key will serverly limit search and tagging
+ times. A personal API key will allow for a 5 times increase in online search speed. See the
+ Wiki page
+ for more information.
Have fun!
""",
)
diff --git a/comictaggerlib/ui/autotagstartwindow.ui b/comictaggerlib/ui/autotagstartwindow.ui
index dcf341d..a0bb361 100644
--- a/comictaggerlib/ui/autotagstartwindow.ui
+++ b/comictaggerlib/ui/autotagstartwindow.ui
@@ -10,7 +10,7 @@
0
0
519
- 440
+ 448
@@ -26,84 +26,19 @@
false
- -
-
-
-
- 0
- 0
-
+
-
+
+
+ Qt::Horizontal
-
-
-
-
- true
+
+ QDialogButtonBox::Cancel|QDialogButtonBox::Ok
-
-
-
-
-
- Checks the publisher against a list of imprints.
-
-
- Auto Imprint
-
-
-
- -
-
-
-
- 0
- 0
-
-
-
- Specify series search string for all selected archives:
-
-
-
- -
-
-
-
- 0
- 0
-
-
-
- Ignore leading (sequence) numbers in filename
-
-
-
- -
-
-
-
- 0
- 0
-
-
-
- Save on low confidence match
-
-
-
-
-
-
-
- 0
- 0
-
-
-
-
- -
@@ -116,7 +51,7 @@
- -
+
-
@@ -129,19 +64,6 @@
- -
-
-
-
- 0
- 0
-
-
-
- Don't use publication year in identification process
-
-
-
-
@@ -155,7 +77,7 @@
- -
+
-
Removes existing metadata before applying retrieved metadata
@@ -165,7 +87,66 @@
+ -
+
+
+
+ 0
+ 0
+
+
+
+
+ -
+
+
+
+ 0
+ 0
+
+
+
+ Don't use publication year in identification process
+
+
+
-
+
+
+ Checks the publisher against a list of imprints.
+
+
+ Auto Imprint
+
+
+
+ -
+
+
+
+ 0
+ 0
+
+
+
+ Specify series search string for all selected archives:
+
+
+
+ -
+
+
+
+ 0
+ 0
+
+
+
+ Save on low confidence match
+
+
+
+ -
@@ -179,13 +160,19 @@
-
-
+
+
+
+ 0
+ 0
+
+
- Wait and retry when Comic Vine rate limit is exceeded (experimental)
+ Ignore leading (sequence) numbers in filename
- -
+
-
@@ -225,13 +212,19 @@
- -
-
-
- Qt::Horizontal
+
-
+
+
+
+ 0
+ 0
+
-
- QDialogButtonBox::Cancel|QDialogButtonBox::Ok
+
+
+
+
+ true
diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py
index 4981700..0aff3cb 100644
--- a/comictalker/talkers/comicvine.py
+++ b/comictalker/talkers/comicvine.py
@@ -26,6 +26,7 @@ from urllib.parse import urljoin
import requests
import settngs
+from pyrate_limiter import Limiter, RequestRate
from typing_extensions import Required, TypedDict
import comictalker.talker_utils as talker_utils
@@ -150,7 +151,10 @@ class CVResult(TypedDict, Generic[T]):
version: str
-CV_STATUS_RATELIMIT = 107
+# https://comicvine.gamespot.com/forums/api-developers-2334/api-rate-limiting-1746419/
+# "Space out your requests so AT LEAST one second passes between each and you can make requests all day."
+custom_limiter = Limiter(RequestRate(10, 10))
+default_limiter = Limiter(RequestRate(1, 5))
class ComicVineTalker(ComicTalker):
@@ -162,15 +166,12 @@ class ComicVineTalker(ComicTalker):
def __init__(self, version: str, cache_folder: pathlib.Path):
super().__init__(version, cache_folder)
+ self.limiter = default_limiter
# Default settings
self.default_api_url = self.api_url = f"{self.website}/api/"
self.default_api_key = self.api_key = "27431e6787042105bd3e47e169a624521f89f3a4"
self.remove_html_tables: bool = False
self.use_series_start_as_volume: bool = False
- self.wait_on_ratelimit: bool = False
-
- # NOTE: This was hardcoded before which is why it isn't in settings
- self.wait_on_ratelimit_time: int = 20
def register_settings(self, parser: settngs.Manager) -> None:
parser.add_setting(
@@ -180,13 +181,6 @@ class ComicVineTalker(ComicTalker):
display_name="Use series start as volume",
help="Use the series start year as the volume number",
)
- parser.add_setting(
- "--cv-wait-on-ratelimit",
- default=False,
- action=argparse.BooleanOptionalAction,
- display_name="Wait on ratelimit",
- help="Wait when the rate limit is hit",
- )
parser.add_setting(
"--cv-remove-html-tables",
default=False,
@@ -212,8 +206,14 @@ class ComicVineTalker(ComicTalker):
settings = super().parse_settings(settings)
self.use_series_start_as_volume = settings["cv_use_series_start_as_volume"]
- self.wait_on_ratelimit = settings["cv_wait_on_ratelimit"]
self.remove_html_tables = settings["cv_remove_html_tables"]
+
+ # Set a different limit if using the default API key
+ if self.api_key == self.default_api_key:
+ self.limiter = default_limiter
+ else:
+ self.limiter = custom_limiter
+
return settings
def check_api_key(self, url: str, key: str) -> tuple[str, bool]:
@@ -430,40 +430,24 @@ class ComicVineTalker(ComicTalker):
def _get_cv_content(self, url: str, params: dict[str, Any]) -> CVResult:
"""
- Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error
- sleep for a bit and retry.
+ Get the content from the CV server.
"""
- total_time_waited = 0
- limit_wait_time = 1
- counter = 0
- wait_times = [1, 2, 3, 4]
- while True:
+ with self.limiter.ratelimit("cv", delay=True):
cv_response: CVResult = self._get_url_content(url, params)
- if self.wait_on_ratelimit and cv_response["status_code"] == CV_STATUS_RATELIMIT:
- logger.info(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n")
- time.sleep(limit_wait_time * 60)
- total_time_waited += limit_wait_time
- limit_wait_time = wait_times[counter]
- if counter < 3:
- counter += 1
- # don't wait much more than 20 minutes
- if total_time_waited < self.wait_on_ratelimit_time:
- continue
+
if cv_response["status_code"] != 1:
logger.debug(
f"{self.name} query failed with error #{cv_response['status_code']}: [{cv_response['error']}]."
)
raise TalkerNetworkError(self.name, 0, f"{cv_response['status_code']}: {cv_response['error']}")
- # it's all good
- break
- return cv_response
+ return cv_response
def _get_url_content(self, url: str, params: dict[str, Any]) -> Any:
- # connect to server:
# if there is a 500 error, try a few more times before giving up
- # any other error, just bail
- for tries in range(3):
+ limit_counter = 0
+ tries = 0
+ while tries < 4:
try:
resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + self.version})
if resp.status_code == 200:
@@ -472,6 +456,19 @@ class ComicVineTalker(ComicTalker):
logger.debug(f"Try #{tries + 1}: ")
time.sleep(1)
logger.debug(str(resp.status_code))
+ tries += 1
+ if resp.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS:
+ logger.info(f"{self.name} rate limit encountered. Waiting for 10 seconds\n")
+ time.sleep(10)
+ limit_counter += 1
+ if limit_counter > 3:
+ # Tried 3 times, inform user to check CV website.
+ logger.error(f"{self.name} rate limit error. Exceeded 3 retires.")
+ raise TalkerNetworkError(
+ self.name,
+ 3,
+ "Rate Limit Error: Check your current API usage limit at https://comicvine.gamespot.com/api/",
+ )
else:
break
diff --git a/setup.cfg b/setup.cfg
index 4fdc2b6..8f51483 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -40,6 +40,7 @@ install_requires =
pathvalidate
pillow>=9.1.0,<10
pycountry
+ pyrate-limiter
rapidfuzz>=2.12.0
requests==2.*
settngs==0.7.1