lordwelch/comictagger
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'mizaki-rate_limit_cv' into develop

Browse Source
This commit is contained in:
Timmy Welch 2023-07-01 18:04:24 -07:00
commit f90f373d20
7 changed files with 127 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
comictaggerlib/autotagstartwindow.py
View File

@ -48,7 +48,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.cbxAssumeIssueOne.setChecked(self.config.autotag_assume_1_if_no_issue_num)
self.cbxIgnoreLeadingDigitsInFilename.setChecked(self.config.autotag_ignore_leading_numbers_in_filename)
self.cbxRemoveAfterSuccess.setChecked(self.config.autotag_remove_archive_after_successful_match)
self.cbxWaitForRateLimit.setChecked(self.config.autotag_wait_and_retry_on_rate_limit)
self.cbxAutoImprint.setChecked(self.config.identifier_auto_imprint)
nlmt_tip = """<html>The <b>Name Match Ratio Threshold: Auto-Identify</b> is for eliminating automatic
@ -73,7 +72,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.assume_issue_one = False
self.ignore_leading_digits_in_filename = False
self.remove_after_success = False
self.wait_and_retry_on_rate_limit = False
self.search_string = ""
self.name_length_match_tolerance = self.config.identifier_series_match_search_thresh
self.split_words = self.cbxSplitWords.isChecked()
@ -91,7 +89,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.ignore_leading_digits_in_filename = self.cbxIgnoreLeadingDigitsInFilename.isChecked()
self.remove_after_success = self.cbxRemoveAfterSuccess.isChecked()
self.name_length_match_tolerance = self.sbNameMatchSearchThresh.value()
self.wait_and_retry_on_rate_limit = self.cbxWaitForRateLimit.isChecked()
self.split_words = self.cbxSplitWords.isChecked()
# persist some settings
@ -100,7 +97,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.config.autotag_assume_1_if_no_issue_num = self.assume_issue_one
self.config.autotag_ignore_leading_numbers_in_filename = self.ignore_leading_digits_in_filename
self.config.autotag_remove_archive_after_successful_match = self.remove_after_success
self.config.autotag_wait_and_retry_on_rate_limit = self.wait_and_retry_on_rate_limit
if self.cbxSpecifySearchString.isChecked():
self.search_string = self.leSearchString.text()

8
comictaggerlib/ctsettings/file.py
View File

@ -207,14 +207,6 @@ def autotag(parser: settngs.Manager) -> None:
help="When searching ignore leading numbers in the filename",
)
parser.add_setting("remove_archive_after_successful_match", default=False, cmdline=False)
parser.add_setting(
"-w",
"--wait-on-rate-limit",
dest="wait_and_retry_on_rate_limit",
action=argparse.BooleanOptionalAction,
default=True,
help="When encountering a Comic Vine rate limit\nerror, wait and retry query.\n\n",
)
def validate_file_settings(config: settngs.Config[ct_ns]) -> settngs.Config[ct_ns]:

1
comictaggerlib/ctsettings/settngs_namespace.py
View File

@ -102,4 +102,3 @@ class settngs_namespace(settngs.TypedNS):
autotag_assume_1_if_no_issue_num: bool
autotag_ignore_leading_numbers_in_filename: bool
autotag_remove_archive_after_successful_match: bool
autotag_wait_and_retry_on_rate_limit: bool

4
comictaggerlib/taggerwindow.py
View File

@ -257,6 +257,10 @@ class TaggerWindow(QtWidgets.QMainWindow):
Also, be aware that writing tags to comic archives will change their file hashes,
which has implications with respect to other software packages. It's best to
use ComicTagger on local copies of your comics.<br><br>
COMIC VINE NOTE: Using the default API key will serverly limit search and tagging
times. A personal API key will allow for a <b>5 times increase</b> in online search speed. See the
<a href='https://github.com/comictagger/comictagger/wiki/UserGuide#comic-vine'>Wiki page</a>
for more information.<br><br>
Have fun!
""",
)

185
comictaggerlib/ui/autotagstartwindow.ui
View File

@ -10,7 +10,7 @@
<x>0</x>
<y>0</y>
<width>519</width>
<height>440</height>
<height>448</height>
</rect>
</property>
<property name="sizePolicy">
@ -26,84 +26,19 @@
<bool>false</bool>
</property>
<layout class="QGridLayout" name="gridLayout_2">
<item row="0" column="0">
<widget class="QLabel" name="label">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
<item row="4" column="0">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="text">
<string/>
</property>
<property name="wordWrap">
<bool>true</bool>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
</property>
</widget>
</item>
<item row="1" column="0">
<layout class="QGridLayout" name="gridLayout">
<item row="6" column="0">
<widget class="QCheckBox" name="cbxAutoImprint">
<property name="toolTip">
<string>Checks the publisher against a list of imprints.</string>
</property>
<property name="text">
<string>Auto Imprint</string>
</property>
</widget>
</item>
<item row="9" column="0">
<widget class="QCheckBox" name="cbxSpecifySearchString">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Specify series search string for all selected archives:</string>
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QCheckBox" name="cbxIgnoreLeadingDigitsInFilename">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Ignore leading (sequence) numbers in filename</string>
</property>
</widget>
</item>
<item row="0" column="0">
<widget class="QCheckBox" name="cbxSaveOnLowConfidence">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Save on low confidence match</string>
</property>
</widget>
</item>
<item row="10" column="0">
<widget class="QLineEdit" name="leSearchString">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
</widget>
</item>
<item row="11" column="0">
<widget class="QLabel" name="label_3">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Fixed">
@ -116,7 +51,7 @@
</property>
</widget>
</item>
<item row="8" column="0">
<item row="7" column="0">
<widget class="QCheckBox" name="cbxSplitWords">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
@ -129,19 +64,6 @@
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QCheckBox" name="cbxDontUseYear">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Don't use publication year in identification process</string>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QCheckBox" name="cbxAssumeIssueOne">
<property name="sizePolicy">
@ -155,7 +77,7 @@
</property>
</widget>
</item>
<item row="7" column="0">
<item row="6" column="0">
<widget class="QCheckBox" name="cbxRemoveMetadata">
<property name="toolTip">
<string>Removes existing metadata before applying retrieved metadata</string>
@ -165,7 +87,66 @@
</property>
</widget>
</item>
<item row="9" column="0">
<widget class="QLineEdit" name="leSearchString">
<property name="sizePolicy">
<sizepolicy hsizetype="Expanding" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QCheckBox" name="cbxDontUseYear">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Don't use publication year in identification process</string>
</property>
</widget>
</item>
<item row="5" column="0">
<widget class="QCheckBox" name="cbxAutoImprint">
<property name="toolTip">
<string>Checks the publisher against a list of imprints.</string>
</property>
<property name="text">
<string>Auto Imprint</string>
</property>
</widget>
</item>
<item row="8" column="0">
<widget class="QCheckBox" name="cbxSpecifySearchString">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Specify series search string for all selected archives:</string>
</property>
</widget>
</item>
<item row="0" column="0">
<widget class="QCheckBox" name="cbxSaveOnLowConfidence">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Save on low confidence match</string>
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QCheckBox" name="cbxRemoveAfterSuccess">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
@ -179,13 +160,19 @@
</widget>
</item>
<item row="3" column="0">
<widget class="QCheckBox" name="cbxWaitForRateLimit">
<widget class="QCheckBox" name="cbxIgnoreLeadingDigitsInFilename">
<property name="sizePolicy">
<sizepolicy hsizetype="Minimum" vsizetype="Fixed">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="text">
<string>Wait and retry when Comic Vine rate limit is exceeded (experimental)</string>
<string>Ignore leading (sequence) numbers in filename</string>
</property>
</widget>
</item>
<item row="12" column="0">
<item row="11" column="0">
<widget class="QSpinBox" name="sbNameMatchSearchThresh">
<property name="maximumSize">
<size>
@ -225,13 +212,19 @@
</property>
</widget>
</item>
<item row="4" column="0">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
<item row="0" column="0">
<widget class="QLabel" name="label">
<property name="sizePolicy">
<sizepolicy hsizetype="Preferred" vsizetype="Preferred">
<horstretch>0</horstretch>
<verstretch>0</verstretch>
</sizepolicy>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
<property name="text">
<string/>
</property>
<property name="wordWrap">
<bool>true</bool>
</property>
</widget>
</item>

69
comictalker/talkers/comicvine.py
View File

@ -26,6 +26,7 @@ from urllib.parse import urljoin
import requests
import settngs
from pyrate_limiter import Limiter, RequestRate
from typing_extensions import Required, TypedDict
import comictalker.talker_utils as talker_utils
@ -150,7 +151,10 @@ class CVResult(TypedDict, Generic[T]):
version: str
CV_STATUS_RATELIMIT = 107
# https://comicvine.gamespot.com/forums/api-developers-2334/api-rate-limiting-1746419/
# "Space out your requests so AT LEAST one second passes between each and you can make requests all day."
custom_limiter = Limiter(RequestRate(10, 10))
default_limiter = Limiter(RequestRate(1, 5))
class ComicVineTalker(ComicTalker):
@ -162,15 +166,12 @@ class ComicVineTalker(ComicTalker):
def __init__(self, version: str, cache_folder: pathlib.Path):
super().__init__(version, cache_folder)
self.limiter = default_limiter
# Default settings
self.default_api_url = self.api_url = f"{self.website}/api/"
self.default_api_key = self.api_key = "27431e6787042105bd3e47e169a624521f89f3a4"
self.remove_html_tables: bool = False
self.use_series_start_as_volume: bool = False
self.wait_on_ratelimit: bool = False
# NOTE: This was hardcoded before which is why it isn't in settings
self.wait_on_ratelimit_time: int = 20
def register_settings(self, parser: settngs.Manager) -> None:
parser.add_setting(
@ -180,13 +181,6 @@ class ComicVineTalker(ComicTalker):
display_name="Use series start as volume",
help="Use the series start year as the volume number",
)
parser.add_setting(
"--cv-wait-on-ratelimit",
default=False,
action=argparse.BooleanOptionalAction,
display_name="Wait on ratelimit",
help="Wait when the rate limit is hit",
)
parser.add_setting(
"--cv-remove-html-tables",
default=False,
@ -212,8 +206,14 @@ class ComicVineTalker(ComicTalker):
settings = super().parse_settings(settings)
self.use_series_start_as_volume = settings["cv_use_series_start_as_volume"]
self.wait_on_ratelimit = settings["cv_wait_on_ratelimit"]
self.remove_html_tables = settings["cv_remove_html_tables"]
# Set a different limit if using the default API key
if self.api_key == self.default_api_key:
self.limiter = default_limiter
else:
self.limiter = custom_limiter
return settings
def check_api_key(self, url: str, key: str) -> tuple[str, bool]:
@ -430,40 +430,24 @@ class ComicVineTalker(ComicTalker):
def _get_cv_content(self, url: str, params: dict[str, Any]) -> CVResult:
"""
Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error
sleep for a bit and retry.
Get the content from the CV server.
"""
total_time_waited = 0
limit_wait_time = 1
counter = 0
wait_times = [1, 2, 3, 4]
while True:
with self.limiter.ratelimit("cv", delay=True):
cv_response: CVResult = self._get_url_content(url, params)
if self.wait_on_ratelimit and cv_response["status_code"] == CV_STATUS_RATELIMIT:
logger.info(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n")
time.sleep(limit_wait_time * 60)
total_time_waited += limit_wait_time
limit_wait_time = wait_times[counter]
if counter < 3:
counter += 1
# don't wait much more than 20 minutes
if total_time_waited < self.wait_on_ratelimit_time:
continue
if cv_response["status_code"] != 1:
logger.debug(
f"{self.name} query failed with error #{cv_response['status_code']}: [{cv_response['error']}]."
)
raise TalkerNetworkError(self.name, 0, f"{cv_response['status_code']}: {cv_response['error']}")
# it's all good
break
return cv_response
return cv_response
def _get_url_content(self, url: str, params: dict[str, Any]) -> Any:
# connect to server:
# if there is a 500 error, try a few more times before giving up
# any other error, just bail
for tries in range(3):
limit_counter = 0
tries = 0
while tries < 4:
try:
resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + self.version})
if resp.status_code == 200:
@ -472,6 +456,19 @@ class ComicVineTalker(ComicTalker):
logger.debug(f"Try #{tries + 1}: ")
time.sleep(1)
logger.debug(str(resp.status_code))
tries += 1
if resp.status_code == requests.status_codes.codes.TOO_MANY_REQUESTS:
logger.info(f"{self.name} rate limit encountered. Waiting for 10 seconds\n")
time.sleep(10)
limit_counter += 1
if limit_counter > 3:
# Tried 3 times, inform user to check CV website.
logger.error(f"{self.name} rate limit error. Exceeded 3 retires.")
raise TalkerNetworkError(
self.name,
3,
"Rate Limit Error: Check your current API usage limit at https://comicvine.gamespot.com/api/",
)
else:
break

1
setup.cfg
View File

@ -40,6 +40,7 @@ install_requires =
pathvalidate
pillow>=9.1.0,<10
pycountry
pyrate-limiter
rapidfuzz>=2.12.0
requests==2.*
settngs==0.7.1