lordwelch/comictagger
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'mizaki-infosources' into dev

Browse Source
This commit is contained in:
Timmy Welch 2022-11-25 19:21:25 -08:00
commit 91b863fcb1
No known key found for this signature in database
34 changed files with 1778 additions and 1595 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
comictaggerlib/autotagmatchwindow.py
View File

@ -28,6 +28,7 @@ from comictaggerlib.resulttypes import IssueResult, MultipleMatch
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@ -42,6 +43,7 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
style: int,
fetch_func: Callable[[IssueResult], GenericMetadata],
settings: ComicTaggerSettings,
talker_api: ComicTalker,
) -> None:
super().__init__(parent)
@ -51,12 +53,12 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
self.current_match_set: MultipleMatch = match_set_list[0]
self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode)
self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode)
gridlayout = QtWidgets.QGridLayout(self.altCoverContainer)
gridlayout.addWidget(self.altCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode)
self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer)
gridlayout.addWidget(self.archiveCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@ -178,7 +180,10 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
if prev is not None and prev.row() == curr.row():
return None
self.altCoverWidget.set_issue_id(self.current_match()["issue_id"])
self.altCoverWidget.set_issue_details(
self.current_match()["issue_id"],
[self.current_match()["image_url"], *self.current_match()["alt_image_urls"]],
)
if self.current_match()["description"] is None:
self.teDescription.setText("")
else:
@ -242,15 +247,13 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
)
# now get the particular issue data
cv_md = self.fetch_func(match)
if cv_md is None:
QtWidgets.QMessageBox.critical(
self, "Network Issue", "Could not connect to Comic Vine to get issue details!"
)
ct_md = self.fetch_func(match)
if ct_md is None:
QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not retrieve issue details!")
return
QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
md.overlay(cv_md)
md.overlay(ct_md)
success = ca.write_metadata(md, self._style)
ca.load_cache([MetaDataStyle.CBI, MetaDataStyle.CIX])

9
comictaggerlib/autotagprogresswindow.py
View File

@ -22,22 +22,25 @@ from PyQt5 import QtCore, QtWidgets, uic
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
class AutoTagProgressWindow(QtWidgets.QDialog):
def __init__(self, parent: QtWidgets.QWidget) -> None:
def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "autotagprogresswindow.ui", self)
self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.DataMode, False)
self.archiveCoverWidget = CoverImageWidget(
self.archiveCoverContainer, talker_api, CoverImageWidget.DataMode, False
)
gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer)
gridlayout.addWidget(self.archiveCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
self.testCoverWidget = CoverImageWidget(self.testCoverContainer, CoverImageWidget.DataMode, False)
self.testCoverWidget = CoverImageWidget(self.testCoverContainer, talker_api, CoverImageWidget.DataMode, False)
gridlayout = QtWidgets.QGridLayout(self.testCoverContainer)
gridlayout.addWidget(self.testCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)

4
comictaggerlib/autotagstartwindow.py
View File

@ -48,7 +48,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.cbxAssumeIssueOne.setChecked(self.settings.assume_1_if_no_issue_num)
self.cbxIgnoreLeadingDigitsInFilename.setChecked(self.settings.ignore_leading_numbers_in_filename)
self.cbxRemoveAfterSuccess.setChecked(self.settings.remove_archive_after_successful_match)
self.cbxWaitForRateLimit.setChecked(self.settings.wait_and_retry_on_rate_limit)
self.cbxAutoImprint.setChecked(self.settings.auto_imprint)
nlmt_tip = """<html>The <b>Name Match Ratio Threshold: Auto-Identify</b> is for eliminating automatic
@ -73,7 +72,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.assume_issue_one = False
self.ignore_leading_digits_in_filename = False
self.remove_after_success = False
self.wait_and_retry_on_rate_limit = False
self.search_string = ""
self.name_length_match_tolerance = self.settings.id_series_match_search_thresh
self.split_words = self.cbxSplitWords.isChecked()
@ -91,7 +89,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.ignore_leading_digits_in_filename = self.cbxIgnoreLeadingDigitsInFilename.isChecked()
self.remove_after_success = self.cbxRemoveAfterSuccess.isChecked()
self.name_length_match_tolerance = self.sbNameMatchSearchThresh.value()
self.wait_and_retry_on_rate_limit = self.cbxWaitForRateLimit.isChecked()
self.split_words = self.cbxSplitWords.isChecked()
# persist some settings
@ -100,7 +97,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.settings.assume_1_if_no_issue_num = self.assume_issue_one
self.settings.ignore_leading_numbers_in_filename = self.ignore_leading_digits_in_filename
self.settings.remove_archive_after_successful_match = self.remove_after_success
self.settings.wait_and_retry_on_rate_limit = self.wait_and_retry_on_rate_limit
if self.cbxSpecifySearchString.isChecked():
self.search_string = self.leSearchString.text()

79
comictaggerlib/cli.py
View File

@ -30,32 +30,30 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle
from comicapi.genericmetadata import GenericMetadata
from comictaggerlib import ctversion
from comictaggerlib.cbltransformer import CBLTransformer
from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.filerenamer import FileRenamer, get_rename_dir
from comictaggerlib.graphics import graphics_path
from comictaggerlib.issueidentifier import IssueIdentifier
from comictaggerlib.resulttypes import IssueResult, MultipleMatch, OnlineMatchResults
from comictaggerlib.resulttypes import MultipleMatch, OnlineMatchResults
from comictaggerlib.settings import ComicTaggerSettings
from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
def actual_issue_data_fetch(
match: IssueResult, settings: ComicTaggerSettings, opts: argparse.Namespace
issue_id: int, settings: ComicTaggerSettings, opts: argparse.Namespace, talker_api: ComicTalker
) -> GenericMetadata:
# now get the particular issue data
try:
comic_vine = ComicVineTalker(settings.id_series_match_search_thresh)
comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit
cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], settings)
except ComicVineTalkerException:
logger.exception("Network error while getting issue details. Save aborted")
ct_md = talker_api.fetch_comic_data(issue_id)
except TalkerError as e:
logger.exception(f"Error retrieving issue details. Save aborted.\n{e}")
return GenericMetadata()
if settings.apply_cbl_transform_on_cv_import:
cv_md = CBLTransformer(cv_md, settings).apply()
ct_md = CBLTransformer(ct_md, settings).apply()
return cv_md
return ct_md
def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: GenericMetadata) -> bool:
@ -80,7 +78,11 @@ def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: Generic
def display_match_set_for_choice(
label: str, match_set: MultipleMatch, opts: argparse.Namespace, settings: ComicTaggerSettings
label: str,
match_set: MultipleMatch,
opts: argparse.Namespace,
settings: ComicTaggerSettings,
talker_api: ComicTalker,
) -> None:
print(f"{match_set.ca.path} -- {label}:")
@ -110,15 +112,15 @@ def display_match_set_for_choice(
# we know at this point, that the file is all good to go
ca = match_set.ca
md = create_local_metadata(opts, ca, settings)
cv_md = actual_issue_data_fetch(match_set.matches[int(i) - 1], settings, opts)
ct_md = actual_issue_data_fetch(match_set.matches[int(i) - 1]["issue_id"], settings, opts, talker_api)
if opts.overwrite:
md = cv_md
md = ct_md
else:
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]"
)
md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
if opts.auto_imprint:
md.fix_publisher()
@ -127,7 +129,7 @@ def display_match_set_for_choice(
def post_process_matches(
match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings
match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker
) -> None:
# now go through the match results
if opts.show_save_summary:
@ -158,7 +160,7 @@ def post_process_matches(
if len(match_results.multiple_matches) > 0:
print("\nArchives with multiple high-confidence matches:\n------------------")
for match_set in match_results.multiple_matches:
display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings)
display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings, talker_api)
if len(match_results.low_confidence_matches) > 0:
print("\nArchives with low-confidence matches:\n------------------")
@ -168,10 +170,10 @@ def post_process_matches(
else:
label = "Multiple low-confidence matches"
display_match_set_for_choice(label, match_set, opts, settings)
display_match_set_for_choice(label, match_set, opts, settings, talker_api)
def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None:
def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None:
if len(opts.file_list) < 1:
logger.error("You must specify at least one filename. Use the -h option for more info")
return
@ -179,10 +181,10 @@ def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None:
match_results = OnlineMatchResults()
for f in opts.file_list:
process_file_cli(f, opts, settings, match_results)
process_file_cli(f, opts, settings, talker_api, match_results)
sys.stdout.flush()
post_process_matches(match_results, opts, settings)
post_process_matches(match_results, opts, settings, talker_api)
def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings: ComicTaggerSettings) -> GenericMetadata:
@ -217,7 +219,11 @@ def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings:
def process_file_cli(
filename: str, opts: argparse.Namespace, settings: ComicTaggerSettings, match_results: OnlineMatchResults
filename: str,
opts: argparse.Namespace,
settings: ComicTaggerSettings,
talker_api: ComicTalker,
match_results: OnlineMatchResults,
) -> None:
batch_mode = len(opts.file_list) > 1
@ -380,31 +386,29 @@ def process_file_cli(
# now, search online
if opts.online:
if opts.issue_id is not None:
# we were given the actual ID to search with
# we were given the actual issue ID to search with
try:
comic_vine = ComicVineTalker(settings.id_series_match_search_thresh)
comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit
cv_md = comic_vine.fetch_issue_data_by_issue_id(opts.issue_id, settings)
except ComicVineTalkerException:
logger.exception("Network error while getting issue details. Save aborted")
ct_md = talker_api.fetch_comic_data(opts.issue_id)
except TalkerError as e:
logger.exception(f"Error retrieving issue details. Save aborted.\n{e}")
match_results.fetch_data_failures.append(str(ca.path.absolute()))
return
if cv_md is None:
if ct_md is None:
logger.error("No match for ID %s was found.", opts.issue_id)
match_results.no_matches.append(str(ca.path.absolute()))
return
if settings.apply_cbl_transform_on_cv_import:
cv_md = CBLTransformer(cv_md, settings).apply()
ct_md = CBLTransformer(ct_md, settings).apply()
else:
ii = IssueIdentifier(ca, settings)
if md is None or md.is_empty:
logger.error("No metadata given to search online with!")
match_results.no_matches.append(str(ca.path.absolute()))
return
ii = IssueIdentifier(ca, settings, talker_api)
def myoutput(text: str) -> None:
if opts.verbose:
IssueIdentifier.default_write_output(text)
@ -412,7 +416,6 @@ def process_file_cli(
# use our overlaid MD struct to search
ii.set_additional_metadata(md)
ii.only_use_additional_meta_data = True
ii.wait_and_retry_on_rate_limit = opts.wait_on_cv_rate_limit
ii.set_output_function(myoutput)
ii.cover_page_index = md.get_cover_page_index_list()[0]
matches = ii.search()
@ -459,19 +462,19 @@ def process_file_cli(
# we got here, so we have a single match
# now get the particular issue data
cv_md = actual_issue_data_fetch(matches[0], settings, opts)
if cv_md.is_empty:
ct_md = actual_issue_data_fetch(matches[0]["issue_id"], settings, opts, talker_api)
if ct_md.is_empty:
match_results.fetch_data_failures.append(str(ca.path.absolute()))
return
if opts.overwrite:
md = cv_md
md = ct_md
else:
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]"
)
md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
if opts.auto_imprint:
md.fix_publisher()

805
comictaggerlib/comicvinetalker.py
View File

@ -1,805 +0,0 @@
"""A python class to manage communication with Comic Vine's REST API"""
#
# Copyright 2012-2014 Anthony Beville
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
import logging
import re
import time
from typing import Any, Callable, cast
from urllib.parse import urlencode, urljoin, urlsplit
import requests
from bs4 import BeautifulSoup
from comicapi import utils
from comicapi.genericmetadata import GenericMetadata
from comicapi.issuestring import IssueString
from comictaggerlib import ctversion, resulttypes
from comictaggerlib.comiccacher import ComicCacher
from comictaggerlib.settings import ComicTaggerSettings
logger = logging.getLogger(__name__)
try:
from PyQt5 import QtCore, QtNetwork
qt_available = True
except ImportError:
qt_available = False
logger = logging.getLogger(__name__)
class CVTypeID:
Volume = "4050"
Issue = "4000"
class ComicVineTalkerException(Exception):
Unknown = -1
Network = -2
InvalidKey = 100
RateLimit = 107
def __init__(self, code: int = -1, desc: str = "") -> None:
super().__init__()
self.desc = desc
self.code = code
def __str__(self) -> str:
if self.code in (ComicVineTalkerException.Unknown, ComicVineTalkerException.Network):
return self.desc
return f"CV error #{self.code}: [{self.desc}]. \n"
def list_fetch_complete(url_list: list[str]) -> None:
...
def url_fetch_complete(image_url: str, thumb_url: str | None) -> None:
...
class ComicVineTalker:
logo_url = "http://static.comicvine.com/bundles/comicvinesite/images/logo.png"
api_key = ""
api_base_url = ""
alt_url_list_fetch_complete = list_fetch_complete
url_fetch_complete = url_fetch_complete
@staticmethod
def get_rate_limit_message() -> str:
if ComicVineTalker.api_key == "":
return "Comic Vine rate limit exceeded. You should configure your own Comic Vine API key."
return "Comic Vine rate limit exceeded. Please wait a bit."
def __init__(self, series_match_thresh: int = 90) -> None:
# Identity name for the information source
self.source_name = "comicvine"
self.wait_for_rate_limit = False
self.series_match_thresh = series_match_thresh
# key that is registered to comictagger
default_api_key = "27431e6787042105bd3e47e169a624521f89f3a4"
default_url = "https://comicvine.gamespot.com/api"
self.issue_id: int | None = None
self.api_key = ComicVineTalker.api_key or default_api_key
tmp_url = urlsplit(ComicVineTalker.api_base_url or default_url)
# joinurl only works properly if there is a trailing slash
if tmp_url.path and tmp_url.path[-1] != "/":
tmp_url = tmp_url._replace(path=tmp_url.path + "/")
self.api_base_url = tmp_url.geturl()
self.log_func: Callable[[str], None] | None = None
if qt_available:
self.nam = QtNetwork.QNetworkAccessManager()
def set_log_func(self, log_func: Callable[[str], None]) -> None:
self.log_func = log_func
def write_log(self, text: str) -> None:
if self.log_func is None:
logger.info(text)
else:
self.log_func(text)
def parse_date_str(self, date_str: str) -> tuple[int | None, int | None, int | None]:
return utils.parse_date_str(date_str)
def test_key(self, key: str, url: str) -> bool:
if not url:
url = self.api_base_url
try:
test_url = urljoin(url, "issue/1/")
cv_response: resulttypes.CVResult = requests.get(
test_url,
headers={"user-agent": "comictagger/" + ctversion.version},
params={
"api_key": key,
"format": "json",
"field_list": "name",
},
).json()
# Bogus request, but if the key is wrong, you get error 100: "Invalid API Key"
return cv_response["status_code"] != 100
except Exception:
return False
def get_cv_content(self, url: str, params: dict[str, Any]) -> resulttypes.CVResult:
"""
Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error
sleep for a bit and retry.
"""
total_time_waited = 0
limit_wait_time = 1
counter = 0
wait_times = [1, 2, 3, 4]
while True:
cv_response: resulttypes.CVResult = self.get_url_content(url, params)
if self.wait_for_rate_limit and cv_response["status_code"] == ComicVineTalkerException.RateLimit:
self.write_log(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n")
time.sleep(limit_wait_time * 60)
total_time_waited += limit_wait_time
limit_wait_time = wait_times[counter]
if counter < 3:
counter += 1
# don't wait much more than 20 minutes
if total_time_waited < 20:
continue
if cv_response["status_code"] != 1:
self.write_log(
f"Comic Vine query failed with error #{cv_response['status_code']}: [{cv_response['error']}]. \n"
)
raise ComicVineTalkerException(cv_response["status_code"], cv_response["error"])
# it's all good
break
return cv_response
def get_url_content(self, url: str, params: dict[str, Any]) -> Any:
# connect to server:
# if there is a 500 error, try a few more times before giving up
# any other error, just bail
for tries in range(3):
try:
resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + ctversion.version})
if resp.status_code == 200:
return resp.json()
if resp.status_code == 500:
self.write_log(f"Try #{tries + 1}: ")
time.sleep(1)
self.write_log(str(resp.status_code) + "\n")
else:
break
except requests.exceptions.RequestException as e:
self.write_log(f"{e}\n")
raise ComicVineTalkerException(ComicVineTalkerException.Network, "Network Error!") from e
except json.JSONDecodeError as e:
self.write_log(f"{e}\n")
raise ComicVineTalkerException(ComicVineTalkerException.Unknown, "ComicVine did not provide json")
raise ComicVineTalkerException(
ComicVineTalkerException.Unknown, f"Error on Comic Vine server: {resp.status_code}"
)
def search_for_series(
self,
series_name: str,
callback: Callable[[int, int], None] | None = None,
refresh_cache: bool = False,
literal: bool = False,
) -> list[resulttypes.CVVolumeResults]:
# Sanitize the series name for comicvine searching, comicvine search ignore symbols
search_series_name = utils.sanitize_title(series_name, literal)
logger.info("Searching: %s", search_series_name)
# Before we search online, look in our cache, since we might have done this same search recently
# For literal searches always retrieve from online
cvc = ComicCacher()
if not refresh_cache and not literal:
cached_search_results = cvc.get_search_results(self.source_name, series_name)
if len(cached_search_results) > 0:
return cached_search_results
params = {
"api_key": self.api_key,
"format": "json",
"resources": "volume",
"query": search_series_name,
"field_list": "volume,name,id,start_year,publisher,image,description,count_of_issues,aliases",
"page": 1,
"limit": 100,
}
cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
search_results: list[resulttypes.CVVolumeResults] = []
# see http://api.comicvine.com/documentation/#handling_responses
current_result_count = cv_response["number_of_page_results"]
total_result_count = cv_response["number_of_total_results"]
# 8 Dec 2018 - Comic Vine changed query results again. Terms are now
# ORed together, and we get thousands of results. Good news is the
# results are sorted by relevance, so we can be smart about halting the search.
# 1. Don't fetch more than some sane amount of pages.
# 2. Halt when any result on the current page is less than or equal to a set ratio using rapidfuzz
max_results = 500 # 5 pages
total_result_count = min(total_result_count, max_results)
if callback is None:
self.write_log(
f"Found {cv_response['number_of_page_results']} of {cv_response['number_of_total_results']} results\n"
)
search_results.extend(cast(list[resulttypes.CVVolumeResults], cv_response["results"]))
page = 1
if callback is not None:
callback(current_result_count, total_result_count)
# see if we need to keep asking for more pages...
while current_result_count < total_result_count:
if not literal:
# Stop searching once any entry falls below the threshold
stop_searching = any(
not utils.titles_match(search_series_name, volume["name"], self.series_match_thresh)
for volume in cast(list[resulttypes.CVVolumeResults], cv_response["results"])
)
if stop_searching:
break
if callback is None:
self.write_log(f"getting another page of results {current_result_count} of {total_result_count}...\n")
page += 1
params["page"] = page
cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
search_results.extend(cast(list[resulttypes.CVVolumeResults], cv_response["results"]))
current_result_count += cv_response["number_of_page_results"]
if callback is not None:
callback(current_result_count, total_result_count)
# Cache these search results, even if it's literal we cache the results
# The most it will cause is extra processing time
cvc.add_search_results(self.source_name, series_name, search_results)
return search_results
def fetch_volume_data(self, series_id: int) -> resulttypes.CVVolumeResults:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
cached_volume_result = cvc.get_volume_info(series_id, self.source_name)
if cached_volume_result is not None:
return cached_volume_result
volume_url = urljoin(self.api_base_url, f"volume/{CVTypeID.Volume}-{series_id}")
params = {
"api_key": self.api_key,
"format": "json",
"field_list": "name,id,start_year,publisher,count_of_issues,aliases",
}
cv_response = self.get_cv_content(volume_url, params)
volume_results = cast(resulttypes.CVVolumeResults, cv_response["results"])
if volume_results:
cvc.add_volume_info(self.source_name, volume_results)
return volume_results
def fetch_issues_by_volume(self, series_id: int) -> list[resulttypes.CVIssuesResults]:
# before we search online, look in our cache, since we might already have this info
volume_data = self.fetch_volume_data(series_id)
cvc = ComicCacher()
cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name)
if len(cached_volume_issues_result) >= volume_data["count_of_issues"]:
return cached_volume_issues_result
params = {
"api_key": self.api_key,
"filter": f"volume:{series_id}",
"format": "json",
"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases",
"offset": 0,
}
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
current_result_count = cv_response["number_of_page_results"]
total_result_count = cv_response["number_of_total_results"]
volume_issues_result = cast(list[resulttypes.CVIssuesResults], cv_response["results"])
page = 1
offset = 0
# see if we need to keep asking for more pages...
while current_result_count < total_result_count:
page += 1
offset += cv_response["number_of_page_results"]
params["offset"] = offset
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
volume_issues_result.extend(cast(list[resulttypes.CVIssuesResults], cv_response["results"]))
current_result_count += cv_response["number_of_page_results"]
self.repair_urls(volume_issues_result)
cvc.add_volume_issues_info(self.source_name, series_id, volume_issues_result)
return volume_issues_result
def fetch_issues_by_volume_issue_num_and_year(
self, volume_id_list: list[int], issue_number: str, year: str | int | None
) -> list[resulttypes.CVIssuesResults]:
volume_filter = ""
for vid in volume_id_list:
volume_filter += str(vid) + "|"
flt = f"volume:{volume_filter},issue_number:{issue_number}"
int_year = utils.xlate(year, True)
if int_year is not None:
flt += f",cover_date:{int_year}-1-1|{int_year+1}-1-1"
params: dict[str, str | int] = {
"api_key": self.api_key,
"format": "json",
"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases",
"filter": flt,
}
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
current_result_count = cv_response["number_of_page_results"]
total_result_count = cv_response["number_of_total_results"]
filtered_issues_result = cast(list[resulttypes.CVIssuesResults], cv_response["results"])
page = 1
offset = 0
# see if we need to keep asking for more pages...
while current_result_count < total_result_count:
page += 1
offset += cv_response["number_of_page_results"]
params["offset"] = offset
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
filtered_issues_result.extend(cast(list[resulttypes.CVIssuesResults], cv_response["results"]))
current_result_count += cv_response["number_of_page_results"]
self.repair_urls(filtered_issues_result)
cvc = ComicCacher()
for c in filtered_issues_result:
cvc.add_volume_issues_info(self.source_name, c["volume"]["id"], [c])
return filtered_issues_result
def fetch_issue_data(self, series_id: int, issue_number: str, settings: ComicTaggerSettings) -> GenericMetadata:
volume_results = self.fetch_volume_data(series_id)
issues_list_results = self.fetch_issues_by_volume(series_id)
f_record = None
for record in issues_list_results:
if not IssueString(issue_number).as_string():
issue_number = "1"
if (
IssueString(record["issue_number"]).as_string().casefold()
== IssueString(issue_number).as_string().casefold()
):
f_record = record
break
if f_record is not None:
issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{f_record['id']}")
params = {"api_key": self.api_key, "format": "json"}
cv_response = self.get_cv_content(issue_url, params)
issue_results = cast(resulttypes.CVIssueDetailResults, cv_response["results"])
else:
return GenericMetadata()
# Now, map the Comic Vine data to generic metadata
return self.map_cv_data_to_metadata(volume_results, issue_results, settings)
def fetch_issue_data_by_issue_id(self, issue_id: int, settings: ComicTaggerSettings) -> GenericMetadata:
issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
params = {"api_key": self.api_key, "format": "json"}
cv_response = self.get_cv_content(issue_url, params)
issue_results = cast(resulttypes.CVIssueDetailResults, cv_response["results"])
volume_results = self.fetch_volume_data(issue_results["volume"]["id"])
# Now, map the Comic Vine data to generic metadata
md = self.map_cv_data_to_metadata(volume_results, issue_results, settings)
md.is_empty = False
return md
def map_cv_data_to_metadata(
self,
volume_results: resulttypes.CVVolumeResults,
issue_results: resulttypes.CVIssueDetailResults,
settings: ComicTaggerSettings,
) -> GenericMetadata:
# Now, map the Comic Vine data to generic metadata
metadata = GenericMetadata()
metadata.is_empty = False
metadata.tag_origin = "Comic Vine"
metadata.issue_id = issue_results["id"]
metadata.series = utils.xlate(issue_results["volume"]["name"])
metadata.issue = IssueString(issue_results["issue_number"]).as_string()
metadata.title = utils.xlate(issue_results["name"])
if volume_results["publisher"] is not None:
metadata.publisher = utils.xlate(volume_results["publisher"]["name"])
metadata.day, metadata.month, metadata.year = self.parse_date_str(issue_results["cover_date"])
metadata.comments = self.cleanup_html(issue_results["description"], settings.remove_html_tables)
if settings.use_series_start_as_volume:
metadata.volume = int(volume_results["start_year"])
metadata.web_link = issue_results["site_detail_url"]
person_credits = issue_results["person_credits"]
for person in person_credits:
if "role" in person:
roles = person["role"].split(",")
for role in roles:
# can we determine 'primary' from CV??
metadata.add_credit(person["name"], role.title().strip(), False)
character_credits = issue_results["character_credits"]
character_list = []
for character in character_credits:
character_list.append(character["name"])
metadata.characters = ", ".join(character_list)
team_credits = issue_results["team_credits"]
team_list = []
for team in team_credits:
team_list.append(team["name"])
metadata.teams = ", ".join(team_list)
location_credits = issue_results["location_credits"]
location_list = []
for location in location_credits:
location_list.append(location["name"])
metadata.locations = ", ".join(location_list)
story_arc_credits = issue_results["story_arc_credits"]
arc_list = []
for arc in story_arc_credits:
arc_list.append(arc["name"])
if len(arc_list) > 0:
metadata.story_arc = ", ".join(arc_list)
return metadata
def cleanup_html(self, string: str, remove_html_tables: bool) -> str:
if string is None:
return ""
# find any tables
soup = BeautifulSoup(string, "html.parser")
tables = soup.findAll("table")
# remove all newlines first
string = string.replace("\n", "")
# put in our own
string = string.replace("<br>", "\n")
string = string.replace("</li>", "\n")
string = string.replace("</p>", "\n\n")
string = string.replace("<h1>", "*")
string = string.replace("</h1>", "*\n")
string = string.replace("<h2>", "*")
string = string.replace("</h2>", "*\n")
string = string.replace("<h3>", "*")
string = string.replace("</h3>", "*\n")
string = string.replace("<h4>", "*")
string = string.replace("</h4>", "*\n")
string = string.replace("<h5>", "*")
string = string.replace("</h5>", "*\n")
string = string.replace("<h6>", "*")
string = string.replace("</h6>", "*\n")
# remove the tables
p = re.compile(r"<table[^<]*?>.*?</table>")
if remove_html_tables:
string = p.sub("", string)
string = string.replace("*List of covers and their creators:*", "")
else:
string = p.sub("{}", string)
# now strip all other tags
p = re.compile(r"<[^<]*?>")
newstring = p.sub("", string)
newstring = newstring.replace("&nbsp;", " ")
newstring = newstring.replace("&amp;", "&")
newstring = newstring.strip()
if not remove_html_tables:
# now rebuild the tables into text from BSoup
try:
table_strings = []
for table in tables:
rows = []
hdrs = []
col_widths = []
for hdr in table.findAll("th"):
item = hdr.string.strip()
hdrs.append(item)
col_widths.append(len(item))
rows.append(hdrs)
for row in table.findAll("tr"):
cols = []
col = row.findAll("td")
i = 0
for c in col:
item = c.string.strip()
cols.append(item)
if len(item) > col_widths[i]:
col_widths[i] = len(item)
i += 1
if len(cols) != 0:
rows.append(cols)
# now we have the data, make it into text
fmtstr = ""
for w in col_widths:
fmtstr += f" {{:{w + 1}}}|"
width = sum(col_widths) + len(col_widths) * 2
table_text = ""
counter = 0
for row in rows:
table_text += fmtstr.format(*row) + "\n"
if counter == 0 and len(hdrs) != 0:
table_text += "-" * width + "\n"
counter += 1
table_strings.append(table_text)
newstring = newstring.format(*table_strings)
except Exception:
# we caught an error rebuilding the table.
# just bail and remove the formatting
logger.exception("table parse error")
newstring.replace("{}", "")
return newstring
def fetch_issue_date(self, issue_id: int) -> tuple[int | None, int | None]:
details = self.fetch_issue_select_details(issue_id)
_, month, year = self.parse_date_str(details["cover_date"] or "")
return month, year
def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]:
details = self.fetch_issue_select_details(issue_id)
return details["image_url"], details["thumb_image_url"]
def fetch_issue_page_url(self, issue_id: int) -> str | None:
details = self.fetch_issue_select_details(issue_id)
return details["site_detail_url"]
def fetch_issue_select_details(self, issue_id: int) -> resulttypes.SelectDetails:
cached_details = self.fetch_cached_issue_select_details(issue_id)
if cached_details["image_url"] is not None:
return cached_details
issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
logger.error("%s, %s", self.api_base_url, issue_url)
params = {"api_key": self.api_key, "format": "json", "field_list": "image,cover_date,site_detail_url"}
cv_response = self.get_cv_content(issue_url, params)
results = cast(resulttypes.CVIssueDetailResults, cv_response["results"])
details = resulttypes.SelectDetails(
image_url=results["image"]["super_url"],
thumb_image_url=results["image"]["thumb_url"],
cover_date=results["cover_date"],
site_detail_url=results["site_detail_url"],
)
if (
details["image_url"] is not None
and details["thumb_image_url"] is not None
and details["cover_date"] is not None
and details["site_detail_url"] is not None
):
self.cache_issue_select_details(
issue_id,
details["image_url"],
details["thumb_image_url"],
details["cover_date"],
details["site_detail_url"],
)
return details
def fetch_cached_issue_select_details(self, issue_id: int) -> resulttypes.SelectDetails:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
return cvc.get_issue_select_details(issue_id, self.source_name)
def cache_issue_select_details(
self, issue_id: int, image_url: str, thumb_url: str, cover_date: str, page_url: str
) -> None:
cvc = ComicCacher()
cvc.add_issue_select_details(self.source_name, issue_id, image_url, thumb_url, cover_date, page_url)
def fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> list[str]:
url_list = self.fetch_cached_alternate_cover_urls(issue_id)
if url_list:
return url_list
# scrape the CV issue page URL to get the alternate cover URLs
content = requests.get(issue_page_url, headers={"user-agent": "comictagger/" + ctversion.version}).text
alt_cover_url_list = self.parse_out_alt_cover_urls(content)
# cache this alt cover URL list
self.cache_alternate_cover_urls(issue_id, alt_cover_url_list)
return alt_cover_url_list
def parse_out_alt_cover_urls(self, page_html: str) -> list[str]:
soup = BeautifulSoup(page_html, "html.parser")
alt_cover_url_list = []
# Using knowledge of the layout of the Comic Vine issue page here:
# look for the divs that are in the classes 'imgboxart' and 'issue-cover'
div_list = soup.find_all("div")
covers_found = 0
for d in div_list:
if "class" in d.attrs:
c = d["class"]
if "imgboxart" in c and "issue-cover" in c:
if d.img["src"].startswith("http"):
covers_found += 1
if covers_found != 1:
alt_cover_url_list.append(d.img["src"])
elif d.img["data-src"].startswith("http"):
covers_found += 1
if covers_found != 1:
alt_cover_url_list.append(d.img["data-src"])
return alt_cover_url_list
def fetch_cached_alternate_cover_urls(self, issue_id: int) -> list[str]:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
url_list = cvc.get_alt_covers(self.source_name, issue_id)
return url_list
def cache_alternate_cover_urls(self, issue_id: int, url_list: list[str]) -> None:
cvc = ComicCacher()
cvc.add_alt_covers(self.source_name, issue_id, url_list)
def async_fetch_issue_cover_urls(self, issue_id: int) -> None:
self.issue_id = issue_id
details = self.fetch_cached_issue_select_details(issue_id)
if details["image_url"] is not None:
ComicVineTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"])
return
issue_url = urlsplit(self.api_base_url)
issue_url = issue_url._replace(
query=urlencode(
{
"api_key": self.api_key,
"format": "json",
"field_list": "image,cover_date,site_detail_url",
}
),
path=f"issue/{CVTypeID.Issue}-{issue_id}",
)
self.nam.finished.connect(self.async_fetch_issue_cover_url_complete)
self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(issue_url.geturl())))
def async_fetch_issue_cover_url_complete(self, reply: QtNetwork.QNetworkReply) -> None:
# read in the response
data = reply.readAll()
try:
cv_response = cast(resulttypes.CVResult, json.loads(bytes(data)))
except Exception:
logger.exception("Comic Vine query failed to get JSON data\n%s", str(data))
return
if cv_response["status_code"] != 1:
logger.error("Comic Vine query failed with error: [%s]. ", cv_response["error"])
return
result = cast(resulttypes.CVIssuesResults, cv_response["results"])
image_url = result["image"]["super_url"]
thumb_url = result["image"]["thumb_url"]
cover_date = result["cover_date"]
page_url = result["site_detail_url"]
self.cache_issue_select_details(cast(int, self.issue_id), image_url, thumb_url, cover_date, page_url)
ComicVineTalker.url_fetch_complete(image_url, thumb_url)
def async_fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> None:
# This async version requires the issue page url to be provided!
self.issue_id = issue_id
url_list = self.fetch_cached_alternate_cover_urls(issue_id)
if url_list:
ComicVineTalker.alt_url_list_fetch_complete(url_list)
return
self.nam.finished.connect(self.async_fetch_alternate_cover_urls_complete)
self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_page_url))))
def async_fetch_alternate_cover_urls_complete(self, reply: QtNetwork.QNetworkReply) -> None:
# read in the response
html = str(reply.readAll())
alt_cover_url_list = self.parse_out_alt_cover_urls(html)
# cache this alt cover URL list
self.cache_alternate_cover_urls(cast(int, self.issue_id), alt_cover_url_list)
ComicVineTalker.alt_url_list_fetch_complete(alt_cover_url_list)
def repair_urls(
self,
issue_list: list[resulttypes.CVIssuesResults]
| list[resulttypes.CVVolumeResults]
| list[resulttypes.CVIssueDetailResults],
) -> None:
# make sure there are URLs for the image fields
for issue in issue_list:
if issue["image"] is None:
issue["image"] = resulttypes.CVImage(
super_url=ComicVineTalker.logo_url,
thumb_url=ComicVineTalker.logo_url,
)

76
comictaggerlib/coverimagewidget.py
View File

@ -20,19 +20,17 @@ TODO: This should be re-factored using subclasses!
from __future__ import annotations
import logging
from typing import Callable, cast
from PyQt5 import QtCore, QtGui, QtWidgets, uic
from comicapi import utils
from comicapi.comicarchive import ComicArchive
from comictaggerlib.comicvinetalker import ComicVineTalker
from comictaggerlib.graphics import graphics_path
from comictaggerlib.imagefetcher import ImageFetcher
from comictaggerlib.imagepopup import ImagePopup
from comictaggerlib.pageloader import PageLoader
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import get_qimage_from_data, reduce_widget_font_size
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@ -56,39 +54,17 @@ def clickable(widget: QtWidgets.QWidget) -> QtCore.pyqtBoundSignal:
return flt.dblclicked
class Signal(QtCore.QObject):
alt_url_list_fetch_complete = QtCore.pyqtSignal(list)
url_fetch_complete = QtCore.pyqtSignal(str, str)
image_fetch_complete = QtCore.pyqtSignal(QtCore.QByteArray)
def __init__(
self,
list_fetch: Callable[[list[str]], None],
url_fetch: Callable[[str, str], None],
image_fetch: Callable[[bytes], None],
) -> None:
super().__init__()
self.alt_url_list_fetch_complete.connect(list_fetch)
self.url_fetch_complete.connect(url_fetch)
self.image_fetch_complete.connect(image_fetch)
def emit_list(self, url_list: list[str]) -> None:
self.alt_url_list_fetch_complete.emit(url_list)
def emit_url(self, image_url: str, thumb_url: str | None) -> None:
self.url_fetch_complete.emit(image_url, thumb_url)
def emit_image(self, image_data: bytes | QtCore.QByteArray) -> None:
self.image_fetch_complete.emit(image_data)
class CoverImageWidget(QtWidgets.QWidget):
ArchiveMode = 0
AltCoverMode = 1
URLMode = 1
DataMode = 3
def __init__(self, parent: QtWidgets.QWidget, mode: int, expand_on_click: bool = True) -> None:
image_fetch_complete = QtCore.pyqtSignal(QtCore.QByteArray)
def __init__(
self, parent: QtWidgets.QWidget, talker_api: ComicTalker, mode: int, expand_on_click: bool = True
) -> None:
super().__init__(parent)
self.cover_fetcher = ImageFetcher()
@ -96,10 +72,7 @@ class CoverImageWidget(QtWidgets.QWidget):
reduce_widget_font_size(self.label)
self.sig = Signal(
self.alt_cover_url_list_fetch_complete, self.primary_url_fetch_complete, self.cover_remote_fetch_complete
)
self.talker_api = talker_api
self.mode: int = mode
self.page_loader: PageLoader | None = None
self.showControls = True
@ -108,6 +81,7 @@ class CoverImageWidget(QtWidgets.QWidget):
self.comic_archive: ComicArchive | None = None
self.issue_id: int | None = None
self.issue_url: str | None = None
self.url_list: list[str] = []
if self.page_loader is not None:
self.page_loader.abandoned = True
@ -121,6 +95,7 @@ class CoverImageWidget(QtWidgets.QWidget):
self.btnLeft.clicked.connect(self.decrement_image)
self.btnRight.clicked.connect(self.increment_image)
self.image_fetch_complete.connect(self.cover_remote_fetch_complete)
if expand_on_click:
clickable(self.lblImage).connect(self.show_popup)
else:
@ -131,6 +106,7 @@ class CoverImageWidget(QtWidgets.QWidget):
def reset_widget(self) -> None:
self.comic_archive = None
self.issue_id = None
self.issue_url = None
self.url_list = []
if self.page_loader is not None:
self.page_loader.abandoned = True
@ -173,15 +149,13 @@ class CoverImageWidget(QtWidgets.QWidget):
self.imageCount = 1
self.update_content()
def set_issue_id(self, issue_id: int) -> None:
def set_issue_details(self, issue_id: int, url_list: list[str]) -> None:
if self.mode == CoverImageWidget.AltCoverMode:
self.reset_widget()
self.update_content()
self.issue_id = issue_id
comic_vine = ComicVineTalker()
ComicVineTalker.url_fetch_complete = self.sig.emit_url
comic_vine.async_fetch_issue_cover_urls(self.issue_id)
self.set_url_list(url_list)
def set_image_data(self, image_data: bytes) -> None:
if self.mode == CoverImageWidget.DataMode:
@ -195,31 +169,11 @@ class CoverImageWidget(QtWidgets.QWidget):
self.update_content()
def primary_url_fetch_complete(self, primary_url: str, thumb_url: str | None = None) -> None:
self.url_list.append(str(primary_url))
def set_url_list(self, url_list: list[str]) -> None:
self.url_list = url_list
self.imageIndex = 0
self.imageCount = len(self.url_list)
self.update_content()
# defer the alt cover search
QtCore.QTimer.singleShot(1, self.start_alt_cover_search)
def start_alt_cover_search(self) -> None:
if self.issue_id is not None:
# now we need to get the list of alt cover URLs
self.label.setText("Searching for alt. covers...")
# page URL should already be cached, so no need to defer
comic_vine = ComicVineTalker()
issue_page_url = comic_vine.fetch_issue_page_url(self.issue_id)
ComicVineTalker.alt_url_list_fetch_complete = self.sig.emit_list
comic_vine.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), cast(str, issue_page_url))
def alt_cover_url_list_fetch_complete(self, url_list: list[str]) -> None:
if url_list:
self.url_list.extend(url_list)
self.imageCount = len(self.url_list)
self.update_controls()
def set_page(self, pagenum: int) -> None:
@ -269,7 +223,7 @@ class CoverImageWidget(QtWidgets.QWidget):
def load_url(self) -> None:
self.load_default()
self.cover_fetcher = ImageFetcher()
ImageFetcher.image_fetch_complete = self.sig.emit_image
ImageFetcher.image_fetch_complete = self.image_fetch_complete.emit
self.cover_fetcher.fetch(self.url_list[self.imageIndex])
# called when the image is done loading from internet

3
comictaggerlib/imagefetcher.py
View File

@ -81,7 +81,8 @@ class ImageFetcher:
# first look in the DB
image_data = self.get_image_from_cache(url)
if blocking or not qt_available:
# Async for retrieving covers seems to work well
if blocking: # if blocking or not qt_available:
if not image_data:
try:
image_data = requests.get(url, headers={"user-agent": "comictagger/" + ctversion.version}).content

62
comictaggerlib/issueidentifier.py
View File

@ -26,11 +26,12 @@ from comicapi import utils
from comicapi.comicarchive import ComicArchive
from comicapi.genericmetadata import GenericMetadata
from comicapi.issuestring import IssueString
from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.imagefetcher import ImageFetcher, ImageFetcherException
from comictaggerlib.imagehasher import ImageHasher
from comictaggerlib.resulttypes import IssueResult
from comictaggerlib.settings import ComicTaggerSettings
from comictalker.talker_utils import parse_date_str
from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@ -72,8 +73,9 @@ class IssueIdentifier:
result_one_good_match = 4
result_multiple_good_matches = 5
def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings) -> None:
def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None:
self.settings = settings
self.talker_api = talker_api
self.comic_archive: ComicArchive = comic_archive
self.image_hasher = 1
@ -107,7 +109,6 @@ class IssueIdentifier:
self.search_result = self.result_no_matches
self.cover_page_index = 0
self.cancel = False
self.wait_and_retry_on_rate_limit = False
self.match_list: list[IssueResult] = []
@ -239,11 +240,10 @@ class IssueIdentifier:
def get_issue_cover_match_score(
self,
comic_vine: ComicVineTalker,
issue_id: int,
primary_img_url: str,
primary_thumb_url: str,
page_url: str,
alt_urls: list[str],
local_cover_hash_list: list[int],
use_remote_alternates: bool = False,
use_log: bool = True,
@ -274,8 +274,7 @@ class IssueIdentifier:
raise IssueIdentifierCancelled
if use_remote_alternates:
alt_img_url_list = comic_vine.fetch_alternate_cover_urls(issue_id, page_url)
for alt_url in alt_img_url_list:
for alt_url in alt_urls:
try:
alt_url_image_data = ImageFetcher().fetch(alt_url, blocking=True)
except ImageFetcherException as e:
@ -369,27 +368,22 @@ class IssueIdentifier:
if keys["month"] is not None:
self.log_msg("\tMonth: " + str(keys["month"]))
comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
comic_vine.wait_for_rate_limit = self.wait_and_retry_on_rate_limit
comic_vine.set_log_func(self.output_function)
self.log_msg(f"Searching for {keys['series']} #{keys['issue_number']} ...")
try:
cv_search_results = comic_vine.search_for_series(keys["series"])
except ComicVineTalkerException:
self.log_msg("Network issue while searching for series. Aborting...")
ct_search_results = self.talker_api.search_for_series(keys["series"])
except TalkerError as e:
self.log_msg(f"Error searching for series.\n{e}")
return []
if self.cancel:
return []
if cv_search_results is None:
if ct_search_results is None:
return []
series_second_round_list = []
for item in cv_search_results:
for item in ct_search_results:
length_approved = False
publisher_approved = True
date_approved = True
@ -404,16 +398,13 @@ class IssueIdentifier:
if int(keys["year"]) < int(item["start_year"]):
date_approved = False
aliases = []
if item["aliases"]:
aliases = item["aliases"].split("\n")
for name in [item["name"], *aliases]:
for name in [item["name"], *item["aliases"]]:
if utils.titles_match(keys["series"], name, self.series_match_thresh):
length_approved = True
break
# remove any series from publishers on the filter
if item["publisher"] is not None:
publisher = item["publisher"]["name"]
publisher = item["publisher"]
if publisher is not None and publisher.casefold() in self.publisher_filter:
publisher_approved = False
@ -436,12 +427,11 @@ class IssueIdentifier:
issue_list = None
try:
if len(volume_id_list) > 0:
issue_list = comic_vine.fetch_issues_by_volume_issue_num_and_year(
issue_list = self.talker_api.fetch_issues_by_volume_issue_num_and_year(
volume_id_list, keys["issue_number"], keys["year"]
)
except ComicVineTalkerException:
self.log_msg("Network issue while searching for series details. Aborting...")
except TalkerError as e:
self.log_msg(f"Issue with while searching for series details. Aborting...\n{e}")
return []
if issue_list is None:
@ -476,7 +466,7 @@ class IssueIdentifier:
)
# parse out the cover date
_, month, year = comic_vine.parse_date_str(issue["cover_date"])
_, month, year = parse_date_str(issue["cover_date"])
# Now check the cover match against the primary image
hash_list = [cover_hash]
@ -484,16 +474,15 @@ class IssueIdentifier:
hash_list.append(narrow_cover_hash)
try:
image_url = issue["image"]["super_url"]
thumb_url = issue["image"]["thumb_url"]
page_url = issue["site_detail_url"]
image_url = issue["image_url"]
thumb_url = issue["image_thumb_url"]
alt_urls = issue["alt_image_urls"]
score_item = self.get_issue_cover_match_score(
comic_vine,
issue["id"],
image_url,
thumb_url,
page_url,
alt_urls,
hash_list,
use_remote_alternates=False,
)
@ -515,11 +504,12 @@ class IssueIdentifier:
"publisher": None,
"image_url": image_url,
"thumb_url": thumb_url,
"page_url": page_url,
# "page_url": page_url,
"alt_image_urls": alt_urls,
"description": issue["description"],
}
if series["publisher"] is not None:
match["publisher"] = series["publisher"]["name"]
match["publisher"] = series["publisher"]
self.match_list.append(match)
@ -577,15 +567,15 @@ class IssueIdentifier:
self.log_msg(f"Examining alternate covers for ID: {m['volume_id']} {m['series']} ...", newline=False)
try:
score_item = self.get_issue_cover_match_score(
comic_vine,
m["issue_id"],
m["image_url"],
m["thumb_url"],
m["page_url"],
m["alt_image_urls"],
hash_list,
use_remote_alternates=True,
)
except Exception:
logger.exception("failed examining alt covers")
self.match_list = []
return self.match_list
self.log_msg(f"--->{score_item['score']}")

33
comictaggerlib/issueselectionwindow.py
View File

@ -20,12 +20,12 @@ import logging
from PyQt5 import QtCore, QtGui, QtWidgets, uic
from comicapi.issuestring import IssueString
from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.resulttypes import CVIssuesResults
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
from comictalker.resulttypes import ComicIssue
from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@ -42,13 +42,18 @@ class IssueSelectionWindow(QtWidgets.QDialog):
volume_id = 0
def __init__(
self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, series_id: int, issue_number: str
self,
parent: QtWidgets.QWidget,
settings: ComicTaggerSettings,
talker_api: ComicTalker,
series_id: int,
issue_number: str,
) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "issueselectionwindow.ui", self)
self.coverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.AltCoverMode)
self.coverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.AltCoverMode)
gridlayout = QtWidgets.QGridLayout(self.coverImageContainer)
gridlayout.addWidget(self.coverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@ -67,8 +72,9 @@ class IssueSelectionWindow(QtWidgets.QDialog):
self.series_id = series_id
self.issue_id: int | None = None
self.settings = settings
self.talker_api = talker_api
self.url_fetch_thread = None
self.issue_list: list[CVIssuesResults] = []
self.issue_list: list[ComicIssue] = []
if issue_number is None or issue_number == "":
self.issue_number = "1"
@ -98,15 +104,14 @@ class IssueSelectionWindow(QtWidgets.QDialog):
QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
try:
comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
comic_vine.fetch_volume_data(self.series_id)
self.issue_list = comic_vine.fetch_issues_by_volume(self.series_id)
except ComicVineTalkerException as e:
self.issue_list = self.talker_api.fetch_issues_by_volume(self.series_id)
except TalkerError as e:
QtWidgets.QApplication.restoreOverrideCursor()
if e.code == ComicVineTalkerException.RateLimit:
QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message())
else:
QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not connect to Comic Vine to list issues!")
QtWidgets.QMessageBox.critical(
self,
f"{e.source} {e.code_name} Error",
f"{e}",
)
return
self.twList.setRowCount(0)
@ -175,7 +180,7 @@ class IssueSelectionWindow(QtWidgets.QDialog):
for record in self.issue_list:
if record["id"] == self.issue_id:
self.issue_number = record["issue_number"]
self.coverWidget.set_issue_id(self.issue_id)
self.coverWidget.set_issue_details(self.issue_id, [record["image_url"], *record["alt_image_urls"]])
if record["description"] is None:
self.teDescription.setText("")
else:

50
comictaggerlib/main.py
View File

@ -25,13 +25,14 @@ import sys
import traceback
import types
import comictalker.comictalkerapi as ct_api
from comicapi import utils
from comictaggerlib import cli
from comictaggerlib.comicvinetalker import ComicVineTalker
from comictaggerlib.ctversion import version
from comictaggerlib.graphics import graphics_path
from comictaggerlib.options import parse_cmd_line
from comictaggerlib.settings import ComicTaggerSettings
from comictalker.talkerbase import TalkerError
if sys.version_info < (3, 10):
import importlib_metadata
@ -41,6 +42,7 @@ else:
logger = logging.getLogger("comictagger")
logging.getLogger("comicapi").setLevel(logging.DEBUG)
logging.getLogger("comictaggerlib").setLevel(logging.DEBUG)
logging.getLogger("comictalker").setLevel(logging.DEBUG)
logger.setLevel(logging.DEBUG)
try:
@ -157,19 +159,6 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
file=sys.stderr,
)
# manage the CV API key
# None comparison is used so that the empty string can unset the value
if opts.cv_api_key is not None or opts.cv_url is not None:
settings.cv_api_key = opts.cv_api_key if opts.cv_api_key is not None else settings.cv_api_key
settings.cv_url = opts.cv_url if opts.cv_url is not None else settings.cv_url
settings.save()
if opts.only_set_cv_key:
print("Key set") # noqa: T201
return
ComicVineTalker.api_key = settings.cv_api_key
ComicVineTalker.api_base_url = settings.cv_url
signal.signal(signal.SIGINT, signal.SIG_DFL)
logger.info(
@ -183,6 +172,28 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
for pkg in sorted(importlib_metadata.distributions(), key=lambda x: x.name):
logger.debug("%s\t%s", pkg.metadata["Name"], pkg.metadata["Version"])
talker_failed = False
try:
talker_api = ct_api.get_comic_talker("comicvine")(
settings.cv_url,
settings.cv_api_key,
settings.id_series_match_search_thresh,
settings.remove_html_tables,
settings.use_series_start_as_volume,
settings.wait_and_retry_on_rate_limit,
)
except TalkerError as te:
talker_failed = True
logger.warning(f"Unable to load talker {te.source}. Error: {te.desc}. Defaulting to Comic Vine.")
talker_api = ct_api.get_comic_talker("comicvine")(
settings.cv_url,
settings.cv_api_key,
settings.id_series_match_search_thresh,
settings.remove_html_tables,
settings.use_series_start_as_volume,
settings.wait_and_retry_on_rate_limit,
)
utils.load_publishers()
update_publishers()
@ -192,7 +203,7 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
if opts.no_gui:
try:
cli.cli_mode(opts, settings)
cli.cli_mode(opts, settings, talker_api)
except Exception:
logger.exception("CLI mode failed")
else:
@ -232,7 +243,7 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
QtWidgets.QApplication.processEvents()
try:
tagger_window = TaggerWindow(opts.files, settings, opts=opts)
tagger_window = TaggerWindow(opts.files, settings, talker_api, opts=opts)
tagger_window.setWindowIcon(QtGui.QIcon(str(graphics_path / "app.png")))
tagger_window.show()
@ -242,6 +253,13 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
if platform.system() != "Linux":
splash.finish(tagger_window)
if talker_failed:
QtWidgets.QMessageBox.warning(
QtWidgets.QMainWindow(),
"Warning",
"Unable to load configured information source, see log for details. Defaulting to Comic Vine",
)
sys.exit(app.exec())
except Exception:
logger.exception("GUI mode failed")

18
comictaggerlib/matchselectionwindow.py
View File

@ -25,6 +25,7 @@ from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.resulttypes import IssueResult
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@ -32,17 +33,23 @@ logger = logging.getLogger(__name__)
class MatchSelectionWindow(QtWidgets.QDialog):
volume_id = 0
def __init__(self, parent: QtWidgets.QWidget, matches: list[IssueResult], comic_archive: ComicArchive) -> None:
def __init__(
self,
parent: QtWidgets.QWidget,
matches: list[IssueResult],
comic_archive: ComicArchive,
talker_api: ComicTalker,
) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "matchselectionwindow.ui", self)
self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode)
self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode)
gridlayout = QtWidgets.QGridLayout(self.altCoverContainer)
gridlayout.addWidget(self.altCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode)
self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer)
gridlayout.addWidget(self.archiveCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@ -142,7 +149,10 @@ class MatchSelectionWindow(QtWidgets.QDialog):
if prev is not None and prev.row() == curr.row():
return
self.altCoverWidget.set_issue_id(self.current_match()["issue_id"])
self.altCoverWidget.set_issue_details(
self.current_match()["issue_id"],
[self.current_match()["image_url"], *self.current_match()["alt_image_urls"]],
)
if self.current_match()["description"] is None:
self.teDescription.setText("")
else:

8
comictaggerlib/options.py
View File

@ -80,6 +80,7 @@ def define_args() -> argparse.ArgumentParser:
action="store_true",
help="Export RAR archive to Zip format.",
)
# TODO: update for new api
commands.add_argument(
"--only-set-cv-key",
action="store_true",
@ -107,10 +108,12 @@ def define_args() -> argparse.ArgumentParser:
dest="config_path",
help="""Config directory defaults to ~/.ComicTagger\non Linux/Mac and %%APPDATA%% on Windows\n""",
)
# TODO: update for new api
parser.add_argument(
"--cv-api-key",
help="Use the given Comic Vine API Key (persisted in settings).",
)
# TODO: update for new api
parser.add_argument(
"--cv-url",
help="Use the given Comic Vine URL (persisted in settings).",
@ -214,6 +217,7 @@ def define_args() -> argparse.ArgumentParser:
action="store_true",
help="Be noisy when doing what it does.",
)
# TODO: update for new api
parser.add_argument(
"-w",
"--wait-on-cv-rate-limit",
@ -369,7 +373,7 @@ def parse_cmd_line() -> argparse.Namespace:
opts.copy,
opts.rename,
opts.export_to_zip,
opts.only_set_cv_key,
opts.only_set_cv_key, # TODO: update for new api
]
)
@ -385,9 +389,11 @@ def parse_cmd_line() -> argparse.Namespace:
for item in globs:
opts.files.extend(glob.glob(item))
# TODO: update for new api
if opts.only_set_cv_key and opts.cv_api_key is None and opts.cv_url is None:
parser.exit(message="Key not given!\n", status=1)
# TODO: update for new api
if not opts.only_set_cv_key and opts.no_gui and not opts.files:
parser.exit(message="Command requires at least one filename!\n", status=1)

5
comictaggerlib/pagebrowser.py
View File

@ -25,17 +25,18 @@ from comicapi.genericmetadata import GenericMetadata
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.graphics import graphics_path
from comictaggerlib.ui import ui_path
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
class PageBrowserWindow(QtWidgets.QDialog):
def __init__(self, parent: QtWidgets.QWidget, metadata: GenericMetadata) -> None:
def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker, metadata: GenericMetadata) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "pagebrowser.ui", self)
self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode)
self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.pageContainer)
gridlayout.addWidget(self.pageWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)

5
comictaggerlib/pagelisteditor.py
View File

@ -23,6 +23,7 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle
from comicapi.genericmetadata import ImageMetadata, PageType
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.ui import ui_path
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@ -67,12 +68,12 @@ class PageListEditor(QtWidgets.QWidget):
PageType.Deleted: "Deleted",
}
def __init__(self, parent: QtWidgets.QWidget) -> None:
def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "pagelisteditor.ui", self)
self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode)
self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.pageContainer)
gridlayout.addWidget(self.pageWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)

5
comictaggerlib/renamewindow.py
View File

@ -27,6 +27,7 @@ from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.settingswindow import SettingsWindow
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import center_window_on_parent
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@ -38,6 +39,7 @@ class RenameWindow(QtWidgets.QDialog):
comic_archive_list: list[ComicArchive],
data_style: int,
settings: ComicTaggerSettings,
talker_api: ComicTalker,
) -> None:
super().__init__(parent)
@ -53,6 +55,7 @@ class RenameWindow(QtWidgets.QDialog):
)
self.settings = settings
self.talker_api = talker_api
self.comic_archive_list = comic_archive_list
self.data_style = data_style
self.rename_list: list[str] = []
@ -161,7 +164,7 @@ class RenameWindow(QtWidgets.QDialog):
self.twList.setSortingEnabled(True)
def modify_settings(self) -> None:
settingswin = SettingsWindow(self, self.settings)
settingswin = SettingsWindow(self, self.settings, self.talker_api)
settingswin.setModal(True)
settingswin.show_rename_tab()
settingswin.exec()

127
comictaggerlib/resulttypes.py
View File

@ -1,6 +1,6 @@
from __future__ import annotations
from typing_extensions import NotRequired, Required, TypedDict
from typing_extensions import TypedDict
from comicapi.comicarchive import ComicArchive
@ -19,7 +19,7 @@ class IssueResult(TypedDict):
publisher: str | None
image_url: str
thumb_url: str
page_url: str
alt_image_urls: list[str]
description: str
@ -37,126 +37,3 @@ class MultipleMatch:
def __init__(self, ca: ComicArchive, match_list: list[IssueResult]) -> None:
self.ca: ComicArchive = ca
self.matches: list[IssueResult] = match_list
class SelectDetails(TypedDict):
image_url: str | None
thumb_image_url: str | None
cover_date: str | None
site_detail_url: str | None
class CVResult(TypedDict):
error: str
limit: int
offset: int
number_of_page_results: int
number_of_total_results: int
status_code: int
results: (
CVIssuesResults
| CVIssueDetailResults
| CVVolumeResults
| list[CVIssuesResults]
| list[CVVolumeResults]
| list[CVIssueDetailResults]
)
version: str
class CVImage(TypedDict, total=False):
icon_url: str
medium_url: str
screen_url: str
screen_large_url: str
small_url: str
super_url: Required[str]
thumb_url: str
tiny_url: str
original_url: str
image_tags: str
class CVVolume(TypedDict):
api_detail_url: str
id: int
name: str
site_detail_url: str
class CVIssuesResults(TypedDict):
cover_date: str
description: str
id: int
image: CVImage
issue_number: str
name: str
site_detail_url: str
volume: NotRequired[CVVolume]
aliases: str
class CVPublisher(TypedDict, total=False):
api_detail_url: str
id: int
name: Required[str]
class CVVolumeResults(TypedDict):
count_of_issues: int
description: NotRequired[str]
id: int
image: NotRequired[CVImage]
name: str
publisher: CVPublisher
start_year: str
resource_type: NotRequired[str]
aliases: NotRequired[str | None]
class CVCredits(TypedDict):
api_detail_url: str
id: int
name: str
site_detail_url: str
class CVPersonCredits(TypedDict):
api_detail_url: str
id: int
name: str
site_detail_url: str
role: str
class CVIssueDetailResults(TypedDict):
aliases: None
api_detail_url: str
character_credits: list[CVCredits]
character_died_in: None
concept_credits: list[CVCredits]
cover_date: str
date_added: str
date_last_updated: str
deck: None
description: str
first_appearance_characters: None
first_appearance_concepts: None
first_appearance_locations: None
first_appearance_objects: None
first_appearance_storyarcs: None
first_appearance_teams: None
has_staff_review: bool
id: int
image: CVImage
issue_number: str
location_credits: list[CVCredits]
name: str
object_credits: list[CVCredits]
person_credits: list[CVPersonCredits]
site_detail_url: str
store_date: str
story_arc_credits: list[CVCredits]
team_credits: list[CVCredits]
team_disbanded_in: None
volume: CVVolume

20
comictaggerlib/settingswindow.py
View File

@ -25,12 +25,12 @@ from PyQt5 import QtCore, QtGui, QtWidgets, uic
from comicapi import utils
from comicapi.genericmetadata import md_test
from comictaggerlib.comiccacher import ComicCacher
from comictaggerlib.comicvinetalker import ComicVineTalker
from comictaggerlib.filerenamer import FileRenamer
from comictaggerlib.imagefetcher import ImageFetcher
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictalker.comiccacher import ComicCacher
from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@ -128,7 +128,7 @@ Spider-Geddon #1 - New Players; Check In
class SettingsWindow(QtWidgets.QDialog):
def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings) -> None:
def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "settingswindow.ui", self)
@ -138,6 +138,7 @@ class SettingsWindow(QtWidgets.QDialog):
)
self.settings = settings
self.talker_api = talker_api
self.name = "Settings"
if platform.system() == "Windows":
@ -326,10 +327,13 @@ class SettingsWindow(QtWidgets.QDialog):
self.settings.sort_series_by_year = self.cbxSortByYear.isChecked()
self.settings.exact_series_matches_first = self.cbxExactMatches.isChecked()
self.settings.cv_api_key = self.leKey.text().strip()
ComicVineTalker.api_key = self.settings.cv_api_key
self.settings.cv_url = self.leURL.text().strip()
ComicVineTalker.api_base_url = self.settings.cv_url
# Ignore empty field
if self.leKey.text().strip():
self.settings.cv_api_key = self.leKey.text().strip()
self.talker_api.api_key = self.settings.cv_api_key
if self.leURL.text().strip():
self.settings.cv_url = self.leURL.text().strip()
self.talker_api.api_base_url = self.settings.cv_url
self.settings.assume_lone_credit_is_primary = self.cbxAssumeLoneCreditIsPrimary.isChecked()
self.settings.copy_characters_to_tags = self.cbxCopyCharactersToTags.isChecked()
self.settings.copy_teams_to_tags = self.cbxCopyTeamsToTags.isChecked()
@ -361,7 +365,7 @@ class SettingsWindow(QtWidgets.QDialog):
QtWidgets.QMessageBox.information(self, self.name, "Cache has been cleared.")
def test_api_key(self) -> None:
if ComicVineTalker().test_key(self.leKey.text().strip(), self.leURL.text().strip()):
if self.talker_api.check_api_key(self.leKey.text().strip(), self.leURL.text().strip()):
QtWidgets.QMessageBox.information(self, "API Key Test", "Key is valid!")
else:
QtWidgets.QMessageBox.warning(self, "API Key Test", "Key is NOT valid.")

85
comictaggerlib/taggerwindow.py
View File

@ -46,7 +46,6 @@ from comictaggerlib.autotagmatchwindow import AutoTagMatchWindow
from comictaggerlib.autotagprogresswindow import AutoTagProgressWindow
from comictaggerlib.autotagstartwindow import AutoTagStartWindow
from comictaggerlib.cbltransformer import CBLTransformer
from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.crediteditorwindow import CreditEditorWindow
from comictaggerlib.exportwindow import ExportConflictOpts, ExportWindow
@ -65,6 +64,7 @@ from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import center_window_on_parent, reduce_widget_font_size
from comictaggerlib.versionchecker import VersionChecker
from comictaggerlib.volumeselectionwindow import VolumeSelectionWindow
from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@ -81,6 +81,7 @@ class TaggerWindow(QtWidgets.QMainWindow):
self,
file_list: list[str],
settings: ComicTaggerSettings,
talker_api: ComicTalker,
parent: QtWidgets.QWidget | None = None,
opts: argparse.Namespace | None = None,
) -> None:
@ -88,6 +89,7 @@ class TaggerWindow(QtWidgets.QMainWindow):
uic.loadUi(ui_path / "taggerwindow.ui", self)
self.settings = settings
self.talker_api = talker_api
self.log_window = self.setup_logger()
# prevent multiple instances
@ -121,12 +123,12 @@ class TaggerWindow(QtWidgets.QMainWindow):
)
sys.exit()
self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.ArchiveMode)
self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.ArchiveMode)
grid_layout = QtWidgets.QGridLayout(self.coverImageContainer)
grid_layout.addWidget(self.archiveCoverWidget)
grid_layout.setContentsMargins(0, 0, 0, 0)
self.page_list_editor = PageListEditor(self.tabPages)
self.page_list_editor = PageListEditor(self.tabPages, self.talker_api)
grid_layout = QtWidgets.QGridLayout(self.tabPages)
grid_layout.addWidget(self.page_list_editor)
@ -1046,6 +1048,7 @@ You have {4-self.settings.settings_warning} warnings left.
issue_number = str(self.leIssueNum.text()).strip()
# Only need this check is the source has issue level data.
if autoselect and issue_number == "":
QtWidgets.QMessageBox.information(
self, "Automatic Identify Search", "Can't auto-identify without an issue number (yet!)"
@ -1072,6 +1075,7 @@ You have {4-self.settings.settings_warning} warnings left.
cover_index_list,
cast(ComicArchive, self.comic_archive),
self.settings,
self.talker_api,
autoselect,
literal,
)
@ -1089,16 +1093,23 @@ You have {4-self.settings.settings_warning} warnings left.
self.form_to_metadata()
try:
comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
new_metadata = comic_vine.fetch_issue_data(selector.volume_id, selector.issue_number, self.settings)
except ComicVineTalkerException as e:
QtWidgets.QApplication.restoreOverrideCursor()
if e.code == ComicVineTalkerException.RateLimit:
QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message())
else:
QtWidgets.QMessageBox.critical(
self, "Network Issue", "Could not connect to Comic Vine to get issue details.!"
if selector.issue_id:
new_metadata = self.talker_api.fetch_comic_data(selector.issue_id)
elif selector.volume_id and selector.issue_number:
# Would this ever be needed?
new_metadata = self.talker_api.fetch_comic_data(
series_id=selector.volume_id, issue_number=selector.issue_number
)
else:
# Only left with series? Isn't series only handled elsewhere?
new_metadata = self.talker_api.fetch_comic_data(series_id=selector.volume_id)
except TalkerError as e:
QtWidgets.QApplication.restoreOverrideCursor()
QtWidgets.QMessageBox.critical(
self,
f"{e.source} {e.code_name} Error",
f"{e}",
)
else:
QtWidgets.QApplication.restoreOverrideCursor()
if new_metadata is not None:
@ -1109,7 +1120,7 @@ You have {4-self.settings.settings_warning} warnings left.
self.clear_form()
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
f"Tagged with ComicTagger {ctversion.version} using info from {self.talker_api.source_details.name} on"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {new_metadata.issue_id}]"
)
self.metadata.overlay(
@ -1390,7 +1401,7 @@ You have {4-self.settings.settings_warning} warnings left.
def show_settings(self) -> None:
settingswin = SettingsWindow(self, self.settings)
settingswin = SettingsWindow(self, self.settings, self.talker_api)
settingswin.setModal(True)
settingswin.exec()
settingswin.result()
@ -1694,24 +1705,22 @@ You have {4-self.settings.settings_warning} warnings left.
def actual_issue_data_fetch(self, match: IssueResult) -> GenericMetadata:
# now get the particular issue data
cv_md = GenericMetadata()
# now get the particular issue data OR series data
ct_md = GenericMetadata()
QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
try:
comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
comic_vine.wait_for_rate_limit = self.settings.wait_and_retry_on_rate_limit
cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], self.settings)
except ComicVineTalkerException:
logger.exception("Network error while getting issue details. Save aborted")
ct_md = self.talker_api.fetch_comic_data(match["issue_id"])
except TalkerError as e:
logger.exception(f"Save aborted.\n{e}")
if not cv_md.is_empty:
if not ct_md.is_empty:
if self.settings.apply_cbl_transform_on_cv_import:
cv_md = CBLTransformer(cv_md, self.settings).apply()
ct_md = CBLTransformer(ct_md, self.settings).apply()
QtWidgets.QApplication.restoreOverrideCursor()
return cv_md
return ct_md
def auto_tag_log(self, text: str) -> None:
IssueIdentifier.default_write_output(text)
@ -1726,7 +1735,7 @@ You have {4-self.settings.settings_warning} warnings left.
self, ca: ComicArchive, match_results: OnlineMatchResults, dlg: AutoTagStartWindow
) -> tuple[bool, OnlineMatchResults]:
success = False
ii = IssueIdentifier(ca, self.settings)
ii = IssueIdentifier(ca, self.settings, self.talker_api)
# read in metadata, and parse file name if not there
try:
@ -1763,7 +1772,6 @@ You have {4-self.settings.settings_warning} warnings left.
md.issue = utils.xlate(md.volume)
ii.set_additional_metadata(md)
ii.only_use_additional_meta_data = True
ii.wait_and_retry_on_rate_limit = dlg.wait_and_retry_on_rate_limit
ii.set_output_function(self.auto_tag_log)
ii.cover_page_index = md.get_cover_page_index_list()[0]
if self.atprogdialog is not None:
@ -1812,19 +1820,19 @@ You have {4-self.settings.settings_warning} warnings left.
self.auto_tag_log("Online search: Low confidence match, but saving anyways, as indicated...\n")
# now get the particular issue data
cv_md = self.actual_issue_data_fetch(matches[0])
if cv_md is None:
ct_md = self.actual_issue_data_fetch(matches[0])
if ct_md is None:
match_results.fetch_data_failures.append(str(ca.path.absolute()))
if cv_md is not None:
if ct_md is not None:
if dlg.cbxRemoveMetadata.isChecked():
md = cv_md
md = ct_md
else:
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]"
)
md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
if self.settings.auto_imprint:
md.fix_publisher()
@ -1867,7 +1875,7 @@ You have {4-self.settings.settings_warning} warnings left.
if not atstartdlg.exec():
return
self.atprogdialog = AutoTagProgressWindow(self)
self.atprogdialog = AutoTagProgressWindow(self, self.talker_api)
self.atprogdialog.setModal(True)
self.atprogdialog.show()
self.atprogdialog.progressBar.setMaximum(len(ca_list))
@ -1954,7 +1962,12 @@ You have {4-self.settings.settings_warning} warnings left.
match_results.multiple_matches.extend(match_results.low_confidence_matches)
if reply == QtWidgets.QMessageBox.StandardButton.Yes:
matchdlg = AutoTagMatchWindow(
self, match_results.multiple_matches, style, self.actual_issue_data_fetch, self.settings
self,
match_results.multiple_matches,
style,
self.actual_issue_data_fetch,
self.settings,
self.talker_api,
)
matchdlg.setModal(True)
matchdlg.exec()
@ -2017,7 +2030,7 @@ You have {4-self.settings.settings_warning} warnings left.
def show_page_browser(self) -> None:
if self.page_browser is None:
self.page_browser = PageBrowserWindow(self, self.metadata)
self.page_browser = PageBrowserWindow(self, self.talker_api, self.metadata)
if self.comic_archive is not None:
self.page_browser.set_comic_archive(self.comic_archive)
self.page_browser.finished.connect(self.page_browser_closed)
@ -2084,7 +2097,7 @@ You have {4-self.settings.settings_warning} warnings left.
"File Rename", "If you rename files now, unsaved data in the form will be lost. Are you sure?"
):
dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings)
dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings, self.talker_api)
dlg.setModal(True)
if dlg.exec() and self.comic_archive is not None:
self.fileSelectionList.update_selected_rows()

275
comictaggerlib/volumeselectionwindow.py
View File

@ -25,16 +25,16 @@ from PyQt5.QtCore import pyqtSignal
from comicapi import utils
from comicapi.comicarchive import ComicArchive
from comicapi.genericmetadata import GenericMetadata
from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.issueidentifier import IssueIdentifier
from comictaggerlib.issueselectionwindow import IssueSelectionWindow
from comictaggerlib.matchselectionwindow import MatchSelectionWindow
from comictaggerlib.progresswindow import IDProgressWindow
from comictaggerlib.resulttypes import CVVolumeResults
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
from comictalker.resulttypes import ComicVolume
from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@ -43,27 +43,34 @@ class SearchThread(QtCore.QThread):
searchComplete = pyqtSignal()
progressUpdate = pyqtSignal(int, int)
def __init__(self, series_name: str, refresh: bool, literal: bool = False, series_match_thresh: int = 90) -> None:
def __init__(
self,
talker_api: ComicTalker,
series_name: str,
refresh: bool,
literal: bool = False,
series_match_thresh: int = 90,
) -> None:
QtCore.QThread.__init__(self)
self.talker_api = talker_api
self.series_name = series_name
self.refresh: bool = refresh
self.error_code: int | None = None
self.cv_error = False
self.cv_search_results: list[CVVolumeResults] = []
self.error_e: TalkerError
self.ct_error = False
self.ct_search_results: list[ComicVolume] = []
self.literal = literal
self.series_match_thresh = series_match_thresh
def run(self) -> None:
comic_vine = ComicVineTalker(self.series_match_thresh)
try:
self.cv_error = False
self.cv_search_results = comic_vine.search_for_series(
self.ct_error = False
self.ct_search_results = self.talker_api.search_for_series(
self.series_name, self.prog_callback, self.refresh, self.literal
)
except ComicVineTalkerException as e:
self.cv_search_results = []
self.cv_error = True
self.error_code = e.code
except TalkerError as e:
self.ct_search_results = []
self.ct_error = True
self.error_e = e
finally:
self.searchComplete.emit()
@ -105,6 +112,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
cover_index_list: list[int],
comic_archive: ComicArchive,
settings: ComicTaggerSettings,
talker_api: ComicTalker,
autoselect: bool = False,
literal: bool = False,
) -> None:
@ -112,7 +120,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
uic.loadUi(ui_path / "volumeselectionwindow.ui", self)
self.imageWidget = CoverImageWidget(self.imageContainer, CoverImageWidget.URLMode)
self.imageWidget = CoverImageWidget(self.imageContainer, talker_api, CoverImageWidget.URLMode)
gridlayout = QtWidgets.QGridLayout(self.imageContainer)
gridlayout.addWidget(self.imageWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@ -131,13 +139,14 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.settings = settings
self.series_name = series_name
self.issue_number = issue_number
self.issue_id: int | None = None
self.year = year
self.issue_count = issue_count
self.volume_id = 0
self.comic_archive = comic_archive
self.immediate_autoselect = autoselect
self.cover_index_list = cover_index_list
self.cv_search_results: list[CVVolumeResults] = []
self.ct_search_results: list[ComicVolume] = []
self.literal = literal
self.ii: IssueIdentifier | None = None
self.iddialog: IDProgressWindow | None = None
@ -147,6 +156,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.use_filter = self.settings.always_use_publisher_filter
# Load to retrieve settings
self.talker_api = talker_api
self.twList.resizeColumnsToContents()
self.twList.currentItemChanged.connect(self.current_item_changed)
self.twList.cellDoubleClicked.connect(self.cell_double_clicked)
@ -158,15 +170,16 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.cbxFilter.toggled.connect(self.filter_toggled)
self.update_buttons()
self.perform_query()
self.twList.selectRow(0)
def update_buttons(self) -> None:
enabled = bool(self.cv_search_results)
enabled = bool(self.ct_search_results)
self.btnRequery.setEnabled(enabled)
self.btnIssues.setEnabled(enabled)
self.btnAutoSelect.setEnabled(enabled)
self.buttonBox.button(QtWidgets.QDialogButtonBox.StandardButton.Ok).setEnabled(enabled)
def requery(self) -> None:
@ -178,6 +191,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.perform_query(refresh=False)
def auto_select(self) -> None:
if self.comic_archive is None:
QtWidgets.QMessageBox.information(self, "Auto-Select", "You need to load a comic first!")
return
@ -191,7 +205,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.iddialog.rejected.connect(self.identify_cancel)
self.iddialog.show()
self.ii = IssueIdentifier(self.comic_archive, self.settings)
self.ii = IssueIdentifier(self.comic_archive, self.settings, self.talker_api)
md = GenericMetadata()
md.series = self.series_name
@ -264,7 +278,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
choices = True
if choices:
selector = MatchSelectionWindow(self, matches, self.comic_archive)
selector = MatchSelectionWindow(self, matches, self.comic_archive, self.talker_api)
selector.setModal(True)
selector.exec()
if selector.result():
@ -280,9 +294,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.show_issues()
def show_issues(self) -> None:
selector = IssueSelectionWindow(self, self.settings, self.volume_id, self.issue_number)
selector = IssueSelectionWindow(self, self.settings, self.talker_api, self.volume_id, self.issue_number)
title = ""
for record in self.cv_search_results:
for record in self.ct_search_results:
if record["id"] == self.volume_id:
title = record["name"]
title += " (" + str(record["start_year"]) + ")"
@ -295,6 +309,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
if selector.result():
# we should now have a volume ID
self.issue_number = selector.issue_number
self.issue_id = selector.issue_id
self.accept()
def select_by_id(self) -> None:
@ -306,18 +321,23 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
def perform_query(self, refresh: bool = False) -> None:
self.search_thread = SearchThread(
self.talker_api, self.series_name, refresh, self.literal, self.settings.id_series_match_search_thresh
)
self.search_thread.searchComplete.connect(self.search_complete)
self.search_thread.progressUpdate.connect(self.search_progress_update)
self.search_thread.start()
self.progdialog = QtWidgets.QProgressDialog("Searching Online", "Cancel", 0, 100, self)
self.progdialog.setWindowTitle("Online Search")
self.progdialog.canceled.connect(self.search_canceled)
self.progdialog.setModal(True)
self.progdialog.setMinimumDuration(300)
self.search_thread = SearchThread(
self.series_name, refresh, self.literal, self.settings.id_series_match_search_thresh
)
self.search_thread.searchComplete.connect(self.search_complete)
self.search_thread.progressUpdate.connect(self.search_progress_update)
self.search_thread.start()
self.progdialog.exec()
if refresh or self.search_thread.isRunning():
self.progdialog.exec()
else:
self.progdialog = None
def search_canceled(self) -> None:
if self.progdialog is not None:
@ -340,123 +360,124 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
def search_complete(self) -> None:
if self.progdialog is not None:
self.progdialog.accept()
del self.progdialog
if self.search_thread is not None and self.search_thread.cv_error:
if self.search_thread.error_code == ComicVineTalkerException.RateLimit:
QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message())
else:
QtWidgets.QMessageBox.critical(
self, "Network Issue", "Could not connect to Comic Vine to search for series!"
self.progdialog = None
if self.search_thread is not None and self.search_thread.ct_error:
# TODO Currently still opens the window
QtWidgets.QMessageBox.critical(
self,
f"{self.search_thread.error_e.source} {self.search_thread.error_e.code_name} Error",
f"{self.search_thread.error_e}",
)
return
self.ct_search_results = self.search_thread.ct_search_results if self.search_thread is not None else []
# filter the publishers if enabled set
if self.use_filter:
try:
publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")}
# use '' as publisher name if None
self.ct_search_results = list(
filter(
lambda d: ("" if d["publisher"] is None else str(d["publisher"]).casefold())
not in publisher_filter,
self.ct_search_results,
)
return
)
except Exception:
logger.exception("bad data error filtering publishers")
self.cv_search_results = self.search_thread.cv_search_results if self.search_thread is not None else []
# filter the publishers if enabled set
if self.use_filter:
try:
publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")}
# use '' as publisher name if None
self.cv_search_results = list(
filter(
lambda d: ("" if d["publisher"] is None else str(d["publisher"]["name"]).casefold())
not in publisher_filter,
self.cv_search_results,
)
)
except Exception:
logger.exception("bad data error filtering publishers")
# pre sort the data - so that we can put exact matches first afterwards
# compare as str in case extra chars ie. '1976?'
# - missing (none) values being converted to 'None' - consistent with prior behaviour in v1.2.3
# sort by start_year if set
if self.settings.sort_series_by_year:
try:
self.ct_search_results = sorted(
self.ct_search_results,
key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])),
reverse=True,
)
except Exception:
logger.exception("bad data error sorting results by start_year,count_of_issues")
else:
try:
self.ct_search_results = sorted(
self.ct_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True
)
except Exception:
logger.exception("bad data error sorting results by count_of_issues")
# pre sort the data - so that we can put exact matches first afterwards
# compare as str in case extra chars ie. '1976?'
# - missing (none) values being converted to 'None' - consistent with prior behaviour in v1.2.3
# sort by start_year if set
if self.settings.sort_series_by_year:
try:
self.cv_search_results = sorted(
self.cv_search_results,
key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])),
reverse=True,
)
except Exception:
logger.exception("bad data error sorting results by start_year,count_of_issues")
else:
try:
self.cv_search_results = sorted(
self.cv_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True
)
except Exception:
logger.exception("bad data error sorting results by count_of_issues")
# move sanitized matches to the front
if self.settings.exact_series_matches_first:
try:
sanitized = utils.sanitize_title(self.series_name, False).casefold()
sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold()
# move sanitized matches to the front
if self.settings.exact_series_matches_first:
try:
sanitized = utils.sanitize_title(self.series_name, False).casefold()
sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold()
deques: list[deque[ComicVolume]] = [deque(), deque(), deque()]
deques: list[deque[CVVolumeResults]] = [deque(), deque(), deque()]
def categorize(result: ComicVolume) -> int:
# We don't remove anything on this one so that we only get exact matches
if utils.sanitize_title(result["name"], True).casefold() == sanitized_no_articles:
return 0
def categorize(result: CVVolumeResults) -> int:
# We don't remove anything on this one so that we only get exact matches
if utils.sanitize_title(result["name"], True).casefold() == sanitized_no_articles:
return 0
# this ensures that 'The Joker' is near the top even if you search 'Joker'
if utils.sanitize_title(result["name"], False).casefold() in sanitized:
return 1
return 2
# this ensures that 'The Joker' is near the top even if you search 'Joker'
if utils.sanitize_title(result["name"], False).casefold() in sanitized:
return 1
return 2
for comic in self.ct_search_results:
deques[categorize(comic)].append(comic)
logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2]))
self.ct_search_results = list(itertools.chain.from_iterable(deques))
except Exception:
logger.exception("bad data error filtering exact/near matches")
for comic in self.cv_search_results:
deques[categorize(comic)].append(comic)
logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2]))
self.cv_search_results = list(itertools.chain.from_iterable(deques))
except Exception:
logger.exception("bad data error filtering exact/near matches")
self.update_buttons()
self.update_buttons()
self.twList.setSortingEnabled(False)
self.twList.setSortingEnabled(False)
self.twList.setRowCount(0)
self.twList.setRowCount(0)
row = 0
for record in self.ct_search_results:
self.twList.insertRow(row)
row = 0
for record in self.cv_search_results:
self.twList.insertRow(row)
item_text = record["name"]
item = QtWidgets.QTableWidgetItem(item_text)
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setData(QtCore.Qt.ItemDataRole.UserRole, record["id"])
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 0, item)
item_text = record["name"]
item = QtWidgets.QTableWidgetItem(item_text)
item_text = str(record["start_year"])
item = QtWidgets.QTableWidgetItem(item_text)
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 1, item)
item_text = str(record["count_of_issues"])
item = QtWidgets.QTableWidgetItem(item_text)
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setData(QtCore.Qt.ItemDataRole.DisplayRole, record["count_of_issues"])
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 2, item)
if record["publisher"] is not None:
item_text = record["publisher"]
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setData(QtCore.Qt.ItemDataRole.UserRole, record["id"])
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 0, item)
item_text = str(record["start_year"])
item = QtWidgets.QTableWidgetItem(item_text)
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 1, item)
self.twList.setItem(row, 3, item)
item_text = str(record["count_of_issues"])
item = QtWidgets.QTableWidgetItem(item_text)
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setData(QtCore.Qt.ItemDataRole.DisplayRole, record["count_of_issues"])
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 2, item)
row += 1
if record["publisher"] is not None:
item_text = record["publisher"]["name"]
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item = QtWidgets.QTableWidgetItem(item_text)
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
self.twList.setItem(row, 3, item)
row += 1
self.twList.setSortingEnabled(True)
self.twList.selectRow(0)
self.twList.resizeColumnsToContents()
self.twList.setSortingEnabled(True)
self.twList.selectRow(0)
self.twList.resizeColumnsToContents()
def showEvent(self, event: QtGui.QShowEvent) -> None:
if not self.cv_search_results:
self.perform_query()
if not self.ct_search_results:
QtCore.QCoreApplication.processEvents()
QtWidgets.QMessageBox.information(self, "Search Result", "No matches found!")
QtCore.QTimer.singleShot(200, self.close_me)
@ -483,11 +504,11 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.volume_id = self.twList.item(curr.row(), 0).data(QtCore.Qt.ItemDataRole.UserRole)
# list selection was changed, update the info on the volume
for record in self.cv_search_results:
for record in self.ct_search_results:
if record["id"] == self.volume_id:
if record["description"] is None:
self.teDetails.setText("")
else:
self.teDetails.setText(record["description"])
self.imageWidget.set_url(record["image"]["super_url"])
self.imageWidget.set_url(record["image_url"])
break

1
comictalker/__init__.py Normal file
View File

@ -0,0 +1 @@
from __future__ import annotations

329
comictaggerlib/comiccacher.py → comictalker/comiccacher.py
View File

@ -16,14 +16,15 @@
from __future__ import annotations
import datetime
import json
import logging
import os
import sqlite3 as lite
from typing import Any
from comictaggerlib import ctversion
from comictaggerlib.resulttypes import CVImage, CVIssuesResults, CVPublisher, CVVolumeResults, SelectDetails
from comictaggerlib.settings import ComicTaggerSettings
from comictalker.resulttypes import ComicIssue, ComicVolume
logger = logging.getLogger(__name__)
@ -77,15 +78,8 @@ class ComicCacher:
"CREATE TABLE VolumeSearchCache("
+ "search_term TEXT,"
+ "id INT NOT NULL,"
+ "name TEXT,"
+ "start_year INT,"
+ "publisher TEXT,"
+ "count_of_issues INT,"
+ "image_url TEXT,"
+ "description TEXT,"
+ "timestamp DATE DEFAULT (datetime('now','localtime')),"
+ "source_name TEXT NOT NULL,"
+ "aliases TEXT)" # Newline separated
+ "source_name TEXT NOT NULL)"
)
cur.execute(
@ -95,29 +89,21 @@ class ComicCacher:
+ "publisher TEXT,"
+ "count_of_issues INT,"
+ "start_year INT,"
+ "image_url TEXT,"
+ "aliases TEXT," # Newline separated
+ "description TEXT,"
+ "timestamp DATE DEFAULT (datetime('now','localtime')), "
+ "source_name TEXT NOT NULL,"
+ "aliases TEXT," # Newline separated
+ "PRIMARY KEY (id, source_name))"
)
cur.execute(
"CREATE TABLE AltCovers("
+ "issue_id INT NOT NULL,"
+ "url_list TEXT,"
+ "timestamp DATE DEFAULT (datetime('now','localtime')), "
+ "source_name TEXT NOT NULL,"
+ "aliases TEXT," # Newline separated
+ "PRIMARY KEY (issue_id, source_name))"
)
cur.execute(
"CREATE TABLE Issues("
+ "id INT NOT NULL,"
+ "volume_id INT,"
+ "name TEXT,"
+ "issue_number TEXT,"
+ "super_url TEXT,"
+ "image_url TEXT,"
+ "thumb_url TEXT,"
+ "cover_date TEXT,"
+ "site_detail_url TEXT,"
@ -125,11 +111,17 @@ class ComicCacher:
+ "timestamp DATE DEFAULT (datetime('now','localtime')), "
+ "source_name TEXT NOT NULL,"
+ "aliases TEXT," # Newline separated
+ "alt_image_urls TEXT," # Newline separated URLs
+ "characters TEXT," # Newline separated
+ "locations TEXT," # Newline separated
+ "credits TEXT," # JSON: "{"name": "Bob Shakespeare", "role": "Writer"}"
+ "teams TEXT," # Newline separated
+ "story_arcs TEXT," # Newline separated
+ "complete BOOL," # Is the data complete? Includes characters, locations, credits.
+ "PRIMARY KEY (id, source_name))"
)
def add_search_results(self, source_name: str, search_term: str, cv_search_results: list[CVVolumeResults]) -> None:
def add_search_results(self, source_name: str, search_term: str, ct_search_results: list[ComicVolume]) -> None:
con = lite.connect(self.db_file)
with con:
@ -143,114 +135,63 @@ class ComicCacher:
)
# now add in new results
for record in cv_search_results:
if record["publisher"] is None:
pub_name = ""
else:
pub_name = record["publisher"]["name"]
if record["image"] is None:
url = ""
else:
url = record["image"]["super_url"]
for record in ct_search_results:
cur.execute(
"INSERT INTO VolumeSearchCache "
+ "(source_name, search_term, id, name, start_year, publisher, count_of_issues, image_url, description, aliases) "
+ "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO VolumeSearchCache " + "(source_name, search_term, id) " + "VALUES(?, ?, ?)",
(
source_name,
search_term.casefold(),
record["id"],
record["name"],
record["start_year"],
pub_name,
record["count_of_issues"],
url,
record["description"],
record["aliases"],
),
)
def get_search_results(self, source_name: str, search_term: str) -> list[CVVolumeResults]:
data = {
"id": record["id"],
"source_name": source_name,
"name": record["name"],
"publisher": record.get("publisher", ""),
"count_of_issues": record.get("count_of_issues"),
"start_year": record.get("start_year"),
"image_url": record.get("image_url", ""),
"description": record.get("description", ""),
"timestamp": datetime.datetime.now(),
"aliases": "\n".join(record.get("aliases", [])),
}
self.upsert(cur, "volumes", data)
def get_search_results(self, source_name: str, search_term: str) -> list[ComicVolume]:
results = []
con = lite.connect(self.db_file)
with con:
con.text_factory = str
cur = con.cursor()
# purge stale search results
a_day_ago = datetime.datetime.today() - datetime.timedelta(days=1)
cur.execute("DELETE FROM VolumeSearchCache WHERE timestamp < ?", [str(a_day_ago)])
# fetch
cur.execute(
"SELECT * FROM VolumeSearchCache WHERE search_term=? AND source_name=?",
"SELECT * FROM VolumeSearchCache INNER JOIN Volumes on"
" VolumeSearchCache.id=Volumes.id AND VolumeSearchCache.source_name=Volumes.source_name"
" WHERE search_term=? AND VolumeSearchCache.source_name=?",
[search_term.casefold(), source_name],
)
rows = cur.fetchall()
# now process the results
for record in rows:
result = CVVolumeResults(
id=record[1],
name=record[2],
start_year=record[3],
count_of_issues=record[5],
description=record[7],
publisher=CVPublisher(name=record[4]),
image=CVImage(super_url=record[6]),
aliases=record[10],
result = ComicVolume(
id=record[4],
name=record[5],
publisher=record[6],
count_of_issues=record[7],
start_year=record[8],
image_url=record[9],
aliases=record[10].strip().splitlines(),
description=record[11],
)
results.append(result)
return results
def add_alt_covers(self, source_name: str, issue_id: int, url_list: list[str]) -> None:
con = lite.connect(self.db_file)
with con:
con.text_factory = str
cur = con.cursor()
# remove all previous entries with this search term
cur.execute("DELETE FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name])
url_list_str = ",".join(url_list)
# now add in new record
cur.execute(
"INSERT INTO AltCovers (source_name, issue_id, url_list) VALUES(?, ?, ?)",
(source_name, issue_id, url_list_str),
)
def get_alt_covers(self, source_name: str, issue_id: int) -> list[str]:
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
# purge stale issue info - probably issue data won't change
# much....
a_month_ago = datetime.datetime.today() - datetime.timedelta(days=30)
cur.execute("DELETE FROM AltCovers WHERE timestamp < ?", [str(a_month_ago)])
cur.execute("SELECT url_list FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name])
row = cur.fetchone()
if row is None:
return []
url_list_str = row[0]
if not url_list_str:
return []
url_list = str(url_list_str).split(",")
return url_list
def add_volume_info(self, source_name: str, cv_volume_record: CVVolumeResults) -> None:
def add_volume_info(self, source_name: str, volume_record: ComicVolume) -> None:
con = lite.connect(self.db_file)
with con:
@ -259,25 +200,21 @@ class ComicCacher:
timestamp = datetime.datetime.now()
if cv_volume_record["publisher"] is None:
pub_name = ""
else:
pub_name = cv_volume_record["publisher"]["name"]
data = {
"id": cv_volume_record["id"],
"id": volume_record["id"],
"source_name": source_name,
"name": cv_volume_record["name"],
"publisher": pub_name,
"count_of_issues": cv_volume_record["count_of_issues"],
"start_year": cv_volume_record["start_year"],
"name": volume_record["name"],
"publisher": volume_record.get("publisher", ""),
"count_of_issues": volume_record.get("count_of_issues"),
"start_year": volume_record.get("start_year"),
"image_url": volume_record.get("image_url", ""),
"description": volume_record.get("description", ""),
"timestamp": timestamp,
"aliases": cv_volume_record["aliases"],
"aliases": "\n".join(volume_record.get("aliases", [])),
}
self.upsert(cur, "volumes", data)
def add_volume_issues_info(self, source_name: str, volume_id: int, cv_volume_issues: list[CVIssuesResults]) -> None:
def add_volume_issues_info(self, source_name: str, volume_issues: list[ComicIssue]) -> None:
con = lite.connect(self.db_file)
with con:
@ -287,40 +224,46 @@ class ComicCacher:
# add in issues
for issue in cv_volume_issues:
for issue in volume_issues:
data = {
"id": issue["id"],
"volume_id": volume_id,
"volume_id": issue["volume"]["id"],
"source_name": source_name,
"name": issue["name"],
"issue_number": issue["issue_number"],
"site_detail_url": issue["site_detail_url"],
"cover_date": issue["cover_date"],
"super_url": issue["image"]["super_url"],
"thumb_url": issue["image"]["thumb_url"],
"description": issue["description"],
"site_detail_url": issue.get("site_detail_url"),
"cover_date": issue.get("cover_date"),
"image_url": issue.get("image_url", ""),
"thumb_url": issue.get("image_thumb_url", ""),
"description": issue.get("description", ""),
"timestamp": timestamp,
"aliases": issue["aliases"],
"aliases": "\n".join(issue.get("aliases", [])),
"alt_image_urls": "\n".join(issue.get("alt_image_urls", [])),
"characters": "\n".join(issue.get("characters", [])),
"locations": "\n".join(issue.get("locations", [])),
"teams": "\n".join(issue.get("teams", [])),
"story_arcs": "\n".join(issue.get("story_arcs", [])),
"credits": json.dumps(issue.get("credits")),
"complete": issue["complete"],
}
self.upsert(cur, "issues", data)
def get_volume_info(self, volume_id: int, source_name: str) -> CVVolumeResults | None:
result: CVVolumeResults | None = None
def get_volume_info(self, volume_id: int, source_name: str, purge: bool = True) -> ComicVolume | None:
result: ComicVolume | None = None
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
# purge stale volume info
a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7)
cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)])
if purge:
# purge stale volume info
a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7)
cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)])
# fetch
cur.execute(
"SELECT source_name,id,name,publisher,count_of_issues,start_year,aliases FROM Volumes"
" WHERE id=? AND source_name=?",
"SELECT * FROM Volumes" " WHERE id=? AND source_name=?",
[volume_id, source_name],
)
@ -330,19 +273,22 @@ class ComicCacher:
return result
# since ID is primary key, there is only one row
result = CVVolumeResults(
id=row[1],
name=row[2],
count_of_issues=row[4],
start_year=row[5],
publisher=CVPublisher(name=row[3]),
aliases=row[6],
result = ComicVolume(
id=row[0],
name=row[1],
publisher=row[2],
count_of_issues=row[3],
start_year=row[4],
image_url=row[5],
aliases=row[6].strip().splitlines(),
description=row[7],
)
return result
def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[CVIssuesResults]:
def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[ComicIssue]:
# get_volume_info should only fail if someone is doing something weird
volume = self.get_volume_info(volume_id, source_name, False) or ComicVolume(id=volume_id, name="")
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
@ -354,11 +300,11 @@ class ComicCacher:
cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)])
# fetch
results: list[CVIssuesResults] = []
results: list[ComicIssue] = []
cur.execute(
(
"SELECT source_name,id,name,issue_number,site_detail_url,cover_date,super_url,thumb_url,description,aliases"
"SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,alt_image_urls,characters,locations,credits,teams,story_arcs,complete"
" FROM Issues WHERE volume_id=? AND source_name=?"
),
[volume_id, source_name],
@ -367,75 +313,78 @@ class ComicCacher:
# now process the results
for row in rows:
record = CVIssuesResults(
record = ComicIssue(
id=row[1],
name=row[2],
issue_number=row[3],
site_detail_url=row[4],
cover_date=row[5],
image=CVImage(super_url=row[6], thumb_url=row[7]),
image_url=row[6],
description=row[8],
aliases=row[9],
volume=volume,
aliases=row[9].strip().splitlines(),
alt_image_urls=row[10].strip().splitlines(),
characters=row[11].strip().splitlines(),
locations=row[12].strip().splitlines(),
credits=json.loads(row[13]),
teams=row[14].strip().splitlines(),
story_arcs=row[15].strip().splitlines(),
complete=bool(row[16]),
)
results.append(record)
return results
def add_issue_select_details(
self,
source_name: str,
issue_id: int,
image_url: str,
thumb_image_url: str,
cover_date: str,
site_detail_url: str,
) -> None:
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
timestamp = datetime.datetime.now()
data = {
"id": issue_id,
"source_name": source_name,
"super_url": image_url,
"thumb_url": thumb_image_url,
"cover_date": cover_date,
"site_detail_url": site_detail_url,
"timestamp": timestamp,
}
self.upsert(cur, "issues", data)
def get_issue_select_details(self, issue_id: int, source_name: str) -> SelectDetails:
def get_issue_info(self, issue_id: int, source_name: str) -> ComicIssue | None:
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
# purge stale issue info - probably issue data won't change
# much....
a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7)
cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)])
cur.execute(
"SELECT super_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=? AND source_name=?",
(
"SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,volume_id,alt_image_urls,characters,locations,credits,teams,story_arcs,complete"
" FROM Issues WHERE id=? AND source_name=?"
),
[issue_id, source_name],
)
row = cur.fetchone()
details = SelectDetails(
image_url=None,
thumb_image_url=None,
cover_date=None,
site_detail_url=None,
)
if row is not None and row[0] is not None:
details["image_url"] = row[0]
details["thumb_image_url"] = row[1]
details["cover_date"] = row[2]
details["site_detail_url"] = row[3]
record = None
return details
if row:
# get_volume_info should only fail if someone is doing something weird
volume = self.get_volume_info(row[10], source_name, False) or ComicVolume(id=row[10], name="")
# now process the results
record = ComicIssue(
id=row[1],
name=row[2],
issue_number=row[3],
site_detail_url=row[4],
cover_date=row[5],
image_url=row[6],
image_thumb_url=row[7],
description=row[8],
volume=volume,
aliases=row[9].strip().splitlines(),
alt_image_urls=row[11].strip().splitlines(),
characters=row[12].strip().splitlines(),
locations=row[13].strip().splitlines(),
credits=json.loads(row[14]),
teams=row[15].strip().splitlines(),
story_arcs=row[16].strip().splitlines(),
complete=bool(row[17]),
)
return record
def upsert(self, cur: lite.Cursor, tablename: str, data: dict[str, Any]) -> None:
"""This does an insert if the given PK doesn't exist, and an

38
comictalker/comictalkerapi.py Normal file
View File

@ -0,0 +1,38 @@
"""Handles collecting data from source talkers."""
# Copyright 2012-2014 Anthony Beville
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import logging
import comictalker.talkers.comicvine
from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
def get_comic_talker(source_name: str) -> type[ComicTalker]:
"""Retrieve the available sources modules"""
sources = get_talkers()
if source_name not in sources:
raise TalkerError(source=source_name, code=4, desc="The talker does not exist")
talker = sources[source_name]
return talker
def get_talkers():
"""Returns all comic talker modules NOT objects"""
return {"comicvine": comictalker.talkers.comicvine.ComicVineTalker}

39
comictalker/resulttypes.py Normal file
View File

@ -0,0 +1,39 @@
from __future__ import annotations
from typing_extensions import Required, TypedDict
class Credits(TypedDict):
name: str
role: str
class ComicVolume(TypedDict, total=False):
aliases: list[str]
count_of_issues: int
description: str
id: Required[int]
image_url: str
name: Required[str]
publisher: str
start_year: int
class ComicIssue(TypedDict, total=False):
aliases: list[str]
cover_date: str
description: str
id: int
image_url: str
image_thumb_url: str
issue_number: Required[str]
name: Required[str]
site_detail_url: str
volume: ComicVolume
alt_image_urls: list[str]
characters: list[str]
locations: list[str]
credits: list[Credits]
teams: list[str]
story_arcs: list[str]
complete: bool # Is this a complete ComicIssue? or is there more data to fetch

199
comictalker/talker_utils.py Normal file
View File

@ -0,0 +1,199 @@
"""Generic sources utils to format API data and the like.
"""
# Copyright 2012-2014 Anthony Beville
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import logging
import re
from datetime import datetime
from bs4 import BeautifulSoup
from comicapi import utils
from comicapi.genericmetadata import GenericMetadata
from comicapi.issuestring import IssueString
from comictaggerlib import ctversion
from comictalker.talkerbase import ComicIssue
logger = logging.getLogger(__name__)
def map_comic_issue_to_metadata(
issue_results: ComicIssue, source: str, remove_html_tables: bool = False, use_year_volume: bool = False
) -> GenericMetadata:
"""Maps ComicIssue to generic metadata"""
metadata = GenericMetadata()
metadata.is_empty = False
# Is this best way to go about checking?
if issue_results["volume"].get("name"):
metadata.series = utils.xlate(issue_results["volume"]["name"])
if issue_results.get("issue_number"):
metadata.issue = IssueString(issue_results["issue_number"]).as_string()
if issue_results.get("name"):
metadata.title = utils.xlate(issue_results["name"])
if issue_results.get("image_url"):
metadata.cover_image = issue_results["image_url"]
if issue_results["volume"].get("publisher"):
metadata.publisher = utils.xlate(issue_results["volume"]["publisher"])
if issue_results.get("cover_date"):
metadata.day, metadata.month, metadata.year = utils.parse_date_str(issue_results["cover_date"])
elif issue_results["volume"].get("start_year"):
metadata.year = utils.xlate(issue_results["volume"]["start_year"], True)
metadata.comments = cleanup_html(issue_results["description"], remove_html_tables)
if use_year_volume:
metadata.volume = issue_results["volume"]["start_year"]
metadata.tag_origin = source
metadata.issue_id = issue_results["id"]
metadata.notes = (
f"Tagged with ComicTagger {ctversion.version} using info from {source} on"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {issue_results['id']}]"
)
metadata.web_link = issue_results["site_detail_url"]
for person in issue_results["credits"]:
if "role" in person:
roles = person["role"].split(",")
for role in roles:
# can we determine 'primary' from CV??
metadata.add_credit(person["name"], role.title().strip(), False)
if issue_results.get("characters"):
metadata.characters = ", ".join(issue_results["characters"])
if issue_results.get("teams"):
metadata.teams = ", ".join(issue_results["teams"])
if issue_results.get("locations"):
metadata.locations = ", ".join(issue_results["locations"])
if issue_results.get("story_arcs"):
metadata.story_arc = ", ".join(issue_results["story_arcs"])
return metadata
def parse_date_str(date_str: str) -> tuple[int | None, int | None, int | None]:
day = None
month = None
year = None
if date_str:
parts = date_str.split("-")
year = utils.xlate(parts[0], True)
if len(parts) > 1:
month = utils.xlate(parts[1], True)
if len(parts) > 2:
day = utils.xlate(parts[2], True)
return day, month, year
def cleanup_html(string: str, remove_html_tables: bool = False) -> str:
"""Cleans HTML code from any text. Will remove any HTML tables with remove_html_tables"""
if string is None:
return ""
# find any tables
soup = BeautifulSoup(string, "html.parser")
tables = soup.findAll("table")
# remove all newlines first
string = string.replace("\n", "")
# put in our own
string = string.replace("<br>", "\n")
string = string.replace("</li>", "\n")
string = string.replace("</p>", "\n\n")
string = string.replace("<h1>", "*")
string = string.replace("</h1>", "*\n")
string = string.replace("<h2>", "*")
string = string.replace("</h2>", "*\n")
string = string.replace("<h3>", "*")
string = string.replace("</h3>", "*\n")
string = string.replace("<h4>", "*")
string = string.replace("</h4>", "*\n")
string = string.replace("<h5>", "*")
string = string.replace("</h5>", "*\n")
string = string.replace("<h6>", "*")
string = string.replace("</h6>", "*\n")
# remove the tables
p = re.compile(r"<table[^<]*?>.*?</table>")
if remove_html_tables:
string = p.sub("", string)
string = string.replace("*List of covers and their creators:*", "")
else:
string = p.sub("{}", string)
# now strip all other tags
p = re.compile(r"<[^<]*?>")
newstring = p.sub("", string)
newstring = newstring.replace("&nbsp;", " ")
newstring = newstring.replace("&amp;", "&")
newstring = newstring.strip()
if not remove_html_tables:
# now rebuild the tables into text from BSoup
try:
table_strings = []
for table in tables:
rows = []
hdrs = []
col_widths = []
for hdr in table.findAll("th"):
item = hdr.string.strip()
hdrs.append(item)
col_widths.append(len(item))
rows.append(hdrs)
for row in table.findAll("tr"):
cols = []
col = row.findAll("td")
i = 0
for c in col:
item = c.string.strip()
cols.append(item)
if len(item) > col_widths[i]:
col_widths[i] = len(item)
i += 1
if len(cols) != 0:
rows.append(cols)
# now we have the data, make it into text
fmtstr = "|"
for w in col_widths:
fmtstr += f" {{:{w + 1}}}|"
table_text = ""
counter = 0
for row in rows:
table_text += fmtstr.format(*row) + "\n"
if counter == 0 and len(hdrs) != 0:
table_text += "|"
for w in col_widths:
table_text += "-" * (w + 2) + "|"
table_text += "\n"
counter += 1
table_strings.append(table_text + "\n")
newstring = newstring.format(*table_strings)
except Exception:
# we caught an error rebuilding the table.
# just bail and remove the formatting
logger.exception("table parse error")
newstring.replace("{}", "")
return newstring

186
comictalker/talkerbase.py Normal file
View File

@ -0,0 +1,186 @@
"""A template for an information source
"""
# Copyright 2012-2014 Anthony Beville
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import logging
from typing import Callable
from comicapi.genericmetadata import GenericMetadata
from comictalker.resulttypes import ComicIssue, ComicVolume
logger = logging.getLogger(__name__)
# NOTE: Series and Volume are synonymous. Some sources (ComicVine) use "volume" and others (MangaUpdates) use "series".
class SourceDetails:
def __init__(
self,
name: str = "",
ident: str = "",
logo: str = "", # Will be scaled if greater than 100px width and 250px height in comictalker/talkers/logos
):
self.name = name
self.id = ident
self.logo = logo
class SourceStaticOptions:
def __init__(
self,
website: str = "",
has_issues: bool = False,
has_alt_covers: bool = False,
requires_apikey: bool = False,
has_nsfw: bool = False,
has_censored_covers: bool = False,
) -> None:
self.website = website
self.has_issues = has_issues
self.has_alt_covers = has_alt_covers
self.requires_apikey = requires_apikey
self.has_nsfw = has_nsfw
self.has_censored_covers = has_censored_covers
class TalkerError(Exception):
"""Base class exception for information sources.
Attributes:
code -- a numerical code
1 - General
2 - Network
3 - Data
desc -- description of the error
source -- the name of the source producing the error
"""
codes = {
1: "General",
2: "Network",
3: "Data",
4: "Other",
}
def __init__(self, source: str = "", code: int = 4, desc: str = "", sub_code: int = 0) -> None:
super().__init__()
if desc == "":
desc = "Unknown"
self.desc = desc
self.code = code
self.code_name = self.codes[code]
self.sub_code = sub_code
self.source = source
def __str__(self):
return f"{self.source} encountered a {self.code_name} error. {self.desc}"
class TalkerNetworkError(TalkerError):
"""Network class exception for information sources
Attributes:
sub_code -- numerical code for finer detail
1 -- connected refused
2 -- api key
3 -- rate limit
4 -- timeout
"""
net_codes = {
0: "General network error.",
1: "The connection was refused.",
2: "An API key error occurred.",
3: "Rate limit exceeded. Please wait a bit or enter a personal key if using the default.",
4: "The connection timed out.",
5: "Number of retries exceeded.",
}
def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None:
if desc == "":
desc = self.net_codes[sub_code]
super().__init__(source, 2, desc, sub_code)
class TalkerDataError(TalkerError):
"""Data class exception for information sources
Attributes:
sub_code -- numerical code for finer detail
1 -- unexpected data
2 -- malformed data
3 -- missing data
"""
data_codes = {
0: "General data error.",
1: "Unexpected data encountered.",
2: "Malformed data encountered.",
3: "Missing data encountered.",
}
def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None:
if desc == "":
desc = self.data_codes[sub_code]
super().__init__(source, 3, desc, sub_code)
# Class talkers instance
class ComicTalker:
"""The base class for all comic source talkers"""
def __init__(self) -> None:
# Identity name for the information source etc.
self.source_details: SourceDetails = (
SourceDetails()
) # Can use this to test if custom talker has been configured
self.static_options: SourceStaticOptions = SourceStaticOptions()
def check_api_key(self, key: str, url: str) -> bool:
"""If the talker has or requires an API key, this function should test its validity"""
raise NotImplementedError
def search_for_series(
self,
series_name: str,
callback: Callable[[int, int], None] | None = None,
refresh_cache: bool = False,
literal: bool = False,
) -> list[ComicVolume]:
"""Searches for the series/volumes with the given series_name
callback is used for...
refresh_cache signals if the data in the cache should be used
literal indicates that no articles (a, the, etc.) should be removed when searching"""
raise NotImplementedError
def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]:
"""Expected to return a list of issues with a given series/volume ID"""
raise NotImplementedError
def fetch_comic_data(self, issue_id: int = 0, series_id: int = 0, issue_number: str = "") -> GenericMetadata:
"""This function is expected to handle a few possibilities:
1. Only series_id passed in: Retrieve the SERIES/VOLUME information only
2. series_id and issue_number: Retrieve the ISSUE information
3. Only issue_id: Retrieve the ISSUE information"""
raise NotImplementedError
def fetch_issues_by_volume_issue_num_and_year(
self, volume_id_list: list[int], issue_number: str, year: str | int | None
) -> list[ComicIssue]:
"""Searches for a list of issues within the given year. Used solely by issueidentifer"""
raise NotImplementedError

1
comictalker/talkers/__init__.py Normal file
View File

@ -0,0 +1 @@
from __future__ import annotations

674
comictalker/talkers/comicvine.py Normal file
View File

@ -0,0 +1,674 @@
"""ComicVine information source
"""
# Copyright 2012-2014 Anthony Beville
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
import logging
import time
from typing import Any, Callable, cast
from urllib.parse import urljoin, urlsplit
import requests
from typing_extensions import Required, TypedDict
import comictalker.talker_utils as talker_utils
from comicapi import utils
from comicapi.genericmetadata import GenericMetadata
from comicapi.issuestring import IssueString
from comictaggerlib import ctversion
from comictalker.comiccacher import ComicCacher
from comictalker.resulttypes import ComicIssue, ComicVolume, Credits
from comictalker.talkerbase import ComicTalker, SourceDetails, SourceStaticOptions, TalkerDataError, TalkerNetworkError
logger = logging.getLogger(__name__)
class CVTypeID:
Volume = "4050"
Issue = "4000"
class CVImage(TypedDict, total=False):
icon_url: str
medium_url: str
screen_url: str
screen_large_url: str
small_url: str
super_url: Required[str]
thumb_url: str
tiny_url: str
original_url: str
image_tags: str
class CVAltImages(TypedDict):
original_url: str
id: int
caption: str
image_tags: str
class CVPublisher(TypedDict, total=False):
api_detail_url: str
id: int
name: Required[str]
class CVVolume(TypedDict):
api_detail_url: str
id: int
name: str
site_detail_url: str
class CVCredits(TypedDict):
api_detail_url: str
id: int
name: str
site_detail_url: str
class CVPersonCredits(TypedDict):
api_detail_url: str
id: int
name: str
site_detail_url: str
role: str
class CVVolumeResults(TypedDict):
aliases: str
count_of_issues: int
description: str
id: int
image: CVImage
name: str
publisher: CVPublisher
start_year: str
resource_type: str
class CVResult(TypedDict):
error: str
limit: int
offset: int
number_of_page_results: int
number_of_total_results: int
status_code: int
results: (CVIssueDetailResults | CVVolumeResults | list[CVVolumeResults] | list[CVIssueDetailResults])
version: str
class CVVolumeFullResult(TypedDict):
characters: list[CVCredits]
locations: list[CVCredits]
people: list[CVPersonCredits]
site_detail_url: str
count_of_issues: int
description: str
id: int
name: str
publisher: CVPublisher
start_year: str
resource_type: str
class CVIssueDetailResults(TypedDict, total=False):
aliases: str
api_detail_url: str
associated_images: list[CVAltImages]
character_credits: list[CVCredits]
character_died_in: None
concept_credits: list[CVCredits]
cover_date: str
date_added: str
date_last_updated: str
deck: None
description: str
first_appearance_characters: None
first_appearance_concepts: None
first_appearance_locations: None
first_appearance_objects: None
first_appearance_storyarcs: None
first_appearance_teams: None
has_staff_review: bool
id: int
image: CVImage
issue_number: str
location_credits: list[CVCredits]
name: str
object_credits: list[CVCredits]
person_credits: list[CVPersonCredits]
site_detail_url: str
store_date: str
story_arc_credits: list[CVCredits]
team_credits: list[CVCredits]
team_disbanded_in: None
volume: CVVolume
CV_RATE_LIMIT_STATUS = 107
class ComicVineTalker(ComicTalker):
def __init__(
self, api_url, api_key, series_match_thresh, remove_html_tables, use_series_start_as_volume, wait_on_ratelimit
):
super().__init__()
self.source_details = SourceDetails(
name="Comic Vine",
ident="comicvine",
)
self.static_options = SourceStaticOptions(
website="https://comicvine.gamespot.com/",
has_issues=True,
has_alt_covers=True,
requires_apikey=True,
has_nsfw=False,
has_censored_covers=False,
)
# Identity name for the information source
self.source_name = self.source_details.id
self.source_name_friendly = self.source_details.name
self.wait_for_rate_limit = wait_on_ratelimit
# NOTE: This was hardcoded before which is why it isn't passed in
self.wait_for_rate_limit_time = 20
self.issue_id: int | None = None
self.api_key = api_key if api_key else "27431e6787042105bd3e47e169a624521f89f3a4"
self.api_base_url = api_url if api_url else "https://comicvine.gamespot.com/api"
self.remove_html_tables = remove_html_tables
self.use_series_start_as_volume = use_series_start_as_volume
tmp_url = urlsplit(self.api_base_url)
# joinurl only works properly if there is a trailing slash
if tmp_url.path and tmp_url.path[-1] != "/":
tmp_url = tmp_url._replace(path=tmp_url.path + "/")
self.api_base_url = tmp_url.geturl()
self.series_match_thresh = series_match_thresh
def check_api_key(self, key: str, url: str) -> bool:
if not url:
url = self.api_base_url
try:
tmp_url = urlsplit(url)
if tmp_url.path and tmp_url.path[-1] != "/":
tmp_url = tmp_url._replace(path=tmp_url.path + "/")
url = tmp_url.geturl()
test_url = urljoin(url, "issue/1/")
cv_response: CVResult = requests.get(
test_url,
headers={"user-agent": "comictagger/" + ctversion.version},
params={
"api_key": key,
"format": "json",
"field_list": "name",
},
).json()
# Bogus request, but if the key is wrong, you get error 100: "Invalid API Key"
return cv_response["status_code"] != 100
except Exception:
return False
def get_cv_content(self, url: str, params: dict[str, Any]) -> CVResult:
"""
Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error
sleep for a bit and retry.
"""
total_time_waited = 0
limit_wait_time = 1
counter = 0
wait_times = [1, 2, 3, 4]
while True:
cv_response: CVResult = self.get_url_content(url, params)
if self.wait_for_rate_limit and cv_response["status_code"] == CV_RATE_LIMIT_STATUS:
logger.info(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n")
time.sleep(limit_wait_time * 60)
total_time_waited += limit_wait_time
limit_wait_time = wait_times[counter]
if counter < 3:
counter += 1
# don't wait much more than 20 minutes
if total_time_waited < self.wait_for_rate_limit_time:
continue
if cv_response["status_code"] != 1:
logger.debug(
f"{self.source_name_friendly} query failed with error #{cv_response['status_code']}: [{cv_response['error']}]."
)
raise TalkerNetworkError(
self.source_name_friendly, 0, f"{cv_response['status_code']}: {cv_response['error']}"
)
# it's all good
break
return cv_response
def get_url_content(self, url: str, params: dict[str, Any]) -> Any:
# connect to server:
# if there is a 500 error, try a few more times before giving up
# any other error, just bail
for tries in range(3):
try:
resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + ctversion.version})
if resp.status_code == 200:
return resp.json()
if resp.status_code == 500:
logger.debug(f"Try #{tries + 1}: ")
time.sleep(1)
logger.debug(str(resp.status_code))
else:
break
except requests.exceptions.Timeout:
logger.debug(f"Connection to {self.source_name_friendly} timed out.")
raise TalkerNetworkError(self.source_name_friendly, 4)
except requests.exceptions.RequestException as e:
logger.debug(f"Request exception: {e}")
raise TalkerNetworkError(self.source_name_friendly, 0, str(e)) from e
except json.JSONDecodeError as e:
logger.debug(f"JSON decode error: {e}")
raise TalkerDataError(self.source_name_friendly, 2, "ComicVine did not provide json")
raise TalkerNetworkError(self.source_name_friendly, 5)
def format_search_results(self, search_results: list[CVVolumeResults]) -> list[ComicVolume]:
formatted_results = []
for record in search_results:
# Flatten publisher to name only
if record.get("publisher") is None:
pub_name = ""
else:
pub_name = record["publisher"].get("name", "")
if record.get("image") is None:
image_url = ""
else:
image_url = record["image"].get("super_url", "")
if record.get("start_year") is None:
start_year = 0
else:
start_year = utils.xlate(record["start_year"], True)
formatted_results.append(
ComicVolume(
aliases=record["aliases"].split("\n") if record["aliases"] else [], # CV returns a null because...?
count_of_issues=record.get("count_of_issues", 0),
description=record.get("description", ""),
id=record["id"],
image_url=image_url,
name=record["name"],
publisher=pub_name,
start_year=start_year,
)
)
return formatted_results
def format_issue_results(
self, issue_results: list[CVIssueDetailResults], complete: bool = False
) -> list[ComicIssue]:
formatted_results = []
for record in issue_results:
# Extract image super and thumb to name only
if record.get("image") is None:
image_url = ""
image_thumb_url = ""
else:
image_url = record["image"].get("super_url", "")
image_thumb_url = record["image"].get("thumb_url", "")
alt_images_list = []
for alt in record["associated_images"]:
alt_images_list.append(alt["original_url"])
character_list = []
if record.get("character_credits"):
for char in record["character_credits"]:
character_list.append(char["name"])
location_list = []
if record.get("location_credits"):
for loc in record["location_credits"]:
location_list.append(loc["name"])
teams_list = []
if record.get("team_credits"):
for loc in record["team_credits"]:
teams_list.append(loc["name"])
story_list = []
if record.get("story_arc_credits"):
for loc in record["story_arc_credits"]:
story_list.append(loc["name"])
persons_list = []
if record.get("person_credits"):
for person in record["person_credits"]:
persons_list.append(Credits(name=person["name"], role=person["role"]))
formatted_results.append(
ComicIssue(
aliases=record["aliases"].split("\n") if record["aliases"] else [],
cover_date=record.get("cover_date", ""),
description=record.get("description", ""),
id=record["id"],
image_url=image_url,
image_thumb_url=image_thumb_url,
issue_number=record["issue_number"],
name=record["name"],
site_detail_url=record.get("site_detail_url", ""),
volume=cast(ComicVolume, record["volume"]),
alt_image_urls=alt_images_list,
characters=character_list,
locations=location_list,
teams=teams_list,
story_arcs=story_list,
credits=persons_list,
complete=complete,
)
)
return formatted_results
def search_for_series(
self,
series_name: str,
callback: Callable[[int, int], None] | None = None,
refresh_cache: bool = False,
literal: bool = False,
) -> list[ComicVolume]:
# Sanitize the series name for comicvine searching, comicvine search ignore symbols
search_series_name = utils.sanitize_title(series_name, literal)
logger.info(f"{self.source_name_friendly} searching: {search_series_name}")
# Before we search online, look in our cache, since we might have done this same search recently
# For literal searches always retrieve from online
cvc = ComicCacher()
if not refresh_cache and not literal:
cached_search_results = cvc.get_search_results(self.source_name, series_name)
if len(cached_search_results) > 0:
return cached_search_results
params = {
"api_key": self.api_key,
"format": "json",
"resources": "volume",
"query": search_series_name,
"field_list": "volume,name,id,start_year,publisher,image,description,count_of_issues,aliases",
"page": 1,
"limit": 100,
}
cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
search_results: list[CVVolumeResults] = []
# see http://api.comicvine.com/documentation/#handling_responses
current_result_count = cv_response["number_of_page_results"]
total_result_count = cv_response["number_of_total_results"]
# 8 Dec 2018 - Comic Vine changed query results again. Terms are now
# ORed together, and we get thousands of results. Good news is the
# results are sorted by relevance, so we can be smart about halting the search.
# 1. Don't fetch more than some sane amount of pages.
# 2. Halt when any result on the current page is less than or equal to a set ratio using thefuzz
max_results = 500 # 5 pages
total_result_count = min(total_result_count, max_results)
if callback is None:
logger.debug(
f"Found {cv_response['number_of_page_results']} of {cv_response['number_of_total_results']} results"
)
search_results.extend(cast(list[CVVolumeResults], cv_response["results"]))
page = 1
if callback is not None:
callback(current_result_count, total_result_count)
# see if we need to keep asking for more pages...
while current_result_count < total_result_count:
if not literal:
# Stop searching once any entry falls below the threshold
stop_searching = any(
not utils.titles_match(search_series_name, volume["name"], self.series_match_thresh)
for volume in cast(list[CVVolumeResults], cv_response["results"])
)
if stop_searching:
break
if callback is None:
logger.debug(f"getting another page of results {current_result_count} of {total_result_count}...")
page += 1
params["page"] = page
cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
search_results.extend(cast(list[CVVolumeResults], cv_response["results"]))
current_result_count += cv_response["number_of_page_results"]
if callback is not None:
callback(current_result_count, total_result_count)
# Format result to ComicIssue
formatted_search_results = self.format_search_results(search_results)
# Cache these search results, even if it's literal we cache the results
# The most it will cause is extra processing time
cvc.add_search_results(self.source_name, series_name, formatted_search_results)
return formatted_search_results
# Get issue or volume information
def fetch_comic_data(self, issue_id: int = 0, series_id: int = 0, issue_number: str = "") -> GenericMetadata:
comic_data = GenericMetadata()
if issue_number and series_id:
comic_data = self.fetch_issue_data(series_id, issue_number)
elif issue_id:
comic_data = self.fetch_issue_data_by_issue_id(issue_id)
return comic_data
def fetch_partial_volume_data(self, series_id: int) -> ComicVolume:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
cached_volume_result = cvc.get_volume_info(series_id, self.source_name)
if cached_volume_result is not None:
return cached_volume_result
volume_url = urljoin(self.api_base_url, f"volume/{CVTypeID.Volume}-{series_id}")
params = {
"api_key": self.api_key,
"format": "json",
"field_list": "name,id,start_year,publisher,count_of_issues,aliases",
}
cv_response = self.get_cv_content(volume_url, params)
volume_results = cast(CVVolumeResults, cv_response["results"])
formatted_volume_results = self.format_search_results([volume_results])
if volume_results:
cvc.add_volume_info(self.source_name, formatted_volume_results[0])
return formatted_volume_results[0]
def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name)
volume_data = self.fetch_partial_volume_data(series_id)
if len(cached_volume_issues_result) == volume_data["count_of_issues"]:
return cached_volume_issues_result
params = {
"api_key": self.api_key,
"filter": f"volume:{series_id}",
"format": "json",
"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases,associated_images",
"offset": 0,
}
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
current_result_count = cv_response["number_of_page_results"]
total_result_count = cv_response["number_of_total_results"]
volume_issues_result = cast(list[CVIssueDetailResults], cv_response["results"])
page = 1
offset = 0
# see if we need to keep asking for more pages...
while current_result_count < total_result_count:
page += 1
offset += cv_response["number_of_page_results"]
params["offset"] = offset
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
volume_issues_result.extend(cast(list[CVIssueDetailResults], cv_response["results"]))
current_result_count += cv_response["number_of_page_results"]
# Format to expected output !! issues/ volume does NOT return publisher!!
formatted_volume_issues_result = self.format_issue_results(volume_issues_result)
cvc.add_volume_issues_info(self.source_name, formatted_volume_issues_result)
return formatted_volume_issues_result
def fetch_issues_by_volume_issue_num_and_year(
self, volume_id_list: list[int], issue_number: str, year: str | int | None
) -> list[ComicIssue]:
volume_filter = ""
for vid in volume_id_list:
volume_filter += str(vid) + "|"
flt = f"volume:{volume_filter},issue_number:{issue_number}"
int_year = utils.xlate(year, True)
if int_year is not None:
flt += f",cover_date:{int_year}-1-1|{int_year + 1}-1-1"
params: dict[str, str | int] = {
"api_key": self.api_key,
"format": "json",
"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases,associated_images",
"filter": flt,
}
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
current_result_count = cv_response["number_of_page_results"]
total_result_count = cv_response["number_of_total_results"]
filtered_issues_result = cast(list[CVIssueDetailResults], cv_response["results"])
page = 1
offset = 0
# see if we need to keep asking for more pages...
while current_result_count < total_result_count:
page += 1
offset += cv_response["number_of_page_results"]
params["offset"] = offset
cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
filtered_issues_result.extend(cast(list[CVIssueDetailResults], cv_response["results"]))
current_result_count += cv_response["number_of_page_results"]
formatted_filtered_issues_result = self.format_issue_results(filtered_issues_result)
return formatted_filtered_issues_result
def fetch_issue_data(self, series_id: int, issue_number: str) -> GenericMetadata:
issues_list_results = self.fetch_issues_by_volume(series_id)
# Loop through issue list to find the required issue info
f_record = None
for record in issues_list_results:
if not IssueString(issue_number).as_string():
issue_number = "1"
if (
IssueString(record["issue_number"]).as_string().casefold()
== IssueString(issue_number).as_string().casefold()
):
f_record = record
break
if f_record and f_record["complete"]:
# Cache had full record
return talker_utils.map_comic_issue_to_metadata(
f_record,
self.source_name_friendly,
self.remove_html_tables,
self.use_series_start_as_volume,
)
if f_record is not None:
return self.fetch_issue_data_by_issue_id(f_record["id"])
else:
return GenericMetadata()
def fetch_issue_data_by_issue_id(self, issue_id: int) -> GenericMetadata:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
cached_issues_result = cvc.get_issue_info(issue_id, self.source_name)
if cached_issues_result and cached_issues_result["complete"]:
return talker_utils.map_comic_issue_to_metadata(
cached_issues_result,
self.source_name_friendly,
self.remove_html_tables,
self.use_series_start_as_volume,
)
issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
params = {"api_key": self.api_key, "format": "json"}
cv_response = self.get_cv_content(issue_url, params)
issue_results = cast(CVIssueDetailResults, cv_response["results"])
# Format to expected output
formatted_issues_result = self.format_issue_results([issue_results], True)
# Due to issue/ not returning volume publisher, get it.
volume_info = self.fetch_partial_volume_data(formatted_issues_result[0]["volume"]["id"])
formatted_issues_result[0]["volume"]["publisher"] = volume_info["publisher"]
cvc.add_volume_issues_info(self.source_name, formatted_issues_result)
# Now, map the ComicIssue data to generic metadata
return talker_utils.map_comic_issue_to_metadata(
formatted_issues_result[0],
self.source_name_friendly,
self.remove_html_tables,
self.use_series_start_as_volume,
)

24
testing/comicdata.py
View File

@ -1,29 +1,29 @@
from __future__ import annotations
import comicapi.genericmetadata
import comictaggerlib.resulttypes
import comictalker.resulttypes
from comicapi import utils
search_results = [
comictaggerlib.resulttypes.CVVolumeResults(
comictalker.resulttypes.ComicVolume(
count_of_issues=1,
description="this is a description",
id=1,
image={"super_url": "https://test.org/image/1"},
image_url="https://test.org/image/1",
name="test",
publisher=comictaggerlib.resulttypes.CVPublisher(name="test"),
start_year="", # This is currently submitted as a string and returned as an int
aliases=None,
publisher="test",
start_year=0,
aliases=[],
),
comictaggerlib.resulttypes.CVVolumeResults(
comictalker.resulttypes.ComicVolume(
count_of_issues=1,
description="this is a description",
id=1,
image={"super_url": "https://test.org/image/2"},
id=2,
image_url="https://test.org/image/2",
name="test 2",
publisher=comictaggerlib.resulttypes.CVPublisher(name="test"),
start_year="", # This is currently submitted as a string and returned as an int
aliases=None,
publisher="test",
start_year=0,
aliases=[],
),
]

38
testing/comicvine.py
View File

@ -3,7 +3,8 @@ from __future__ import annotations
from typing import Any
import comicapi.genericmetadata
import comictaggerlib.comicvinetalker
from comicapi import utils
from comictalker.talker_utils import cleanup_html
def filter_field_list(cv_result, kwargs):
@ -21,7 +22,7 @@ cv_issue_result: dict[str, Any] = {
"number_of_total_results": 1,
"status_code": 1,
"results": {
"aliases": None,
"aliases": [],
"api_detail_url": "https://comicvine.gamespot.com/api/issue/4000-140529/",
"associated_images": [],
"character_credits": [],
@ -116,7 +117,7 @@ cv_volume_result: dict[str, Any] = {
"number_of_total_results": 1,
"status_code": 1,
"results": {
"aliases": None,
"aliases": [],
"api_detail_url": "https://comicvine.gamespot.com/api/volume/4050-23437/",
"count_of_issues": 6,
"date_added": "2008-10-16 05:25:47",
@ -156,7 +157,24 @@ cv_not_found = {
"status_code": 101,
"results": [],
}
date = comictaggerlib.comicvinetalker.ComicVineTalker().parse_date_str(cv_issue_result["results"]["cover_date"])
comic_issue_result: dict[str, Any] = {
"aliases": cv_issue_result["results"]["aliases"],
"cover_date": cv_issue_result["results"]["cover_date"],
"description": cv_issue_result["results"]["description"],
"id": cv_issue_result["results"]["id"],
"image_url": cv_issue_result["results"]["image"]["super_url"],
"image_thumb_url": cv_issue_result["results"]["image"]["thumb_url"],
"issue_number": cv_issue_result["results"]["issue_number"],
"name": cv_issue_result["results"]["name"],
"site_detail_url": cv_issue_result["results"]["site_detail_url"],
"volume": {
"api_detail_url": cv_issue_result["results"]["volume"]["api_detail_url"],
"id": cv_issue_result["results"]["volume"]["id"],
"name": cv_issue_result["results"]["volume"]["name"],
"site_detail_url": cv_issue_result["results"]["volume"]["site_detail_url"],
},
}
date = utils.parse_date_str(cv_issue_result["results"]["cover_date"])
cv_md = comicapi.genericmetadata.GenericMetadata(
is_empty=False,
@ -173,9 +191,7 @@ cv_md = comicapi.genericmetadata.GenericMetadata(
volume=None,
genre=None,
language=None,
comments=comictaggerlib.comicvinetalker.ComicVineTalker().cleanup_html(
cv_issue_result["results"]["description"], False
),
comments=cleanup_html(cv_issue_result["results"]["description"], False),
volume_count=None,
critical_rating=None,
country=None,
@ -193,9 +209,9 @@ cv_md = comicapi.genericmetadata.GenericMetadata(
story_arc=None,
series_group=None,
scan_info=None,
characters="",
teams="",
locations="",
characters=None,
teams=None,
locations=None,
credits=[
comicapi.genericmetadata.CreditMetadata(person=x["name"], role=x["role"].title(), primary=False)
for x in cv_issue_result["results"]["person_credits"]
@ -207,7 +223,7 @@ cv_md = comicapi.genericmetadata.GenericMetadata(
rights=None,
identifier=None,
last_mark=None,
cover_image=None,
cover_image=cv_issue_result["results"]["image"]["super_url"],
)

2
tests/comicarchive_test.py
View File

@ -16,7 +16,7 @@ def test_getPageNameList():
pageNameList = c.get_page_name_list()
assert pageNameList == [
"!cover.jpg",
"!cover.jpg", # Depending on locale punctuation or numbers might come first (Linux)
"00.jpg",
"page0.jpg",
"Page1.jpeg",

30
tests/comiccacher_test.py
View File

@ -2,13 +2,12 @@ from __future__ import annotations
import pytest
import comictaggerlib.comiccacher
import comictaggerlib.resulttypes
from testing.comicdata import alt_covers, search_results, select_details
import comictalker.comiccacher
from testing.comicdata import search_results
def test_create_cache(settings):
comictaggerlib.comiccacher.ComicCacher()
comictalker.comiccacher.ComicCacher()
assert (settings.get_settings_folder() / "settings").exists()
@ -21,24 +20,13 @@ def test_search_results(comic_cache):
assert search_results == comic_cache.get_search_results("test", "test search")
@pytest.mark.parametrize("alt_cover", alt_covers)
def test_alt_covers(comic_cache, alt_cover):
comic_cache.add_alt_covers(**alt_cover, source_name="test")
assert alt_cover["url_list"] == comic_cache.get_alt_covers(issue_id=alt_cover["issue_id"], source_name="test")
@pytest.mark.parametrize("volume_info", search_results)
def test_volume_info(comic_cache, volume_info):
comic_cache.add_volume_info(cv_volume_record=volume_info, source_name="test")
comic_cache.add_volume_info(volume_record=volume_info, source_name="test")
vi = volume_info.copy()
del vi["description"]
del vi["image"]
assert vi == comic_cache.get_volume_info(volume_id=volume_info["id"], source_name="test")
@pytest.mark.parametrize("details", select_details)
def test_issue_select_details(comic_cache, details):
comic_cache.add_issue_select_details(**details, source_name="test")
det = details.copy()
del det["issue_id"]
assert det == comic_cache.get_issue_select_details(details["issue_id"], "test")
del vi["image_url"]
cache_result = comic_cache.get_volume_info(volume_id=volume_info["id"], source_name="test")
del cache_result["description"]
del cache_result["image_url"]
assert vi == cache_result

92
tests/comicvinetalker_test.py
View File

@ -3,57 +3,74 @@ from __future__ import annotations
import pytest
import comicapi.genericmetadata
import comictaggerlib.comicvinetalker
import comictalker.talkers.comicvine
import testing.comicvine
from testing.comicdata import select_details
def test_search_for_series(comicvine_api, comic_cache):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.search_for_series("cory doctorows futuristic tales of the here and now")
for r in results:
r["image"] = {"super_url": r["image"]["super_url"]}
r["start_year"] = int(r["start_year"])
del r["publisher"]["id"]
del r["publisher"]["api_detail_url"]
cache_issues = comic_cache.get_search_results(ct.source_name, "cory doctorows futuristic tales of the here and now")
assert results == cache_issues
def test_fetch_volume_data(comicvine_api, comic_cache):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
result = ct.fetch_volume_data(23437)
result["start_year"] = int(result["start_year"])
del result["publisher"]["id"]
del result["publisher"]["api_detail_url"]
assert result == comic_cache.get_volume_info(23437, ct.source_name)
ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
result = ct.fetch_partial_volume_data(23437)
del result["description"]
del result["image_url"]
cache_result = comic_cache.get_volume_info(23437, ct.source_name)
del cache_result["description"]
del cache_result["image_url"]
assert result == cache_result
def test_fetch_issues_by_volume(comicvine_api, comic_cache):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.fetch_issues_by_volume(23437)
cache_issues = comic_cache.get_volume_issues_info(23437, ct.source_name)
for r in results:
r["image"] = {"super_url": r["image"]["super_url"], "thumb_url": r["image"]["thumb_url"]}
del r["volume"]
del r["image_thumb_url"]
del r["characters"]
del r["locations"]
del r["story_arcs"]
del r["teams"]
for c in cache_issues:
del c["volume"]
del c["characters"]
del c["locations"]
del c["story_arcs"]
del c["teams"]
assert results == cache_issues
def test_fetch_issue_data_by_issue_id(comicvine_api, settings, mock_version):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
result = ct.fetch_issue_data_by_issue_id(140529, settings)
ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
result = ct.fetch_comic_data(140529)
result.notes = None
assert result == testing.comicvine.cv_md
def test_fetch_issues_by_volume_issue_num_and_year(comicvine_api):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.fetch_issues_by_volume_issue_num_and_year([23437], "1", None)
cv_expected = testing.comicvine.cv_issue_result["results"].copy()
cv_expected = testing.comicvine.comic_issue_result.copy()
testing.comicvine.filter_field_list(
cv_expected,
{"params": {"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases"}},
)
for r, e in zip(results, [cv_expected]):
del r["image_thumb_url"]
del r["image_url"]
del r["alt_image_urls"]
del r["characters"]
del r["credits"]
del r["locations"]
del r["story_arcs"]
del r["teams"]
del r["complete"]
del r["volume"]["publisher"]
assert r == e
@ -66,36 +83,7 @@ cv_issue = [
@pytest.mark.parametrize("volume_id, issue_number, expected", cv_issue)
def test_fetch_issue_data(comicvine_api, settings, mock_version, volume_id, issue_number, expected):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
results = ct.fetch_issue_data(volume_id, issue_number, settings)
ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.fetch_issue_data(volume_id, issue_number)
results.notes = None
assert results == expected
def test_fetch_issue_select_details(comicvine_api, mock_version):
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
result = ct.fetch_issue_select_details(140529)
expected = {
"cover_date": testing.comicvine.cv_issue_result["results"]["cover_date"],
"site_detail_url": testing.comicvine.cv_issue_result["results"]["site_detail_url"],
"image_url": testing.comicvine.cv_issue_result["results"]["image"]["super_url"],
"thumb_image_url": testing.comicvine.cv_issue_result["results"]["image"]["thumb_url"],
}
assert result == expected
@pytest.mark.parametrize("details", select_details)
def test_issue_select_details(comic_cache, details):
expected = details.copy()
del expected["issue_id"]
ct = comictaggerlib.comicvinetalker.ComicVineTalker()
ct.cache_issue_select_details(
issue_id=details["issue_id"],
image_url=details["image_url"],
thumb_url=details["thumb_image_url"],
cover_date=details["cover_date"],
page_url=details["site_detail_url"],
)
result = comic_cache.get_issue_select_details(details["issue_id"], ct.source_name)
assert result == expected

14
tests/conftest.py
View File

@ -13,9 +13,9 @@ from PIL import Image
import comicapi.comicarchive
import comicapi.genericmetadata
import comictaggerlib.comiccacher
import comictaggerlib.comicvinetalker
import comictaggerlib.settings
import comictalker.comiccacher
import comictalker.talkers.comicvine
from comicapi import utils
from testing import comicvine, filenames
from testing.comicdata import all_seed_imprints, seed_imprints
@ -54,7 +54,7 @@ def no_requests(monkeypatch) -> None:
@pytest.fixture
def comicvine_api(monkeypatch, cbz, comic_cache) -> unittest.mock.Mock:
def comicvine_api(monkeypatch, cbz, comic_cache) -> comictalker.talkers.comicvine.ComicVineTalker:
# Any arguments may be passed and mock_get() will always return our
# mocked object, which only has the .json() method or None for invalid urls.
@ -111,7 +111,9 @@ def comicvine_api(monkeypatch, cbz, comic_cache) -> unittest.mock.Mock:
# apply the monkeypatch for requests.get to mock_get
monkeypatch.setattr(requests, "get", m_get)
return m_get
cv = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
return cv
@pytest.fixture
@ -153,5 +155,5 @@ def settings(tmp_path):
@pytest.fixture
def comic_cache(settings) -> Generator[comictaggerlib.comiccacher.ComicCacher, Any, None]:
yield comictaggerlib.comiccacher.ComicCacher()
def comic_cache(settings) -> Generator[comictalker.comiccacher.ComicCacher, Any, None]:
yield comictalker.comiccacher.ComicCacher()

16
tests/issueidentifier_test.py
View File

@ -4,14 +4,13 @@ import pytest
import comicapi.comicarchive
import comicapi.issuestring
import comictaggerlib.comicvinetalker
import comictaggerlib.issueidentifier
import testing.comicdata
import testing.comicvine
def test_crop(cbz_double_cover, settings, tmp_path):
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz_double_cover, settings)
def test_crop(cbz_double_cover, settings, tmp_path, comicvine_api):
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz_double_cover, settings, comicvine_api)
cropped = ii.crop_cover(cbz_double_cover.archiver.read_file("double_cover.jpg"))
original_cover = cbz_double_cover.get_page(0)
@ -22,17 +21,16 @@ def test_crop(cbz_double_cover, settings, tmp_path):
@pytest.mark.parametrize("additional_md, expected", testing.comicdata.metadata_keys)
def test_get_search_keys(cbz, settings, additional_md, expected):
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings)
def test_get_search_keys(cbz, settings, additional_md, expected, comicvine_api):
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api)
ii.set_additional_metadata(additional_md)
assert expected == ii.get_search_keys()
def test_get_issue_cover_match_score(cbz, settings, comicvine_api):
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings)
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api)
score = ii.get_issue_cover_match_score(
comictaggerlib.comicvinetalker.ComicVineTalker(),
int(
comicapi.issuestring.IssueString(
cbz.read_metadata(comicapi.comicarchive.MetaDataStyle.CIX).issue
@ -52,12 +50,13 @@ def test_get_issue_cover_match_score(cbz, settings, comicvine_api):
def test_search(cbz, settings, comicvine_api):
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings)
ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api)
results = ii.search()
cv_expected = {
"series": f"{testing.comicvine.cv_volume_result['results']['name']} ({testing.comicvine.cv_volume_result['results']['start_year']})",
"distance": 0,
"issue_number": testing.comicvine.cv_issue_result["results"]["issue_number"],
"alt_image_urls": [],
"cv_issue_count": testing.comicvine.cv_volume_result["results"]["count_of_issues"],
"issue_title": testing.comicvine.cv_issue_result["results"]["name"],
"issue_id": testing.comicvine.cv_issue_result["results"]["id"],
@ -67,7 +66,6 @@ def test_search(cbz, settings, comicvine_api):
"publisher": testing.comicvine.cv_volume_result["results"]["publisher"]["name"],
"image_url": testing.comicvine.cv_issue_result["results"]["image"]["super_url"],
"thumb_url": testing.comicvine.cv_issue_result["results"]["image"]["thumb_url"],
"page_url": testing.comicvine.cv_issue_result["results"]["site_detail_url"],
"description": testing.comicvine.cv_issue_result["results"]["description"],
}
for r, e in zip(results, [cv_expected]):