diff --git a/comictaggerlib/autotagmatchwindow.py b/comictaggerlib/autotagmatchwindow.py index 16d58c2..516dc04 100644 --- a/comictaggerlib/autotagmatchwindow.py +++ b/comictaggerlib/autotagmatchwindow.py @@ -28,6 +28,7 @@ from comictaggerlib.resulttypes import IssueResult, MultipleMatch from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) @@ -42,6 +43,7 @@ class AutoTagMatchWindow(QtWidgets.QDialog): style: int, fetch_func: Callable[[IssueResult], GenericMetadata], settings: ComicTaggerSettings, + talker_api: ComicTalker, ) -> None: super().__init__(parent) @@ -51,12 +53,12 @@ class AutoTagMatchWindow(QtWidgets.QDialog): self.current_match_set: MultipleMatch = match_set_list[0] - self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode) + self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode) gridlayout = QtWidgets.QGridLayout(self.altCoverContainer) gridlayout.addWidget(self.altCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) - self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode) + self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer) gridlayout.addWidget(self.archiveCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -178,7 +180,10 @@ class AutoTagMatchWindow(QtWidgets.QDialog): if prev is not None and prev.row() == curr.row(): return None - self.altCoverWidget.set_issue_id(self.current_match()["issue_id"]) + self.altCoverWidget.set_issue_details( + self.current_match()["issue_id"], + [self.current_match()["image_url"], *self.current_match()["alt_image_urls"]], + ) if self.current_match()["description"] is None: self.teDescription.setText("") else: @@ -242,15 +247,13 @@ class AutoTagMatchWindow(QtWidgets.QDialog): ) # now get the particular issue data - cv_md = self.fetch_func(match) - if cv_md is None: - QtWidgets.QMessageBox.critical( - self, "Network Issue", "Could not connect to Comic Vine to get issue details!" - ) + ct_md = self.fetch_func(match) + if ct_md is None: + QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not retrieve issue details!") return QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor)) - md.overlay(cv_md) + md.overlay(ct_md) success = ca.write_metadata(md, self._style) ca.load_cache([MetaDataStyle.CBI, MetaDataStyle.CIX]) diff --git a/comictaggerlib/autotagprogresswindow.py b/comictaggerlib/autotagprogresswindow.py index dd49740..477381b 100644 --- a/comictaggerlib/autotagprogresswindow.py +++ b/comictaggerlib/autotagprogresswindow.py @@ -22,22 +22,25 @@ from PyQt5 import QtCore, QtWidgets, uic from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) class AutoTagProgressWindow(QtWidgets.QDialog): - def __init__(self, parent: QtWidgets.QWidget) -> None: + def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None: super().__init__(parent) uic.loadUi(ui_path / "autotagprogresswindow.ui", self) - self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.DataMode, False) + self.archiveCoverWidget = CoverImageWidget( + self.archiveCoverContainer, talker_api, CoverImageWidget.DataMode, False + ) gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer) gridlayout.addWidget(self.archiveCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) - self.testCoverWidget = CoverImageWidget(self.testCoverContainer, CoverImageWidget.DataMode, False) + self.testCoverWidget = CoverImageWidget(self.testCoverContainer, talker_api, CoverImageWidget.DataMode, False) gridlayout = QtWidgets.QGridLayout(self.testCoverContainer) gridlayout.addWidget(self.testCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/autotagstartwindow.py b/comictaggerlib/autotagstartwindow.py index d818719..bae9f19 100644 --- a/comictaggerlib/autotagstartwindow.py +++ b/comictaggerlib/autotagstartwindow.py @@ -48,7 +48,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.cbxAssumeIssueOne.setChecked(self.settings.assume_1_if_no_issue_num) self.cbxIgnoreLeadingDigitsInFilename.setChecked(self.settings.ignore_leading_numbers_in_filename) self.cbxRemoveAfterSuccess.setChecked(self.settings.remove_archive_after_successful_match) - self.cbxWaitForRateLimit.setChecked(self.settings.wait_and_retry_on_rate_limit) self.cbxAutoImprint.setChecked(self.settings.auto_imprint) nlmt_tip = """The Name Match Ratio Threshold: Auto-Identify is for eliminating automatic @@ -73,7 +72,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.assume_issue_one = False self.ignore_leading_digits_in_filename = False self.remove_after_success = False - self.wait_and_retry_on_rate_limit = False self.search_string = "" self.name_length_match_tolerance = self.settings.id_series_match_search_thresh self.split_words = self.cbxSplitWords.isChecked() @@ -91,7 +89,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.ignore_leading_digits_in_filename = self.cbxIgnoreLeadingDigitsInFilename.isChecked() self.remove_after_success = self.cbxRemoveAfterSuccess.isChecked() self.name_length_match_tolerance = self.sbNameMatchSearchThresh.value() - self.wait_and_retry_on_rate_limit = self.cbxWaitForRateLimit.isChecked() self.split_words = self.cbxSplitWords.isChecked() # persist some settings @@ -100,7 +97,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.settings.assume_1_if_no_issue_num = self.assume_issue_one self.settings.ignore_leading_numbers_in_filename = self.ignore_leading_digits_in_filename self.settings.remove_archive_after_successful_match = self.remove_after_success - self.settings.wait_and_retry_on_rate_limit = self.wait_and_retry_on_rate_limit if self.cbxSpecifySearchString.isChecked(): self.search_string = self.leSearchString.text() diff --git a/comictaggerlib/cli.py b/comictaggerlib/cli.py index b73b75d..3a13e2c 100644 --- a/comictaggerlib/cli.py +++ b/comictaggerlib/cli.py @@ -30,32 +30,30 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle from comicapi.genericmetadata import GenericMetadata from comictaggerlib import ctversion from comictaggerlib.cbltransformer import CBLTransformer -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.filerenamer import FileRenamer, get_rename_dir from comictaggerlib.graphics import graphics_path from comictaggerlib.issueidentifier import IssueIdentifier -from comictaggerlib.resulttypes import IssueResult, MultipleMatch, OnlineMatchResults +from comictaggerlib.resulttypes import MultipleMatch, OnlineMatchResults from comictaggerlib.settings import ComicTaggerSettings +from comictalker.talkerbase import ComicTalker, TalkerError logger = logging.getLogger(__name__) def actual_issue_data_fetch( - match: IssueResult, settings: ComicTaggerSettings, opts: argparse.Namespace + issue_id: int, settings: ComicTaggerSettings, opts: argparse.Namespace, talker_api: ComicTalker ) -> GenericMetadata: # now get the particular issue data try: - comic_vine = ComicVineTalker(settings.id_series_match_search_thresh) - comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit - cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], settings) - except ComicVineTalkerException: - logger.exception("Network error while getting issue details. Save aborted") + ct_md = talker_api.fetch_comic_data(issue_id) + except TalkerError as e: + logger.exception(f"Error retrieving issue details. Save aborted.\n{e}") return GenericMetadata() if settings.apply_cbl_transform_on_cv_import: - cv_md = CBLTransformer(cv_md, settings).apply() + ct_md = CBLTransformer(ct_md, settings).apply() - return cv_md + return ct_md def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: GenericMetadata) -> bool: @@ -80,7 +78,11 @@ def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: Generic def display_match_set_for_choice( - label: str, match_set: MultipleMatch, opts: argparse.Namespace, settings: ComicTaggerSettings + label: str, + match_set: MultipleMatch, + opts: argparse.Namespace, + settings: ComicTaggerSettings, + talker_api: ComicTalker, ) -> None: print(f"{match_set.ca.path} -- {label}:") @@ -110,15 +112,15 @@ def display_match_set_for_choice( # we know at this point, that the file is all good to go ca = match_set.ca md = create_local_metadata(opts, ca, settings) - cv_md = actual_issue_data_fetch(match_set.matches[int(i) - 1], settings, opts) + ct_md = actual_issue_data_fetch(match_set.matches[int(i) - 1]["issue_id"], settings, opts, talker_api) if opts.overwrite: - md = cv_md + md = ct_md else: notes = ( f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on" - f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]" + f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]" ) - md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger"))) + md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger"))) if opts.auto_imprint: md.fix_publisher() @@ -127,7 +129,7 @@ def display_match_set_for_choice( def post_process_matches( - match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings + match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker ) -> None: # now go through the match results if opts.show_save_summary: @@ -158,7 +160,7 @@ def post_process_matches( if len(match_results.multiple_matches) > 0: print("\nArchives with multiple high-confidence matches:\n------------------") for match_set in match_results.multiple_matches: - display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings) + display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings, talker_api) if len(match_results.low_confidence_matches) > 0: print("\nArchives with low-confidence matches:\n------------------") @@ -168,10 +170,10 @@ def post_process_matches( else: label = "Multiple low-confidence matches" - display_match_set_for_choice(label, match_set, opts, settings) + display_match_set_for_choice(label, match_set, opts, settings, talker_api) -def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None: +def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None: if len(opts.file_list) < 1: logger.error("You must specify at least one filename. Use the -h option for more info") return @@ -179,10 +181,10 @@ def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None: match_results = OnlineMatchResults() for f in opts.file_list: - process_file_cli(f, opts, settings, match_results) + process_file_cli(f, opts, settings, talker_api, match_results) sys.stdout.flush() - post_process_matches(match_results, opts, settings) + post_process_matches(match_results, opts, settings, talker_api) def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings: ComicTaggerSettings) -> GenericMetadata: @@ -217,7 +219,11 @@ def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings: def process_file_cli( - filename: str, opts: argparse.Namespace, settings: ComicTaggerSettings, match_results: OnlineMatchResults + filename: str, + opts: argparse.Namespace, + settings: ComicTaggerSettings, + talker_api: ComicTalker, + match_results: OnlineMatchResults, ) -> None: batch_mode = len(opts.file_list) > 1 @@ -380,31 +386,29 @@ def process_file_cli( # now, search online if opts.online: if opts.issue_id is not None: - # we were given the actual ID to search with + # we were given the actual issue ID to search with try: - comic_vine = ComicVineTalker(settings.id_series_match_search_thresh) - comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit - cv_md = comic_vine.fetch_issue_data_by_issue_id(opts.issue_id, settings) - except ComicVineTalkerException: - logger.exception("Network error while getting issue details. Save aborted") + ct_md = talker_api.fetch_comic_data(opts.issue_id) + except TalkerError as e: + logger.exception(f"Error retrieving issue details. Save aborted.\n{e}") match_results.fetch_data_failures.append(str(ca.path.absolute())) return - if cv_md is None: + if ct_md is None: logger.error("No match for ID %s was found.", opts.issue_id) match_results.no_matches.append(str(ca.path.absolute())) return if settings.apply_cbl_transform_on_cv_import: - cv_md = CBLTransformer(cv_md, settings).apply() + ct_md = CBLTransformer(ct_md, settings).apply() else: - ii = IssueIdentifier(ca, settings) - if md is None or md.is_empty: logger.error("No metadata given to search online with!") match_results.no_matches.append(str(ca.path.absolute())) return + ii = IssueIdentifier(ca, settings, talker_api) + def myoutput(text: str) -> None: if opts.verbose: IssueIdentifier.default_write_output(text) @@ -412,7 +416,6 @@ def process_file_cli( # use our overlaid MD struct to search ii.set_additional_metadata(md) ii.only_use_additional_meta_data = True - ii.wait_and_retry_on_rate_limit = opts.wait_on_cv_rate_limit ii.set_output_function(myoutput) ii.cover_page_index = md.get_cover_page_index_list()[0] matches = ii.search() @@ -459,19 +462,19 @@ def process_file_cli( # we got here, so we have a single match # now get the particular issue data - cv_md = actual_issue_data_fetch(matches[0], settings, opts) - if cv_md.is_empty: + ct_md = actual_issue_data_fetch(matches[0]["issue_id"], settings, opts, talker_api) + if ct_md.is_empty: match_results.fetch_data_failures.append(str(ca.path.absolute())) return if opts.overwrite: - md = cv_md + md = ct_md else: notes = ( f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on" - f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]" + f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]" ) - md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger"))) + md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger"))) if opts.auto_imprint: md.fix_publisher() diff --git a/comictaggerlib/comicvinetalker.py b/comictaggerlib/comicvinetalker.py deleted file mode 100644 index 13fedaf..0000000 --- a/comictaggerlib/comicvinetalker.py +++ /dev/null @@ -1,805 +0,0 @@ -"""A python class to manage communication with Comic Vine's REST API""" -# -# Copyright 2012-2014 Anthony Beville -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from __future__ import annotations - -import json -import logging -import re -import time -from typing import Any, Callable, cast -from urllib.parse import urlencode, urljoin, urlsplit - -import requests -from bs4 import BeautifulSoup - -from comicapi import utils -from comicapi.genericmetadata import GenericMetadata -from comicapi.issuestring import IssueString -from comictaggerlib import ctversion, resulttypes -from comictaggerlib.comiccacher import ComicCacher -from comictaggerlib.settings import ComicTaggerSettings - -logger = logging.getLogger(__name__) - -try: - from PyQt5 import QtCore, QtNetwork - - qt_available = True -except ImportError: - qt_available = False - -logger = logging.getLogger(__name__) - - -class CVTypeID: - Volume = "4050" - Issue = "4000" - - -class ComicVineTalkerException(Exception): - Unknown = -1 - Network = -2 - InvalidKey = 100 - RateLimit = 107 - - def __init__(self, code: int = -1, desc: str = "") -> None: - super().__init__() - self.desc = desc - self.code = code - - def __str__(self) -> str: - if self.code in (ComicVineTalkerException.Unknown, ComicVineTalkerException.Network): - return self.desc - - return f"CV error #{self.code}: [{self.desc}]. \n" - - -def list_fetch_complete(url_list: list[str]) -> None: - ... - - -def url_fetch_complete(image_url: str, thumb_url: str | None) -> None: - ... - - -class ComicVineTalker: - logo_url = "http://static.comicvine.com/bundles/comicvinesite/images/logo.png" - api_key = "" - api_base_url = "" - - alt_url_list_fetch_complete = list_fetch_complete - url_fetch_complete = url_fetch_complete - - @staticmethod - def get_rate_limit_message() -> str: - if ComicVineTalker.api_key == "": - return "Comic Vine rate limit exceeded. You should configure your own Comic Vine API key." - - return "Comic Vine rate limit exceeded. Please wait a bit." - - def __init__(self, series_match_thresh: int = 90) -> None: - # Identity name for the information source - self.source_name = "comicvine" - - self.wait_for_rate_limit = False - self.series_match_thresh = series_match_thresh - - # key that is registered to comictagger - default_api_key = "27431e6787042105bd3e47e169a624521f89f3a4" - default_url = "https://comicvine.gamespot.com/api" - - self.issue_id: int | None = None - - self.api_key = ComicVineTalker.api_key or default_api_key - tmp_url = urlsplit(ComicVineTalker.api_base_url or default_url) - - # joinurl only works properly if there is a trailing slash - if tmp_url.path and tmp_url.path[-1] != "/": - tmp_url = tmp_url._replace(path=tmp_url.path + "/") - - self.api_base_url = tmp_url.geturl() - - self.log_func: Callable[[str], None] | None = None - - if qt_available: - self.nam = QtNetwork.QNetworkAccessManager() - - def set_log_func(self, log_func: Callable[[str], None]) -> None: - self.log_func = log_func - - def write_log(self, text: str) -> None: - if self.log_func is None: - logger.info(text) - else: - self.log_func(text) - - def parse_date_str(self, date_str: str) -> tuple[int | None, int | None, int | None]: - return utils.parse_date_str(date_str) - - def test_key(self, key: str, url: str) -> bool: - if not url: - url = self.api_base_url - try: - test_url = urljoin(url, "issue/1/") - - cv_response: resulttypes.CVResult = requests.get( - test_url, - headers={"user-agent": "comictagger/" + ctversion.version}, - params={ - "api_key": key, - "format": "json", - "field_list": "name", - }, - ).json() - - # Bogus request, but if the key is wrong, you get error 100: "Invalid API Key" - return cv_response["status_code"] != 100 - except Exception: - return False - - def get_cv_content(self, url: str, params: dict[str, Any]) -> resulttypes.CVResult: - """ - Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error - sleep for a bit and retry. - """ - total_time_waited = 0 - limit_wait_time = 1 - counter = 0 - wait_times = [1, 2, 3, 4] - while True: - cv_response: resulttypes.CVResult = self.get_url_content(url, params) - if self.wait_for_rate_limit and cv_response["status_code"] == ComicVineTalkerException.RateLimit: - self.write_log(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n") - time.sleep(limit_wait_time * 60) - total_time_waited += limit_wait_time - limit_wait_time = wait_times[counter] - if counter < 3: - counter += 1 - # don't wait much more than 20 minutes - if total_time_waited < 20: - continue - if cv_response["status_code"] != 1: - self.write_log( - f"Comic Vine query failed with error #{cv_response['status_code']}: [{cv_response['error']}]. \n" - ) - raise ComicVineTalkerException(cv_response["status_code"], cv_response["error"]) - - # it's all good - break - return cv_response - - def get_url_content(self, url: str, params: dict[str, Any]) -> Any: - # connect to server: - # if there is a 500 error, try a few more times before giving up - # any other error, just bail - for tries in range(3): - try: - resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + ctversion.version}) - if resp.status_code == 200: - return resp.json() - if resp.status_code == 500: - self.write_log(f"Try #{tries + 1}: ") - time.sleep(1) - self.write_log(str(resp.status_code) + "\n") - else: - break - - except requests.exceptions.RequestException as e: - self.write_log(f"{e}\n") - raise ComicVineTalkerException(ComicVineTalkerException.Network, "Network Error!") from e - except json.JSONDecodeError as e: - self.write_log(f"{e}\n") - raise ComicVineTalkerException(ComicVineTalkerException.Unknown, "ComicVine did not provide json") - - raise ComicVineTalkerException( - ComicVineTalkerException.Unknown, f"Error on Comic Vine server: {resp.status_code}" - ) - - def search_for_series( - self, - series_name: str, - callback: Callable[[int, int], None] | None = None, - refresh_cache: bool = False, - literal: bool = False, - ) -> list[resulttypes.CVVolumeResults]: - - # Sanitize the series name for comicvine searching, comicvine search ignore symbols - search_series_name = utils.sanitize_title(series_name, literal) - logger.info("Searching: %s", search_series_name) - - # Before we search online, look in our cache, since we might have done this same search recently - # For literal searches always retrieve from online - cvc = ComicCacher() - if not refresh_cache and not literal: - cached_search_results = cvc.get_search_results(self.source_name, series_name) - - if len(cached_search_results) > 0: - return cached_search_results - - params = { - "api_key": self.api_key, - "format": "json", - "resources": "volume", - "query": search_series_name, - "field_list": "volume,name,id,start_year,publisher,image,description,count_of_issues,aliases", - "page": 1, - "limit": 100, - } - - cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params) - - search_results: list[resulttypes.CVVolumeResults] = [] - - # see http://api.comicvine.com/documentation/#handling_responses - - current_result_count = cv_response["number_of_page_results"] - total_result_count = cv_response["number_of_total_results"] - - # 8 Dec 2018 - Comic Vine changed query results again. Terms are now - # ORed together, and we get thousands of results. Good news is the - # results are sorted by relevance, so we can be smart about halting the search. - # 1. Don't fetch more than some sane amount of pages. - # 2. Halt when any result on the current page is less than or equal to a set ratio using rapidfuzz - max_results = 500 # 5 pages - - total_result_count = min(total_result_count, max_results) - - if callback is None: - self.write_log( - f"Found {cv_response['number_of_page_results']} of {cv_response['number_of_total_results']} results\n" - ) - search_results.extend(cast(list[resulttypes.CVVolumeResults], cv_response["results"])) - page = 1 - - if callback is not None: - callback(current_result_count, total_result_count) - - # see if we need to keep asking for more pages... - while current_result_count < total_result_count: - - if not literal: - # Stop searching once any entry falls below the threshold - stop_searching = any( - not utils.titles_match(search_series_name, volume["name"], self.series_match_thresh) - for volume in cast(list[resulttypes.CVVolumeResults], cv_response["results"]) - ) - - if stop_searching: - break - - if callback is None: - self.write_log(f"getting another page of results {current_result_count} of {total_result_count}...\n") - page += 1 - - params["page"] = page - cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params) - - search_results.extend(cast(list[resulttypes.CVVolumeResults], cv_response["results"])) - current_result_count += cv_response["number_of_page_results"] - - if callback is not None: - callback(current_result_count, total_result_count) - - # Cache these search results, even if it's literal we cache the results - # The most it will cause is extra processing time - cvc.add_search_results(self.source_name, series_name, search_results) - - return search_results - - def fetch_volume_data(self, series_id: int) -> resulttypes.CVVolumeResults: - - # before we search online, look in our cache, since we might already have this info - cvc = ComicCacher() - cached_volume_result = cvc.get_volume_info(series_id, self.source_name) - - if cached_volume_result is not None: - return cached_volume_result - - volume_url = urljoin(self.api_base_url, f"volume/{CVTypeID.Volume}-{series_id}") - - params = { - "api_key": self.api_key, - "format": "json", - "field_list": "name,id,start_year,publisher,count_of_issues,aliases", - } - cv_response = self.get_cv_content(volume_url, params) - - volume_results = cast(resulttypes.CVVolumeResults, cv_response["results"]) - - if volume_results: - cvc.add_volume_info(self.source_name, volume_results) - - return volume_results - - def fetch_issues_by_volume(self, series_id: int) -> list[resulttypes.CVIssuesResults]: - # before we search online, look in our cache, since we might already have this info - volume_data = self.fetch_volume_data(series_id) - cvc = ComicCacher() - cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name) - - if len(cached_volume_issues_result) >= volume_data["count_of_issues"]: - return cached_volume_issues_result - - params = { - "api_key": self.api_key, - "filter": f"volume:{series_id}", - "format": "json", - "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases", - "offset": 0, - } - cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) - - current_result_count = cv_response["number_of_page_results"] - total_result_count = cv_response["number_of_total_results"] - - volume_issues_result = cast(list[resulttypes.CVIssuesResults], cv_response["results"]) - page = 1 - offset = 0 - - # see if we need to keep asking for more pages... - while current_result_count < total_result_count: - page += 1 - offset += cv_response["number_of_page_results"] - - params["offset"] = offset - cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) - - volume_issues_result.extend(cast(list[resulttypes.CVIssuesResults], cv_response["results"])) - current_result_count += cv_response["number_of_page_results"] - - self.repair_urls(volume_issues_result) - - cvc.add_volume_issues_info(self.source_name, series_id, volume_issues_result) - - return volume_issues_result - - def fetch_issues_by_volume_issue_num_and_year( - self, volume_id_list: list[int], issue_number: str, year: str | int | None - ) -> list[resulttypes.CVIssuesResults]: - volume_filter = "" - for vid in volume_id_list: - volume_filter += str(vid) + "|" - flt = f"volume:{volume_filter},issue_number:{issue_number}" - - int_year = utils.xlate(year, True) - if int_year is not None: - flt += f",cover_date:{int_year}-1-1|{int_year+1}-1-1" - - params: dict[str, str | int] = { - "api_key": self.api_key, - "format": "json", - "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases", - "filter": flt, - } - - cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) - - current_result_count = cv_response["number_of_page_results"] - total_result_count = cv_response["number_of_total_results"] - - filtered_issues_result = cast(list[resulttypes.CVIssuesResults], cv_response["results"]) - page = 1 - offset = 0 - - # see if we need to keep asking for more pages... - while current_result_count < total_result_count: - page += 1 - offset += cv_response["number_of_page_results"] - - params["offset"] = offset - cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) - - filtered_issues_result.extend(cast(list[resulttypes.CVIssuesResults], cv_response["results"])) - current_result_count += cv_response["number_of_page_results"] - - self.repair_urls(filtered_issues_result) - - cvc = ComicCacher() - for c in filtered_issues_result: - cvc.add_volume_issues_info(self.source_name, c["volume"]["id"], [c]) - - return filtered_issues_result - - def fetch_issue_data(self, series_id: int, issue_number: str, settings: ComicTaggerSettings) -> GenericMetadata: - volume_results = self.fetch_volume_data(series_id) - issues_list_results = self.fetch_issues_by_volume(series_id) - - f_record = None - for record in issues_list_results: - if not IssueString(issue_number).as_string(): - issue_number = "1" - if ( - IssueString(record["issue_number"]).as_string().casefold() - == IssueString(issue_number).as_string().casefold() - ): - f_record = record - break - - if f_record is not None: - issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{f_record['id']}") - params = {"api_key": self.api_key, "format": "json"} - cv_response = self.get_cv_content(issue_url, params) - issue_results = cast(resulttypes.CVIssueDetailResults, cv_response["results"]) - - else: - return GenericMetadata() - - # Now, map the Comic Vine data to generic metadata - return self.map_cv_data_to_metadata(volume_results, issue_results, settings) - - def fetch_issue_data_by_issue_id(self, issue_id: int, settings: ComicTaggerSettings) -> GenericMetadata: - - issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}") - params = {"api_key": self.api_key, "format": "json"} - cv_response = self.get_cv_content(issue_url, params) - - issue_results = cast(resulttypes.CVIssueDetailResults, cv_response["results"]) - - volume_results = self.fetch_volume_data(issue_results["volume"]["id"]) - - # Now, map the Comic Vine data to generic metadata - md = self.map_cv_data_to_metadata(volume_results, issue_results, settings) - md.is_empty = False - return md - - def map_cv_data_to_metadata( - self, - volume_results: resulttypes.CVVolumeResults, - issue_results: resulttypes.CVIssueDetailResults, - settings: ComicTaggerSettings, - ) -> GenericMetadata: - - # Now, map the Comic Vine data to generic metadata - metadata = GenericMetadata() - metadata.is_empty = False - metadata.tag_origin = "Comic Vine" - metadata.issue_id = issue_results["id"] - - metadata.series = utils.xlate(issue_results["volume"]["name"]) - metadata.issue = IssueString(issue_results["issue_number"]).as_string() - metadata.title = utils.xlate(issue_results["name"]) - - if volume_results["publisher"] is not None: - metadata.publisher = utils.xlate(volume_results["publisher"]["name"]) - metadata.day, metadata.month, metadata.year = self.parse_date_str(issue_results["cover_date"]) - - metadata.comments = self.cleanup_html(issue_results["description"], settings.remove_html_tables) - if settings.use_series_start_as_volume: - metadata.volume = int(volume_results["start_year"]) - - metadata.web_link = issue_results["site_detail_url"] - - person_credits = issue_results["person_credits"] - for person in person_credits: - if "role" in person: - roles = person["role"].split(",") - for role in roles: - # can we determine 'primary' from CV?? - metadata.add_credit(person["name"], role.title().strip(), False) - - character_credits = issue_results["character_credits"] - character_list = [] - for character in character_credits: - character_list.append(character["name"]) - metadata.characters = ", ".join(character_list) - - team_credits = issue_results["team_credits"] - team_list = [] - for team in team_credits: - team_list.append(team["name"]) - metadata.teams = ", ".join(team_list) - - location_credits = issue_results["location_credits"] - location_list = [] - for location in location_credits: - location_list.append(location["name"]) - metadata.locations = ", ".join(location_list) - - story_arc_credits = issue_results["story_arc_credits"] - arc_list = [] - for arc in story_arc_credits: - arc_list.append(arc["name"]) - if len(arc_list) > 0: - metadata.story_arc = ", ".join(arc_list) - - return metadata - - def cleanup_html(self, string: str, remove_html_tables: bool) -> str: - if string is None: - return "" - # find any tables - soup = BeautifulSoup(string, "html.parser") - tables = soup.findAll("table") - - # remove all newlines first - string = string.replace("\n", "") - - # put in our own - string = string.replace("
", "\n") - string = string.replace("", "\n") - string = string.replace("

", "\n\n") - string = string.replace("

", "*") - string = string.replace("

", "*\n") - string = string.replace("

", "*") - string = string.replace("

", "*\n") - string = string.replace("

", "*") - string = string.replace("

", "*\n") - string = string.replace("

", "*") - string = string.replace("

", "*\n") - string = string.replace("
", "*") - string = string.replace("
", "*\n") - string = string.replace("
", "*") - string = string.replace("
", "*\n") - - # remove the tables - p = re.compile(r".*?") - if remove_html_tables: - string = p.sub("", string) - string = string.replace("*List of covers and their creators:*", "") - else: - string = p.sub("{}", string) - - # now strip all other tags - p = re.compile(r"<[^<]*?>") - newstring = p.sub("", string) - - newstring = newstring.replace(" ", " ") - newstring = newstring.replace("&", "&") - - newstring = newstring.strip() - - if not remove_html_tables: - # now rebuild the tables into text from BSoup - try: - table_strings = [] - for table in tables: - rows = [] - hdrs = [] - col_widths = [] - for hdr in table.findAll("th"): - item = hdr.string.strip() - hdrs.append(item) - col_widths.append(len(item)) - rows.append(hdrs) - - for row in table.findAll("tr"): - cols = [] - col = row.findAll("td") - i = 0 - for c in col: - item = c.string.strip() - cols.append(item) - if len(item) > col_widths[i]: - col_widths[i] = len(item) - i += 1 - if len(cols) != 0: - rows.append(cols) - # now we have the data, make it into text - fmtstr = "" - for w in col_widths: - fmtstr += f" {{:{w + 1}}}|" - width = sum(col_widths) + len(col_widths) * 2 - table_text = "" - counter = 0 - for row in rows: - table_text += fmtstr.format(*row) + "\n" - if counter == 0 and len(hdrs) != 0: - table_text += "-" * width + "\n" - counter += 1 - - table_strings.append(table_text) - - newstring = newstring.format(*table_strings) - except Exception: - # we caught an error rebuilding the table. - # just bail and remove the formatting - logger.exception("table parse error") - newstring.replace("{}", "") - - return newstring - - def fetch_issue_date(self, issue_id: int) -> tuple[int | None, int | None]: - details = self.fetch_issue_select_details(issue_id) - _, month, year = self.parse_date_str(details["cover_date"] or "") - return month, year - - def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]: - details = self.fetch_issue_select_details(issue_id) - return details["image_url"], details["thumb_image_url"] - - def fetch_issue_page_url(self, issue_id: int) -> str | None: - details = self.fetch_issue_select_details(issue_id) - return details["site_detail_url"] - - def fetch_issue_select_details(self, issue_id: int) -> resulttypes.SelectDetails: - cached_details = self.fetch_cached_issue_select_details(issue_id) - if cached_details["image_url"] is not None: - return cached_details - - issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}") - logger.error("%s, %s", self.api_base_url, issue_url) - - params = {"api_key": self.api_key, "format": "json", "field_list": "image,cover_date,site_detail_url"} - - cv_response = self.get_cv_content(issue_url, params) - results = cast(resulttypes.CVIssueDetailResults, cv_response["results"]) - - details = resulttypes.SelectDetails( - image_url=results["image"]["super_url"], - thumb_image_url=results["image"]["thumb_url"], - cover_date=results["cover_date"], - site_detail_url=results["site_detail_url"], - ) - - if ( - details["image_url"] is not None - and details["thumb_image_url"] is not None - and details["cover_date"] is not None - and details["site_detail_url"] is not None - ): - self.cache_issue_select_details( - issue_id, - details["image_url"], - details["thumb_image_url"], - details["cover_date"], - details["site_detail_url"], - ) - return details - - def fetch_cached_issue_select_details(self, issue_id: int) -> resulttypes.SelectDetails: - - # before we search online, look in our cache, since we might already have this info - cvc = ComicCacher() - return cvc.get_issue_select_details(issue_id, self.source_name) - - def cache_issue_select_details( - self, issue_id: int, image_url: str, thumb_url: str, cover_date: str, page_url: str - ) -> None: - cvc = ComicCacher() - cvc.add_issue_select_details(self.source_name, issue_id, image_url, thumb_url, cover_date, page_url) - - def fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> list[str]: - url_list = self.fetch_cached_alternate_cover_urls(issue_id) - if url_list: - return url_list - - # scrape the CV issue page URL to get the alternate cover URLs - content = requests.get(issue_page_url, headers={"user-agent": "comictagger/" + ctversion.version}).text - alt_cover_url_list = self.parse_out_alt_cover_urls(content) - - # cache this alt cover URL list - self.cache_alternate_cover_urls(issue_id, alt_cover_url_list) - - return alt_cover_url_list - - def parse_out_alt_cover_urls(self, page_html: str) -> list[str]: - soup = BeautifulSoup(page_html, "html.parser") - - alt_cover_url_list = [] - - # Using knowledge of the layout of the Comic Vine issue page here: - # look for the divs that are in the classes 'imgboxart' and 'issue-cover' - div_list = soup.find_all("div") - covers_found = 0 - for d in div_list: - if "class" in d.attrs: - c = d["class"] - if "imgboxart" in c and "issue-cover" in c: - if d.img["src"].startswith("http"): - covers_found += 1 - if covers_found != 1: - alt_cover_url_list.append(d.img["src"]) - elif d.img["data-src"].startswith("http"): - covers_found += 1 - if covers_found != 1: - alt_cover_url_list.append(d.img["data-src"]) - - return alt_cover_url_list - - def fetch_cached_alternate_cover_urls(self, issue_id: int) -> list[str]: - - # before we search online, look in our cache, since we might already have this info - cvc = ComicCacher() - url_list = cvc.get_alt_covers(self.source_name, issue_id) - - return url_list - - def cache_alternate_cover_urls(self, issue_id: int, url_list: list[str]) -> None: - cvc = ComicCacher() - cvc.add_alt_covers(self.source_name, issue_id, url_list) - - def async_fetch_issue_cover_urls(self, issue_id: int) -> None: - - self.issue_id = issue_id - details = self.fetch_cached_issue_select_details(issue_id) - if details["image_url"] is not None: - ComicVineTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"]) - return - - issue_url = urlsplit(self.api_base_url) - issue_url = issue_url._replace( - query=urlencode( - { - "api_key": self.api_key, - "format": "json", - "field_list": "image,cover_date,site_detail_url", - } - ), - path=f"issue/{CVTypeID.Issue}-{issue_id}", - ) - - self.nam.finished.connect(self.async_fetch_issue_cover_url_complete) - self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(issue_url.geturl()))) - - def async_fetch_issue_cover_url_complete(self, reply: QtNetwork.QNetworkReply) -> None: - # read in the response - data = reply.readAll() - - try: - cv_response = cast(resulttypes.CVResult, json.loads(bytes(data))) - except Exception: - logger.exception("Comic Vine query failed to get JSON data\n%s", str(data)) - return - - if cv_response["status_code"] != 1: - logger.error("Comic Vine query failed with error: [%s]. ", cv_response["error"]) - return - - result = cast(resulttypes.CVIssuesResults, cv_response["results"]) - - image_url = result["image"]["super_url"] - thumb_url = result["image"]["thumb_url"] - cover_date = result["cover_date"] - page_url = result["site_detail_url"] - - self.cache_issue_select_details(cast(int, self.issue_id), image_url, thumb_url, cover_date, page_url) - - ComicVineTalker.url_fetch_complete(image_url, thumb_url) - - def async_fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> None: - # This async version requires the issue page url to be provided! - self.issue_id = issue_id - url_list = self.fetch_cached_alternate_cover_urls(issue_id) - if url_list: - ComicVineTalker.alt_url_list_fetch_complete(url_list) - return - - self.nam.finished.connect(self.async_fetch_alternate_cover_urls_complete) - self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_page_url)))) - - def async_fetch_alternate_cover_urls_complete(self, reply: QtNetwork.QNetworkReply) -> None: - # read in the response - html = str(reply.readAll()) - alt_cover_url_list = self.parse_out_alt_cover_urls(html) - - # cache this alt cover URL list - self.cache_alternate_cover_urls(cast(int, self.issue_id), alt_cover_url_list) - - ComicVineTalker.alt_url_list_fetch_complete(alt_cover_url_list) - - def repair_urls( - self, - issue_list: list[resulttypes.CVIssuesResults] - | list[resulttypes.CVVolumeResults] - | list[resulttypes.CVIssueDetailResults], - ) -> None: - # make sure there are URLs for the image fields - for issue in issue_list: - if issue["image"] is None: - issue["image"] = resulttypes.CVImage( - super_url=ComicVineTalker.logo_url, - thumb_url=ComicVineTalker.logo_url, - ) diff --git a/comictaggerlib/coverimagewidget.py b/comictaggerlib/coverimagewidget.py index 6c3d8f7..ecd19a4 100644 --- a/comictaggerlib/coverimagewidget.py +++ b/comictaggerlib/coverimagewidget.py @@ -20,19 +20,17 @@ TODO: This should be re-factored using subclasses! from __future__ import annotations import logging -from typing import Callable, cast from PyQt5 import QtCore, QtGui, QtWidgets, uic -from comicapi import utils from comicapi.comicarchive import ComicArchive -from comictaggerlib.comicvinetalker import ComicVineTalker from comictaggerlib.graphics import graphics_path from comictaggerlib.imagefetcher import ImageFetcher from comictaggerlib.imagepopup import ImagePopup from comictaggerlib.pageloader import PageLoader from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import get_qimage_from_data, reduce_widget_font_size +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) @@ -56,39 +54,17 @@ def clickable(widget: QtWidgets.QWidget) -> QtCore.pyqtBoundSignal: return flt.dblclicked -class Signal(QtCore.QObject): - alt_url_list_fetch_complete = QtCore.pyqtSignal(list) - url_fetch_complete = QtCore.pyqtSignal(str, str) - image_fetch_complete = QtCore.pyqtSignal(QtCore.QByteArray) - - def __init__( - self, - list_fetch: Callable[[list[str]], None], - url_fetch: Callable[[str, str], None], - image_fetch: Callable[[bytes], None], - ) -> None: - super().__init__() - self.alt_url_list_fetch_complete.connect(list_fetch) - self.url_fetch_complete.connect(url_fetch) - self.image_fetch_complete.connect(image_fetch) - - def emit_list(self, url_list: list[str]) -> None: - self.alt_url_list_fetch_complete.emit(url_list) - - def emit_url(self, image_url: str, thumb_url: str | None) -> None: - self.url_fetch_complete.emit(image_url, thumb_url) - - def emit_image(self, image_data: bytes | QtCore.QByteArray) -> None: - self.image_fetch_complete.emit(image_data) - - class CoverImageWidget(QtWidgets.QWidget): ArchiveMode = 0 AltCoverMode = 1 URLMode = 1 DataMode = 3 - def __init__(self, parent: QtWidgets.QWidget, mode: int, expand_on_click: bool = True) -> None: + image_fetch_complete = QtCore.pyqtSignal(QtCore.QByteArray) + + def __init__( + self, parent: QtWidgets.QWidget, talker_api: ComicTalker, mode: int, expand_on_click: bool = True + ) -> None: super().__init__(parent) self.cover_fetcher = ImageFetcher() @@ -96,10 +72,7 @@ class CoverImageWidget(QtWidgets.QWidget): reduce_widget_font_size(self.label) - self.sig = Signal( - self.alt_cover_url_list_fetch_complete, self.primary_url_fetch_complete, self.cover_remote_fetch_complete - ) - + self.talker_api = talker_api self.mode: int = mode self.page_loader: PageLoader | None = None self.showControls = True @@ -108,6 +81,7 @@ class CoverImageWidget(QtWidgets.QWidget): self.comic_archive: ComicArchive | None = None self.issue_id: int | None = None + self.issue_url: str | None = None self.url_list: list[str] = [] if self.page_loader is not None: self.page_loader.abandoned = True @@ -121,6 +95,7 @@ class CoverImageWidget(QtWidgets.QWidget): self.btnLeft.clicked.connect(self.decrement_image) self.btnRight.clicked.connect(self.increment_image) + self.image_fetch_complete.connect(self.cover_remote_fetch_complete) if expand_on_click: clickable(self.lblImage).connect(self.show_popup) else: @@ -131,6 +106,7 @@ class CoverImageWidget(QtWidgets.QWidget): def reset_widget(self) -> None: self.comic_archive = None self.issue_id = None + self.issue_url = None self.url_list = [] if self.page_loader is not None: self.page_loader.abandoned = True @@ -173,15 +149,13 @@ class CoverImageWidget(QtWidgets.QWidget): self.imageCount = 1 self.update_content() - def set_issue_id(self, issue_id: int) -> None: + def set_issue_details(self, issue_id: int, url_list: list[str]) -> None: if self.mode == CoverImageWidget.AltCoverMode: self.reset_widget() self.update_content() self.issue_id = issue_id - comic_vine = ComicVineTalker() - ComicVineTalker.url_fetch_complete = self.sig.emit_url - comic_vine.async_fetch_issue_cover_urls(self.issue_id) + self.set_url_list(url_list) def set_image_data(self, image_data: bytes) -> None: if self.mode == CoverImageWidget.DataMode: @@ -195,31 +169,11 @@ class CoverImageWidget(QtWidgets.QWidget): self.update_content() - def primary_url_fetch_complete(self, primary_url: str, thumb_url: str | None = None) -> None: - self.url_list.append(str(primary_url)) + def set_url_list(self, url_list: list[str]) -> None: + self.url_list = url_list self.imageIndex = 0 self.imageCount = len(self.url_list) self.update_content() - - # defer the alt cover search - QtCore.QTimer.singleShot(1, self.start_alt_cover_search) - - def start_alt_cover_search(self) -> None: - - if self.issue_id is not None: - # now we need to get the list of alt cover URLs - self.label.setText("Searching for alt. covers...") - - # page URL should already be cached, so no need to defer - comic_vine = ComicVineTalker() - issue_page_url = comic_vine.fetch_issue_page_url(self.issue_id) - ComicVineTalker.alt_url_list_fetch_complete = self.sig.emit_list - comic_vine.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), cast(str, issue_page_url)) - - def alt_cover_url_list_fetch_complete(self, url_list: list[str]) -> None: - if url_list: - self.url_list.extend(url_list) - self.imageCount = len(self.url_list) self.update_controls() def set_page(self, pagenum: int) -> None: @@ -269,7 +223,7 @@ class CoverImageWidget(QtWidgets.QWidget): def load_url(self) -> None: self.load_default() self.cover_fetcher = ImageFetcher() - ImageFetcher.image_fetch_complete = self.sig.emit_image + ImageFetcher.image_fetch_complete = self.image_fetch_complete.emit self.cover_fetcher.fetch(self.url_list[self.imageIndex]) # called when the image is done loading from internet diff --git a/comictaggerlib/imagefetcher.py b/comictaggerlib/imagefetcher.py index b4c55f3..214f2b8 100644 --- a/comictaggerlib/imagefetcher.py +++ b/comictaggerlib/imagefetcher.py @@ -81,7 +81,8 @@ class ImageFetcher: # first look in the DB image_data = self.get_image_from_cache(url) - if blocking or not qt_available: + # Async for retrieving covers seems to work well + if blocking: # if blocking or not qt_available: if not image_data: try: image_data = requests.get(url, headers={"user-agent": "comictagger/" + ctversion.version}).content diff --git a/comictaggerlib/issueidentifier.py b/comictaggerlib/issueidentifier.py index 4f6d316..3ed87c3 100644 --- a/comictaggerlib/issueidentifier.py +++ b/comictaggerlib/issueidentifier.py @@ -26,11 +26,12 @@ from comicapi import utils from comicapi.comicarchive import ComicArchive from comicapi.genericmetadata import GenericMetadata from comicapi.issuestring import IssueString -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.imagefetcher import ImageFetcher, ImageFetcherException from comictaggerlib.imagehasher import ImageHasher from comictaggerlib.resulttypes import IssueResult from comictaggerlib.settings import ComicTaggerSettings +from comictalker.talker_utils import parse_date_str +from comictalker.talkerbase import ComicTalker, TalkerError logger = logging.getLogger(__name__) @@ -72,8 +73,9 @@ class IssueIdentifier: result_one_good_match = 4 result_multiple_good_matches = 5 - def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings) -> None: + def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None: self.settings = settings + self.talker_api = talker_api self.comic_archive: ComicArchive = comic_archive self.image_hasher = 1 @@ -107,7 +109,6 @@ class IssueIdentifier: self.search_result = self.result_no_matches self.cover_page_index = 0 self.cancel = False - self.wait_and_retry_on_rate_limit = False self.match_list: list[IssueResult] = [] @@ -239,11 +240,10 @@ class IssueIdentifier: def get_issue_cover_match_score( self, - comic_vine: ComicVineTalker, issue_id: int, primary_img_url: str, primary_thumb_url: str, - page_url: str, + alt_urls: list[str], local_cover_hash_list: list[int], use_remote_alternates: bool = False, use_log: bool = True, @@ -274,8 +274,7 @@ class IssueIdentifier: raise IssueIdentifierCancelled if use_remote_alternates: - alt_img_url_list = comic_vine.fetch_alternate_cover_urls(issue_id, page_url) - for alt_url in alt_img_url_list: + for alt_url in alt_urls: try: alt_url_image_data = ImageFetcher().fetch(alt_url, blocking=True) except ImageFetcherException as e: @@ -369,27 +368,22 @@ class IssueIdentifier: if keys["month"] is not None: self.log_msg("\tMonth: " + str(keys["month"])) - comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh) - comic_vine.wait_for_rate_limit = self.wait_and_retry_on_rate_limit - - comic_vine.set_log_func(self.output_function) - self.log_msg(f"Searching for {keys['series']} #{keys['issue_number']} ...") try: - cv_search_results = comic_vine.search_for_series(keys["series"]) - except ComicVineTalkerException: - self.log_msg("Network issue while searching for series. Aborting...") + ct_search_results = self.talker_api.search_for_series(keys["series"]) + except TalkerError as e: + self.log_msg(f"Error searching for series.\n{e}") return [] if self.cancel: return [] - if cv_search_results is None: + if ct_search_results is None: return [] series_second_round_list = [] - for item in cv_search_results: + for item in ct_search_results: length_approved = False publisher_approved = True date_approved = True @@ -404,16 +398,13 @@ class IssueIdentifier: if int(keys["year"]) < int(item["start_year"]): date_approved = False - aliases = [] - if item["aliases"]: - aliases = item["aliases"].split("\n") - for name in [item["name"], *aliases]: + for name in [item["name"], *item["aliases"]]: if utils.titles_match(keys["series"], name, self.series_match_thresh): length_approved = True break # remove any series from publishers on the filter if item["publisher"] is not None: - publisher = item["publisher"]["name"] + publisher = item["publisher"] if publisher is not None and publisher.casefold() in self.publisher_filter: publisher_approved = False @@ -436,12 +427,11 @@ class IssueIdentifier: issue_list = None try: if len(volume_id_list) > 0: - issue_list = comic_vine.fetch_issues_by_volume_issue_num_and_year( + issue_list = self.talker_api.fetch_issues_by_volume_issue_num_and_year( volume_id_list, keys["issue_number"], keys["year"] ) - - except ComicVineTalkerException: - self.log_msg("Network issue while searching for series details. Aborting...") + except TalkerError as e: + self.log_msg(f"Issue with while searching for series details. Aborting...\n{e}") return [] if issue_list is None: @@ -476,7 +466,7 @@ class IssueIdentifier: ) # parse out the cover date - _, month, year = comic_vine.parse_date_str(issue["cover_date"]) + _, month, year = parse_date_str(issue["cover_date"]) # Now check the cover match against the primary image hash_list = [cover_hash] @@ -484,16 +474,15 @@ class IssueIdentifier: hash_list.append(narrow_cover_hash) try: - image_url = issue["image"]["super_url"] - thumb_url = issue["image"]["thumb_url"] - page_url = issue["site_detail_url"] + image_url = issue["image_url"] + thumb_url = issue["image_thumb_url"] + alt_urls = issue["alt_image_urls"] score_item = self.get_issue_cover_match_score( - comic_vine, issue["id"], image_url, thumb_url, - page_url, + alt_urls, hash_list, use_remote_alternates=False, ) @@ -515,11 +504,12 @@ class IssueIdentifier: "publisher": None, "image_url": image_url, "thumb_url": thumb_url, - "page_url": page_url, + # "page_url": page_url, + "alt_image_urls": alt_urls, "description": issue["description"], } if series["publisher"] is not None: - match["publisher"] = series["publisher"]["name"] + match["publisher"] = series["publisher"] self.match_list.append(match) @@ -577,15 +567,15 @@ class IssueIdentifier: self.log_msg(f"Examining alternate covers for ID: {m['volume_id']} {m['series']} ...", newline=False) try: score_item = self.get_issue_cover_match_score( - comic_vine, m["issue_id"], m["image_url"], m["thumb_url"], - m["page_url"], + m["alt_image_urls"], hash_list, use_remote_alternates=True, ) except Exception: + logger.exception("failed examining alt covers") self.match_list = [] return self.match_list self.log_msg(f"--->{score_item['score']}") diff --git a/comictaggerlib/issueselectionwindow.py b/comictaggerlib/issueselectionwindow.py index b8015ff..e1b6e0c 100644 --- a/comictaggerlib/issueselectionwindow.py +++ b/comictaggerlib/issueselectionwindow.py @@ -20,12 +20,12 @@ import logging from PyQt5 import QtCore, QtGui, QtWidgets, uic from comicapi.issuestring import IssueString -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.coverimagewidget import CoverImageWidget -from comictaggerlib.resulttypes import CVIssuesResults from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.resulttypes import ComicIssue +from comictalker.talkerbase import ComicTalker, TalkerError logger = logging.getLogger(__name__) @@ -42,13 +42,18 @@ class IssueSelectionWindow(QtWidgets.QDialog): volume_id = 0 def __init__( - self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, series_id: int, issue_number: str + self, + parent: QtWidgets.QWidget, + settings: ComicTaggerSettings, + talker_api: ComicTalker, + series_id: int, + issue_number: str, ) -> None: super().__init__(parent) uic.loadUi(ui_path / "issueselectionwindow.ui", self) - self.coverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.AltCoverMode) + self.coverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.AltCoverMode) gridlayout = QtWidgets.QGridLayout(self.coverImageContainer) gridlayout.addWidget(self.coverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -67,8 +72,9 @@ class IssueSelectionWindow(QtWidgets.QDialog): self.series_id = series_id self.issue_id: int | None = None self.settings = settings + self.talker_api = talker_api self.url_fetch_thread = None - self.issue_list: list[CVIssuesResults] = [] + self.issue_list: list[ComicIssue] = [] if issue_number is None or issue_number == "": self.issue_number = "1" @@ -98,15 +104,14 @@ class IssueSelectionWindow(QtWidgets.QDialog): QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor)) try: - comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh) - comic_vine.fetch_volume_data(self.series_id) - self.issue_list = comic_vine.fetch_issues_by_volume(self.series_id) - except ComicVineTalkerException as e: + self.issue_list = self.talker_api.fetch_issues_by_volume(self.series_id) + except TalkerError as e: QtWidgets.QApplication.restoreOverrideCursor() - if e.code == ComicVineTalkerException.RateLimit: - QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message()) - else: - QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not connect to Comic Vine to list issues!") + QtWidgets.QMessageBox.critical( + self, + f"{e.source} {e.code_name} Error", + f"{e}", + ) return self.twList.setRowCount(0) @@ -175,7 +180,7 @@ class IssueSelectionWindow(QtWidgets.QDialog): for record in self.issue_list: if record["id"] == self.issue_id: self.issue_number = record["issue_number"] - self.coverWidget.set_issue_id(self.issue_id) + self.coverWidget.set_issue_details(self.issue_id, [record["image_url"], *record["alt_image_urls"]]) if record["description"] is None: self.teDescription.setText("") else: diff --git a/comictaggerlib/main.py b/comictaggerlib/main.py index 6657685..60fb8e7 100755 --- a/comictaggerlib/main.py +++ b/comictaggerlib/main.py @@ -25,13 +25,14 @@ import sys import traceback import types +import comictalker.comictalkerapi as ct_api from comicapi import utils from comictaggerlib import cli -from comictaggerlib.comicvinetalker import ComicVineTalker from comictaggerlib.ctversion import version from comictaggerlib.graphics import graphics_path from comictaggerlib.options import parse_cmd_line from comictaggerlib.settings import ComicTaggerSettings +from comictalker.talkerbase import TalkerError if sys.version_info < (3, 10): import importlib_metadata @@ -41,6 +42,7 @@ else: logger = logging.getLogger("comictagger") logging.getLogger("comicapi").setLevel(logging.DEBUG) logging.getLogger("comictaggerlib").setLevel(logging.DEBUG) +logging.getLogger("comictalker").setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG) try: @@ -157,19 +159,6 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati file=sys.stderr, ) - # manage the CV API key - # None comparison is used so that the empty string can unset the value - if opts.cv_api_key is not None or opts.cv_url is not None: - settings.cv_api_key = opts.cv_api_key if opts.cv_api_key is not None else settings.cv_api_key - settings.cv_url = opts.cv_url if opts.cv_url is not None else settings.cv_url - settings.save() - if opts.only_set_cv_key: - print("Key set") # noqa: T201 - return - - ComicVineTalker.api_key = settings.cv_api_key - ComicVineTalker.api_base_url = settings.cv_url - signal.signal(signal.SIGINT, signal.SIG_DFL) logger.info( @@ -183,6 +172,28 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati for pkg in sorted(importlib_metadata.distributions(), key=lambda x: x.name): logger.debug("%s\t%s", pkg.metadata["Name"], pkg.metadata["Version"]) + talker_failed = False + try: + talker_api = ct_api.get_comic_talker("comicvine")( + settings.cv_url, + settings.cv_api_key, + settings.id_series_match_search_thresh, + settings.remove_html_tables, + settings.use_series_start_as_volume, + settings.wait_and_retry_on_rate_limit, + ) + except TalkerError as te: + talker_failed = True + logger.warning(f"Unable to load talker {te.source}. Error: {te.desc}. Defaulting to Comic Vine.") + talker_api = ct_api.get_comic_talker("comicvine")( + settings.cv_url, + settings.cv_api_key, + settings.id_series_match_search_thresh, + settings.remove_html_tables, + settings.use_series_start_as_volume, + settings.wait_and_retry_on_rate_limit, + ) + utils.load_publishers() update_publishers() @@ -192,7 +203,7 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati if opts.no_gui: try: - cli.cli_mode(opts, settings) + cli.cli_mode(opts, settings, talker_api) except Exception: logger.exception("CLI mode failed") else: @@ -232,7 +243,7 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati QtWidgets.QApplication.processEvents() try: - tagger_window = TaggerWindow(opts.files, settings, opts=opts) + tagger_window = TaggerWindow(opts.files, settings, talker_api, opts=opts) tagger_window.setWindowIcon(QtGui.QIcon(str(graphics_path / "app.png"))) tagger_window.show() @@ -242,6 +253,13 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati if platform.system() != "Linux": splash.finish(tagger_window) + if talker_failed: + QtWidgets.QMessageBox.warning( + QtWidgets.QMainWindow(), + "Warning", + "Unable to load configured information source, see log for details. Defaulting to Comic Vine", + ) + sys.exit(app.exec()) except Exception: logger.exception("GUI mode failed") diff --git a/comictaggerlib/matchselectionwindow.py b/comictaggerlib/matchselectionwindow.py index 27bb16a..330318c 100644 --- a/comictaggerlib/matchselectionwindow.py +++ b/comictaggerlib/matchselectionwindow.py @@ -25,6 +25,7 @@ from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.resulttypes import IssueResult from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) @@ -32,17 +33,23 @@ logger = logging.getLogger(__name__) class MatchSelectionWindow(QtWidgets.QDialog): volume_id = 0 - def __init__(self, parent: QtWidgets.QWidget, matches: list[IssueResult], comic_archive: ComicArchive) -> None: + def __init__( + self, + parent: QtWidgets.QWidget, + matches: list[IssueResult], + comic_archive: ComicArchive, + talker_api: ComicTalker, + ) -> None: super().__init__(parent) uic.loadUi(ui_path / "matchselectionwindow.ui", self) - self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode) + self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode) gridlayout = QtWidgets.QGridLayout(self.altCoverContainer) gridlayout.addWidget(self.altCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) - self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode) + self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer) gridlayout.addWidget(self.archiveCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -142,7 +149,10 @@ class MatchSelectionWindow(QtWidgets.QDialog): if prev is not None and prev.row() == curr.row(): return - self.altCoverWidget.set_issue_id(self.current_match()["issue_id"]) + self.altCoverWidget.set_issue_details( + self.current_match()["issue_id"], + [self.current_match()["image_url"], *self.current_match()["alt_image_urls"]], + ) if self.current_match()["description"] is None: self.teDescription.setText("") else: diff --git a/comictaggerlib/options.py b/comictaggerlib/options.py index 884979c..be9aecf 100644 --- a/comictaggerlib/options.py +++ b/comictaggerlib/options.py @@ -80,6 +80,7 @@ def define_args() -> argparse.ArgumentParser: action="store_true", help="Export RAR archive to Zip format.", ) + # TODO: update for new api commands.add_argument( "--only-set-cv-key", action="store_true", @@ -107,10 +108,12 @@ def define_args() -> argparse.ArgumentParser: dest="config_path", help="""Config directory defaults to ~/.ComicTagger\non Linux/Mac and %%APPDATA%% on Windows\n""", ) + # TODO: update for new api parser.add_argument( "--cv-api-key", help="Use the given Comic Vine API Key (persisted in settings).", ) + # TODO: update for new api parser.add_argument( "--cv-url", help="Use the given Comic Vine URL (persisted in settings).", @@ -214,6 +217,7 @@ def define_args() -> argparse.ArgumentParser: action="store_true", help="Be noisy when doing what it does.", ) + # TODO: update for new api parser.add_argument( "-w", "--wait-on-cv-rate-limit", @@ -369,7 +373,7 @@ def parse_cmd_line() -> argparse.Namespace: opts.copy, opts.rename, opts.export_to_zip, - opts.only_set_cv_key, + opts.only_set_cv_key, # TODO: update for new api ] ) @@ -385,9 +389,11 @@ def parse_cmd_line() -> argparse.Namespace: for item in globs: opts.files.extend(glob.glob(item)) + # TODO: update for new api if opts.only_set_cv_key and opts.cv_api_key is None and opts.cv_url is None: parser.exit(message="Key not given!\n", status=1) + # TODO: update for new api if not opts.only_set_cv_key and opts.no_gui and not opts.files: parser.exit(message="Command requires at least one filename!\n", status=1) diff --git a/comictaggerlib/pagebrowser.py b/comictaggerlib/pagebrowser.py index d536fc5..f2e2387 100644 --- a/comictaggerlib/pagebrowser.py +++ b/comictaggerlib/pagebrowser.py @@ -25,17 +25,18 @@ from comicapi.genericmetadata import GenericMetadata from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.graphics import graphics_path from comictaggerlib.ui import ui_path +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) class PageBrowserWindow(QtWidgets.QDialog): - def __init__(self, parent: QtWidgets.QWidget, metadata: GenericMetadata) -> None: + def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker, metadata: GenericMetadata) -> None: super().__init__(parent) uic.loadUi(ui_path / "pagebrowser.ui", self) - self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode) + self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.pageContainer) gridlayout.addWidget(self.pageWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/pagelisteditor.py b/comictaggerlib/pagelisteditor.py index 88df8e0..39b71a9 100644 --- a/comictaggerlib/pagelisteditor.py +++ b/comictaggerlib/pagelisteditor.py @@ -23,6 +23,7 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle from comicapi.genericmetadata import ImageMetadata, PageType from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.ui import ui_path +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) @@ -67,12 +68,12 @@ class PageListEditor(QtWidgets.QWidget): PageType.Deleted: "Deleted", } - def __init__(self, parent: QtWidgets.QWidget) -> None: + def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None: super().__init__(parent) uic.loadUi(ui_path / "pagelisteditor.ui", self) - self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode) + self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.pageContainer) gridlayout.addWidget(self.pageWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/renamewindow.py b/comictaggerlib/renamewindow.py index b752faa..b732579 100644 --- a/comictaggerlib/renamewindow.py +++ b/comictaggerlib/renamewindow.py @@ -27,6 +27,7 @@ from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.settingswindow import SettingsWindow from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import center_window_on_parent +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) @@ -38,6 +39,7 @@ class RenameWindow(QtWidgets.QDialog): comic_archive_list: list[ComicArchive], data_style: int, settings: ComicTaggerSettings, + talker_api: ComicTalker, ) -> None: super().__init__(parent) @@ -53,6 +55,7 @@ class RenameWindow(QtWidgets.QDialog): ) self.settings = settings + self.talker_api = talker_api self.comic_archive_list = comic_archive_list self.data_style = data_style self.rename_list: list[str] = [] @@ -161,7 +164,7 @@ class RenameWindow(QtWidgets.QDialog): self.twList.setSortingEnabled(True) def modify_settings(self) -> None: - settingswin = SettingsWindow(self, self.settings) + settingswin = SettingsWindow(self, self.settings, self.talker_api) settingswin.setModal(True) settingswin.show_rename_tab() settingswin.exec() diff --git a/comictaggerlib/resulttypes.py b/comictaggerlib/resulttypes.py index 0f53ae8..4261fa5 100644 --- a/comictaggerlib/resulttypes.py +++ b/comictaggerlib/resulttypes.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing_extensions import NotRequired, Required, TypedDict +from typing_extensions import TypedDict from comicapi.comicarchive import ComicArchive @@ -19,7 +19,7 @@ class IssueResult(TypedDict): publisher: str | None image_url: str thumb_url: str - page_url: str + alt_image_urls: list[str] description: str @@ -37,126 +37,3 @@ class MultipleMatch: def __init__(self, ca: ComicArchive, match_list: list[IssueResult]) -> None: self.ca: ComicArchive = ca self.matches: list[IssueResult] = match_list - - -class SelectDetails(TypedDict): - image_url: str | None - thumb_image_url: str | None - cover_date: str | None - site_detail_url: str | None - - -class CVResult(TypedDict): - error: str - limit: int - offset: int - number_of_page_results: int - number_of_total_results: int - status_code: int - results: ( - CVIssuesResults - | CVIssueDetailResults - | CVVolumeResults - | list[CVIssuesResults] - | list[CVVolumeResults] - | list[CVIssueDetailResults] - ) - version: str - - -class CVImage(TypedDict, total=False): - icon_url: str - medium_url: str - screen_url: str - screen_large_url: str - small_url: str - super_url: Required[str] - thumb_url: str - tiny_url: str - original_url: str - image_tags: str - - -class CVVolume(TypedDict): - api_detail_url: str - id: int - name: str - site_detail_url: str - - -class CVIssuesResults(TypedDict): - cover_date: str - description: str - id: int - image: CVImage - issue_number: str - name: str - site_detail_url: str - volume: NotRequired[CVVolume] - aliases: str - - -class CVPublisher(TypedDict, total=False): - api_detail_url: str - id: int - name: Required[str] - - -class CVVolumeResults(TypedDict): - count_of_issues: int - description: NotRequired[str] - id: int - image: NotRequired[CVImage] - name: str - publisher: CVPublisher - start_year: str - resource_type: NotRequired[str] - aliases: NotRequired[str | None] - - -class CVCredits(TypedDict): - api_detail_url: str - id: int - name: str - site_detail_url: str - - -class CVPersonCredits(TypedDict): - api_detail_url: str - id: int - name: str - site_detail_url: str - role: str - - -class CVIssueDetailResults(TypedDict): - aliases: None - api_detail_url: str - character_credits: list[CVCredits] - character_died_in: None - concept_credits: list[CVCredits] - cover_date: str - date_added: str - date_last_updated: str - deck: None - description: str - first_appearance_characters: None - first_appearance_concepts: None - first_appearance_locations: None - first_appearance_objects: None - first_appearance_storyarcs: None - first_appearance_teams: None - has_staff_review: bool - id: int - image: CVImage - issue_number: str - location_credits: list[CVCredits] - name: str - object_credits: list[CVCredits] - person_credits: list[CVPersonCredits] - site_detail_url: str - store_date: str - story_arc_credits: list[CVCredits] - team_credits: list[CVCredits] - team_disbanded_in: None - volume: CVVolume diff --git a/comictaggerlib/settingswindow.py b/comictaggerlib/settingswindow.py index c829be6..7cdfa09 100644 --- a/comictaggerlib/settingswindow.py +++ b/comictaggerlib/settingswindow.py @@ -25,12 +25,12 @@ from PyQt5 import QtCore, QtGui, QtWidgets, uic from comicapi import utils from comicapi.genericmetadata import md_test -from comictaggerlib.comiccacher import ComicCacher -from comictaggerlib.comicvinetalker import ComicVineTalker from comictaggerlib.filerenamer import FileRenamer from comictaggerlib.imagefetcher import ImageFetcher from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui import ui_path +from comictalker.comiccacher import ComicCacher +from comictalker.talkerbase import ComicTalker logger = logging.getLogger(__name__) @@ -128,7 +128,7 @@ Spider-Geddon #1 - New Players; Check In class SettingsWindow(QtWidgets.QDialog): - def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings) -> None: + def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None: super().__init__(parent) uic.loadUi(ui_path / "settingswindow.ui", self) @@ -138,6 +138,7 @@ class SettingsWindow(QtWidgets.QDialog): ) self.settings = settings + self.talker_api = talker_api self.name = "Settings" if platform.system() == "Windows": @@ -326,10 +327,13 @@ class SettingsWindow(QtWidgets.QDialog): self.settings.sort_series_by_year = self.cbxSortByYear.isChecked() self.settings.exact_series_matches_first = self.cbxExactMatches.isChecked() - self.settings.cv_api_key = self.leKey.text().strip() - ComicVineTalker.api_key = self.settings.cv_api_key - self.settings.cv_url = self.leURL.text().strip() - ComicVineTalker.api_base_url = self.settings.cv_url + # Ignore empty field + if self.leKey.text().strip(): + self.settings.cv_api_key = self.leKey.text().strip() + self.talker_api.api_key = self.settings.cv_api_key + if self.leURL.text().strip(): + self.settings.cv_url = self.leURL.text().strip() + self.talker_api.api_base_url = self.settings.cv_url self.settings.assume_lone_credit_is_primary = self.cbxAssumeLoneCreditIsPrimary.isChecked() self.settings.copy_characters_to_tags = self.cbxCopyCharactersToTags.isChecked() self.settings.copy_teams_to_tags = self.cbxCopyTeamsToTags.isChecked() @@ -361,7 +365,7 @@ class SettingsWindow(QtWidgets.QDialog): QtWidgets.QMessageBox.information(self, self.name, "Cache has been cleared.") def test_api_key(self) -> None: - if ComicVineTalker().test_key(self.leKey.text().strip(), self.leURL.text().strip()): + if self.talker_api.check_api_key(self.leKey.text().strip(), self.leURL.text().strip()): QtWidgets.QMessageBox.information(self, "API Key Test", "Key is valid!") else: QtWidgets.QMessageBox.warning(self, "API Key Test", "Key is NOT valid.") diff --git a/comictaggerlib/taggerwindow.py b/comictaggerlib/taggerwindow.py index 74c2212..52bade0 100644 --- a/comictaggerlib/taggerwindow.py +++ b/comictaggerlib/taggerwindow.py @@ -46,7 +46,6 @@ from comictaggerlib.autotagmatchwindow import AutoTagMatchWindow from comictaggerlib.autotagprogresswindow import AutoTagProgressWindow from comictaggerlib.autotagstartwindow import AutoTagStartWindow from comictaggerlib.cbltransformer import CBLTransformer -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.crediteditorwindow import CreditEditorWindow from comictaggerlib.exportwindow import ExportConflictOpts, ExportWindow @@ -65,6 +64,7 @@ from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import center_window_on_parent, reduce_widget_font_size from comictaggerlib.versionchecker import VersionChecker from comictaggerlib.volumeselectionwindow import VolumeSelectionWindow +from comictalker.talkerbase import ComicTalker, TalkerError logger = logging.getLogger(__name__) @@ -81,6 +81,7 @@ class TaggerWindow(QtWidgets.QMainWindow): self, file_list: list[str], settings: ComicTaggerSettings, + talker_api: ComicTalker, parent: QtWidgets.QWidget | None = None, opts: argparse.Namespace | None = None, ) -> None: @@ -88,6 +89,7 @@ class TaggerWindow(QtWidgets.QMainWindow): uic.loadUi(ui_path / "taggerwindow.ui", self) self.settings = settings + self.talker_api = talker_api self.log_window = self.setup_logger() # prevent multiple instances @@ -121,12 +123,12 @@ class TaggerWindow(QtWidgets.QMainWindow): ) sys.exit() - self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.ArchiveMode) + self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.ArchiveMode) grid_layout = QtWidgets.QGridLayout(self.coverImageContainer) grid_layout.addWidget(self.archiveCoverWidget) grid_layout.setContentsMargins(0, 0, 0, 0) - self.page_list_editor = PageListEditor(self.tabPages) + self.page_list_editor = PageListEditor(self.tabPages, self.talker_api) grid_layout = QtWidgets.QGridLayout(self.tabPages) grid_layout.addWidget(self.page_list_editor) @@ -1046,6 +1048,7 @@ You have {4-self.settings.settings_warning} warnings left. issue_number = str(self.leIssueNum.text()).strip() + # Only need this check is the source has issue level data. if autoselect and issue_number == "": QtWidgets.QMessageBox.information( self, "Automatic Identify Search", "Can't auto-identify without an issue number (yet!)" @@ -1072,6 +1075,7 @@ You have {4-self.settings.settings_warning} warnings left. cover_index_list, cast(ComicArchive, self.comic_archive), self.settings, + self.talker_api, autoselect, literal, ) @@ -1089,16 +1093,23 @@ You have {4-self.settings.settings_warning} warnings left. self.form_to_metadata() try: - comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh) - new_metadata = comic_vine.fetch_issue_data(selector.volume_id, selector.issue_number, self.settings) - except ComicVineTalkerException as e: - QtWidgets.QApplication.restoreOverrideCursor() - if e.code == ComicVineTalkerException.RateLimit: - QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message()) - else: - QtWidgets.QMessageBox.critical( - self, "Network Issue", "Could not connect to Comic Vine to get issue details.!" + if selector.issue_id: + new_metadata = self.talker_api.fetch_comic_data(selector.issue_id) + elif selector.volume_id and selector.issue_number: + # Would this ever be needed? + new_metadata = self.talker_api.fetch_comic_data( + series_id=selector.volume_id, issue_number=selector.issue_number ) + else: + # Only left with series? Isn't series only handled elsewhere? + new_metadata = self.talker_api.fetch_comic_data(series_id=selector.volume_id) + except TalkerError as e: + QtWidgets.QApplication.restoreOverrideCursor() + QtWidgets.QMessageBox.critical( + self, + f"{e.source} {e.code_name} Error", + f"{e}", + ) else: QtWidgets.QApplication.restoreOverrideCursor() if new_metadata is not None: @@ -1109,7 +1120,7 @@ You have {4-self.settings.settings_warning} warnings left. self.clear_form() notes = ( - f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on" + f"Tagged with ComicTagger {ctversion.version} using info from {self.talker_api.source_details.name} on" f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {new_metadata.issue_id}]" ) self.metadata.overlay( @@ -1390,7 +1401,7 @@ You have {4-self.settings.settings_warning} warnings left. def show_settings(self) -> None: - settingswin = SettingsWindow(self, self.settings) + settingswin = SettingsWindow(self, self.settings, self.talker_api) settingswin.setModal(True) settingswin.exec() settingswin.result() @@ -1694,24 +1705,22 @@ You have {4-self.settings.settings_warning} warnings left. def actual_issue_data_fetch(self, match: IssueResult) -> GenericMetadata: - # now get the particular issue data - cv_md = GenericMetadata() + # now get the particular issue data OR series data + ct_md = GenericMetadata() QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor)) try: - comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh) - comic_vine.wait_for_rate_limit = self.settings.wait_and_retry_on_rate_limit - cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], self.settings) - except ComicVineTalkerException: - logger.exception("Network error while getting issue details. Save aborted") + ct_md = self.talker_api.fetch_comic_data(match["issue_id"]) + except TalkerError as e: + logger.exception(f"Save aborted.\n{e}") - if not cv_md.is_empty: + if not ct_md.is_empty: if self.settings.apply_cbl_transform_on_cv_import: - cv_md = CBLTransformer(cv_md, self.settings).apply() + ct_md = CBLTransformer(ct_md, self.settings).apply() QtWidgets.QApplication.restoreOverrideCursor() - return cv_md + return ct_md def auto_tag_log(self, text: str) -> None: IssueIdentifier.default_write_output(text) @@ -1726,7 +1735,7 @@ You have {4-self.settings.settings_warning} warnings left. self, ca: ComicArchive, match_results: OnlineMatchResults, dlg: AutoTagStartWindow ) -> tuple[bool, OnlineMatchResults]: success = False - ii = IssueIdentifier(ca, self.settings) + ii = IssueIdentifier(ca, self.settings, self.talker_api) # read in metadata, and parse file name if not there try: @@ -1763,7 +1772,6 @@ You have {4-self.settings.settings_warning} warnings left. md.issue = utils.xlate(md.volume) ii.set_additional_metadata(md) ii.only_use_additional_meta_data = True - ii.wait_and_retry_on_rate_limit = dlg.wait_and_retry_on_rate_limit ii.set_output_function(self.auto_tag_log) ii.cover_page_index = md.get_cover_page_index_list()[0] if self.atprogdialog is not None: @@ -1812,19 +1820,19 @@ You have {4-self.settings.settings_warning} warnings left. self.auto_tag_log("Online search: Low confidence match, but saving anyways, as indicated...\n") # now get the particular issue data - cv_md = self.actual_issue_data_fetch(matches[0]) - if cv_md is None: + ct_md = self.actual_issue_data_fetch(matches[0]) + if ct_md is None: match_results.fetch_data_failures.append(str(ca.path.absolute())) - if cv_md is not None: + if ct_md is not None: if dlg.cbxRemoveMetadata.isChecked(): - md = cv_md + md = ct_md else: notes = ( f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on" - f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]" + f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]" ) - md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger"))) + md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger"))) if self.settings.auto_imprint: md.fix_publisher() @@ -1867,7 +1875,7 @@ You have {4-self.settings.settings_warning} warnings left. if not atstartdlg.exec(): return - self.atprogdialog = AutoTagProgressWindow(self) + self.atprogdialog = AutoTagProgressWindow(self, self.talker_api) self.atprogdialog.setModal(True) self.atprogdialog.show() self.atprogdialog.progressBar.setMaximum(len(ca_list)) @@ -1954,7 +1962,12 @@ You have {4-self.settings.settings_warning} warnings left. match_results.multiple_matches.extend(match_results.low_confidence_matches) if reply == QtWidgets.QMessageBox.StandardButton.Yes: matchdlg = AutoTagMatchWindow( - self, match_results.multiple_matches, style, self.actual_issue_data_fetch, self.settings + self, + match_results.multiple_matches, + style, + self.actual_issue_data_fetch, + self.settings, + self.talker_api, ) matchdlg.setModal(True) matchdlg.exec() @@ -2017,7 +2030,7 @@ You have {4-self.settings.settings_warning} warnings left. def show_page_browser(self) -> None: if self.page_browser is None: - self.page_browser = PageBrowserWindow(self, self.metadata) + self.page_browser = PageBrowserWindow(self, self.talker_api, self.metadata) if self.comic_archive is not None: self.page_browser.set_comic_archive(self.comic_archive) self.page_browser.finished.connect(self.page_browser_closed) @@ -2084,7 +2097,7 @@ You have {4-self.settings.settings_warning} warnings left. "File Rename", "If you rename files now, unsaved data in the form will be lost. Are you sure?" ): - dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings) + dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings, self.talker_api) dlg.setModal(True) if dlg.exec() and self.comic_archive is not None: self.fileSelectionList.update_selected_rows() diff --git a/comictaggerlib/volumeselectionwindow.py b/comictaggerlib/volumeselectionwindow.py index 21dfe6d..f4bcd9a 100644 --- a/comictaggerlib/volumeselectionwindow.py +++ b/comictaggerlib/volumeselectionwindow.py @@ -25,16 +25,16 @@ from PyQt5.QtCore import pyqtSignal from comicapi import utils from comicapi.comicarchive import ComicArchive from comicapi.genericmetadata import GenericMetadata -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.issueidentifier import IssueIdentifier from comictaggerlib.issueselectionwindow import IssueSelectionWindow from comictaggerlib.matchselectionwindow import MatchSelectionWindow from comictaggerlib.progresswindow import IDProgressWindow -from comictaggerlib.resulttypes import CVVolumeResults from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui import ui_path from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.resulttypes import ComicVolume +from comictalker.talkerbase import ComicTalker, TalkerError logger = logging.getLogger(__name__) @@ -43,27 +43,34 @@ class SearchThread(QtCore.QThread): searchComplete = pyqtSignal() progressUpdate = pyqtSignal(int, int) - def __init__(self, series_name: str, refresh: bool, literal: bool = False, series_match_thresh: int = 90) -> None: + def __init__( + self, + talker_api: ComicTalker, + series_name: str, + refresh: bool, + literal: bool = False, + series_match_thresh: int = 90, + ) -> None: QtCore.QThread.__init__(self) + self.talker_api = talker_api self.series_name = series_name self.refresh: bool = refresh - self.error_code: int | None = None - self.cv_error = False - self.cv_search_results: list[CVVolumeResults] = [] + self.error_e: TalkerError + self.ct_error = False + self.ct_search_results: list[ComicVolume] = [] self.literal = literal self.series_match_thresh = series_match_thresh def run(self) -> None: - comic_vine = ComicVineTalker(self.series_match_thresh) try: - self.cv_error = False - self.cv_search_results = comic_vine.search_for_series( + self.ct_error = False + self.ct_search_results = self.talker_api.search_for_series( self.series_name, self.prog_callback, self.refresh, self.literal ) - except ComicVineTalkerException as e: - self.cv_search_results = [] - self.cv_error = True - self.error_code = e.code + except TalkerError as e: + self.ct_search_results = [] + self.ct_error = True + self.error_e = e finally: self.searchComplete.emit() @@ -105,6 +112,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): cover_index_list: list[int], comic_archive: ComicArchive, settings: ComicTaggerSettings, + talker_api: ComicTalker, autoselect: bool = False, literal: bool = False, ) -> None: @@ -112,7 +120,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): uic.loadUi(ui_path / "volumeselectionwindow.ui", self) - self.imageWidget = CoverImageWidget(self.imageContainer, CoverImageWidget.URLMode) + self.imageWidget = CoverImageWidget(self.imageContainer, talker_api, CoverImageWidget.URLMode) gridlayout = QtWidgets.QGridLayout(self.imageContainer) gridlayout.addWidget(self.imageWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -131,13 +139,14 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.settings = settings self.series_name = series_name self.issue_number = issue_number + self.issue_id: int | None = None self.year = year self.issue_count = issue_count self.volume_id = 0 self.comic_archive = comic_archive self.immediate_autoselect = autoselect self.cover_index_list = cover_index_list - self.cv_search_results: list[CVVolumeResults] = [] + self.ct_search_results: list[ComicVolume] = [] self.literal = literal self.ii: IssueIdentifier | None = None self.iddialog: IDProgressWindow | None = None @@ -147,6 +156,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.use_filter = self.settings.always_use_publisher_filter + # Load to retrieve settings + self.talker_api = talker_api + self.twList.resizeColumnsToContents() self.twList.currentItemChanged.connect(self.current_item_changed) self.twList.cellDoubleClicked.connect(self.cell_double_clicked) @@ -158,15 +170,16 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.cbxFilter.toggled.connect(self.filter_toggled) self.update_buttons() - self.perform_query() self.twList.selectRow(0) def update_buttons(self) -> None: - enabled = bool(self.cv_search_results) + enabled = bool(self.ct_search_results) self.btnRequery.setEnabled(enabled) + self.btnIssues.setEnabled(enabled) self.btnAutoSelect.setEnabled(enabled) + self.buttonBox.button(QtWidgets.QDialogButtonBox.StandardButton.Ok).setEnabled(enabled) def requery(self) -> None: @@ -178,6 +191,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.perform_query(refresh=False) def auto_select(self) -> None: + if self.comic_archive is None: QtWidgets.QMessageBox.information(self, "Auto-Select", "You need to load a comic first!") return @@ -191,7 +205,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.iddialog.rejected.connect(self.identify_cancel) self.iddialog.show() - self.ii = IssueIdentifier(self.comic_archive, self.settings) + self.ii = IssueIdentifier(self.comic_archive, self.settings, self.talker_api) md = GenericMetadata() md.series = self.series_name @@ -264,7 +278,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): choices = True if choices: - selector = MatchSelectionWindow(self, matches, self.comic_archive) + selector = MatchSelectionWindow(self, matches, self.comic_archive, self.talker_api) selector.setModal(True) selector.exec() if selector.result(): @@ -280,9 +294,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.show_issues() def show_issues(self) -> None: - selector = IssueSelectionWindow(self, self.settings, self.volume_id, self.issue_number) + selector = IssueSelectionWindow(self, self.settings, self.talker_api, self.volume_id, self.issue_number) title = "" - for record in self.cv_search_results: + for record in self.ct_search_results: if record["id"] == self.volume_id: title = record["name"] title += " (" + str(record["start_year"]) + ")" @@ -295,6 +309,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): if selector.result(): # we should now have a volume ID self.issue_number = selector.issue_number + self.issue_id = selector.issue_id self.accept() def select_by_id(self) -> None: @@ -306,18 +321,23 @@ class VolumeSelectionWindow(QtWidgets.QDialog): def perform_query(self, refresh: bool = False) -> None: + self.search_thread = SearchThread( + self.talker_api, self.series_name, refresh, self.literal, self.settings.id_series_match_search_thresh + ) + self.search_thread.searchComplete.connect(self.search_complete) + self.search_thread.progressUpdate.connect(self.search_progress_update) + self.search_thread.start() + self.progdialog = QtWidgets.QProgressDialog("Searching Online", "Cancel", 0, 100, self) self.progdialog.setWindowTitle("Online Search") self.progdialog.canceled.connect(self.search_canceled) self.progdialog.setModal(True) self.progdialog.setMinimumDuration(300) - self.search_thread = SearchThread( - self.series_name, refresh, self.literal, self.settings.id_series_match_search_thresh - ) - self.search_thread.searchComplete.connect(self.search_complete) - self.search_thread.progressUpdate.connect(self.search_progress_update) - self.search_thread.start() - self.progdialog.exec() + + if refresh or self.search_thread.isRunning(): + self.progdialog.exec() + else: + self.progdialog = None def search_canceled(self) -> None: if self.progdialog is not None: @@ -340,123 +360,124 @@ class VolumeSelectionWindow(QtWidgets.QDialog): def search_complete(self) -> None: if self.progdialog is not None: self.progdialog.accept() - del self.progdialog - if self.search_thread is not None and self.search_thread.cv_error: - if self.search_thread.error_code == ComicVineTalkerException.RateLimit: - QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message()) - else: - QtWidgets.QMessageBox.critical( - self, "Network Issue", "Could not connect to Comic Vine to search for series!" + self.progdialog = None + if self.search_thread is not None and self.search_thread.ct_error: + # TODO Currently still opens the window + QtWidgets.QMessageBox.critical( + self, + f"{self.search_thread.error_e.source} {self.search_thread.error_e.code_name} Error", + f"{self.search_thread.error_e}", + ) + return + + self.ct_search_results = self.search_thread.ct_search_results if self.search_thread is not None else [] + # filter the publishers if enabled set + if self.use_filter: + try: + publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")} + # use '' as publisher name if None + self.ct_search_results = list( + filter( + lambda d: ("" if d["publisher"] is None else str(d["publisher"]).casefold()) + not in publisher_filter, + self.ct_search_results, ) - return + ) + except Exception: + logger.exception("bad data error filtering publishers") - self.cv_search_results = self.search_thread.cv_search_results if self.search_thread is not None else [] - # filter the publishers if enabled set - if self.use_filter: - try: - publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")} - # use '' as publisher name if None - self.cv_search_results = list( - filter( - lambda d: ("" if d["publisher"] is None else str(d["publisher"]["name"]).casefold()) - not in publisher_filter, - self.cv_search_results, - ) - ) - except Exception: - logger.exception("bad data error filtering publishers") + # pre sort the data - so that we can put exact matches first afterwards + # compare as str in case extra chars ie. '1976?' + # - missing (none) values being converted to 'None' - consistent with prior behaviour in v1.2.3 + # sort by start_year if set + if self.settings.sort_series_by_year: + try: + self.ct_search_results = sorted( + self.ct_search_results, + key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])), + reverse=True, + ) + except Exception: + logger.exception("bad data error sorting results by start_year,count_of_issues") + else: + try: + self.ct_search_results = sorted( + self.ct_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True + ) + except Exception: + logger.exception("bad data error sorting results by count_of_issues") - # pre sort the data - so that we can put exact matches first afterwards - # compare as str in case extra chars ie. '1976?' - # - missing (none) values being converted to 'None' - consistent with prior behaviour in v1.2.3 - # sort by start_year if set - if self.settings.sort_series_by_year: - try: - self.cv_search_results = sorted( - self.cv_search_results, - key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])), - reverse=True, - ) - except Exception: - logger.exception("bad data error sorting results by start_year,count_of_issues") - else: - try: - self.cv_search_results = sorted( - self.cv_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True - ) - except Exception: - logger.exception("bad data error sorting results by count_of_issues") + # move sanitized matches to the front + if self.settings.exact_series_matches_first: + try: + sanitized = utils.sanitize_title(self.series_name, False).casefold() + sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold() - # move sanitized matches to the front - if self.settings.exact_series_matches_first: - try: - sanitized = utils.sanitize_title(self.series_name, False).casefold() - sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold() + deques: list[deque[ComicVolume]] = [deque(), deque(), deque()] - deques: list[deque[CVVolumeResults]] = [deque(), deque(), deque()] + def categorize(result: ComicVolume) -> int: + # We don't remove anything on this one so that we only get exact matches + if utils.sanitize_title(result["name"], True).casefold() == sanitized_no_articles: + return 0 - def categorize(result: CVVolumeResults) -> int: - # We don't remove anything on this one so that we only get exact matches - if utils.sanitize_title(result["name"], True).casefold() == sanitized_no_articles: - return 0 + # this ensures that 'The Joker' is near the top even if you search 'Joker' + if utils.sanitize_title(result["name"], False).casefold() in sanitized: + return 1 + return 2 - # this ensures that 'The Joker' is near the top even if you search 'Joker' - if utils.sanitize_title(result["name"], False).casefold() in sanitized: - return 1 - return 2 + for comic in self.ct_search_results: + deques[categorize(comic)].append(comic) + logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2])) + self.ct_search_results = list(itertools.chain.from_iterable(deques)) + except Exception: + logger.exception("bad data error filtering exact/near matches") - for comic in self.cv_search_results: - deques[categorize(comic)].append(comic) - logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2])) - self.cv_search_results = list(itertools.chain.from_iterable(deques)) - except Exception: - logger.exception("bad data error filtering exact/near matches") + self.update_buttons() - self.update_buttons() + self.twList.setSortingEnabled(False) - self.twList.setSortingEnabled(False) + self.twList.setRowCount(0) - self.twList.setRowCount(0) + row = 0 + for record in self.ct_search_results: + self.twList.insertRow(row) - row = 0 - for record in self.cv_search_results: - self.twList.insertRow(row) + item_text = record["name"] + item = QtWidgets.QTableWidgetItem(item_text) + item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) + item.setData(QtCore.Qt.ItemDataRole.UserRole, record["id"]) + item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) + self.twList.setItem(row, 0, item) - item_text = record["name"] - item = QtWidgets.QTableWidgetItem(item_text) + item_text = str(record["start_year"]) + item = QtWidgets.QTableWidgetItem(item_text) + item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) + item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) + self.twList.setItem(row, 1, item) + + item_text = str(record["count_of_issues"]) + item = QtWidgets.QTableWidgetItem(item_text) + item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) + item.setData(QtCore.Qt.ItemDataRole.DisplayRole, record["count_of_issues"]) + item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) + self.twList.setItem(row, 2, item) + + if record["publisher"] is not None: + item_text = record["publisher"] item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) - item.setData(QtCore.Qt.ItemDataRole.UserRole, record["id"]) - item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) - self.twList.setItem(row, 0, item) - - item_text = str(record["start_year"]) item = QtWidgets.QTableWidgetItem(item_text) - item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) - self.twList.setItem(row, 1, item) + self.twList.setItem(row, 3, item) - item_text = str(record["count_of_issues"]) - item = QtWidgets.QTableWidgetItem(item_text) - item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) - item.setData(QtCore.Qt.ItemDataRole.DisplayRole, record["count_of_issues"]) - item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) - self.twList.setItem(row, 2, item) + row += 1 - if record["publisher"] is not None: - item_text = record["publisher"]["name"] - item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) - item = QtWidgets.QTableWidgetItem(item_text) - item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) - self.twList.setItem(row, 3, item) - - row += 1 - - self.twList.setSortingEnabled(True) - self.twList.selectRow(0) - self.twList.resizeColumnsToContents() + self.twList.setSortingEnabled(True) + self.twList.selectRow(0) + self.twList.resizeColumnsToContents() def showEvent(self, event: QtGui.QShowEvent) -> None: - if not self.cv_search_results: + self.perform_query() + if not self.ct_search_results: QtCore.QCoreApplication.processEvents() QtWidgets.QMessageBox.information(self, "Search Result", "No matches found!") QtCore.QTimer.singleShot(200, self.close_me) @@ -483,11 +504,11 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.volume_id = self.twList.item(curr.row(), 0).data(QtCore.Qt.ItemDataRole.UserRole) # list selection was changed, update the info on the volume - for record in self.cv_search_results: + for record in self.ct_search_results: if record["id"] == self.volume_id: if record["description"] is None: self.teDetails.setText("") else: self.teDetails.setText(record["description"]) - self.imageWidget.set_url(record["image"]["super_url"]) + self.imageWidget.set_url(record["image_url"]) break diff --git a/comictalker/__init__.py b/comictalker/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/comictalker/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/comictaggerlib/comiccacher.py b/comictalker/comiccacher.py similarity index 53% rename from comictaggerlib/comiccacher.py rename to comictalker/comiccacher.py index a501122..7625edf 100644 --- a/comictaggerlib/comiccacher.py +++ b/comictalker/comiccacher.py @@ -16,14 +16,15 @@ from __future__ import annotations import datetime +import json import logging import os import sqlite3 as lite from typing import Any from comictaggerlib import ctversion -from comictaggerlib.resulttypes import CVImage, CVIssuesResults, CVPublisher, CVVolumeResults, SelectDetails from comictaggerlib.settings import ComicTaggerSettings +from comictalker.resulttypes import ComicIssue, ComicVolume logger = logging.getLogger(__name__) @@ -77,15 +78,8 @@ class ComicCacher: "CREATE TABLE VolumeSearchCache(" + "search_term TEXT," + "id INT NOT NULL," - + "name TEXT," - + "start_year INT," - + "publisher TEXT," - + "count_of_issues INT," - + "image_url TEXT," - + "description TEXT," + "timestamp DATE DEFAULT (datetime('now','localtime'))," - + "source_name TEXT NOT NULL," - + "aliases TEXT)" # Newline separated + + "source_name TEXT NOT NULL)" ) cur.execute( @@ -95,29 +89,21 @@ class ComicCacher: + "publisher TEXT," + "count_of_issues INT," + "start_year INT," + + "image_url TEXT," + + "aliases TEXT," # Newline separated + + "description TEXT," + "timestamp DATE DEFAULT (datetime('now','localtime')), " + "source_name TEXT NOT NULL," - + "aliases TEXT," # Newline separated + "PRIMARY KEY (id, source_name))" ) - cur.execute( - "CREATE TABLE AltCovers(" - + "issue_id INT NOT NULL," - + "url_list TEXT," - + "timestamp DATE DEFAULT (datetime('now','localtime')), " - + "source_name TEXT NOT NULL," - + "aliases TEXT," # Newline separated - + "PRIMARY KEY (issue_id, source_name))" - ) - cur.execute( "CREATE TABLE Issues(" + "id INT NOT NULL," + "volume_id INT," + "name TEXT," + "issue_number TEXT," - + "super_url TEXT," + + "image_url TEXT," + "thumb_url TEXT," + "cover_date TEXT," + "site_detail_url TEXT," @@ -125,11 +111,17 @@ class ComicCacher: + "timestamp DATE DEFAULT (datetime('now','localtime')), " + "source_name TEXT NOT NULL," + "aliases TEXT," # Newline separated + + "alt_image_urls TEXT," # Newline separated URLs + + "characters TEXT," # Newline separated + + "locations TEXT," # Newline separated + + "credits TEXT," # JSON: "{"name": "Bob Shakespeare", "role": "Writer"}" + + "teams TEXT," # Newline separated + + "story_arcs TEXT," # Newline separated + + "complete BOOL," # Is the data complete? Includes characters, locations, credits. + "PRIMARY KEY (id, source_name))" ) - def add_search_results(self, source_name: str, search_term: str, cv_search_results: list[CVVolumeResults]) -> None: - + def add_search_results(self, source_name: str, search_term: str, ct_search_results: list[ComicVolume]) -> None: con = lite.connect(self.db_file) with con: @@ -143,114 +135,63 @@ class ComicCacher: ) # now add in new results - for record in cv_search_results: - - if record["publisher"] is None: - pub_name = "" - else: - pub_name = record["publisher"]["name"] - - if record["image"] is None: - url = "" - else: - url = record["image"]["super_url"] - + for record in ct_search_results: cur.execute( - "INSERT INTO VolumeSearchCache " - + "(source_name, search_term, id, name, start_year, publisher, count_of_issues, image_url, description, aliases) " - + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + "INSERT INTO VolumeSearchCache " + "(source_name, search_term, id) " + "VALUES(?, ?, ?)", ( source_name, search_term.casefold(), record["id"], - record["name"], - record["start_year"], - pub_name, - record["count_of_issues"], - url, - record["description"], - record["aliases"], ), ) - def get_search_results(self, source_name: str, search_term: str) -> list[CVVolumeResults]: + data = { + "id": record["id"], + "source_name": source_name, + "name": record["name"], + "publisher": record.get("publisher", ""), + "count_of_issues": record.get("count_of_issues"), + "start_year": record.get("start_year"), + "image_url": record.get("image_url", ""), + "description": record.get("description", ""), + "timestamp": datetime.datetime.now(), + "aliases": "\n".join(record.get("aliases", [])), + } + self.upsert(cur, "volumes", data) + def get_search_results(self, source_name: str, search_term: str) -> list[ComicVolume]: results = [] con = lite.connect(self.db_file) with con: con.text_factory = str cur = con.cursor() - # purge stale search results - a_day_ago = datetime.datetime.today() - datetime.timedelta(days=1) - cur.execute("DELETE FROM VolumeSearchCache WHERE timestamp < ?", [str(a_day_ago)]) - - # fetch cur.execute( - "SELECT * FROM VolumeSearchCache WHERE search_term=? AND source_name=?", + "SELECT * FROM VolumeSearchCache INNER JOIN Volumes on" + " VolumeSearchCache.id=Volumes.id AND VolumeSearchCache.source_name=Volumes.source_name" + " WHERE search_term=? AND VolumeSearchCache.source_name=?", [search_term.casefold(), source_name], ) + rows = cur.fetchall() # now process the results for record in rows: - result = CVVolumeResults( - id=record[1], - name=record[2], - start_year=record[3], - count_of_issues=record[5], - description=record[7], - publisher=CVPublisher(name=record[4]), - image=CVImage(super_url=record[6]), - aliases=record[10], + result = ComicVolume( + id=record[4], + name=record[5], + publisher=record[6], + count_of_issues=record[7], + start_year=record[8], + image_url=record[9], + aliases=record[10].strip().splitlines(), + description=record[11], ) results.append(result) return results - def add_alt_covers(self, source_name: str, issue_id: int, url_list: list[str]) -> None: - - con = lite.connect(self.db_file) - - with con: - con.text_factory = str - cur = con.cursor() - - # remove all previous entries with this search term - cur.execute("DELETE FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name]) - - url_list_str = ",".join(url_list) - # now add in new record - cur.execute( - "INSERT INTO AltCovers (source_name, issue_id, url_list) VALUES(?, ?, ?)", - (source_name, issue_id, url_list_str), - ) - - def get_alt_covers(self, source_name: str, issue_id: int) -> list[str]: - - con = lite.connect(self.db_file) - with con: - cur = con.cursor() - con.text_factory = str - - # purge stale issue info - probably issue data won't change - # much.... - a_month_ago = datetime.datetime.today() - datetime.timedelta(days=30) - cur.execute("DELETE FROM AltCovers WHERE timestamp < ?", [str(a_month_ago)]) - - cur.execute("SELECT url_list FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name]) - row = cur.fetchone() - if row is None: - return [] - - url_list_str = row[0] - if not url_list_str: - return [] - url_list = str(url_list_str).split(",") - return url_list - - def add_volume_info(self, source_name: str, cv_volume_record: CVVolumeResults) -> None: - + def add_volume_info(self, source_name: str, volume_record: ComicVolume) -> None: con = lite.connect(self.db_file) with con: @@ -259,25 +200,21 @@ class ComicCacher: timestamp = datetime.datetime.now() - if cv_volume_record["publisher"] is None: - pub_name = "" - else: - pub_name = cv_volume_record["publisher"]["name"] - data = { - "id": cv_volume_record["id"], + "id": volume_record["id"], "source_name": source_name, - "name": cv_volume_record["name"], - "publisher": pub_name, - "count_of_issues": cv_volume_record["count_of_issues"], - "start_year": cv_volume_record["start_year"], + "name": volume_record["name"], + "publisher": volume_record.get("publisher", ""), + "count_of_issues": volume_record.get("count_of_issues"), + "start_year": volume_record.get("start_year"), + "image_url": volume_record.get("image_url", ""), + "description": volume_record.get("description", ""), "timestamp": timestamp, - "aliases": cv_volume_record["aliases"], + "aliases": "\n".join(volume_record.get("aliases", [])), } self.upsert(cur, "volumes", data) - def add_volume_issues_info(self, source_name: str, volume_id: int, cv_volume_issues: list[CVIssuesResults]) -> None: - + def add_volume_issues_info(self, source_name: str, volume_issues: list[ComicIssue]) -> None: con = lite.connect(self.db_file) with con: @@ -287,40 +224,46 @@ class ComicCacher: # add in issues - for issue in cv_volume_issues: + for issue in volume_issues: data = { "id": issue["id"], - "volume_id": volume_id, + "volume_id": issue["volume"]["id"], "source_name": source_name, "name": issue["name"], "issue_number": issue["issue_number"], - "site_detail_url": issue["site_detail_url"], - "cover_date": issue["cover_date"], - "super_url": issue["image"]["super_url"], - "thumb_url": issue["image"]["thumb_url"], - "description": issue["description"], + "site_detail_url": issue.get("site_detail_url"), + "cover_date": issue.get("cover_date"), + "image_url": issue.get("image_url", ""), + "thumb_url": issue.get("image_thumb_url", ""), + "description": issue.get("description", ""), "timestamp": timestamp, - "aliases": issue["aliases"], + "aliases": "\n".join(issue.get("aliases", [])), + "alt_image_urls": "\n".join(issue.get("alt_image_urls", [])), + "characters": "\n".join(issue.get("characters", [])), + "locations": "\n".join(issue.get("locations", [])), + "teams": "\n".join(issue.get("teams", [])), + "story_arcs": "\n".join(issue.get("story_arcs", [])), + "credits": json.dumps(issue.get("credits")), + "complete": issue["complete"], } self.upsert(cur, "issues", data) - def get_volume_info(self, volume_id: int, source_name: str) -> CVVolumeResults | None: - - result: CVVolumeResults | None = None + def get_volume_info(self, volume_id: int, source_name: str, purge: bool = True) -> ComicVolume | None: + result: ComicVolume | None = None con = lite.connect(self.db_file) with con: cur = con.cursor() con.text_factory = str - # purge stale volume info - a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7) - cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)]) + if purge: + # purge stale volume info + a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7) + cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)]) # fetch cur.execute( - "SELECT source_name,id,name,publisher,count_of_issues,start_year,aliases FROM Volumes" - " WHERE id=? AND source_name=?", + "SELECT * FROM Volumes" " WHERE id=? AND source_name=?", [volume_id, source_name], ) @@ -330,19 +273,22 @@ class ComicCacher: return result # since ID is primary key, there is only one row - result = CVVolumeResults( - id=row[1], - name=row[2], - count_of_issues=row[4], - start_year=row[5], - publisher=CVPublisher(name=row[3]), - aliases=row[6], + result = ComicVolume( + id=row[0], + name=row[1], + publisher=row[2], + count_of_issues=row[3], + start_year=row[4], + image_url=row[5], + aliases=row[6].strip().splitlines(), + description=row[7], ) return result - def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[CVIssuesResults]: - + def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[ComicIssue]: + # get_volume_info should only fail if someone is doing something weird + volume = self.get_volume_info(volume_id, source_name, False) or ComicVolume(id=volume_id, name="") con = lite.connect(self.db_file) with con: cur = con.cursor() @@ -354,11 +300,11 @@ class ComicCacher: cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)]) # fetch - results: list[CVIssuesResults] = [] + results: list[ComicIssue] = [] cur.execute( ( - "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,super_url,thumb_url,description,aliases" + "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,alt_image_urls,characters,locations,credits,teams,story_arcs,complete" " FROM Issues WHERE volume_id=? AND source_name=?" ), [volume_id, source_name], @@ -367,75 +313,78 @@ class ComicCacher: # now process the results for row in rows: - record = CVIssuesResults( + record = ComicIssue( id=row[1], name=row[2], issue_number=row[3], site_detail_url=row[4], cover_date=row[5], - image=CVImage(super_url=row[6], thumb_url=row[7]), + image_url=row[6], description=row[8], - aliases=row[9], + volume=volume, + aliases=row[9].strip().splitlines(), + alt_image_urls=row[10].strip().splitlines(), + characters=row[11].strip().splitlines(), + locations=row[12].strip().splitlines(), + credits=json.loads(row[13]), + teams=row[14].strip().splitlines(), + story_arcs=row[15].strip().splitlines(), + complete=bool(row[16]), ) results.append(record) return results - def add_issue_select_details( - self, - source_name: str, - issue_id: int, - image_url: str, - thumb_image_url: str, - cover_date: str, - site_detail_url: str, - ) -> None: - - con = lite.connect(self.db_file) - - with con: - cur = con.cursor() - con.text_factory = str - timestamp = datetime.datetime.now() - - data = { - "id": issue_id, - "source_name": source_name, - "super_url": image_url, - "thumb_url": thumb_image_url, - "cover_date": cover_date, - "site_detail_url": site_detail_url, - "timestamp": timestamp, - } - self.upsert(cur, "issues", data) - - def get_issue_select_details(self, issue_id: int, source_name: str) -> SelectDetails: - + def get_issue_info(self, issue_id: int, source_name: str) -> ComicIssue | None: con = lite.connect(self.db_file) with con: cur = con.cursor() con.text_factory = str + # purge stale issue info - probably issue data won't change + # much.... + a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7) + cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)]) + cur.execute( - "SELECT super_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=? AND source_name=?", + ( + "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,volume_id,alt_image_urls,characters,locations,credits,teams,story_arcs,complete" + " FROM Issues WHERE id=? AND source_name=?" + ), [issue_id, source_name], ) row = cur.fetchone() - details = SelectDetails( - image_url=None, - thumb_image_url=None, - cover_date=None, - site_detail_url=None, - ) - if row is not None and row[0] is not None: - details["image_url"] = row[0] - details["thumb_image_url"] = row[1] - details["cover_date"] = row[2] - details["site_detail_url"] = row[3] + record = None - return details + if row: + # get_volume_info should only fail if someone is doing something weird + volume = self.get_volume_info(row[10], source_name, False) or ComicVolume(id=row[10], name="") + + # now process the results + + record = ComicIssue( + id=row[1], + name=row[2], + issue_number=row[3], + site_detail_url=row[4], + cover_date=row[5], + image_url=row[6], + image_thumb_url=row[7], + description=row[8], + volume=volume, + aliases=row[9].strip().splitlines(), + alt_image_urls=row[11].strip().splitlines(), + characters=row[12].strip().splitlines(), + locations=row[13].strip().splitlines(), + credits=json.loads(row[14]), + teams=row[15].strip().splitlines(), + story_arcs=row[16].strip().splitlines(), + complete=bool(row[17]), + ) + + return record def upsert(self, cur: lite.Cursor, tablename: str, data: dict[str, Any]) -> None: """This does an insert if the given PK doesn't exist, and an diff --git a/comictalker/comictalkerapi.py b/comictalker/comictalkerapi.py new file mode 100644 index 0000000..e9c3bbe --- /dev/null +++ b/comictalker/comictalkerapi.py @@ -0,0 +1,38 @@ +"""Handles collecting data from source talkers.""" + +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging + +import comictalker.talkers.comicvine +from comictalker.talkerbase import ComicTalker, TalkerError + +logger = logging.getLogger(__name__) + + +def get_comic_talker(source_name: str) -> type[ComicTalker]: + """Retrieve the available sources modules""" + sources = get_talkers() + if source_name not in sources: + raise TalkerError(source=source_name, code=4, desc="The talker does not exist") + + talker = sources[source_name] + return talker + + +def get_talkers(): + """Returns all comic talker modules NOT objects""" + return {"comicvine": comictalker.talkers.comicvine.ComicVineTalker} diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py new file mode 100644 index 0000000..ca2b013 --- /dev/null +++ b/comictalker/resulttypes.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing_extensions import Required, TypedDict + + +class Credits(TypedDict): + name: str + role: str + + +class ComicVolume(TypedDict, total=False): + aliases: list[str] + count_of_issues: int + description: str + id: Required[int] + image_url: str + name: Required[str] + publisher: str + start_year: int + + +class ComicIssue(TypedDict, total=False): + aliases: list[str] + cover_date: str + description: str + id: int + image_url: str + image_thumb_url: str + issue_number: Required[str] + name: Required[str] + site_detail_url: str + volume: ComicVolume + alt_image_urls: list[str] + characters: list[str] + locations: list[str] + credits: list[Credits] + teams: list[str] + story_arcs: list[str] + complete: bool # Is this a complete ComicIssue? or is there more data to fetch diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py new file mode 100644 index 0000000..a039cfc --- /dev/null +++ b/comictalker/talker_utils.py @@ -0,0 +1,199 @@ +"""Generic sources utils to format API data and the like. +""" +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging +import re +from datetime import datetime + +from bs4 import BeautifulSoup + +from comicapi import utils +from comicapi.genericmetadata import GenericMetadata +from comicapi.issuestring import IssueString +from comictaggerlib import ctversion +from comictalker.talkerbase import ComicIssue + +logger = logging.getLogger(__name__) + + +def map_comic_issue_to_metadata( + issue_results: ComicIssue, source: str, remove_html_tables: bool = False, use_year_volume: bool = False +) -> GenericMetadata: + """Maps ComicIssue to generic metadata""" + metadata = GenericMetadata() + metadata.is_empty = False + + # Is this best way to go about checking? + if issue_results["volume"].get("name"): + metadata.series = utils.xlate(issue_results["volume"]["name"]) + if issue_results.get("issue_number"): + metadata.issue = IssueString(issue_results["issue_number"]).as_string() + if issue_results.get("name"): + metadata.title = utils.xlate(issue_results["name"]) + if issue_results.get("image_url"): + metadata.cover_image = issue_results["image_url"] + + if issue_results["volume"].get("publisher"): + metadata.publisher = utils.xlate(issue_results["volume"]["publisher"]) + + if issue_results.get("cover_date"): + metadata.day, metadata.month, metadata.year = utils.parse_date_str(issue_results["cover_date"]) + elif issue_results["volume"].get("start_year"): + metadata.year = utils.xlate(issue_results["volume"]["start_year"], True) + + metadata.comments = cleanup_html(issue_results["description"], remove_html_tables) + if use_year_volume: + metadata.volume = issue_results["volume"]["start_year"] + + metadata.tag_origin = source + metadata.issue_id = issue_results["id"] + + metadata.notes = ( + f"Tagged with ComicTagger {ctversion.version} using info from {source} on" + f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {issue_results['id']}]" + ) + metadata.web_link = issue_results["site_detail_url"] + + for person in issue_results["credits"]: + if "role" in person: + roles = person["role"].split(",") + for role in roles: + # can we determine 'primary' from CV?? + metadata.add_credit(person["name"], role.title().strip(), False) + + if issue_results.get("characters"): + metadata.characters = ", ".join(issue_results["characters"]) + if issue_results.get("teams"): + metadata.teams = ", ".join(issue_results["teams"]) + if issue_results.get("locations"): + metadata.locations = ", ".join(issue_results["locations"]) + if issue_results.get("story_arcs"): + metadata.story_arc = ", ".join(issue_results["story_arcs"]) + + return metadata + + +def parse_date_str(date_str: str) -> tuple[int | None, int | None, int | None]: + day = None + month = None + year = None + if date_str: + parts = date_str.split("-") + year = utils.xlate(parts[0], True) + if len(parts) > 1: + month = utils.xlate(parts[1], True) + if len(parts) > 2: + day = utils.xlate(parts[2], True) + return day, month, year + + +def cleanup_html(string: str, remove_html_tables: bool = False) -> str: + """Cleans HTML code from any text. Will remove any HTML tables with remove_html_tables""" + if string is None: + return "" + # find any tables + soup = BeautifulSoup(string, "html.parser") + tables = soup.findAll("table") + + # remove all newlines first + string = string.replace("\n", "") + + # put in our own + string = string.replace("
", "\n") + string = string.replace("", "\n") + string = string.replace("

", "\n\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("
", "*") + string = string.replace("
", "*\n") + string = string.replace("
", "*") + string = string.replace("
", "*\n") + + # remove the tables + p = re.compile(r".*?") + if remove_html_tables: + string = p.sub("", string) + string = string.replace("*List of covers and their creators:*", "") + else: + string = p.sub("{}", string) + + # now strip all other tags + p = re.compile(r"<[^<]*?>") + newstring = p.sub("", string) + + newstring = newstring.replace(" ", " ") + newstring = newstring.replace("&", "&") + + newstring = newstring.strip() + + if not remove_html_tables: + # now rebuild the tables into text from BSoup + try: + table_strings = [] + for table in tables: + rows = [] + hdrs = [] + col_widths = [] + for hdr in table.findAll("th"): + item = hdr.string.strip() + hdrs.append(item) + col_widths.append(len(item)) + rows.append(hdrs) + + for row in table.findAll("tr"): + cols = [] + col = row.findAll("td") + i = 0 + for c in col: + item = c.string.strip() + cols.append(item) + if len(item) > col_widths[i]: + col_widths[i] = len(item) + i += 1 + if len(cols) != 0: + rows.append(cols) + # now we have the data, make it into text + fmtstr = "|" + for w in col_widths: + fmtstr += f" {{:{w + 1}}}|" + table_text = "" + counter = 0 + for row in rows: + table_text += fmtstr.format(*row) + "\n" + if counter == 0 and len(hdrs) != 0: + table_text += "|" + for w in col_widths: + table_text += "-" * (w + 2) + "|" + table_text += "\n" + counter += 1 + + table_strings.append(table_text + "\n") + + newstring = newstring.format(*table_strings) + except Exception: + # we caught an error rebuilding the table. + # just bail and remove the formatting + logger.exception("table parse error") + newstring.replace("{}", "") + + return newstring diff --git a/comictalker/talkerbase.py b/comictalker/talkerbase.py new file mode 100644 index 0000000..c624374 --- /dev/null +++ b/comictalker/talkerbase.py @@ -0,0 +1,186 @@ +"""A template for an information source +""" +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging +from typing import Callable + +from comicapi.genericmetadata import GenericMetadata +from comictalker.resulttypes import ComicIssue, ComicVolume + +logger = logging.getLogger(__name__) + +# NOTE: Series and Volume are synonymous. Some sources (ComicVine) use "volume" and others (MangaUpdates) use "series". + + +class SourceDetails: + def __init__( + self, + name: str = "", + ident: str = "", + logo: str = "", # Will be scaled if greater than 100px width and 250px height in comictalker/talkers/logos + ): + self.name = name + self.id = ident + self.logo = logo + + +class SourceStaticOptions: + def __init__( + self, + website: str = "", + has_issues: bool = False, + has_alt_covers: bool = False, + requires_apikey: bool = False, + has_nsfw: bool = False, + has_censored_covers: bool = False, + ) -> None: + self.website = website + self.has_issues = has_issues + self.has_alt_covers = has_alt_covers + self.requires_apikey = requires_apikey + self.has_nsfw = has_nsfw + self.has_censored_covers = has_censored_covers + + +class TalkerError(Exception): + """Base class exception for information sources. + + Attributes: + code -- a numerical code + 1 - General + 2 - Network + 3 - Data + desc -- description of the error + source -- the name of the source producing the error + """ + + codes = { + 1: "General", + 2: "Network", + 3: "Data", + 4: "Other", + } + + def __init__(self, source: str = "", code: int = 4, desc: str = "", sub_code: int = 0) -> None: + super().__init__() + if desc == "": + desc = "Unknown" + self.desc = desc + self.code = code + self.code_name = self.codes[code] + self.sub_code = sub_code + self.source = source + + def __str__(self): + return f"{self.source} encountered a {self.code_name} error. {self.desc}" + + +class TalkerNetworkError(TalkerError): + """Network class exception for information sources + + Attributes: + sub_code -- numerical code for finer detail + 1 -- connected refused + 2 -- api key + 3 -- rate limit + 4 -- timeout + """ + + net_codes = { + 0: "General network error.", + 1: "The connection was refused.", + 2: "An API key error occurred.", + 3: "Rate limit exceeded. Please wait a bit or enter a personal key if using the default.", + 4: "The connection timed out.", + 5: "Number of retries exceeded.", + } + + def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None: + if desc == "": + desc = self.net_codes[sub_code] + + super().__init__(source, 2, desc, sub_code) + + +class TalkerDataError(TalkerError): + """Data class exception for information sources + + Attributes: + sub_code -- numerical code for finer detail + 1 -- unexpected data + 2 -- malformed data + 3 -- missing data + """ + + data_codes = { + 0: "General data error.", + 1: "Unexpected data encountered.", + 2: "Malformed data encountered.", + 3: "Missing data encountered.", + } + + def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None: + if desc == "": + desc = self.data_codes[sub_code] + + super().__init__(source, 3, desc, sub_code) + + +# Class talkers instance +class ComicTalker: + """The base class for all comic source talkers""" + + def __init__(self) -> None: + # Identity name for the information source etc. + self.source_details: SourceDetails = ( + SourceDetails() + ) # Can use this to test if custom talker has been configured + self.static_options: SourceStaticOptions = SourceStaticOptions() + + def check_api_key(self, key: str, url: str) -> bool: + """If the talker has or requires an API key, this function should test its validity""" + raise NotImplementedError + + def search_for_series( + self, + series_name: str, + callback: Callable[[int, int], None] | None = None, + refresh_cache: bool = False, + literal: bool = False, + ) -> list[ComicVolume]: + """Searches for the series/volumes with the given series_name + callback is used for... + refresh_cache signals if the data in the cache should be used + literal indicates that no articles (a, the, etc.) should be removed when searching""" + raise NotImplementedError + + def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]: + """Expected to return a list of issues with a given series/volume ID""" + raise NotImplementedError + + def fetch_comic_data(self, issue_id: int = 0, series_id: int = 0, issue_number: str = "") -> GenericMetadata: + """This function is expected to handle a few possibilities: + 1. Only series_id passed in: Retrieve the SERIES/VOLUME information only + 2. series_id and issue_number: Retrieve the ISSUE information + 3. Only issue_id: Retrieve the ISSUE information""" + raise NotImplementedError + + def fetch_issues_by_volume_issue_num_and_year( + self, volume_id_list: list[int], issue_number: str, year: str | int | None + ) -> list[ComicIssue]: + """Searches for a list of issues within the given year. Used solely by issueidentifer""" + raise NotImplementedError diff --git a/comictalker/talkers/__init__.py b/comictalker/talkers/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/comictalker/talkers/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py new file mode 100644 index 0000000..0f9a4e0 --- /dev/null +++ b/comictalker/talkers/comicvine.py @@ -0,0 +1,674 @@ +"""ComicVine information source +""" +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Callable, cast +from urllib.parse import urljoin, urlsplit + +import requests +from typing_extensions import Required, TypedDict + +import comictalker.talker_utils as talker_utils +from comicapi import utils +from comicapi.genericmetadata import GenericMetadata +from comicapi.issuestring import IssueString +from comictaggerlib import ctversion +from comictalker.comiccacher import ComicCacher +from comictalker.resulttypes import ComicIssue, ComicVolume, Credits +from comictalker.talkerbase import ComicTalker, SourceDetails, SourceStaticOptions, TalkerDataError, TalkerNetworkError + +logger = logging.getLogger(__name__) + + +class CVTypeID: + Volume = "4050" + Issue = "4000" + + +class CVImage(TypedDict, total=False): + icon_url: str + medium_url: str + screen_url: str + screen_large_url: str + small_url: str + super_url: Required[str] + thumb_url: str + tiny_url: str + original_url: str + image_tags: str + + +class CVAltImages(TypedDict): + original_url: str + id: int + caption: str + image_tags: str + + +class CVPublisher(TypedDict, total=False): + api_detail_url: str + id: int + name: Required[str] + + +class CVVolume(TypedDict): + api_detail_url: str + id: int + name: str + site_detail_url: str + + +class CVCredits(TypedDict): + api_detail_url: str + id: int + name: str + site_detail_url: str + + +class CVPersonCredits(TypedDict): + api_detail_url: str + id: int + name: str + site_detail_url: str + role: str + + +class CVVolumeResults(TypedDict): + aliases: str + count_of_issues: int + description: str + id: int + image: CVImage + name: str + publisher: CVPublisher + start_year: str + resource_type: str + + +class CVResult(TypedDict): + error: str + limit: int + offset: int + number_of_page_results: int + number_of_total_results: int + status_code: int + results: (CVIssueDetailResults | CVVolumeResults | list[CVVolumeResults] | list[CVIssueDetailResults]) + version: str + + +class CVVolumeFullResult(TypedDict): + characters: list[CVCredits] + locations: list[CVCredits] + people: list[CVPersonCredits] + site_detail_url: str + count_of_issues: int + description: str + id: int + name: str + publisher: CVPublisher + start_year: str + resource_type: str + + +class CVIssueDetailResults(TypedDict, total=False): + aliases: str + api_detail_url: str + associated_images: list[CVAltImages] + character_credits: list[CVCredits] + character_died_in: None + concept_credits: list[CVCredits] + cover_date: str + date_added: str + date_last_updated: str + deck: None + description: str + first_appearance_characters: None + first_appearance_concepts: None + first_appearance_locations: None + first_appearance_objects: None + first_appearance_storyarcs: None + first_appearance_teams: None + has_staff_review: bool + id: int + image: CVImage + issue_number: str + location_credits: list[CVCredits] + name: str + object_credits: list[CVCredits] + person_credits: list[CVPersonCredits] + site_detail_url: str + store_date: str + story_arc_credits: list[CVCredits] + team_credits: list[CVCredits] + team_disbanded_in: None + volume: CVVolume + + +CV_RATE_LIMIT_STATUS = 107 + + +class ComicVineTalker(ComicTalker): + def __init__( + self, api_url, api_key, series_match_thresh, remove_html_tables, use_series_start_as_volume, wait_on_ratelimit + ): + super().__init__() + self.source_details = SourceDetails( + name="Comic Vine", + ident="comicvine", + ) + self.static_options = SourceStaticOptions( + website="https://comicvine.gamespot.com/", + has_issues=True, + has_alt_covers=True, + requires_apikey=True, + has_nsfw=False, + has_censored_covers=False, + ) + + # Identity name for the information source + self.source_name = self.source_details.id + self.source_name_friendly = self.source_details.name + + self.wait_for_rate_limit = wait_on_ratelimit + # NOTE: This was hardcoded before which is why it isn't passed in + self.wait_for_rate_limit_time = 20 + + self.issue_id: int | None = None + + self.api_key = api_key if api_key else "27431e6787042105bd3e47e169a624521f89f3a4" + self.api_base_url = api_url if api_url else "https://comicvine.gamespot.com/api" + + self.remove_html_tables = remove_html_tables + self.use_series_start_as_volume = use_series_start_as_volume + + tmp_url = urlsplit(self.api_base_url) + + # joinurl only works properly if there is a trailing slash + if tmp_url.path and tmp_url.path[-1] != "/": + tmp_url = tmp_url._replace(path=tmp_url.path + "/") + + self.api_base_url = tmp_url.geturl() + + self.series_match_thresh = series_match_thresh + + def check_api_key(self, key: str, url: str) -> bool: + if not url: + url = self.api_base_url + try: + tmp_url = urlsplit(url) + if tmp_url.path and tmp_url.path[-1] != "/": + tmp_url = tmp_url._replace(path=tmp_url.path + "/") + url = tmp_url.geturl() + test_url = urljoin(url, "issue/1/") + + cv_response: CVResult = requests.get( + test_url, + headers={"user-agent": "comictagger/" + ctversion.version}, + params={ + "api_key": key, + "format": "json", + "field_list": "name", + }, + ).json() + + # Bogus request, but if the key is wrong, you get error 100: "Invalid API Key" + return cv_response["status_code"] != 100 + except Exception: + return False + + def get_cv_content(self, url: str, params: dict[str, Any]) -> CVResult: + """ + Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error + sleep for a bit and retry. + """ + total_time_waited = 0 + limit_wait_time = 1 + counter = 0 + wait_times = [1, 2, 3, 4] + while True: + cv_response: CVResult = self.get_url_content(url, params) + if self.wait_for_rate_limit and cv_response["status_code"] == CV_RATE_LIMIT_STATUS: + logger.info(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n") + time.sleep(limit_wait_time * 60) + total_time_waited += limit_wait_time + limit_wait_time = wait_times[counter] + if counter < 3: + counter += 1 + # don't wait much more than 20 minutes + if total_time_waited < self.wait_for_rate_limit_time: + continue + if cv_response["status_code"] != 1: + logger.debug( + f"{self.source_name_friendly} query failed with error #{cv_response['status_code']}: [{cv_response['error']}]." + ) + raise TalkerNetworkError( + self.source_name_friendly, 0, f"{cv_response['status_code']}: {cv_response['error']}" + ) + + # it's all good + break + return cv_response + + def get_url_content(self, url: str, params: dict[str, Any]) -> Any: + # connect to server: + # if there is a 500 error, try a few more times before giving up + # any other error, just bail + for tries in range(3): + try: + resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + ctversion.version}) + if resp.status_code == 200: + return resp.json() + if resp.status_code == 500: + logger.debug(f"Try #{tries + 1}: ") + time.sleep(1) + logger.debug(str(resp.status_code)) + else: + break + + except requests.exceptions.Timeout: + logger.debug(f"Connection to {self.source_name_friendly} timed out.") + raise TalkerNetworkError(self.source_name_friendly, 4) + except requests.exceptions.RequestException as e: + logger.debug(f"Request exception: {e}") + raise TalkerNetworkError(self.source_name_friendly, 0, str(e)) from e + except json.JSONDecodeError as e: + logger.debug(f"JSON decode error: {e}") + raise TalkerDataError(self.source_name_friendly, 2, "ComicVine did not provide json") + + raise TalkerNetworkError(self.source_name_friendly, 5) + + def format_search_results(self, search_results: list[CVVolumeResults]) -> list[ComicVolume]: + formatted_results = [] + for record in search_results: + # Flatten publisher to name only + if record.get("publisher") is None: + pub_name = "" + else: + pub_name = record["publisher"].get("name", "") + + if record.get("image") is None: + image_url = "" + else: + image_url = record["image"].get("super_url", "") + + if record.get("start_year") is None: + start_year = 0 + else: + start_year = utils.xlate(record["start_year"], True) + + formatted_results.append( + ComicVolume( + aliases=record["aliases"].split("\n") if record["aliases"] else [], # CV returns a null because...? + count_of_issues=record.get("count_of_issues", 0), + description=record.get("description", ""), + id=record["id"], + image_url=image_url, + name=record["name"], + publisher=pub_name, + start_year=start_year, + ) + ) + + return formatted_results + + def format_issue_results( + self, issue_results: list[CVIssueDetailResults], complete: bool = False + ) -> list[ComicIssue]: + formatted_results = [] + for record in issue_results: + # Extract image super and thumb to name only + if record.get("image") is None: + image_url = "" + image_thumb_url = "" + else: + image_url = record["image"].get("super_url", "") + image_thumb_url = record["image"].get("thumb_url", "") + + alt_images_list = [] + for alt in record["associated_images"]: + alt_images_list.append(alt["original_url"]) + + character_list = [] + if record.get("character_credits"): + for char in record["character_credits"]: + character_list.append(char["name"]) + + location_list = [] + if record.get("location_credits"): + for loc in record["location_credits"]: + location_list.append(loc["name"]) + + teams_list = [] + if record.get("team_credits"): + for loc in record["team_credits"]: + teams_list.append(loc["name"]) + + story_list = [] + if record.get("story_arc_credits"): + for loc in record["story_arc_credits"]: + story_list.append(loc["name"]) + + persons_list = [] + if record.get("person_credits"): + for person in record["person_credits"]: + persons_list.append(Credits(name=person["name"], role=person["role"])) + + formatted_results.append( + ComicIssue( + aliases=record["aliases"].split("\n") if record["aliases"] else [], + cover_date=record.get("cover_date", ""), + description=record.get("description", ""), + id=record["id"], + image_url=image_url, + image_thumb_url=image_thumb_url, + issue_number=record["issue_number"], + name=record["name"], + site_detail_url=record.get("site_detail_url", ""), + volume=cast(ComicVolume, record["volume"]), + alt_image_urls=alt_images_list, + characters=character_list, + locations=location_list, + teams=teams_list, + story_arcs=story_list, + credits=persons_list, + complete=complete, + ) + ) + + return formatted_results + + def search_for_series( + self, + series_name: str, + callback: Callable[[int, int], None] | None = None, + refresh_cache: bool = False, + literal: bool = False, + ) -> list[ComicVolume]: + # Sanitize the series name for comicvine searching, comicvine search ignore symbols + search_series_name = utils.sanitize_title(series_name, literal) + logger.info(f"{self.source_name_friendly} searching: {search_series_name}") + + # Before we search online, look in our cache, since we might have done this same search recently + # For literal searches always retrieve from online + cvc = ComicCacher() + if not refresh_cache and not literal: + cached_search_results = cvc.get_search_results(self.source_name, series_name) + + if len(cached_search_results) > 0: + return cached_search_results + + params = { + "api_key": self.api_key, + "format": "json", + "resources": "volume", + "query": search_series_name, + "field_list": "volume,name,id,start_year,publisher,image,description,count_of_issues,aliases", + "page": 1, + "limit": 100, + } + + cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params) + + search_results: list[CVVolumeResults] = [] + + # see http://api.comicvine.com/documentation/#handling_responses + + current_result_count = cv_response["number_of_page_results"] + total_result_count = cv_response["number_of_total_results"] + + # 8 Dec 2018 - Comic Vine changed query results again. Terms are now + # ORed together, and we get thousands of results. Good news is the + # results are sorted by relevance, so we can be smart about halting the search. + # 1. Don't fetch more than some sane amount of pages. + # 2. Halt when any result on the current page is less than or equal to a set ratio using thefuzz + max_results = 500 # 5 pages + + total_result_count = min(total_result_count, max_results) + + if callback is None: + logger.debug( + f"Found {cv_response['number_of_page_results']} of {cv_response['number_of_total_results']} results" + ) + search_results.extend(cast(list[CVVolumeResults], cv_response["results"])) + page = 1 + + if callback is not None: + callback(current_result_count, total_result_count) + + # see if we need to keep asking for more pages... + while current_result_count < total_result_count: + + if not literal: + # Stop searching once any entry falls below the threshold + stop_searching = any( + not utils.titles_match(search_series_name, volume["name"], self.series_match_thresh) + for volume in cast(list[CVVolumeResults], cv_response["results"]) + ) + + if stop_searching: + break + + if callback is None: + logger.debug(f"getting another page of results {current_result_count} of {total_result_count}...") + page += 1 + + params["page"] = page + cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params) + + search_results.extend(cast(list[CVVolumeResults], cv_response["results"])) + current_result_count += cv_response["number_of_page_results"] + + if callback is not None: + callback(current_result_count, total_result_count) + + # Format result to ComicIssue + formatted_search_results = self.format_search_results(search_results) + + # Cache these search results, even if it's literal we cache the results + # The most it will cause is extra processing time + cvc.add_search_results(self.source_name, series_name, formatted_search_results) + + return formatted_search_results + + # Get issue or volume information + def fetch_comic_data(self, issue_id: int = 0, series_id: int = 0, issue_number: str = "") -> GenericMetadata: + comic_data = GenericMetadata() + if issue_number and series_id: + comic_data = self.fetch_issue_data(series_id, issue_number) + elif issue_id: + comic_data = self.fetch_issue_data_by_issue_id(issue_id) + + return comic_data + + def fetch_partial_volume_data(self, series_id: int) -> ComicVolume: + # before we search online, look in our cache, since we might already have this info + cvc = ComicCacher() + cached_volume_result = cvc.get_volume_info(series_id, self.source_name) + + if cached_volume_result is not None: + return cached_volume_result + + volume_url = urljoin(self.api_base_url, f"volume/{CVTypeID.Volume}-{series_id}") + + params = { + "api_key": self.api_key, + "format": "json", + "field_list": "name,id,start_year,publisher,count_of_issues,aliases", + } + cv_response = self.get_cv_content(volume_url, params) + + volume_results = cast(CVVolumeResults, cv_response["results"]) + formatted_volume_results = self.format_search_results([volume_results]) + + if volume_results: + cvc.add_volume_info(self.source_name, formatted_volume_results[0]) + + return formatted_volume_results[0] + + def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]: + # before we search online, look in our cache, since we might already have this info + cvc = ComicCacher() + cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name) + + volume_data = self.fetch_partial_volume_data(series_id) + + if len(cached_volume_issues_result) == volume_data["count_of_issues"]: + return cached_volume_issues_result + + params = { + "api_key": self.api_key, + "filter": f"volume:{series_id}", + "format": "json", + "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases,associated_images", + "offset": 0, + } + cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) + + current_result_count = cv_response["number_of_page_results"] + total_result_count = cv_response["number_of_total_results"] + + volume_issues_result = cast(list[CVIssueDetailResults], cv_response["results"]) + page = 1 + offset = 0 + + # see if we need to keep asking for more pages... + while current_result_count < total_result_count: + page += 1 + offset += cv_response["number_of_page_results"] + + params["offset"] = offset + cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) + + volume_issues_result.extend(cast(list[CVIssueDetailResults], cv_response["results"])) + current_result_count += cv_response["number_of_page_results"] + + # Format to expected output !! issues/ volume does NOT return publisher!! + formatted_volume_issues_result = self.format_issue_results(volume_issues_result) + + cvc.add_volume_issues_info(self.source_name, formatted_volume_issues_result) + + return formatted_volume_issues_result + + def fetch_issues_by_volume_issue_num_and_year( + self, volume_id_list: list[int], issue_number: str, year: str | int | None + ) -> list[ComicIssue]: + volume_filter = "" + for vid in volume_id_list: + volume_filter += str(vid) + "|" + flt = f"volume:{volume_filter},issue_number:{issue_number}" + + int_year = utils.xlate(year, True) + if int_year is not None: + flt += f",cover_date:{int_year}-1-1|{int_year + 1}-1-1" + + params: dict[str, str | int] = { + "api_key": self.api_key, + "format": "json", + "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases,associated_images", + "filter": flt, + } + + cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) + + current_result_count = cv_response["number_of_page_results"] + total_result_count = cv_response["number_of_total_results"] + + filtered_issues_result = cast(list[CVIssueDetailResults], cv_response["results"]) + page = 1 + offset = 0 + + # see if we need to keep asking for more pages... + while current_result_count < total_result_count: + page += 1 + offset += cv_response["number_of_page_results"] + + params["offset"] = offset + cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params) + + filtered_issues_result.extend(cast(list[CVIssueDetailResults], cv_response["results"])) + current_result_count += cv_response["number_of_page_results"] + + formatted_filtered_issues_result = self.format_issue_results(filtered_issues_result) + + return formatted_filtered_issues_result + + def fetch_issue_data(self, series_id: int, issue_number: str) -> GenericMetadata: + issues_list_results = self.fetch_issues_by_volume(series_id) + + # Loop through issue list to find the required issue info + f_record = None + for record in issues_list_results: + if not IssueString(issue_number).as_string(): + issue_number = "1" + if ( + IssueString(record["issue_number"]).as_string().casefold() + == IssueString(issue_number).as_string().casefold() + ): + f_record = record + break + + if f_record and f_record["complete"]: + # Cache had full record + return talker_utils.map_comic_issue_to_metadata( + f_record, + self.source_name_friendly, + self.remove_html_tables, + self.use_series_start_as_volume, + ) + + if f_record is not None: + return self.fetch_issue_data_by_issue_id(f_record["id"]) + else: + return GenericMetadata() + + def fetch_issue_data_by_issue_id(self, issue_id: int) -> GenericMetadata: + # before we search online, look in our cache, since we might already have this info + cvc = ComicCacher() + cached_issues_result = cvc.get_issue_info(issue_id, self.source_name) + + if cached_issues_result and cached_issues_result["complete"]: + return talker_utils.map_comic_issue_to_metadata( + cached_issues_result, + self.source_name_friendly, + self.remove_html_tables, + self.use_series_start_as_volume, + ) + + issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}") + params = {"api_key": self.api_key, "format": "json"} + cv_response = self.get_cv_content(issue_url, params) + + issue_results = cast(CVIssueDetailResults, cv_response["results"]) + + # Format to expected output + formatted_issues_result = self.format_issue_results([issue_results], True) + + # Due to issue/ not returning volume publisher, get it. + volume_info = self.fetch_partial_volume_data(formatted_issues_result[0]["volume"]["id"]) + formatted_issues_result[0]["volume"]["publisher"] = volume_info["publisher"] + + cvc.add_volume_issues_info(self.source_name, formatted_issues_result) + + # Now, map the ComicIssue data to generic metadata + return talker_utils.map_comic_issue_to_metadata( + formatted_issues_result[0], + self.source_name_friendly, + self.remove_html_tables, + self.use_series_start_as_volume, + ) diff --git a/testing/comicdata.py b/testing/comicdata.py index d2533fe..4340ba7 100644 --- a/testing/comicdata.py +++ b/testing/comicdata.py @@ -1,29 +1,29 @@ from __future__ import annotations import comicapi.genericmetadata -import comictaggerlib.resulttypes +import comictalker.resulttypes from comicapi import utils search_results = [ - comictaggerlib.resulttypes.CVVolumeResults( + comictalker.resulttypes.ComicVolume( count_of_issues=1, description="this is a description", id=1, - image={"super_url": "https://test.org/image/1"}, + image_url="https://test.org/image/1", name="test", - publisher=comictaggerlib.resulttypes.CVPublisher(name="test"), - start_year="", # This is currently submitted as a string and returned as an int - aliases=None, + publisher="test", + start_year=0, + aliases=[], ), - comictaggerlib.resulttypes.CVVolumeResults( + comictalker.resulttypes.ComicVolume( count_of_issues=1, description="this is a description", - id=1, - image={"super_url": "https://test.org/image/2"}, + id=2, + image_url="https://test.org/image/2", name="test 2", - publisher=comictaggerlib.resulttypes.CVPublisher(name="test"), - start_year="", # This is currently submitted as a string and returned as an int - aliases=None, + publisher="test", + start_year=0, + aliases=[], ), ] diff --git a/testing/comicvine.py b/testing/comicvine.py index 82dfca2..9c15978 100644 --- a/testing/comicvine.py +++ b/testing/comicvine.py @@ -3,7 +3,8 @@ from __future__ import annotations from typing import Any import comicapi.genericmetadata -import comictaggerlib.comicvinetalker +from comicapi import utils +from comictalker.talker_utils import cleanup_html def filter_field_list(cv_result, kwargs): @@ -21,7 +22,7 @@ cv_issue_result: dict[str, Any] = { "number_of_total_results": 1, "status_code": 1, "results": { - "aliases": None, + "aliases": [], "api_detail_url": "https://comicvine.gamespot.com/api/issue/4000-140529/", "associated_images": [], "character_credits": [], @@ -116,7 +117,7 @@ cv_volume_result: dict[str, Any] = { "number_of_total_results": 1, "status_code": 1, "results": { - "aliases": None, + "aliases": [], "api_detail_url": "https://comicvine.gamespot.com/api/volume/4050-23437/", "count_of_issues": 6, "date_added": "2008-10-16 05:25:47", @@ -156,7 +157,24 @@ cv_not_found = { "status_code": 101, "results": [], } -date = comictaggerlib.comicvinetalker.ComicVineTalker().parse_date_str(cv_issue_result["results"]["cover_date"]) +comic_issue_result: dict[str, Any] = { + "aliases": cv_issue_result["results"]["aliases"], + "cover_date": cv_issue_result["results"]["cover_date"], + "description": cv_issue_result["results"]["description"], + "id": cv_issue_result["results"]["id"], + "image_url": cv_issue_result["results"]["image"]["super_url"], + "image_thumb_url": cv_issue_result["results"]["image"]["thumb_url"], + "issue_number": cv_issue_result["results"]["issue_number"], + "name": cv_issue_result["results"]["name"], + "site_detail_url": cv_issue_result["results"]["site_detail_url"], + "volume": { + "api_detail_url": cv_issue_result["results"]["volume"]["api_detail_url"], + "id": cv_issue_result["results"]["volume"]["id"], + "name": cv_issue_result["results"]["volume"]["name"], + "site_detail_url": cv_issue_result["results"]["volume"]["site_detail_url"], + }, +} +date = utils.parse_date_str(cv_issue_result["results"]["cover_date"]) cv_md = comicapi.genericmetadata.GenericMetadata( is_empty=False, @@ -173,9 +191,7 @@ cv_md = comicapi.genericmetadata.GenericMetadata( volume=None, genre=None, language=None, - comments=comictaggerlib.comicvinetalker.ComicVineTalker().cleanup_html( - cv_issue_result["results"]["description"], False - ), + comments=cleanup_html(cv_issue_result["results"]["description"], False), volume_count=None, critical_rating=None, country=None, @@ -193,9 +209,9 @@ cv_md = comicapi.genericmetadata.GenericMetadata( story_arc=None, series_group=None, scan_info=None, - characters="", - teams="", - locations="", + characters=None, + teams=None, + locations=None, credits=[ comicapi.genericmetadata.CreditMetadata(person=x["name"], role=x["role"].title(), primary=False) for x in cv_issue_result["results"]["person_credits"] @@ -207,7 +223,7 @@ cv_md = comicapi.genericmetadata.GenericMetadata( rights=None, identifier=None, last_mark=None, - cover_image=None, + cover_image=cv_issue_result["results"]["image"]["super_url"], ) diff --git a/tests/comicarchive_test.py b/tests/comicarchive_test.py index 004758d..b675431 100644 --- a/tests/comicarchive_test.py +++ b/tests/comicarchive_test.py @@ -16,7 +16,7 @@ def test_getPageNameList(): pageNameList = c.get_page_name_list() assert pageNameList == [ - "!cover.jpg", + "!cover.jpg", # Depending on locale punctuation or numbers might come first (Linux) "00.jpg", "page0.jpg", "Page1.jpeg", diff --git a/tests/comiccacher_test.py b/tests/comiccacher_test.py index 5ae7cd1..91b9291 100644 --- a/tests/comiccacher_test.py +++ b/tests/comiccacher_test.py @@ -2,13 +2,12 @@ from __future__ import annotations import pytest -import comictaggerlib.comiccacher -import comictaggerlib.resulttypes -from testing.comicdata import alt_covers, search_results, select_details +import comictalker.comiccacher +from testing.comicdata import search_results def test_create_cache(settings): - comictaggerlib.comiccacher.ComicCacher() + comictalker.comiccacher.ComicCacher() assert (settings.get_settings_folder() / "settings").exists() @@ -21,24 +20,13 @@ def test_search_results(comic_cache): assert search_results == comic_cache.get_search_results("test", "test search") -@pytest.mark.parametrize("alt_cover", alt_covers) -def test_alt_covers(comic_cache, alt_cover): - comic_cache.add_alt_covers(**alt_cover, source_name="test") - assert alt_cover["url_list"] == comic_cache.get_alt_covers(issue_id=alt_cover["issue_id"], source_name="test") - - @pytest.mark.parametrize("volume_info", search_results) def test_volume_info(comic_cache, volume_info): - comic_cache.add_volume_info(cv_volume_record=volume_info, source_name="test") + comic_cache.add_volume_info(volume_record=volume_info, source_name="test") vi = volume_info.copy() del vi["description"] - del vi["image"] - assert vi == comic_cache.get_volume_info(volume_id=volume_info["id"], source_name="test") - - -@pytest.mark.parametrize("details", select_details) -def test_issue_select_details(comic_cache, details): - comic_cache.add_issue_select_details(**details, source_name="test") - det = details.copy() - del det["issue_id"] - assert det == comic_cache.get_issue_select_details(details["issue_id"], "test") + del vi["image_url"] + cache_result = comic_cache.get_volume_info(volume_id=volume_info["id"], source_name="test") + del cache_result["description"] + del cache_result["image_url"] + assert vi == cache_result diff --git a/tests/comicvinetalker_test.py b/tests/comicvinetalker_test.py index 1d4a43c..ae8372f 100644 --- a/tests/comicvinetalker_test.py +++ b/tests/comicvinetalker_test.py @@ -3,57 +3,74 @@ from __future__ import annotations import pytest import comicapi.genericmetadata -import comictaggerlib.comicvinetalker +import comictalker.talkers.comicvine import testing.comicvine -from testing.comicdata import select_details def test_search_for_series(comicvine_api, comic_cache): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() + ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) results = ct.search_for_series("cory doctorows futuristic tales of the here and now") - for r in results: - r["image"] = {"super_url": r["image"]["super_url"]} - r["start_year"] = int(r["start_year"]) - del r["publisher"]["id"] - del r["publisher"]["api_detail_url"] cache_issues = comic_cache.get_search_results(ct.source_name, "cory doctorows futuristic tales of the here and now") assert results == cache_issues def test_fetch_volume_data(comicvine_api, comic_cache): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() - result = ct.fetch_volume_data(23437) - result["start_year"] = int(result["start_year"]) - del result["publisher"]["id"] - del result["publisher"]["api_detail_url"] - assert result == comic_cache.get_volume_info(23437, ct.source_name) + ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) + result = ct.fetch_partial_volume_data(23437) + del result["description"] + del result["image_url"] + cache_result = comic_cache.get_volume_info(23437, ct.source_name) + del cache_result["description"] + del cache_result["image_url"] + assert result == cache_result def test_fetch_issues_by_volume(comicvine_api, comic_cache): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() + ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) results = ct.fetch_issues_by_volume(23437) cache_issues = comic_cache.get_volume_issues_info(23437, ct.source_name) for r in results: - r["image"] = {"super_url": r["image"]["super_url"], "thumb_url": r["image"]["thumb_url"]} del r["volume"] + del r["image_thumb_url"] + del r["characters"] + del r["locations"] + del r["story_arcs"] + del r["teams"] + for c in cache_issues: + del c["volume"] + del c["characters"] + del c["locations"] + del c["story_arcs"] + del c["teams"] assert results == cache_issues def test_fetch_issue_data_by_issue_id(comicvine_api, settings, mock_version): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() - result = ct.fetch_issue_data_by_issue_id(140529, settings) + ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) + result = ct.fetch_comic_data(140529) + result.notes = None assert result == testing.comicvine.cv_md def test_fetch_issues_by_volume_issue_num_and_year(comicvine_api): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() + ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) results = ct.fetch_issues_by_volume_issue_num_and_year([23437], "1", None) - cv_expected = testing.comicvine.cv_issue_result["results"].copy() + cv_expected = testing.comicvine.comic_issue_result.copy() testing.comicvine.filter_field_list( cv_expected, {"params": {"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases"}}, ) for r, e in zip(results, [cv_expected]): + del r["image_thumb_url"] + del r["image_url"] + del r["alt_image_urls"] + del r["characters"] + del r["credits"] + del r["locations"] + del r["story_arcs"] + del r["teams"] + del r["complete"] + del r["volume"]["publisher"] assert r == e @@ -66,36 +83,7 @@ cv_issue = [ @pytest.mark.parametrize("volume_id, issue_number, expected", cv_issue) def test_fetch_issue_data(comicvine_api, settings, mock_version, volume_id, issue_number, expected): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() - results = ct.fetch_issue_data(volume_id, issue_number, settings) + ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) + results = ct.fetch_issue_data(volume_id, issue_number) + results.notes = None assert results == expected - - -def test_fetch_issue_select_details(comicvine_api, mock_version): - ct = comictaggerlib.comicvinetalker.ComicVineTalker() - result = ct.fetch_issue_select_details(140529) - expected = { - "cover_date": testing.comicvine.cv_issue_result["results"]["cover_date"], - "site_detail_url": testing.comicvine.cv_issue_result["results"]["site_detail_url"], - "image_url": testing.comicvine.cv_issue_result["results"]["image"]["super_url"], - "thumb_image_url": testing.comicvine.cv_issue_result["results"]["image"]["thumb_url"], - } - assert result == expected - - -@pytest.mark.parametrize("details", select_details) -def test_issue_select_details(comic_cache, details): - expected = details.copy() - del expected["issue_id"] - - ct = comictaggerlib.comicvinetalker.ComicVineTalker() - ct.cache_issue_select_details( - issue_id=details["issue_id"], - image_url=details["image_url"], - thumb_url=details["thumb_image_url"], - cover_date=details["cover_date"], - page_url=details["site_detail_url"], - ) - result = comic_cache.get_issue_select_details(details["issue_id"], ct.source_name) - - assert result == expected diff --git a/tests/conftest.py b/tests/conftest.py index 1fac14f..7cccda8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,9 +13,9 @@ from PIL import Image import comicapi.comicarchive import comicapi.genericmetadata -import comictaggerlib.comiccacher -import comictaggerlib.comicvinetalker import comictaggerlib.settings +import comictalker.comiccacher +import comictalker.talkers.comicvine from comicapi import utils from testing import comicvine, filenames from testing.comicdata import all_seed_imprints, seed_imprints @@ -54,7 +54,7 @@ def no_requests(monkeypatch) -> None: @pytest.fixture -def comicvine_api(monkeypatch, cbz, comic_cache) -> unittest.mock.Mock: +def comicvine_api(monkeypatch, cbz, comic_cache) -> comictalker.talkers.comicvine.ComicVineTalker: # Any arguments may be passed and mock_get() will always return our # mocked object, which only has the .json() method or None for invalid urls. @@ -111,7 +111,9 @@ def comicvine_api(monkeypatch, cbz, comic_cache) -> unittest.mock.Mock: # apply the monkeypatch for requests.get to mock_get monkeypatch.setattr(requests, "get", m_get) - return m_get + + cv = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False) + return cv @pytest.fixture @@ -153,5 +155,5 @@ def settings(tmp_path): @pytest.fixture -def comic_cache(settings) -> Generator[comictaggerlib.comiccacher.ComicCacher, Any, None]: - yield comictaggerlib.comiccacher.ComicCacher() +def comic_cache(settings) -> Generator[comictalker.comiccacher.ComicCacher, Any, None]: + yield comictalker.comiccacher.ComicCacher() diff --git a/tests/issueidentifier_test.py b/tests/issueidentifier_test.py index 2111839..a4e3498 100644 --- a/tests/issueidentifier_test.py +++ b/tests/issueidentifier_test.py @@ -4,14 +4,13 @@ import pytest import comicapi.comicarchive import comicapi.issuestring -import comictaggerlib.comicvinetalker import comictaggerlib.issueidentifier import testing.comicdata import testing.comicvine -def test_crop(cbz_double_cover, settings, tmp_path): - ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz_double_cover, settings) +def test_crop(cbz_double_cover, settings, tmp_path, comicvine_api): + ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz_double_cover, settings, comicvine_api) cropped = ii.crop_cover(cbz_double_cover.archiver.read_file("double_cover.jpg")) original_cover = cbz_double_cover.get_page(0) @@ -22,17 +21,16 @@ def test_crop(cbz_double_cover, settings, tmp_path): @pytest.mark.parametrize("additional_md, expected", testing.comicdata.metadata_keys) -def test_get_search_keys(cbz, settings, additional_md, expected): - ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings) +def test_get_search_keys(cbz, settings, additional_md, expected, comicvine_api): + ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api) ii.set_additional_metadata(additional_md) assert expected == ii.get_search_keys() def test_get_issue_cover_match_score(cbz, settings, comicvine_api): - ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings) + ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api) score = ii.get_issue_cover_match_score( - comictaggerlib.comicvinetalker.ComicVineTalker(), int( comicapi.issuestring.IssueString( cbz.read_metadata(comicapi.comicarchive.MetaDataStyle.CIX).issue @@ -52,12 +50,13 @@ def test_get_issue_cover_match_score(cbz, settings, comicvine_api): def test_search(cbz, settings, comicvine_api): - ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings) + ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api) results = ii.search() cv_expected = { "series": f"{testing.comicvine.cv_volume_result['results']['name']} ({testing.comicvine.cv_volume_result['results']['start_year']})", "distance": 0, "issue_number": testing.comicvine.cv_issue_result["results"]["issue_number"], + "alt_image_urls": [], "cv_issue_count": testing.comicvine.cv_volume_result["results"]["count_of_issues"], "issue_title": testing.comicvine.cv_issue_result["results"]["name"], "issue_id": testing.comicvine.cv_issue_result["results"]["id"], @@ -67,7 +66,6 @@ def test_search(cbz, settings, comicvine_api): "publisher": testing.comicvine.cv_volume_result["results"]["publisher"]["name"], "image_url": testing.comicvine.cv_issue_result["results"]["image"]["super_url"], "thumb_url": testing.comicvine.cv_issue_result["results"]["image"]["thumb_url"], - "page_url": testing.comicvine.cv_issue_result["results"]["site_detail_url"], "description": testing.comicvine.cv_issue_result["results"]["description"], } for r, e in zip(results, [cv_expected]):