diff --git a/comictaggerlib/autotagmatchwindow.py b/comictaggerlib/autotagmatchwindow.py
index 16d58c2..516dc04 100644
--- a/comictaggerlib/autotagmatchwindow.py
+++ b/comictaggerlib/autotagmatchwindow.py
@@ -28,6 +28,7 @@ from comictaggerlib.resulttypes import IssueResult, MultipleMatch
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@@ -42,6 +43,7 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
style: int,
fetch_func: Callable[[IssueResult], GenericMetadata],
settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
) -> None:
super().__init__(parent)
@@ -51,12 +53,12 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
self.current_match_set: MultipleMatch = match_set_list[0]
- self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode)
+ self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode)
gridlayout = QtWidgets.QGridLayout(self.altCoverContainer)
gridlayout.addWidget(self.altCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
- self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode)
+ self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer)
gridlayout.addWidget(self.archiveCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@@ -178,7 +180,10 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
if prev is not None and prev.row() == curr.row():
return None
- self.altCoverWidget.set_issue_id(self.current_match()["issue_id"])
+ self.altCoverWidget.set_issue_details(
+ self.current_match()["issue_id"],
+ [self.current_match()["image_url"], *self.current_match()["alt_image_urls"]],
+ )
if self.current_match()["description"] is None:
self.teDescription.setText("")
else:
@@ -242,15 +247,13 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
)
# now get the particular issue data
- cv_md = self.fetch_func(match)
- if cv_md is None:
- QtWidgets.QMessageBox.critical(
- self, "Network Issue", "Could not connect to Comic Vine to get issue details!"
- )
+ ct_md = self.fetch_func(match)
+ if ct_md is None:
+ QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not retrieve issue details!")
return
QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
- md.overlay(cv_md)
+ md.overlay(ct_md)
success = ca.write_metadata(md, self._style)
ca.load_cache([MetaDataStyle.CBI, MetaDataStyle.CIX])
diff --git a/comictaggerlib/autotagprogresswindow.py b/comictaggerlib/autotagprogresswindow.py
index dd49740..477381b 100644
--- a/comictaggerlib/autotagprogresswindow.py
+++ b/comictaggerlib/autotagprogresswindow.py
@@ -22,22 +22,25 @@ from PyQt5 import QtCore, QtWidgets, uic
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
class AutoTagProgressWindow(QtWidgets.QDialog):
- def __init__(self, parent: QtWidgets.QWidget) -> None:
+ def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "autotagprogresswindow.ui", self)
- self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.DataMode, False)
+ self.archiveCoverWidget = CoverImageWidget(
+ self.archiveCoverContainer, talker_api, CoverImageWidget.DataMode, False
+ )
gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer)
gridlayout.addWidget(self.archiveCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
- self.testCoverWidget = CoverImageWidget(self.testCoverContainer, CoverImageWidget.DataMode, False)
+ self.testCoverWidget = CoverImageWidget(self.testCoverContainer, talker_api, CoverImageWidget.DataMode, False)
gridlayout = QtWidgets.QGridLayout(self.testCoverContainer)
gridlayout.addWidget(self.testCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
diff --git a/comictaggerlib/autotagstartwindow.py b/comictaggerlib/autotagstartwindow.py
index d818719..bae9f19 100644
--- a/comictaggerlib/autotagstartwindow.py
+++ b/comictaggerlib/autotagstartwindow.py
@@ -48,7 +48,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.cbxAssumeIssueOne.setChecked(self.settings.assume_1_if_no_issue_num)
self.cbxIgnoreLeadingDigitsInFilename.setChecked(self.settings.ignore_leading_numbers_in_filename)
self.cbxRemoveAfterSuccess.setChecked(self.settings.remove_archive_after_successful_match)
- self.cbxWaitForRateLimit.setChecked(self.settings.wait_and_retry_on_rate_limit)
self.cbxAutoImprint.setChecked(self.settings.auto_imprint)
nlmt_tip = """The Name Match Ratio Threshold: Auto-Identify is for eliminating automatic
@@ -73,7 +72,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.assume_issue_one = False
self.ignore_leading_digits_in_filename = False
self.remove_after_success = False
- self.wait_and_retry_on_rate_limit = False
self.search_string = ""
self.name_length_match_tolerance = self.settings.id_series_match_search_thresh
self.split_words = self.cbxSplitWords.isChecked()
@@ -91,7 +89,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.ignore_leading_digits_in_filename = self.cbxIgnoreLeadingDigitsInFilename.isChecked()
self.remove_after_success = self.cbxRemoveAfterSuccess.isChecked()
self.name_length_match_tolerance = self.sbNameMatchSearchThresh.value()
- self.wait_and_retry_on_rate_limit = self.cbxWaitForRateLimit.isChecked()
self.split_words = self.cbxSplitWords.isChecked()
# persist some settings
@@ -100,7 +97,6 @@ class AutoTagStartWindow(QtWidgets.QDialog):
self.settings.assume_1_if_no_issue_num = self.assume_issue_one
self.settings.ignore_leading_numbers_in_filename = self.ignore_leading_digits_in_filename
self.settings.remove_archive_after_successful_match = self.remove_after_success
- self.settings.wait_and_retry_on_rate_limit = self.wait_and_retry_on_rate_limit
if self.cbxSpecifySearchString.isChecked():
self.search_string = self.leSearchString.text()
diff --git a/comictaggerlib/cli.py b/comictaggerlib/cli.py
index b73b75d..3a13e2c 100644
--- a/comictaggerlib/cli.py
+++ b/comictaggerlib/cli.py
@@ -30,32 +30,30 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle
from comicapi.genericmetadata import GenericMetadata
from comictaggerlib import ctversion
from comictaggerlib.cbltransformer import CBLTransformer
-from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.filerenamer import FileRenamer, get_rename_dir
from comictaggerlib.graphics import graphics_path
from comictaggerlib.issueidentifier import IssueIdentifier
-from comictaggerlib.resulttypes import IssueResult, MultipleMatch, OnlineMatchResults
+from comictaggerlib.resulttypes import MultipleMatch, OnlineMatchResults
from comictaggerlib.settings import ComicTaggerSettings
+from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
def actual_issue_data_fetch(
- match: IssueResult, settings: ComicTaggerSettings, opts: argparse.Namespace
+ issue_id: int, settings: ComicTaggerSettings, opts: argparse.Namespace, talker_api: ComicTalker
) -> GenericMetadata:
# now get the particular issue data
try:
- comic_vine = ComicVineTalker(settings.id_series_match_search_thresh)
- comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit
- cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], settings)
- except ComicVineTalkerException:
- logger.exception("Network error while getting issue details. Save aborted")
+ ct_md = talker_api.fetch_comic_data(issue_id)
+ except TalkerError as e:
+ logger.exception(f"Error retrieving issue details. Save aborted.\n{e}")
return GenericMetadata()
if settings.apply_cbl_transform_on_cv_import:
- cv_md = CBLTransformer(cv_md, settings).apply()
+ ct_md = CBLTransformer(ct_md, settings).apply()
- return cv_md
+ return ct_md
def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: GenericMetadata) -> bool:
@@ -80,7 +78,11 @@ def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: Generic
def display_match_set_for_choice(
- label: str, match_set: MultipleMatch, opts: argparse.Namespace, settings: ComicTaggerSettings
+ label: str,
+ match_set: MultipleMatch,
+ opts: argparse.Namespace,
+ settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
) -> None:
print(f"{match_set.ca.path} -- {label}:")
@@ -110,15 +112,15 @@ def display_match_set_for_choice(
# we know at this point, that the file is all good to go
ca = match_set.ca
md = create_local_metadata(opts, ca, settings)
- cv_md = actual_issue_data_fetch(match_set.matches[int(i) - 1], settings, opts)
+ ct_md = actual_issue_data_fetch(match_set.matches[int(i) - 1]["issue_id"], settings, opts, talker_api)
if opts.overwrite:
- md = cv_md
+ md = ct_md
else:
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
- f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]"
+ f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]"
)
- md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
+ md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
if opts.auto_imprint:
md.fix_publisher()
@@ -127,7 +129,7 @@ def display_match_set_for_choice(
def post_process_matches(
- match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings
+ match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker
) -> None:
# now go through the match results
if opts.show_save_summary:
@@ -158,7 +160,7 @@ def post_process_matches(
if len(match_results.multiple_matches) > 0:
print("\nArchives with multiple high-confidence matches:\n------------------")
for match_set in match_results.multiple_matches:
- display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings)
+ display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings, talker_api)
if len(match_results.low_confidence_matches) > 0:
print("\nArchives with low-confidence matches:\n------------------")
@@ -168,10 +170,10 @@ def post_process_matches(
else:
label = "Multiple low-confidence matches"
- display_match_set_for_choice(label, match_set, opts, settings)
+ display_match_set_for_choice(label, match_set, opts, settings, talker_api)
-def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None:
+def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None:
if len(opts.file_list) < 1:
logger.error("You must specify at least one filename. Use the -h option for more info")
return
@@ -179,10 +181,10 @@ def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None:
match_results = OnlineMatchResults()
for f in opts.file_list:
- process_file_cli(f, opts, settings, match_results)
+ process_file_cli(f, opts, settings, talker_api, match_results)
sys.stdout.flush()
- post_process_matches(match_results, opts, settings)
+ post_process_matches(match_results, opts, settings, talker_api)
def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings: ComicTaggerSettings) -> GenericMetadata:
@@ -217,7 +219,11 @@ def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings:
def process_file_cli(
- filename: str, opts: argparse.Namespace, settings: ComicTaggerSettings, match_results: OnlineMatchResults
+ filename: str,
+ opts: argparse.Namespace,
+ settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
+ match_results: OnlineMatchResults,
) -> None:
batch_mode = len(opts.file_list) > 1
@@ -380,31 +386,29 @@ def process_file_cli(
# now, search online
if opts.online:
if opts.issue_id is not None:
- # we were given the actual ID to search with
+ # we were given the actual issue ID to search with
try:
- comic_vine = ComicVineTalker(settings.id_series_match_search_thresh)
- comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit
- cv_md = comic_vine.fetch_issue_data_by_issue_id(opts.issue_id, settings)
- except ComicVineTalkerException:
- logger.exception("Network error while getting issue details. Save aborted")
+ ct_md = talker_api.fetch_comic_data(opts.issue_id)
+ except TalkerError as e:
+ logger.exception(f"Error retrieving issue details. Save aborted.\n{e}")
match_results.fetch_data_failures.append(str(ca.path.absolute()))
return
- if cv_md is None:
+ if ct_md is None:
logger.error("No match for ID %s was found.", opts.issue_id)
match_results.no_matches.append(str(ca.path.absolute()))
return
if settings.apply_cbl_transform_on_cv_import:
- cv_md = CBLTransformer(cv_md, settings).apply()
+ ct_md = CBLTransformer(ct_md, settings).apply()
else:
- ii = IssueIdentifier(ca, settings)
-
if md is None or md.is_empty:
logger.error("No metadata given to search online with!")
match_results.no_matches.append(str(ca.path.absolute()))
return
+ ii = IssueIdentifier(ca, settings, talker_api)
+
def myoutput(text: str) -> None:
if opts.verbose:
IssueIdentifier.default_write_output(text)
@@ -412,7 +416,6 @@ def process_file_cli(
# use our overlaid MD struct to search
ii.set_additional_metadata(md)
ii.only_use_additional_meta_data = True
- ii.wait_and_retry_on_rate_limit = opts.wait_on_cv_rate_limit
ii.set_output_function(myoutput)
ii.cover_page_index = md.get_cover_page_index_list()[0]
matches = ii.search()
@@ -459,19 +462,19 @@ def process_file_cli(
# we got here, so we have a single match
# now get the particular issue data
- cv_md = actual_issue_data_fetch(matches[0], settings, opts)
- if cv_md.is_empty:
+ ct_md = actual_issue_data_fetch(matches[0]["issue_id"], settings, opts, talker_api)
+ if ct_md.is_empty:
match_results.fetch_data_failures.append(str(ca.path.absolute()))
return
if opts.overwrite:
- md = cv_md
+ md = ct_md
else:
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
- f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]"
+ f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]"
)
- md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
+ md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
if opts.auto_imprint:
md.fix_publisher()
diff --git a/comictaggerlib/comicvinetalker.py b/comictaggerlib/comicvinetalker.py
deleted file mode 100644
index 13fedaf..0000000
--- a/comictaggerlib/comicvinetalker.py
+++ /dev/null
@@ -1,805 +0,0 @@
-"""A python class to manage communication with Comic Vine's REST API"""
-#
-# Copyright 2012-2014 Anthony Beville
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-from __future__ import annotations
-
-import json
-import logging
-import re
-import time
-from typing import Any, Callable, cast
-from urllib.parse import urlencode, urljoin, urlsplit
-
-import requests
-from bs4 import BeautifulSoup
-
-from comicapi import utils
-from comicapi.genericmetadata import GenericMetadata
-from comicapi.issuestring import IssueString
-from comictaggerlib import ctversion, resulttypes
-from comictaggerlib.comiccacher import ComicCacher
-from comictaggerlib.settings import ComicTaggerSettings
-
-logger = logging.getLogger(__name__)
-
-try:
- from PyQt5 import QtCore, QtNetwork
-
- qt_available = True
-except ImportError:
- qt_available = False
-
-logger = logging.getLogger(__name__)
-
-
-class CVTypeID:
- Volume = "4050"
- Issue = "4000"
-
-
-class ComicVineTalkerException(Exception):
- Unknown = -1
- Network = -2
- InvalidKey = 100
- RateLimit = 107
-
- def __init__(self, code: int = -1, desc: str = "") -> None:
- super().__init__()
- self.desc = desc
- self.code = code
-
- def __str__(self) -> str:
- if self.code in (ComicVineTalkerException.Unknown, ComicVineTalkerException.Network):
- return self.desc
-
- return f"CV error #{self.code}: [{self.desc}]. \n"
-
-
-def list_fetch_complete(url_list: list[str]) -> None:
- ...
-
-
-def url_fetch_complete(image_url: str, thumb_url: str | None) -> None:
- ...
-
-
-class ComicVineTalker:
- logo_url = "http://static.comicvine.com/bundles/comicvinesite/images/logo.png"
- api_key = ""
- api_base_url = ""
-
- alt_url_list_fetch_complete = list_fetch_complete
- url_fetch_complete = url_fetch_complete
-
- @staticmethod
- def get_rate_limit_message() -> str:
- if ComicVineTalker.api_key == "":
- return "Comic Vine rate limit exceeded. You should configure your own Comic Vine API key."
-
- return "Comic Vine rate limit exceeded. Please wait a bit."
-
- def __init__(self, series_match_thresh: int = 90) -> None:
- # Identity name for the information source
- self.source_name = "comicvine"
-
- self.wait_for_rate_limit = False
- self.series_match_thresh = series_match_thresh
-
- # key that is registered to comictagger
- default_api_key = "27431e6787042105bd3e47e169a624521f89f3a4"
- default_url = "https://comicvine.gamespot.com/api"
-
- self.issue_id: int | None = None
-
- self.api_key = ComicVineTalker.api_key or default_api_key
- tmp_url = urlsplit(ComicVineTalker.api_base_url or default_url)
-
- # joinurl only works properly if there is a trailing slash
- if tmp_url.path and tmp_url.path[-1] != "/":
- tmp_url = tmp_url._replace(path=tmp_url.path + "/")
-
- self.api_base_url = tmp_url.geturl()
-
- self.log_func: Callable[[str], None] | None = None
-
- if qt_available:
- self.nam = QtNetwork.QNetworkAccessManager()
-
- def set_log_func(self, log_func: Callable[[str], None]) -> None:
- self.log_func = log_func
-
- def write_log(self, text: str) -> None:
- if self.log_func is None:
- logger.info(text)
- else:
- self.log_func(text)
-
- def parse_date_str(self, date_str: str) -> tuple[int | None, int | None, int | None]:
- return utils.parse_date_str(date_str)
-
- def test_key(self, key: str, url: str) -> bool:
- if not url:
- url = self.api_base_url
- try:
- test_url = urljoin(url, "issue/1/")
-
- cv_response: resulttypes.CVResult = requests.get(
- test_url,
- headers={"user-agent": "comictagger/" + ctversion.version},
- params={
- "api_key": key,
- "format": "json",
- "field_list": "name",
- },
- ).json()
-
- # Bogus request, but if the key is wrong, you get error 100: "Invalid API Key"
- return cv_response["status_code"] != 100
- except Exception:
- return False
-
- def get_cv_content(self, url: str, params: dict[str, Any]) -> resulttypes.CVResult:
- """
- Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error
- sleep for a bit and retry.
- """
- total_time_waited = 0
- limit_wait_time = 1
- counter = 0
- wait_times = [1, 2, 3, 4]
- while True:
- cv_response: resulttypes.CVResult = self.get_url_content(url, params)
- if self.wait_for_rate_limit and cv_response["status_code"] == ComicVineTalkerException.RateLimit:
- self.write_log(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n")
- time.sleep(limit_wait_time * 60)
- total_time_waited += limit_wait_time
- limit_wait_time = wait_times[counter]
- if counter < 3:
- counter += 1
- # don't wait much more than 20 minutes
- if total_time_waited < 20:
- continue
- if cv_response["status_code"] != 1:
- self.write_log(
- f"Comic Vine query failed with error #{cv_response['status_code']}: [{cv_response['error']}]. \n"
- )
- raise ComicVineTalkerException(cv_response["status_code"], cv_response["error"])
-
- # it's all good
- break
- return cv_response
-
- def get_url_content(self, url: str, params: dict[str, Any]) -> Any:
- # connect to server:
- # if there is a 500 error, try a few more times before giving up
- # any other error, just bail
- for tries in range(3):
- try:
- resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + ctversion.version})
- if resp.status_code == 200:
- return resp.json()
- if resp.status_code == 500:
- self.write_log(f"Try #{tries + 1}: ")
- time.sleep(1)
- self.write_log(str(resp.status_code) + "\n")
- else:
- break
-
- except requests.exceptions.RequestException as e:
- self.write_log(f"{e}\n")
- raise ComicVineTalkerException(ComicVineTalkerException.Network, "Network Error!") from e
- except json.JSONDecodeError as e:
- self.write_log(f"{e}\n")
- raise ComicVineTalkerException(ComicVineTalkerException.Unknown, "ComicVine did not provide json")
-
- raise ComicVineTalkerException(
- ComicVineTalkerException.Unknown, f"Error on Comic Vine server: {resp.status_code}"
- )
-
- def search_for_series(
- self,
- series_name: str,
- callback: Callable[[int, int], None] | None = None,
- refresh_cache: bool = False,
- literal: bool = False,
- ) -> list[resulttypes.CVVolumeResults]:
-
- # Sanitize the series name for comicvine searching, comicvine search ignore symbols
- search_series_name = utils.sanitize_title(series_name, literal)
- logger.info("Searching: %s", search_series_name)
-
- # Before we search online, look in our cache, since we might have done this same search recently
- # For literal searches always retrieve from online
- cvc = ComicCacher()
- if not refresh_cache and not literal:
- cached_search_results = cvc.get_search_results(self.source_name, series_name)
-
- if len(cached_search_results) > 0:
- return cached_search_results
-
- params = {
- "api_key": self.api_key,
- "format": "json",
- "resources": "volume",
- "query": search_series_name,
- "field_list": "volume,name,id,start_year,publisher,image,description,count_of_issues,aliases",
- "page": 1,
- "limit": 100,
- }
-
- cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
-
- search_results: list[resulttypes.CVVolumeResults] = []
-
- # see http://api.comicvine.com/documentation/#handling_responses
-
- current_result_count = cv_response["number_of_page_results"]
- total_result_count = cv_response["number_of_total_results"]
-
- # 8 Dec 2018 - Comic Vine changed query results again. Terms are now
- # ORed together, and we get thousands of results. Good news is the
- # results are sorted by relevance, so we can be smart about halting the search.
- # 1. Don't fetch more than some sane amount of pages.
- # 2. Halt when any result on the current page is less than or equal to a set ratio using rapidfuzz
- max_results = 500 # 5 pages
-
- total_result_count = min(total_result_count, max_results)
-
- if callback is None:
- self.write_log(
- f"Found {cv_response['number_of_page_results']} of {cv_response['number_of_total_results']} results\n"
- )
- search_results.extend(cast(list[resulttypes.CVVolumeResults], cv_response["results"]))
- page = 1
-
- if callback is not None:
- callback(current_result_count, total_result_count)
-
- # see if we need to keep asking for more pages...
- while current_result_count < total_result_count:
-
- if not literal:
- # Stop searching once any entry falls below the threshold
- stop_searching = any(
- not utils.titles_match(search_series_name, volume["name"], self.series_match_thresh)
- for volume in cast(list[resulttypes.CVVolumeResults], cv_response["results"])
- )
-
- if stop_searching:
- break
-
- if callback is None:
- self.write_log(f"getting another page of results {current_result_count} of {total_result_count}...\n")
- page += 1
-
- params["page"] = page
- cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
-
- search_results.extend(cast(list[resulttypes.CVVolumeResults], cv_response["results"]))
- current_result_count += cv_response["number_of_page_results"]
-
- if callback is not None:
- callback(current_result_count, total_result_count)
-
- # Cache these search results, even if it's literal we cache the results
- # The most it will cause is extra processing time
- cvc.add_search_results(self.source_name, series_name, search_results)
-
- return search_results
-
- def fetch_volume_data(self, series_id: int) -> resulttypes.CVVolumeResults:
-
- # before we search online, look in our cache, since we might already have this info
- cvc = ComicCacher()
- cached_volume_result = cvc.get_volume_info(series_id, self.source_name)
-
- if cached_volume_result is not None:
- return cached_volume_result
-
- volume_url = urljoin(self.api_base_url, f"volume/{CVTypeID.Volume}-{series_id}")
-
- params = {
- "api_key": self.api_key,
- "format": "json",
- "field_list": "name,id,start_year,publisher,count_of_issues,aliases",
- }
- cv_response = self.get_cv_content(volume_url, params)
-
- volume_results = cast(resulttypes.CVVolumeResults, cv_response["results"])
-
- if volume_results:
- cvc.add_volume_info(self.source_name, volume_results)
-
- return volume_results
-
- def fetch_issues_by_volume(self, series_id: int) -> list[resulttypes.CVIssuesResults]:
- # before we search online, look in our cache, since we might already have this info
- volume_data = self.fetch_volume_data(series_id)
- cvc = ComicCacher()
- cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name)
-
- if len(cached_volume_issues_result) >= volume_data["count_of_issues"]:
- return cached_volume_issues_result
-
- params = {
- "api_key": self.api_key,
- "filter": f"volume:{series_id}",
- "format": "json",
- "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases",
- "offset": 0,
- }
- cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
-
- current_result_count = cv_response["number_of_page_results"]
- total_result_count = cv_response["number_of_total_results"]
-
- volume_issues_result = cast(list[resulttypes.CVIssuesResults], cv_response["results"])
- page = 1
- offset = 0
-
- # see if we need to keep asking for more pages...
- while current_result_count < total_result_count:
- page += 1
- offset += cv_response["number_of_page_results"]
-
- params["offset"] = offset
- cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
-
- volume_issues_result.extend(cast(list[resulttypes.CVIssuesResults], cv_response["results"]))
- current_result_count += cv_response["number_of_page_results"]
-
- self.repair_urls(volume_issues_result)
-
- cvc.add_volume_issues_info(self.source_name, series_id, volume_issues_result)
-
- return volume_issues_result
-
- def fetch_issues_by_volume_issue_num_and_year(
- self, volume_id_list: list[int], issue_number: str, year: str | int | None
- ) -> list[resulttypes.CVIssuesResults]:
- volume_filter = ""
- for vid in volume_id_list:
- volume_filter += str(vid) + "|"
- flt = f"volume:{volume_filter},issue_number:{issue_number}"
-
- int_year = utils.xlate(year, True)
- if int_year is not None:
- flt += f",cover_date:{int_year}-1-1|{int_year+1}-1-1"
-
- params: dict[str, str | int] = {
- "api_key": self.api_key,
- "format": "json",
- "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases",
- "filter": flt,
- }
-
- cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
-
- current_result_count = cv_response["number_of_page_results"]
- total_result_count = cv_response["number_of_total_results"]
-
- filtered_issues_result = cast(list[resulttypes.CVIssuesResults], cv_response["results"])
- page = 1
- offset = 0
-
- # see if we need to keep asking for more pages...
- while current_result_count < total_result_count:
- page += 1
- offset += cv_response["number_of_page_results"]
-
- params["offset"] = offset
- cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
-
- filtered_issues_result.extend(cast(list[resulttypes.CVIssuesResults], cv_response["results"]))
- current_result_count += cv_response["number_of_page_results"]
-
- self.repair_urls(filtered_issues_result)
-
- cvc = ComicCacher()
- for c in filtered_issues_result:
- cvc.add_volume_issues_info(self.source_name, c["volume"]["id"], [c])
-
- return filtered_issues_result
-
- def fetch_issue_data(self, series_id: int, issue_number: str, settings: ComicTaggerSettings) -> GenericMetadata:
- volume_results = self.fetch_volume_data(series_id)
- issues_list_results = self.fetch_issues_by_volume(series_id)
-
- f_record = None
- for record in issues_list_results:
- if not IssueString(issue_number).as_string():
- issue_number = "1"
- if (
- IssueString(record["issue_number"]).as_string().casefold()
- == IssueString(issue_number).as_string().casefold()
- ):
- f_record = record
- break
-
- if f_record is not None:
- issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{f_record['id']}")
- params = {"api_key": self.api_key, "format": "json"}
- cv_response = self.get_cv_content(issue_url, params)
- issue_results = cast(resulttypes.CVIssueDetailResults, cv_response["results"])
-
- else:
- return GenericMetadata()
-
- # Now, map the Comic Vine data to generic metadata
- return self.map_cv_data_to_metadata(volume_results, issue_results, settings)
-
- def fetch_issue_data_by_issue_id(self, issue_id: int, settings: ComicTaggerSettings) -> GenericMetadata:
-
- issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
- params = {"api_key": self.api_key, "format": "json"}
- cv_response = self.get_cv_content(issue_url, params)
-
- issue_results = cast(resulttypes.CVIssueDetailResults, cv_response["results"])
-
- volume_results = self.fetch_volume_data(issue_results["volume"]["id"])
-
- # Now, map the Comic Vine data to generic metadata
- md = self.map_cv_data_to_metadata(volume_results, issue_results, settings)
- md.is_empty = False
- return md
-
- def map_cv_data_to_metadata(
- self,
- volume_results: resulttypes.CVVolumeResults,
- issue_results: resulttypes.CVIssueDetailResults,
- settings: ComicTaggerSettings,
- ) -> GenericMetadata:
-
- # Now, map the Comic Vine data to generic metadata
- metadata = GenericMetadata()
- metadata.is_empty = False
- metadata.tag_origin = "Comic Vine"
- metadata.issue_id = issue_results["id"]
-
- metadata.series = utils.xlate(issue_results["volume"]["name"])
- metadata.issue = IssueString(issue_results["issue_number"]).as_string()
- metadata.title = utils.xlate(issue_results["name"])
-
- if volume_results["publisher"] is not None:
- metadata.publisher = utils.xlate(volume_results["publisher"]["name"])
- metadata.day, metadata.month, metadata.year = self.parse_date_str(issue_results["cover_date"])
-
- metadata.comments = self.cleanup_html(issue_results["description"], settings.remove_html_tables)
- if settings.use_series_start_as_volume:
- metadata.volume = int(volume_results["start_year"])
-
- metadata.web_link = issue_results["site_detail_url"]
-
- person_credits = issue_results["person_credits"]
- for person in person_credits:
- if "role" in person:
- roles = person["role"].split(",")
- for role in roles:
- # can we determine 'primary' from CV??
- metadata.add_credit(person["name"], role.title().strip(), False)
-
- character_credits = issue_results["character_credits"]
- character_list = []
- for character in character_credits:
- character_list.append(character["name"])
- metadata.characters = ", ".join(character_list)
-
- team_credits = issue_results["team_credits"]
- team_list = []
- for team in team_credits:
- team_list.append(team["name"])
- metadata.teams = ", ".join(team_list)
-
- location_credits = issue_results["location_credits"]
- location_list = []
- for location in location_credits:
- location_list.append(location["name"])
- metadata.locations = ", ".join(location_list)
-
- story_arc_credits = issue_results["story_arc_credits"]
- arc_list = []
- for arc in story_arc_credits:
- arc_list.append(arc["name"])
- if len(arc_list) > 0:
- metadata.story_arc = ", ".join(arc_list)
-
- return metadata
-
- def cleanup_html(self, string: str, remove_html_tables: bool) -> str:
- if string is None:
- return ""
- # find any tables
- soup = BeautifulSoup(string, "html.parser")
- tables = soup.findAll("table")
-
- # remove all newlines first
- string = string.replace("\n", "")
-
- # put in our own
- string = string.replace("
", "\n")
- string = string.replace("", "\n")
- string = string.replace("
", "\n\n")
- string = string.replace("", "*")
- string = string.replace("
", "*\n")
- string = string.replace("", "*")
- string = string.replace("
", "*\n")
- string = string.replace("", "*")
- string = string.replace("
", "*\n")
- string = string.replace("", "*")
- string = string.replace("
", "*\n")
- string = string.replace("", "*")
- string = string.replace("
", "*\n")
- string = string.replace("", "*")
- string = string.replace("
", "*\n")
-
- # remove the tables
- p = re.compile(r"")
- if remove_html_tables:
- string = p.sub("", string)
- string = string.replace("*List of covers and their creators:*", "")
- else:
- string = p.sub("{}", string)
-
- # now strip all other tags
- p = re.compile(r"<[^<]*?>")
- newstring = p.sub("", string)
-
- newstring = newstring.replace(" ", " ")
- newstring = newstring.replace("&", "&")
-
- newstring = newstring.strip()
-
- if not remove_html_tables:
- # now rebuild the tables into text from BSoup
- try:
- table_strings = []
- for table in tables:
- rows = []
- hdrs = []
- col_widths = []
- for hdr in table.findAll("th"):
- item = hdr.string.strip()
- hdrs.append(item)
- col_widths.append(len(item))
- rows.append(hdrs)
-
- for row in table.findAll("tr"):
- cols = []
- col = row.findAll("td")
- i = 0
- for c in col:
- item = c.string.strip()
- cols.append(item)
- if len(item) > col_widths[i]:
- col_widths[i] = len(item)
- i += 1
- if len(cols) != 0:
- rows.append(cols)
- # now we have the data, make it into text
- fmtstr = ""
- for w in col_widths:
- fmtstr += f" {{:{w + 1}}}|"
- width = sum(col_widths) + len(col_widths) * 2
- table_text = ""
- counter = 0
- for row in rows:
- table_text += fmtstr.format(*row) + "\n"
- if counter == 0 and len(hdrs) != 0:
- table_text += "-" * width + "\n"
- counter += 1
-
- table_strings.append(table_text)
-
- newstring = newstring.format(*table_strings)
- except Exception:
- # we caught an error rebuilding the table.
- # just bail and remove the formatting
- logger.exception("table parse error")
- newstring.replace("{}", "")
-
- return newstring
-
- def fetch_issue_date(self, issue_id: int) -> tuple[int | None, int | None]:
- details = self.fetch_issue_select_details(issue_id)
- _, month, year = self.parse_date_str(details["cover_date"] or "")
- return month, year
-
- def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]:
- details = self.fetch_issue_select_details(issue_id)
- return details["image_url"], details["thumb_image_url"]
-
- def fetch_issue_page_url(self, issue_id: int) -> str | None:
- details = self.fetch_issue_select_details(issue_id)
- return details["site_detail_url"]
-
- def fetch_issue_select_details(self, issue_id: int) -> resulttypes.SelectDetails:
- cached_details = self.fetch_cached_issue_select_details(issue_id)
- if cached_details["image_url"] is not None:
- return cached_details
-
- issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
- logger.error("%s, %s", self.api_base_url, issue_url)
-
- params = {"api_key": self.api_key, "format": "json", "field_list": "image,cover_date,site_detail_url"}
-
- cv_response = self.get_cv_content(issue_url, params)
- results = cast(resulttypes.CVIssueDetailResults, cv_response["results"])
-
- details = resulttypes.SelectDetails(
- image_url=results["image"]["super_url"],
- thumb_image_url=results["image"]["thumb_url"],
- cover_date=results["cover_date"],
- site_detail_url=results["site_detail_url"],
- )
-
- if (
- details["image_url"] is not None
- and details["thumb_image_url"] is not None
- and details["cover_date"] is not None
- and details["site_detail_url"] is not None
- ):
- self.cache_issue_select_details(
- issue_id,
- details["image_url"],
- details["thumb_image_url"],
- details["cover_date"],
- details["site_detail_url"],
- )
- return details
-
- def fetch_cached_issue_select_details(self, issue_id: int) -> resulttypes.SelectDetails:
-
- # before we search online, look in our cache, since we might already have this info
- cvc = ComicCacher()
- return cvc.get_issue_select_details(issue_id, self.source_name)
-
- def cache_issue_select_details(
- self, issue_id: int, image_url: str, thumb_url: str, cover_date: str, page_url: str
- ) -> None:
- cvc = ComicCacher()
- cvc.add_issue_select_details(self.source_name, issue_id, image_url, thumb_url, cover_date, page_url)
-
- def fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> list[str]:
- url_list = self.fetch_cached_alternate_cover_urls(issue_id)
- if url_list:
- return url_list
-
- # scrape the CV issue page URL to get the alternate cover URLs
- content = requests.get(issue_page_url, headers={"user-agent": "comictagger/" + ctversion.version}).text
- alt_cover_url_list = self.parse_out_alt_cover_urls(content)
-
- # cache this alt cover URL list
- self.cache_alternate_cover_urls(issue_id, alt_cover_url_list)
-
- return alt_cover_url_list
-
- def parse_out_alt_cover_urls(self, page_html: str) -> list[str]:
- soup = BeautifulSoup(page_html, "html.parser")
-
- alt_cover_url_list = []
-
- # Using knowledge of the layout of the Comic Vine issue page here:
- # look for the divs that are in the classes 'imgboxart' and 'issue-cover'
- div_list = soup.find_all("div")
- covers_found = 0
- for d in div_list:
- if "class" in d.attrs:
- c = d["class"]
- if "imgboxart" in c and "issue-cover" in c:
- if d.img["src"].startswith("http"):
- covers_found += 1
- if covers_found != 1:
- alt_cover_url_list.append(d.img["src"])
- elif d.img["data-src"].startswith("http"):
- covers_found += 1
- if covers_found != 1:
- alt_cover_url_list.append(d.img["data-src"])
-
- return alt_cover_url_list
-
- def fetch_cached_alternate_cover_urls(self, issue_id: int) -> list[str]:
-
- # before we search online, look in our cache, since we might already have this info
- cvc = ComicCacher()
- url_list = cvc.get_alt_covers(self.source_name, issue_id)
-
- return url_list
-
- def cache_alternate_cover_urls(self, issue_id: int, url_list: list[str]) -> None:
- cvc = ComicCacher()
- cvc.add_alt_covers(self.source_name, issue_id, url_list)
-
- def async_fetch_issue_cover_urls(self, issue_id: int) -> None:
-
- self.issue_id = issue_id
- details = self.fetch_cached_issue_select_details(issue_id)
- if details["image_url"] is not None:
- ComicVineTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"])
- return
-
- issue_url = urlsplit(self.api_base_url)
- issue_url = issue_url._replace(
- query=urlencode(
- {
- "api_key": self.api_key,
- "format": "json",
- "field_list": "image,cover_date,site_detail_url",
- }
- ),
- path=f"issue/{CVTypeID.Issue}-{issue_id}",
- )
-
- self.nam.finished.connect(self.async_fetch_issue_cover_url_complete)
- self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(issue_url.geturl())))
-
- def async_fetch_issue_cover_url_complete(self, reply: QtNetwork.QNetworkReply) -> None:
- # read in the response
- data = reply.readAll()
-
- try:
- cv_response = cast(resulttypes.CVResult, json.loads(bytes(data)))
- except Exception:
- logger.exception("Comic Vine query failed to get JSON data\n%s", str(data))
- return
-
- if cv_response["status_code"] != 1:
- logger.error("Comic Vine query failed with error: [%s]. ", cv_response["error"])
- return
-
- result = cast(resulttypes.CVIssuesResults, cv_response["results"])
-
- image_url = result["image"]["super_url"]
- thumb_url = result["image"]["thumb_url"]
- cover_date = result["cover_date"]
- page_url = result["site_detail_url"]
-
- self.cache_issue_select_details(cast(int, self.issue_id), image_url, thumb_url, cover_date, page_url)
-
- ComicVineTalker.url_fetch_complete(image_url, thumb_url)
-
- def async_fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> None:
- # This async version requires the issue page url to be provided!
- self.issue_id = issue_id
- url_list = self.fetch_cached_alternate_cover_urls(issue_id)
- if url_list:
- ComicVineTalker.alt_url_list_fetch_complete(url_list)
- return
-
- self.nam.finished.connect(self.async_fetch_alternate_cover_urls_complete)
- self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_page_url))))
-
- def async_fetch_alternate_cover_urls_complete(self, reply: QtNetwork.QNetworkReply) -> None:
- # read in the response
- html = str(reply.readAll())
- alt_cover_url_list = self.parse_out_alt_cover_urls(html)
-
- # cache this alt cover URL list
- self.cache_alternate_cover_urls(cast(int, self.issue_id), alt_cover_url_list)
-
- ComicVineTalker.alt_url_list_fetch_complete(alt_cover_url_list)
-
- def repair_urls(
- self,
- issue_list: list[resulttypes.CVIssuesResults]
- | list[resulttypes.CVVolumeResults]
- | list[resulttypes.CVIssueDetailResults],
- ) -> None:
- # make sure there are URLs for the image fields
- for issue in issue_list:
- if issue["image"] is None:
- issue["image"] = resulttypes.CVImage(
- super_url=ComicVineTalker.logo_url,
- thumb_url=ComicVineTalker.logo_url,
- )
diff --git a/comictaggerlib/coverimagewidget.py b/comictaggerlib/coverimagewidget.py
index 6c3d8f7..ecd19a4 100644
--- a/comictaggerlib/coverimagewidget.py
+++ b/comictaggerlib/coverimagewidget.py
@@ -20,19 +20,17 @@ TODO: This should be re-factored using subclasses!
from __future__ import annotations
import logging
-from typing import Callable, cast
from PyQt5 import QtCore, QtGui, QtWidgets, uic
-from comicapi import utils
from comicapi.comicarchive import ComicArchive
-from comictaggerlib.comicvinetalker import ComicVineTalker
from comictaggerlib.graphics import graphics_path
from comictaggerlib.imagefetcher import ImageFetcher
from comictaggerlib.imagepopup import ImagePopup
from comictaggerlib.pageloader import PageLoader
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import get_qimage_from_data, reduce_widget_font_size
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@@ -56,39 +54,17 @@ def clickable(widget: QtWidgets.QWidget) -> QtCore.pyqtBoundSignal:
return flt.dblclicked
-class Signal(QtCore.QObject):
- alt_url_list_fetch_complete = QtCore.pyqtSignal(list)
- url_fetch_complete = QtCore.pyqtSignal(str, str)
- image_fetch_complete = QtCore.pyqtSignal(QtCore.QByteArray)
-
- def __init__(
- self,
- list_fetch: Callable[[list[str]], None],
- url_fetch: Callable[[str, str], None],
- image_fetch: Callable[[bytes], None],
- ) -> None:
- super().__init__()
- self.alt_url_list_fetch_complete.connect(list_fetch)
- self.url_fetch_complete.connect(url_fetch)
- self.image_fetch_complete.connect(image_fetch)
-
- def emit_list(self, url_list: list[str]) -> None:
- self.alt_url_list_fetch_complete.emit(url_list)
-
- def emit_url(self, image_url: str, thumb_url: str | None) -> None:
- self.url_fetch_complete.emit(image_url, thumb_url)
-
- def emit_image(self, image_data: bytes | QtCore.QByteArray) -> None:
- self.image_fetch_complete.emit(image_data)
-
-
class CoverImageWidget(QtWidgets.QWidget):
ArchiveMode = 0
AltCoverMode = 1
URLMode = 1
DataMode = 3
- def __init__(self, parent: QtWidgets.QWidget, mode: int, expand_on_click: bool = True) -> None:
+ image_fetch_complete = QtCore.pyqtSignal(QtCore.QByteArray)
+
+ def __init__(
+ self, parent: QtWidgets.QWidget, talker_api: ComicTalker, mode: int, expand_on_click: bool = True
+ ) -> None:
super().__init__(parent)
self.cover_fetcher = ImageFetcher()
@@ -96,10 +72,7 @@ class CoverImageWidget(QtWidgets.QWidget):
reduce_widget_font_size(self.label)
- self.sig = Signal(
- self.alt_cover_url_list_fetch_complete, self.primary_url_fetch_complete, self.cover_remote_fetch_complete
- )
-
+ self.talker_api = talker_api
self.mode: int = mode
self.page_loader: PageLoader | None = None
self.showControls = True
@@ -108,6 +81,7 @@ class CoverImageWidget(QtWidgets.QWidget):
self.comic_archive: ComicArchive | None = None
self.issue_id: int | None = None
+ self.issue_url: str | None = None
self.url_list: list[str] = []
if self.page_loader is not None:
self.page_loader.abandoned = True
@@ -121,6 +95,7 @@ class CoverImageWidget(QtWidgets.QWidget):
self.btnLeft.clicked.connect(self.decrement_image)
self.btnRight.clicked.connect(self.increment_image)
+ self.image_fetch_complete.connect(self.cover_remote_fetch_complete)
if expand_on_click:
clickable(self.lblImage).connect(self.show_popup)
else:
@@ -131,6 +106,7 @@ class CoverImageWidget(QtWidgets.QWidget):
def reset_widget(self) -> None:
self.comic_archive = None
self.issue_id = None
+ self.issue_url = None
self.url_list = []
if self.page_loader is not None:
self.page_loader.abandoned = True
@@ -173,15 +149,13 @@ class CoverImageWidget(QtWidgets.QWidget):
self.imageCount = 1
self.update_content()
- def set_issue_id(self, issue_id: int) -> None:
+ def set_issue_details(self, issue_id: int, url_list: list[str]) -> None:
if self.mode == CoverImageWidget.AltCoverMode:
self.reset_widget()
self.update_content()
self.issue_id = issue_id
- comic_vine = ComicVineTalker()
- ComicVineTalker.url_fetch_complete = self.sig.emit_url
- comic_vine.async_fetch_issue_cover_urls(self.issue_id)
+ self.set_url_list(url_list)
def set_image_data(self, image_data: bytes) -> None:
if self.mode == CoverImageWidget.DataMode:
@@ -195,31 +169,11 @@ class CoverImageWidget(QtWidgets.QWidget):
self.update_content()
- def primary_url_fetch_complete(self, primary_url: str, thumb_url: str | None = None) -> None:
- self.url_list.append(str(primary_url))
+ def set_url_list(self, url_list: list[str]) -> None:
+ self.url_list = url_list
self.imageIndex = 0
self.imageCount = len(self.url_list)
self.update_content()
-
- # defer the alt cover search
- QtCore.QTimer.singleShot(1, self.start_alt_cover_search)
-
- def start_alt_cover_search(self) -> None:
-
- if self.issue_id is not None:
- # now we need to get the list of alt cover URLs
- self.label.setText("Searching for alt. covers...")
-
- # page URL should already be cached, so no need to defer
- comic_vine = ComicVineTalker()
- issue_page_url = comic_vine.fetch_issue_page_url(self.issue_id)
- ComicVineTalker.alt_url_list_fetch_complete = self.sig.emit_list
- comic_vine.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), cast(str, issue_page_url))
-
- def alt_cover_url_list_fetch_complete(self, url_list: list[str]) -> None:
- if url_list:
- self.url_list.extend(url_list)
- self.imageCount = len(self.url_list)
self.update_controls()
def set_page(self, pagenum: int) -> None:
@@ -269,7 +223,7 @@ class CoverImageWidget(QtWidgets.QWidget):
def load_url(self) -> None:
self.load_default()
self.cover_fetcher = ImageFetcher()
- ImageFetcher.image_fetch_complete = self.sig.emit_image
+ ImageFetcher.image_fetch_complete = self.image_fetch_complete.emit
self.cover_fetcher.fetch(self.url_list[self.imageIndex])
# called when the image is done loading from internet
diff --git a/comictaggerlib/imagefetcher.py b/comictaggerlib/imagefetcher.py
index b4c55f3..214f2b8 100644
--- a/comictaggerlib/imagefetcher.py
+++ b/comictaggerlib/imagefetcher.py
@@ -81,7 +81,8 @@ class ImageFetcher:
# first look in the DB
image_data = self.get_image_from_cache(url)
- if blocking or not qt_available:
+ # Async for retrieving covers seems to work well
+ if blocking: # if blocking or not qt_available:
if not image_data:
try:
image_data = requests.get(url, headers={"user-agent": "comictagger/" + ctversion.version}).content
diff --git a/comictaggerlib/issueidentifier.py b/comictaggerlib/issueidentifier.py
index 4f6d316..3ed87c3 100644
--- a/comictaggerlib/issueidentifier.py
+++ b/comictaggerlib/issueidentifier.py
@@ -26,11 +26,12 @@ from comicapi import utils
from comicapi.comicarchive import ComicArchive
from comicapi.genericmetadata import GenericMetadata
from comicapi.issuestring import IssueString
-from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.imagefetcher import ImageFetcher, ImageFetcherException
from comictaggerlib.imagehasher import ImageHasher
from comictaggerlib.resulttypes import IssueResult
from comictaggerlib.settings import ComicTaggerSettings
+from comictalker.talker_utils import parse_date_str
+from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@@ -72,8 +73,9 @@ class IssueIdentifier:
result_one_good_match = 4
result_multiple_good_matches = 5
- def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings) -> None:
+ def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None:
self.settings = settings
+ self.talker_api = talker_api
self.comic_archive: ComicArchive = comic_archive
self.image_hasher = 1
@@ -107,7 +109,6 @@ class IssueIdentifier:
self.search_result = self.result_no_matches
self.cover_page_index = 0
self.cancel = False
- self.wait_and_retry_on_rate_limit = False
self.match_list: list[IssueResult] = []
@@ -239,11 +240,10 @@ class IssueIdentifier:
def get_issue_cover_match_score(
self,
- comic_vine: ComicVineTalker,
issue_id: int,
primary_img_url: str,
primary_thumb_url: str,
- page_url: str,
+ alt_urls: list[str],
local_cover_hash_list: list[int],
use_remote_alternates: bool = False,
use_log: bool = True,
@@ -274,8 +274,7 @@ class IssueIdentifier:
raise IssueIdentifierCancelled
if use_remote_alternates:
- alt_img_url_list = comic_vine.fetch_alternate_cover_urls(issue_id, page_url)
- for alt_url in alt_img_url_list:
+ for alt_url in alt_urls:
try:
alt_url_image_data = ImageFetcher().fetch(alt_url, blocking=True)
except ImageFetcherException as e:
@@ -369,27 +368,22 @@ class IssueIdentifier:
if keys["month"] is not None:
self.log_msg("\tMonth: " + str(keys["month"]))
- comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
- comic_vine.wait_for_rate_limit = self.wait_and_retry_on_rate_limit
-
- comic_vine.set_log_func(self.output_function)
-
self.log_msg(f"Searching for {keys['series']} #{keys['issue_number']} ...")
try:
- cv_search_results = comic_vine.search_for_series(keys["series"])
- except ComicVineTalkerException:
- self.log_msg("Network issue while searching for series. Aborting...")
+ ct_search_results = self.talker_api.search_for_series(keys["series"])
+ except TalkerError as e:
+ self.log_msg(f"Error searching for series.\n{e}")
return []
if self.cancel:
return []
- if cv_search_results is None:
+ if ct_search_results is None:
return []
series_second_round_list = []
- for item in cv_search_results:
+ for item in ct_search_results:
length_approved = False
publisher_approved = True
date_approved = True
@@ -404,16 +398,13 @@ class IssueIdentifier:
if int(keys["year"]) < int(item["start_year"]):
date_approved = False
- aliases = []
- if item["aliases"]:
- aliases = item["aliases"].split("\n")
- for name in [item["name"], *aliases]:
+ for name in [item["name"], *item["aliases"]]:
if utils.titles_match(keys["series"], name, self.series_match_thresh):
length_approved = True
break
# remove any series from publishers on the filter
if item["publisher"] is not None:
- publisher = item["publisher"]["name"]
+ publisher = item["publisher"]
if publisher is not None and publisher.casefold() in self.publisher_filter:
publisher_approved = False
@@ -436,12 +427,11 @@ class IssueIdentifier:
issue_list = None
try:
if len(volume_id_list) > 0:
- issue_list = comic_vine.fetch_issues_by_volume_issue_num_and_year(
+ issue_list = self.talker_api.fetch_issues_by_volume_issue_num_and_year(
volume_id_list, keys["issue_number"], keys["year"]
)
-
- except ComicVineTalkerException:
- self.log_msg("Network issue while searching for series details. Aborting...")
+ except TalkerError as e:
+ self.log_msg(f"Issue with while searching for series details. Aborting...\n{e}")
return []
if issue_list is None:
@@ -476,7 +466,7 @@ class IssueIdentifier:
)
# parse out the cover date
- _, month, year = comic_vine.parse_date_str(issue["cover_date"])
+ _, month, year = parse_date_str(issue["cover_date"])
# Now check the cover match against the primary image
hash_list = [cover_hash]
@@ -484,16 +474,15 @@ class IssueIdentifier:
hash_list.append(narrow_cover_hash)
try:
- image_url = issue["image"]["super_url"]
- thumb_url = issue["image"]["thumb_url"]
- page_url = issue["site_detail_url"]
+ image_url = issue["image_url"]
+ thumb_url = issue["image_thumb_url"]
+ alt_urls = issue["alt_image_urls"]
score_item = self.get_issue_cover_match_score(
- comic_vine,
issue["id"],
image_url,
thumb_url,
- page_url,
+ alt_urls,
hash_list,
use_remote_alternates=False,
)
@@ -515,11 +504,12 @@ class IssueIdentifier:
"publisher": None,
"image_url": image_url,
"thumb_url": thumb_url,
- "page_url": page_url,
+ # "page_url": page_url,
+ "alt_image_urls": alt_urls,
"description": issue["description"],
}
if series["publisher"] is not None:
- match["publisher"] = series["publisher"]["name"]
+ match["publisher"] = series["publisher"]
self.match_list.append(match)
@@ -577,15 +567,15 @@ class IssueIdentifier:
self.log_msg(f"Examining alternate covers for ID: {m['volume_id']} {m['series']} ...", newline=False)
try:
score_item = self.get_issue_cover_match_score(
- comic_vine,
m["issue_id"],
m["image_url"],
m["thumb_url"],
- m["page_url"],
+ m["alt_image_urls"],
hash_list,
use_remote_alternates=True,
)
except Exception:
+ logger.exception("failed examining alt covers")
self.match_list = []
return self.match_list
self.log_msg(f"--->{score_item['score']}")
diff --git a/comictaggerlib/issueselectionwindow.py b/comictaggerlib/issueselectionwindow.py
index b8015ff..e1b6e0c 100644
--- a/comictaggerlib/issueselectionwindow.py
+++ b/comictaggerlib/issueselectionwindow.py
@@ -20,12 +20,12 @@ import logging
from PyQt5 import QtCore, QtGui, QtWidgets, uic
from comicapi.issuestring import IssueString
-from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.coverimagewidget import CoverImageWidget
-from comictaggerlib.resulttypes import CVIssuesResults
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
+from comictalker.resulttypes import ComicIssue
+from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@@ -42,13 +42,18 @@ class IssueSelectionWindow(QtWidgets.QDialog):
volume_id = 0
def __init__(
- self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, series_id: int, issue_number: str
+ self,
+ parent: QtWidgets.QWidget,
+ settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
+ series_id: int,
+ issue_number: str,
) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "issueselectionwindow.ui", self)
- self.coverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.AltCoverMode)
+ self.coverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.AltCoverMode)
gridlayout = QtWidgets.QGridLayout(self.coverImageContainer)
gridlayout.addWidget(self.coverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@@ -67,8 +72,9 @@ class IssueSelectionWindow(QtWidgets.QDialog):
self.series_id = series_id
self.issue_id: int | None = None
self.settings = settings
+ self.talker_api = talker_api
self.url_fetch_thread = None
- self.issue_list: list[CVIssuesResults] = []
+ self.issue_list: list[ComicIssue] = []
if issue_number is None or issue_number == "":
self.issue_number = "1"
@@ -98,15 +104,14 @@ class IssueSelectionWindow(QtWidgets.QDialog):
QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
try:
- comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
- comic_vine.fetch_volume_data(self.series_id)
- self.issue_list = comic_vine.fetch_issues_by_volume(self.series_id)
- except ComicVineTalkerException as e:
+ self.issue_list = self.talker_api.fetch_issues_by_volume(self.series_id)
+ except TalkerError as e:
QtWidgets.QApplication.restoreOverrideCursor()
- if e.code == ComicVineTalkerException.RateLimit:
- QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message())
- else:
- QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not connect to Comic Vine to list issues!")
+ QtWidgets.QMessageBox.critical(
+ self,
+ f"{e.source} {e.code_name} Error",
+ f"{e}",
+ )
return
self.twList.setRowCount(0)
@@ -175,7 +180,7 @@ class IssueSelectionWindow(QtWidgets.QDialog):
for record in self.issue_list:
if record["id"] == self.issue_id:
self.issue_number = record["issue_number"]
- self.coverWidget.set_issue_id(self.issue_id)
+ self.coverWidget.set_issue_details(self.issue_id, [record["image_url"], *record["alt_image_urls"]])
if record["description"] is None:
self.teDescription.setText("")
else:
diff --git a/comictaggerlib/main.py b/comictaggerlib/main.py
index 6657685..60fb8e7 100755
--- a/comictaggerlib/main.py
+++ b/comictaggerlib/main.py
@@ -25,13 +25,14 @@ import sys
import traceback
import types
+import comictalker.comictalkerapi as ct_api
from comicapi import utils
from comictaggerlib import cli
-from comictaggerlib.comicvinetalker import ComicVineTalker
from comictaggerlib.ctversion import version
from comictaggerlib.graphics import graphics_path
from comictaggerlib.options import parse_cmd_line
from comictaggerlib.settings import ComicTaggerSettings
+from comictalker.talkerbase import TalkerError
if sys.version_info < (3, 10):
import importlib_metadata
@@ -41,6 +42,7 @@ else:
logger = logging.getLogger("comictagger")
logging.getLogger("comicapi").setLevel(logging.DEBUG)
logging.getLogger("comictaggerlib").setLevel(logging.DEBUG)
+logging.getLogger("comictalker").setLevel(logging.DEBUG)
logger.setLevel(logging.DEBUG)
try:
@@ -157,19 +159,6 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
file=sys.stderr,
)
- # manage the CV API key
- # None comparison is used so that the empty string can unset the value
- if opts.cv_api_key is not None or opts.cv_url is not None:
- settings.cv_api_key = opts.cv_api_key if opts.cv_api_key is not None else settings.cv_api_key
- settings.cv_url = opts.cv_url if opts.cv_url is not None else settings.cv_url
- settings.save()
- if opts.only_set_cv_key:
- print("Key set") # noqa: T201
- return
-
- ComicVineTalker.api_key = settings.cv_api_key
- ComicVineTalker.api_base_url = settings.cv_url
-
signal.signal(signal.SIGINT, signal.SIG_DFL)
logger.info(
@@ -183,6 +172,28 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
for pkg in sorted(importlib_metadata.distributions(), key=lambda x: x.name):
logger.debug("%s\t%s", pkg.metadata["Name"], pkg.metadata["Version"])
+ talker_failed = False
+ try:
+ talker_api = ct_api.get_comic_talker("comicvine")(
+ settings.cv_url,
+ settings.cv_api_key,
+ settings.id_series_match_search_thresh,
+ settings.remove_html_tables,
+ settings.use_series_start_as_volume,
+ settings.wait_and_retry_on_rate_limit,
+ )
+ except TalkerError as te:
+ talker_failed = True
+ logger.warning(f"Unable to load talker {te.source}. Error: {te.desc}. Defaulting to Comic Vine.")
+ talker_api = ct_api.get_comic_talker("comicvine")(
+ settings.cv_url,
+ settings.cv_api_key,
+ settings.id_series_match_search_thresh,
+ settings.remove_html_tables,
+ settings.use_series_start_as_volume,
+ settings.wait_and_retry_on_rate_limit,
+ )
+
utils.load_publishers()
update_publishers()
@@ -192,7 +203,7 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
if opts.no_gui:
try:
- cli.cli_mode(opts, settings)
+ cli.cli_mode(opts, settings, talker_api)
except Exception:
logger.exception("CLI mode failed")
else:
@@ -232,7 +243,7 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
QtWidgets.QApplication.processEvents()
try:
- tagger_window = TaggerWindow(opts.files, settings, opts=opts)
+ tagger_window = TaggerWindow(opts.files, settings, talker_api, opts=opts)
tagger_window.setWindowIcon(QtGui.QIcon(str(graphics_path / "app.png")))
tagger_window.show()
@@ -242,6 +253,13 @@ See https://github.com/comictagger/comictagger/releases/1.5.5 for more informati
if platform.system() != "Linux":
splash.finish(tagger_window)
+ if talker_failed:
+ QtWidgets.QMessageBox.warning(
+ QtWidgets.QMainWindow(),
+ "Warning",
+ "Unable to load configured information source, see log for details. Defaulting to Comic Vine",
+ )
+
sys.exit(app.exec())
except Exception:
logger.exception("GUI mode failed")
diff --git a/comictaggerlib/matchselectionwindow.py b/comictaggerlib/matchselectionwindow.py
index 27bb16a..330318c 100644
--- a/comictaggerlib/matchselectionwindow.py
+++ b/comictaggerlib/matchselectionwindow.py
@@ -25,6 +25,7 @@ from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.resulttypes import IssueResult
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@@ -32,17 +33,23 @@ logger = logging.getLogger(__name__)
class MatchSelectionWindow(QtWidgets.QDialog):
volume_id = 0
- def __init__(self, parent: QtWidgets.QWidget, matches: list[IssueResult], comic_archive: ComicArchive) -> None:
+ def __init__(
+ self,
+ parent: QtWidgets.QWidget,
+ matches: list[IssueResult],
+ comic_archive: ComicArchive,
+ talker_api: ComicTalker,
+ ) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "matchselectionwindow.ui", self)
- self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode)
+ self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode)
gridlayout = QtWidgets.QGridLayout(self.altCoverContainer)
gridlayout.addWidget(self.altCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
- self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode)
+ self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer)
gridlayout.addWidget(self.archiveCoverWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@@ -142,7 +149,10 @@ class MatchSelectionWindow(QtWidgets.QDialog):
if prev is not None and prev.row() == curr.row():
return
- self.altCoverWidget.set_issue_id(self.current_match()["issue_id"])
+ self.altCoverWidget.set_issue_details(
+ self.current_match()["issue_id"],
+ [self.current_match()["image_url"], *self.current_match()["alt_image_urls"]],
+ )
if self.current_match()["description"] is None:
self.teDescription.setText("")
else:
diff --git a/comictaggerlib/options.py b/comictaggerlib/options.py
index 884979c..be9aecf 100644
--- a/comictaggerlib/options.py
+++ b/comictaggerlib/options.py
@@ -80,6 +80,7 @@ def define_args() -> argparse.ArgumentParser:
action="store_true",
help="Export RAR archive to Zip format.",
)
+ # TODO: update for new api
commands.add_argument(
"--only-set-cv-key",
action="store_true",
@@ -107,10 +108,12 @@ def define_args() -> argparse.ArgumentParser:
dest="config_path",
help="""Config directory defaults to ~/.ComicTagger\non Linux/Mac and %%APPDATA%% on Windows\n""",
)
+ # TODO: update for new api
parser.add_argument(
"--cv-api-key",
help="Use the given Comic Vine API Key (persisted in settings).",
)
+ # TODO: update for new api
parser.add_argument(
"--cv-url",
help="Use the given Comic Vine URL (persisted in settings).",
@@ -214,6 +217,7 @@ def define_args() -> argparse.ArgumentParser:
action="store_true",
help="Be noisy when doing what it does.",
)
+ # TODO: update for new api
parser.add_argument(
"-w",
"--wait-on-cv-rate-limit",
@@ -369,7 +373,7 @@ def parse_cmd_line() -> argparse.Namespace:
opts.copy,
opts.rename,
opts.export_to_zip,
- opts.only_set_cv_key,
+ opts.only_set_cv_key, # TODO: update for new api
]
)
@@ -385,9 +389,11 @@ def parse_cmd_line() -> argparse.Namespace:
for item in globs:
opts.files.extend(glob.glob(item))
+ # TODO: update for new api
if opts.only_set_cv_key and opts.cv_api_key is None and opts.cv_url is None:
parser.exit(message="Key not given!\n", status=1)
+ # TODO: update for new api
if not opts.only_set_cv_key and opts.no_gui and not opts.files:
parser.exit(message="Command requires at least one filename!\n", status=1)
diff --git a/comictaggerlib/pagebrowser.py b/comictaggerlib/pagebrowser.py
index d536fc5..f2e2387 100644
--- a/comictaggerlib/pagebrowser.py
+++ b/comictaggerlib/pagebrowser.py
@@ -25,17 +25,18 @@ from comicapi.genericmetadata import GenericMetadata
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.graphics import graphics_path
from comictaggerlib.ui import ui_path
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
class PageBrowserWindow(QtWidgets.QDialog):
- def __init__(self, parent: QtWidgets.QWidget, metadata: GenericMetadata) -> None:
+ def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker, metadata: GenericMetadata) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "pagebrowser.ui", self)
- self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode)
+ self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.pageContainer)
gridlayout.addWidget(self.pageWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
diff --git a/comictaggerlib/pagelisteditor.py b/comictaggerlib/pagelisteditor.py
index 88df8e0..39b71a9 100644
--- a/comictaggerlib/pagelisteditor.py
+++ b/comictaggerlib/pagelisteditor.py
@@ -23,6 +23,7 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle
from comicapi.genericmetadata import ImageMetadata, PageType
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.ui import ui_path
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@@ -67,12 +68,12 @@ class PageListEditor(QtWidgets.QWidget):
PageType.Deleted: "Deleted",
}
- def __init__(self, parent: QtWidgets.QWidget) -> None:
+ def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "pagelisteditor.ui", self)
- self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode)
+ self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode)
gridlayout = QtWidgets.QGridLayout(self.pageContainer)
gridlayout.addWidget(self.pageWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
diff --git a/comictaggerlib/renamewindow.py b/comictaggerlib/renamewindow.py
index b752faa..b732579 100644
--- a/comictaggerlib/renamewindow.py
+++ b/comictaggerlib/renamewindow.py
@@ -27,6 +27,7 @@ from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.settingswindow import SettingsWindow
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import center_window_on_parent
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@@ -38,6 +39,7 @@ class RenameWindow(QtWidgets.QDialog):
comic_archive_list: list[ComicArchive],
data_style: int,
settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
) -> None:
super().__init__(parent)
@@ -53,6 +55,7 @@ class RenameWindow(QtWidgets.QDialog):
)
self.settings = settings
+ self.talker_api = talker_api
self.comic_archive_list = comic_archive_list
self.data_style = data_style
self.rename_list: list[str] = []
@@ -161,7 +164,7 @@ class RenameWindow(QtWidgets.QDialog):
self.twList.setSortingEnabled(True)
def modify_settings(self) -> None:
- settingswin = SettingsWindow(self, self.settings)
+ settingswin = SettingsWindow(self, self.settings, self.talker_api)
settingswin.setModal(True)
settingswin.show_rename_tab()
settingswin.exec()
diff --git a/comictaggerlib/resulttypes.py b/comictaggerlib/resulttypes.py
index 0f53ae8..4261fa5 100644
--- a/comictaggerlib/resulttypes.py
+++ b/comictaggerlib/resulttypes.py
@@ -1,6 +1,6 @@
from __future__ import annotations
-from typing_extensions import NotRequired, Required, TypedDict
+from typing_extensions import TypedDict
from comicapi.comicarchive import ComicArchive
@@ -19,7 +19,7 @@ class IssueResult(TypedDict):
publisher: str | None
image_url: str
thumb_url: str
- page_url: str
+ alt_image_urls: list[str]
description: str
@@ -37,126 +37,3 @@ class MultipleMatch:
def __init__(self, ca: ComicArchive, match_list: list[IssueResult]) -> None:
self.ca: ComicArchive = ca
self.matches: list[IssueResult] = match_list
-
-
-class SelectDetails(TypedDict):
- image_url: str | None
- thumb_image_url: str | None
- cover_date: str | None
- site_detail_url: str | None
-
-
-class CVResult(TypedDict):
- error: str
- limit: int
- offset: int
- number_of_page_results: int
- number_of_total_results: int
- status_code: int
- results: (
- CVIssuesResults
- | CVIssueDetailResults
- | CVVolumeResults
- | list[CVIssuesResults]
- | list[CVVolumeResults]
- | list[CVIssueDetailResults]
- )
- version: str
-
-
-class CVImage(TypedDict, total=False):
- icon_url: str
- medium_url: str
- screen_url: str
- screen_large_url: str
- small_url: str
- super_url: Required[str]
- thumb_url: str
- tiny_url: str
- original_url: str
- image_tags: str
-
-
-class CVVolume(TypedDict):
- api_detail_url: str
- id: int
- name: str
- site_detail_url: str
-
-
-class CVIssuesResults(TypedDict):
- cover_date: str
- description: str
- id: int
- image: CVImage
- issue_number: str
- name: str
- site_detail_url: str
- volume: NotRequired[CVVolume]
- aliases: str
-
-
-class CVPublisher(TypedDict, total=False):
- api_detail_url: str
- id: int
- name: Required[str]
-
-
-class CVVolumeResults(TypedDict):
- count_of_issues: int
- description: NotRequired[str]
- id: int
- image: NotRequired[CVImage]
- name: str
- publisher: CVPublisher
- start_year: str
- resource_type: NotRequired[str]
- aliases: NotRequired[str | None]
-
-
-class CVCredits(TypedDict):
- api_detail_url: str
- id: int
- name: str
- site_detail_url: str
-
-
-class CVPersonCredits(TypedDict):
- api_detail_url: str
- id: int
- name: str
- site_detail_url: str
- role: str
-
-
-class CVIssueDetailResults(TypedDict):
- aliases: None
- api_detail_url: str
- character_credits: list[CVCredits]
- character_died_in: None
- concept_credits: list[CVCredits]
- cover_date: str
- date_added: str
- date_last_updated: str
- deck: None
- description: str
- first_appearance_characters: None
- first_appearance_concepts: None
- first_appearance_locations: None
- first_appearance_objects: None
- first_appearance_storyarcs: None
- first_appearance_teams: None
- has_staff_review: bool
- id: int
- image: CVImage
- issue_number: str
- location_credits: list[CVCredits]
- name: str
- object_credits: list[CVCredits]
- person_credits: list[CVPersonCredits]
- site_detail_url: str
- store_date: str
- story_arc_credits: list[CVCredits]
- team_credits: list[CVCredits]
- team_disbanded_in: None
- volume: CVVolume
diff --git a/comictaggerlib/settingswindow.py b/comictaggerlib/settingswindow.py
index c829be6..7cdfa09 100644
--- a/comictaggerlib/settingswindow.py
+++ b/comictaggerlib/settingswindow.py
@@ -25,12 +25,12 @@ from PyQt5 import QtCore, QtGui, QtWidgets, uic
from comicapi import utils
from comicapi.genericmetadata import md_test
-from comictaggerlib.comiccacher import ComicCacher
-from comictaggerlib.comicvinetalker import ComicVineTalker
from comictaggerlib.filerenamer import FileRenamer
from comictaggerlib.imagefetcher import ImageFetcher
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
+from comictalker.comiccacher import ComicCacher
+from comictalker.talkerbase import ComicTalker
logger = logging.getLogger(__name__)
@@ -128,7 +128,7 @@ Spider-Geddon #1 - New Players; Check In
class SettingsWindow(QtWidgets.QDialog):
- def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings) -> None:
+ def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None:
super().__init__(parent)
uic.loadUi(ui_path / "settingswindow.ui", self)
@@ -138,6 +138,7 @@ class SettingsWindow(QtWidgets.QDialog):
)
self.settings = settings
+ self.talker_api = talker_api
self.name = "Settings"
if platform.system() == "Windows":
@@ -326,10 +327,13 @@ class SettingsWindow(QtWidgets.QDialog):
self.settings.sort_series_by_year = self.cbxSortByYear.isChecked()
self.settings.exact_series_matches_first = self.cbxExactMatches.isChecked()
- self.settings.cv_api_key = self.leKey.text().strip()
- ComicVineTalker.api_key = self.settings.cv_api_key
- self.settings.cv_url = self.leURL.text().strip()
- ComicVineTalker.api_base_url = self.settings.cv_url
+ # Ignore empty field
+ if self.leKey.text().strip():
+ self.settings.cv_api_key = self.leKey.text().strip()
+ self.talker_api.api_key = self.settings.cv_api_key
+ if self.leURL.text().strip():
+ self.settings.cv_url = self.leURL.text().strip()
+ self.talker_api.api_base_url = self.settings.cv_url
self.settings.assume_lone_credit_is_primary = self.cbxAssumeLoneCreditIsPrimary.isChecked()
self.settings.copy_characters_to_tags = self.cbxCopyCharactersToTags.isChecked()
self.settings.copy_teams_to_tags = self.cbxCopyTeamsToTags.isChecked()
@@ -361,7 +365,7 @@ class SettingsWindow(QtWidgets.QDialog):
QtWidgets.QMessageBox.information(self, self.name, "Cache has been cleared.")
def test_api_key(self) -> None:
- if ComicVineTalker().test_key(self.leKey.text().strip(), self.leURL.text().strip()):
+ if self.talker_api.check_api_key(self.leKey.text().strip(), self.leURL.text().strip()):
QtWidgets.QMessageBox.information(self, "API Key Test", "Key is valid!")
else:
QtWidgets.QMessageBox.warning(self, "API Key Test", "Key is NOT valid.")
diff --git a/comictaggerlib/taggerwindow.py b/comictaggerlib/taggerwindow.py
index 74c2212..52bade0 100644
--- a/comictaggerlib/taggerwindow.py
+++ b/comictaggerlib/taggerwindow.py
@@ -46,7 +46,6 @@ from comictaggerlib.autotagmatchwindow import AutoTagMatchWindow
from comictaggerlib.autotagprogresswindow import AutoTagProgressWindow
from comictaggerlib.autotagstartwindow import AutoTagStartWindow
from comictaggerlib.cbltransformer import CBLTransformer
-from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.crediteditorwindow import CreditEditorWindow
from comictaggerlib.exportwindow import ExportConflictOpts, ExportWindow
@@ -65,6 +64,7 @@ from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import center_window_on_parent, reduce_widget_font_size
from comictaggerlib.versionchecker import VersionChecker
from comictaggerlib.volumeselectionwindow import VolumeSelectionWindow
+from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@@ -81,6 +81,7 @@ class TaggerWindow(QtWidgets.QMainWindow):
self,
file_list: list[str],
settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
parent: QtWidgets.QWidget | None = None,
opts: argparse.Namespace | None = None,
) -> None:
@@ -88,6 +89,7 @@ class TaggerWindow(QtWidgets.QMainWindow):
uic.loadUi(ui_path / "taggerwindow.ui", self)
self.settings = settings
+ self.talker_api = talker_api
self.log_window = self.setup_logger()
# prevent multiple instances
@@ -121,12 +123,12 @@ class TaggerWindow(QtWidgets.QMainWindow):
)
sys.exit()
- self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.ArchiveMode)
+ self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.ArchiveMode)
grid_layout = QtWidgets.QGridLayout(self.coverImageContainer)
grid_layout.addWidget(self.archiveCoverWidget)
grid_layout.setContentsMargins(0, 0, 0, 0)
- self.page_list_editor = PageListEditor(self.tabPages)
+ self.page_list_editor = PageListEditor(self.tabPages, self.talker_api)
grid_layout = QtWidgets.QGridLayout(self.tabPages)
grid_layout.addWidget(self.page_list_editor)
@@ -1046,6 +1048,7 @@ You have {4-self.settings.settings_warning} warnings left.
issue_number = str(self.leIssueNum.text()).strip()
+ # Only need this check is the source has issue level data.
if autoselect and issue_number == "":
QtWidgets.QMessageBox.information(
self, "Automatic Identify Search", "Can't auto-identify without an issue number (yet!)"
@@ -1072,6 +1075,7 @@ You have {4-self.settings.settings_warning} warnings left.
cover_index_list,
cast(ComicArchive, self.comic_archive),
self.settings,
+ self.talker_api,
autoselect,
literal,
)
@@ -1089,16 +1093,23 @@ You have {4-self.settings.settings_warning} warnings left.
self.form_to_metadata()
try:
- comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
- new_metadata = comic_vine.fetch_issue_data(selector.volume_id, selector.issue_number, self.settings)
- except ComicVineTalkerException as e:
- QtWidgets.QApplication.restoreOverrideCursor()
- if e.code == ComicVineTalkerException.RateLimit:
- QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message())
- else:
- QtWidgets.QMessageBox.critical(
- self, "Network Issue", "Could not connect to Comic Vine to get issue details.!"
+ if selector.issue_id:
+ new_metadata = self.talker_api.fetch_comic_data(selector.issue_id)
+ elif selector.volume_id and selector.issue_number:
+ # Would this ever be needed?
+ new_metadata = self.talker_api.fetch_comic_data(
+ series_id=selector.volume_id, issue_number=selector.issue_number
)
+ else:
+ # Only left with series? Isn't series only handled elsewhere?
+ new_metadata = self.talker_api.fetch_comic_data(series_id=selector.volume_id)
+ except TalkerError as e:
+ QtWidgets.QApplication.restoreOverrideCursor()
+ QtWidgets.QMessageBox.critical(
+ self,
+ f"{e.source} {e.code_name} Error",
+ f"{e}",
+ )
else:
QtWidgets.QApplication.restoreOverrideCursor()
if new_metadata is not None:
@@ -1109,7 +1120,7 @@ You have {4-self.settings.settings_warning} warnings left.
self.clear_form()
notes = (
- f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
+ f"Tagged with ComicTagger {ctversion.version} using info from {self.talker_api.source_details.name} on"
f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {new_metadata.issue_id}]"
)
self.metadata.overlay(
@@ -1390,7 +1401,7 @@ You have {4-self.settings.settings_warning} warnings left.
def show_settings(self) -> None:
- settingswin = SettingsWindow(self, self.settings)
+ settingswin = SettingsWindow(self, self.settings, self.talker_api)
settingswin.setModal(True)
settingswin.exec()
settingswin.result()
@@ -1694,24 +1705,22 @@ You have {4-self.settings.settings_warning} warnings left.
def actual_issue_data_fetch(self, match: IssueResult) -> GenericMetadata:
- # now get the particular issue data
- cv_md = GenericMetadata()
+ # now get the particular issue data OR series data
+ ct_md = GenericMetadata()
QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor))
try:
- comic_vine = ComicVineTalker(self.settings.id_series_match_search_thresh)
- comic_vine.wait_for_rate_limit = self.settings.wait_and_retry_on_rate_limit
- cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], self.settings)
- except ComicVineTalkerException:
- logger.exception("Network error while getting issue details. Save aborted")
+ ct_md = self.talker_api.fetch_comic_data(match["issue_id"])
+ except TalkerError as e:
+ logger.exception(f"Save aborted.\n{e}")
- if not cv_md.is_empty:
+ if not ct_md.is_empty:
if self.settings.apply_cbl_transform_on_cv_import:
- cv_md = CBLTransformer(cv_md, self.settings).apply()
+ ct_md = CBLTransformer(ct_md, self.settings).apply()
QtWidgets.QApplication.restoreOverrideCursor()
- return cv_md
+ return ct_md
def auto_tag_log(self, text: str) -> None:
IssueIdentifier.default_write_output(text)
@@ -1726,7 +1735,7 @@ You have {4-self.settings.settings_warning} warnings left.
self, ca: ComicArchive, match_results: OnlineMatchResults, dlg: AutoTagStartWindow
) -> tuple[bool, OnlineMatchResults]:
success = False
- ii = IssueIdentifier(ca, self.settings)
+ ii = IssueIdentifier(ca, self.settings, self.talker_api)
# read in metadata, and parse file name if not there
try:
@@ -1763,7 +1772,6 @@ You have {4-self.settings.settings_warning} warnings left.
md.issue = utils.xlate(md.volume)
ii.set_additional_metadata(md)
ii.only_use_additional_meta_data = True
- ii.wait_and_retry_on_rate_limit = dlg.wait_and_retry_on_rate_limit
ii.set_output_function(self.auto_tag_log)
ii.cover_page_index = md.get_cover_page_index_list()[0]
if self.atprogdialog is not None:
@@ -1812,19 +1820,19 @@ You have {4-self.settings.settings_warning} warnings left.
self.auto_tag_log("Online search: Low confidence match, but saving anyways, as indicated...\n")
# now get the particular issue data
- cv_md = self.actual_issue_data_fetch(matches[0])
- if cv_md is None:
+ ct_md = self.actual_issue_data_fetch(matches[0])
+ if ct_md is None:
match_results.fetch_data_failures.append(str(ca.path.absolute()))
- if cv_md is not None:
+ if ct_md is not None:
if dlg.cbxRemoveMetadata.isChecked():
- md = cv_md
+ md = ct_md
else:
notes = (
f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on"
- f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {cv_md.issue_id}]"
+ f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {ct_md.issue_id}]"
)
- md.overlay(cv_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
+ md.overlay(ct_md.replace(notes=utils.combine_notes(md.notes, notes, "Tagged with ComicTagger")))
if self.settings.auto_imprint:
md.fix_publisher()
@@ -1867,7 +1875,7 @@ You have {4-self.settings.settings_warning} warnings left.
if not atstartdlg.exec():
return
- self.atprogdialog = AutoTagProgressWindow(self)
+ self.atprogdialog = AutoTagProgressWindow(self, self.talker_api)
self.atprogdialog.setModal(True)
self.atprogdialog.show()
self.atprogdialog.progressBar.setMaximum(len(ca_list))
@@ -1954,7 +1962,12 @@ You have {4-self.settings.settings_warning} warnings left.
match_results.multiple_matches.extend(match_results.low_confidence_matches)
if reply == QtWidgets.QMessageBox.StandardButton.Yes:
matchdlg = AutoTagMatchWindow(
- self, match_results.multiple_matches, style, self.actual_issue_data_fetch, self.settings
+ self,
+ match_results.multiple_matches,
+ style,
+ self.actual_issue_data_fetch,
+ self.settings,
+ self.talker_api,
)
matchdlg.setModal(True)
matchdlg.exec()
@@ -2017,7 +2030,7 @@ You have {4-self.settings.settings_warning} warnings left.
def show_page_browser(self) -> None:
if self.page_browser is None:
- self.page_browser = PageBrowserWindow(self, self.metadata)
+ self.page_browser = PageBrowserWindow(self, self.talker_api, self.metadata)
if self.comic_archive is not None:
self.page_browser.set_comic_archive(self.comic_archive)
self.page_browser.finished.connect(self.page_browser_closed)
@@ -2084,7 +2097,7 @@ You have {4-self.settings.settings_warning} warnings left.
"File Rename", "If you rename files now, unsaved data in the form will be lost. Are you sure?"
):
- dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings)
+ dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings, self.talker_api)
dlg.setModal(True)
if dlg.exec() and self.comic_archive is not None:
self.fileSelectionList.update_selected_rows()
diff --git a/comictaggerlib/volumeselectionwindow.py b/comictaggerlib/volumeselectionwindow.py
index 21dfe6d..f4bcd9a 100644
--- a/comictaggerlib/volumeselectionwindow.py
+++ b/comictaggerlib/volumeselectionwindow.py
@@ -25,16 +25,16 @@ from PyQt5.QtCore import pyqtSignal
from comicapi import utils
from comicapi.comicarchive import ComicArchive
from comicapi.genericmetadata import GenericMetadata
-from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException
from comictaggerlib.coverimagewidget import CoverImageWidget
from comictaggerlib.issueidentifier import IssueIdentifier
from comictaggerlib.issueselectionwindow import IssueSelectionWindow
from comictaggerlib.matchselectionwindow import MatchSelectionWindow
from comictaggerlib.progresswindow import IDProgressWindow
-from comictaggerlib.resulttypes import CVVolumeResults
from comictaggerlib.settings import ComicTaggerSettings
from comictaggerlib.ui import ui_path
from comictaggerlib.ui.qtutils import reduce_widget_font_size
+from comictalker.resulttypes import ComicVolume
+from comictalker.talkerbase import ComicTalker, TalkerError
logger = logging.getLogger(__name__)
@@ -43,27 +43,34 @@ class SearchThread(QtCore.QThread):
searchComplete = pyqtSignal()
progressUpdate = pyqtSignal(int, int)
- def __init__(self, series_name: str, refresh: bool, literal: bool = False, series_match_thresh: int = 90) -> None:
+ def __init__(
+ self,
+ talker_api: ComicTalker,
+ series_name: str,
+ refresh: bool,
+ literal: bool = False,
+ series_match_thresh: int = 90,
+ ) -> None:
QtCore.QThread.__init__(self)
+ self.talker_api = talker_api
self.series_name = series_name
self.refresh: bool = refresh
- self.error_code: int | None = None
- self.cv_error = False
- self.cv_search_results: list[CVVolumeResults] = []
+ self.error_e: TalkerError
+ self.ct_error = False
+ self.ct_search_results: list[ComicVolume] = []
self.literal = literal
self.series_match_thresh = series_match_thresh
def run(self) -> None:
- comic_vine = ComicVineTalker(self.series_match_thresh)
try:
- self.cv_error = False
- self.cv_search_results = comic_vine.search_for_series(
+ self.ct_error = False
+ self.ct_search_results = self.talker_api.search_for_series(
self.series_name, self.prog_callback, self.refresh, self.literal
)
- except ComicVineTalkerException as e:
- self.cv_search_results = []
- self.cv_error = True
- self.error_code = e.code
+ except TalkerError as e:
+ self.ct_search_results = []
+ self.ct_error = True
+ self.error_e = e
finally:
self.searchComplete.emit()
@@ -105,6 +112,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
cover_index_list: list[int],
comic_archive: ComicArchive,
settings: ComicTaggerSettings,
+ talker_api: ComicTalker,
autoselect: bool = False,
literal: bool = False,
) -> None:
@@ -112,7 +120,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
uic.loadUi(ui_path / "volumeselectionwindow.ui", self)
- self.imageWidget = CoverImageWidget(self.imageContainer, CoverImageWidget.URLMode)
+ self.imageWidget = CoverImageWidget(self.imageContainer, talker_api, CoverImageWidget.URLMode)
gridlayout = QtWidgets.QGridLayout(self.imageContainer)
gridlayout.addWidget(self.imageWidget)
gridlayout.setContentsMargins(0, 0, 0, 0)
@@ -131,13 +139,14 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.settings = settings
self.series_name = series_name
self.issue_number = issue_number
+ self.issue_id: int | None = None
self.year = year
self.issue_count = issue_count
self.volume_id = 0
self.comic_archive = comic_archive
self.immediate_autoselect = autoselect
self.cover_index_list = cover_index_list
- self.cv_search_results: list[CVVolumeResults] = []
+ self.ct_search_results: list[ComicVolume] = []
self.literal = literal
self.ii: IssueIdentifier | None = None
self.iddialog: IDProgressWindow | None = None
@@ -147,6 +156,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.use_filter = self.settings.always_use_publisher_filter
+ # Load to retrieve settings
+ self.talker_api = talker_api
+
self.twList.resizeColumnsToContents()
self.twList.currentItemChanged.connect(self.current_item_changed)
self.twList.cellDoubleClicked.connect(self.cell_double_clicked)
@@ -158,15 +170,16 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.cbxFilter.toggled.connect(self.filter_toggled)
self.update_buttons()
- self.perform_query()
self.twList.selectRow(0)
def update_buttons(self) -> None:
- enabled = bool(self.cv_search_results)
+ enabled = bool(self.ct_search_results)
self.btnRequery.setEnabled(enabled)
+
self.btnIssues.setEnabled(enabled)
self.btnAutoSelect.setEnabled(enabled)
+
self.buttonBox.button(QtWidgets.QDialogButtonBox.StandardButton.Ok).setEnabled(enabled)
def requery(self) -> None:
@@ -178,6 +191,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.perform_query(refresh=False)
def auto_select(self) -> None:
+
if self.comic_archive is None:
QtWidgets.QMessageBox.information(self, "Auto-Select", "You need to load a comic first!")
return
@@ -191,7 +205,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.iddialog.rejected.connect(self.identify_cancel)
self.iddialog.show()
- self.ii = IssueIdentifier(self.comic_archive, self.settings)
+ self.ii = IssueIdentifier(self.comic_archive, self.settings, self.talker_api)
md = GenericMetadata()
md.series = self.series_name
@@ -264,7 +278,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
choices = True
if choices:
- selector = MatchSelectionWindow(self, matches, self.comic_archive)
+ selector = MatchSelectionWindow(self, matches, self.comic_archive, self.talker_api)
selector.setModal(True)
selector.exec()
if selector.result():
@@ -280,9 +294,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.show_issues()
def show_issues(self) -> None:
- selector = IssueSelectionWindow(self, self.settings, self.volume_id, self.issue_number)
+ selector = IssueSelectionWindow(self, self.settings, self.talker_api, self.volume_id, self.issue_number)
title = ""
- for record in self.cv_search_results:
+ for record in self.ct_search_results:
if record["id"] == self.volume_id:
title = record["name"]
title += " (" + str(record["start_year"]) + ")"
@@ -295,6 +309,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
if selector.result():
# we should now have a volume ID
self.issue_number = selector.issue_number
+ self.issue_id = selector.issue_id
self.accept()
def select_by_id(self) -> None:
@@ -306,18 +321,23 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
def perform_query(self, refresh: bool = False) -> None:
+ self.search_thread = SearchThread(
+ self.talker_api, self.series_name, refresh, self.literal, self.settings.id_series_match_search_thresh
+ )
+ self.search_thread.searchComplete.connect(self.search_complete)
+ self.search_thread.progressUpdate.connect(self.search_progress_update)
+ self.search_thread.start()
+
self.progdialog = QtWidgets.QProgressDialog("Searching Online", "Cancel", 0, 100, self)
self.progdialog.setWindowTitle("Online Search")
self.progdialog.canceled.connect(self.search_canceled)
self.progdialog.setModal(True)
self.progdialog.setMinimumDuration(300)
- self.search_thread = SearchThread(
- self.series_name, refresh, self.literal, self.settings.id_series_match_search_thresh
- )
- self.search_thread.searchComplete.connect(self.search_complete)
- self.search_thread.progressUpdate.connect(self.search_progress_update)
- self.search_thread.start()
- self.progdialog.exec()
+
+ if refresh or self.search_thread.isRunning():
+ self.progdialog.exec()
+ else:
+ self.progdialog = None
def search_canceled(self) -> None:
if self.progdialog is not None:
@@ -340,123 +360,124 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
def search_complete(self) -> None:
if self.progdialog is not None:
self.progdialog.accept()
- del self.progdialog
- if self.search_thread is not None and self.search_thread.cv_error:
- if self.search_thread.error_code == ComicVineTalkerException.RateLimit:
- QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message())
- else:
- QtWidgets.QMessageBox.critical(
- self, "Network Issue", "Could not connect to Comic Vine to search for series!"
+ self.progdialog = None
+ if self.search_thread is not None and self.search_thread.ct_error:
+ # TODO Currently still opens the window
+ QtWidgets.QMessageBox.critical(
+ self,
+ f"{self.search_thread.error_e.source} {self.search_thread.error_e.code_name} Error",
+ f"{self.search_thread.error_e}",
+ )
+ return
+
+ self.ct_search_results = self.search_thread.ct_search_results if self.search_thread is not None else []
+ # filter the publishers if enabled set
+ if self.use_filter:
+ try:
+ publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")}
+ # use '' as publisher name if None
+ self.ct_search_results = list(
+ filter(
+ lambda d: ("" if d["publisher"] is None else str(d["publisher"]).casefold())
+ not in publisher_filter,
+ self.ct_search_results,
)
- return
+ )
+ except Exception:
+ logger.exception("bad data error filtering publishers")
- self.cv_search_results = self.search_thread.cv_search_results if self.search_thread is not None else []
- # filter the publishers if enabled set
- if self.use_filter:
- try:
- publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")}
- # use '' as publisher name if None
- self.cv_search_results = list(
- filter(
- lambda d: ("" if d["publisher"] is None else str(d["publisher"]["name"]).casefold())
- not in publisher_filter,
- self.cv_search_results,
- )
- )
- except Exception:
- logger.exception("bad data error filtering publishers")
+ # pre sort the data - so that we can put exact matches first afterwards
+ # compare as str in case extra chars ie. '1976?'
+ # - missing (none) values being converted to 'None' - consistent with prior behaviour in v1.2.3
+ # sort by start_year if set
+ if self.settings.sort_series_by_year:
+ try:
+ self.ct_search_results = sorted(
+ self.ct_search_results,
+ key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])),
+ reverse=True,
+ )
+ except Exception:
+ logger.exception("bad data error sorting results by start_year,count_of_issues")
+ else:
+ try:
+ self.ct_search_results = sorted(
+ self.ct_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True
+ )
+ except Exception:
+ logger.exception("bad data error sorting results by count_of_issues")
- # pre sort the data - so that we can put exact matches first afterwards
- # compare as str in case extra chars ie. '1976?'
- # - missing (none) values being converted to 'None' - consistent with prior behaviour in v1.2.3
- # sort by start_year if set
- if self.settings.sort_series_by_year:
- try:
- self.cv_search_results = sorted(
- self.cv_search_results,
- key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])),
- reverse=True,
- )
- except Exception:
- logger.exception("bad data error sorting results by start_year,count_of_issues")
- else:
- try:
- self.cv_search_results = sorted(
- self.cv_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True
- )
- except Exception:
- logger.exception("bad data error sorting results by count_of_issues")
+ # move sanitized matches to the front
+ if self.settings.exact_series_matches_first:
+ try:
+ sanitized = utils.sanitize_title(self.series_name, False).casefold()
+ sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold()
- # move sanitized matches to the front
- if self.settings.exact_series_matches_first:
- try:
- sanitized = utils.sanitize_title(self.series_name, False).casefold()
- sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold()
+ deques: list[deque[ComicVolume]] = [deque(), deque(), deque()]
- deques: list[deque[CVVolumeResults]] = [deque(), deque(), deque()]
+ def categorize(result: ComicVolume) -> int:
+ # We don't remove anything on this one so that we only get exact matches
+ if utils.sanitize_title(result["name"], True).casefold() == sanitized_no_articles:
+ return 0
- def categorize(result: CVVolumeResults) -> int:
- # We don't remove anything on this one so that we only get exact matches
- if utils.sanitize_title(result["name"], True).casefold() == sanitized_no_articles:
- return 0
+ # this ensures that 'The Joker' is near the top even if you search 'Joker'
+ if utils.sanitize_title(result["name"], False).casefold() in sanitized:
+ return 1
+ return 2
- # this ensures that 'The Joker' is near the top even if you search 'Joker'
- if utils.sanitize_title(result["name"], False).casefold() in sanitized:
- return 1
- return 2
+ for comic in self.ct_search_results:
+ deques[categorize(comic)].append(comic)
+ logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2]))
+ self.ct_search_results = list(itertools.chain.from_iterable(deques))
+ except Exception:
+ logger.exception("bad data error filtering exact/near matches")
- for comic in self.cv_search_results:
- deques[categorize(comic)].append(comic)
- logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2]))
- self.cv_search_results = list(itertools.chain.from_iterable(deques))
- except Exception:
- logger.exception("bad data error filtering exact/near matches")
+ self.update_buttons()
- self.update_buttons()
+ self.twList.setSortingEnabled(False)
- self.twList.setSortingEnabled(False)
+ self.twList.setRowCount(0)
- self.twList.setRowCount(0)
+ row = 0
+ for record in self.ct_search_results:
+ self.twList.insertRow(row)
- row = 0
- for record in self.cv_search_results:
- self.twList.insertRow(row)
+ item_text = record["name"]
+ item = QtWidgets.QTableWidgetItem(item_text)
+ item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
+ item.setData(QtCore.Qt.ItemDataRole.UserRole, record["id"])
+ item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
+ self.twList.setItem(row, 0, item)
- item_text = record["name"]
- item = QtWidgets.QTableWidgetItem(item_text)
+ item_text = str(record["start_year"])
+ item = QtWidgets.QTableWidgetItem(item_text)
+ item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
+ item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
+ self.twList.setItem(row, 1, item)
+
+ item_text = str(record["count_of_issues"])
+ item = QtWidgets.QTableWidgetItem(item_text)
+ item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
+ item.setData(QtCore.Qt.ItemDataRole.DisplayRole, record["count_of_issues"])
+ item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
+ self.twList.setItem(row, 2, item)
+
+ if record["publisher"] is not None:
+ item_text = record["publisher"]
item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
- item.setData(QtCore.Qt.ItemDataRole.UserRole, record["id"])
- item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
- self.twList.setItem(row, 0, item)
-
- item_text = str(record["start_year"])
item = QtWidgets.QTableWidgetItem(item_text)
- item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
- self.twList.setItem(row, 1, item)
+ self.twList.setItem(row, 3, item)
- item_text = str(record["count_of_issues"])
- item = QtWidgets.QTableWidgetItem(item_text)
- item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
- item.setData(QtCore.Qt.ItemDataRole.DisplayRole, record["count_of_issues"])
- item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
- self.twList.setItem(row, 2, item)
+ row += 1
- if record["publisher"] is not None:
- item_text = record["publisher"]["name"]
- item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text)
- item = QtWidgets.QTableWidgetItem(item_text)
- item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled)
- self.twList.setItem(row, 3, item)
-
- row += 1
-
- self.twList.setSortingEnabled(True)
- self.twList.selectRow(0)
- self.twList.resizeColumnsToContents()
+ self.twList.setSortingEnabled(True)
+ self.twList.selectRow(0)
+ self.twList.resizeColumnsToContents()
def showEvent(self, event: QtGui.QShowEvent) -> None:
- if not self.cv_search_results:
+ self.perform_query()
+ if not self.ct_search_results:
QtCore.QCoreApplication.processEvents()
QtWidgets.QMessageBox.information(self, "Search Result", "No matches found!")
QtCore.QTimer.singleShot(200, self.close_me)
@@ -483,11 +504,11 @@ class VolumeSelectionWindow(QtWidgets.QDialog):
self.volume_id = self.twList.item(curr.row(), 0).data(QtCore.Qt.ItemDataRole.UserRole)
# list selection was changed, update the info on the volume
- for record in self.cv_search_results:
+ for record in self.ct_search_results:
if record["id"] == self.volume_id:
if record["description"] is None:
self.teDetails.setText("")
else:
self.teDetails.setText(record["description"])
- self.imageWidget.set_url(record["image"]["super_url"])
+ self.imageWidget.set_url(record["image_url"])
break
diff --git a/comictalker/__init__.py b/comictalker/__init__.py
new file mode 100644
index 0000000..9d48db4
--- /dev/null
+++ b/comictalker/__init__.py
@@ -0,0 +1 @@
+from __future__ import annotations
diff --git a/comictaggerlib/comiccacher.py b/comictalker/comiccacher.py
similarity index 53%
rename from comictaggerlib/comiccacher.py
rename to comictalker/comiccacher.py
index a501122..7625edf 100644
--- a/comictaggerlib/comiccacher.py
+++ b/comictalker/comiccacher.py
@@ -16,14 +16,15 @@
from __future__ import annotations
import datetime
+import json
import logging
import os
import sqlite3 as lite
from typing import Any
from comictaggerlib import ctversion
-from comictaggerlib.resulttypes import CVImage, CVIssuesResults, CVPublisher, CVVolumeResults, SelectDetails
from comictaggerlib.settings import ComicTaggerSettings
+from comictalker.resulttypes import ComicIssue, ComicVolume
logger = logging.getLogger(__name__)
@@ -77,15 +78,8 @@ class ComicCacher:
"CREATE TABLE VolumeSearchCache("
+ "search_term TEXT,"
+ "id INT NOT NULL,"
- + "name TEXT,"
- + "start_year INT,"
- + "publisher TEXT,"
- + "count_of_issues INT,"
- + "image_url TEXT,"
- + "description TEXT,"
+ "timestamp DATE DEFAULT (datetime('now','localtime')),"
- + "source_name TEXT NOT NULL,"
- + "aliases TEXT)" # Newline separated
+ + "source_name TEXT NOT NULL)"
)
cur.execute(
@@ -95,29 +89,21 @@ class ComicCacher:
+ "publisher TEXT,"
+ "count_of_issues INT,"
+ "start_year INT,"
+ + "image_url TEXT,"
+ + "aliases TEXT," # Newline separated
+ + "description TEXT,"
+ "timestamp DATE DEFAULT (datetime('now','localtime')), "
+ "source_name TEXT NOT NULL,"
- + "aliases TEXT," # Newline separated
+ "PRIMARY KEY (id, source_name))"
)
- cur.execute(
- "CREATE TABLE AltCovers("
- + "issue_id INT NOT NULL,"
- + "url_list TEXT,"
- + "timestamp DATE DEFAULT (datetime('now','localtime')), "
- + "source_name TEXT NOT NULL,"
- + "aliases TEXT," # Newline separated
- + "PRIMARY KEY (issue_id, source_name))"
- )
-
cur.execute(
"CREATE TABLE Issues("
+ "id INT NOT NULL,"
+ "volume_id INT,"
+ "name TEXT,"
+ "issue_number TEXT,"
- + "super_url TEXT,"
+ + "image_url TEXT,"
+ "thumb_url TEXT,"
+ "cover_date TEXT,"
+ "site_detail_url TEXT,"
@@ -125,11 +111,17 @@ class ComicCacher:
+ "timestamp DATE DEFAULT (datetime('now','localtime')), "
+ "source_name TEXT NOT NULL,"
+ "aliases TEXT," # Newline separated
+ + "alt_image_urls TEXT," # Newline separated URLs
+ + "characters TEXT," # Newline separated
+ + "locations TEXT," # Newline separated
+ + "credits TEXT," # JSON: "{"name": "Bob Shakespeare", "role": "Writer"}"
+ + "teams TEXT," # Newline separated
+ + "story_arcs TEXT," # Newline separated
+ + "complete BOOL," # Is the data complete? Includes characters, locations, credits.
+ "PRIMARY KEY (id, source_name))"
)
- def add_search_results(self, source_name: str, search_term: str, cv_search_results: list[CVVolumeResults]) -> None:
-
+ def add_search_results(self, source_name: str, search_term: str, ct_search_results: list[ComicVolume]) -> None:
con = lite.connect(self.db_file)
with con:
@@ -143,114 +135,63 @@ class ComicCacher:
)
# now add in new results
- for record in cv_search_results:
-
- if record["publisher"] is None:
- pub_name = ""
- else:
- pub_name = record["publisher"]["name"]
-
- if record["image"] is None:
- url = ""
- else:
- url = record["image"]["super_url"]
-
+ for record in ct_search_results:
cur.execute(
- "INSERT INTO VolumeSearchCache "
- + "(source_name, search_term, id, name, start_year, publisher, count_of_issues, image_url, description, aliases) "
- + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
+ "INSERT INTO VolumeSearchCache " + "(source_name, search_term, id) " + "VALUES(?, ?, ?)",
(
source_name,
search_term.casefold(),
record["id"],
- record["name"],
- record["start_year"],
- pub_name,
- record["count_of_issues"],
- url,
- record["description"],
- record["aliases"],
),
)
- def get_search_results(self, source_name: str, search_term: str) -> list[CVVolumeResults]:
+ data = {
+ "id": record["id"],
+ "source_name": source_name,
+ "name": record["name"],
+ "publisher": record.get("publisher", ""),
+ "count_of_issues": record.get("count_of_issues"),
+ "start_year": record.get("start_year"),
+ "image_url": record.get("image_url", ""),
+ "description": record.get("description", ""),
+ "timestamp": datetime.datetime.now(),
+ "aliases": "\n".join(record.get("aliases", [])),
+ }
+ self.upsert(cur, "volumes", data)
+ def get_search_results(self, source_name: str, search_term: str) -> list[ComicVolume]:
results = []
con = lite.connect(self.db_file)
with con:
con.text_factory = str
cur = con.cursor()
- # purge stale search results
- a_day_ago = datetime.datetime.today() - datetime.timedelta(days=1)
- cur.execute("DELETE FROM VolumeSearchCache WHERE timestamp < ?", [str(a_day_ago)])
-
- # fetch
cur.execute(
- "SELECT * FROM VolumeSearchCache WHERE search_term=? AND source_name=?",
+ "SELECT * FROM VolumeSearchCache INNER JOIN Volumes on"
+ " VolumeSearchCache.id=Volumes.id AND VolumeSearchCache.source_name=Volumes.source_name"
+ " WHERE search_term=? AND VolumeSearchCache.source_name=?",
[search_term.casefold(), source_name],
)
+
rows = cur.fetchall()
# now process the results
for record in rows:
- result = CVVolumeResults(
- id=record[1],
- name=record[2],
- start_year=record[3],
- count_of_issues=record[5],
- description=record[7],
- publisher=CVPublisher(name=record[4]),
- image=CVImage(super_url=record[6]),
- aliases=record[10],
+ result = ComicVolume(
+ id=record[4],
+ name=record[5],
+ publisher=record[6],
+ count_of_issues=record[7],
+ start_year=record[8],
+ image_url=record[9],
+ aliases=record[10].strip().splitlines(),
+ description=record[11],
)
results.append(result)
return results
- def add_alt_covers(self, source_name: str, issue_id: int, url_list: list[str]) -> None:
-
- con = lite.connect(self.db_file)
-
- with con:
- con.text_factory = str
- cur = con.cursor()
-
- # remove all previous entries with this search term
- cur.execute("DELETE FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name])
-
- url_list_str = ",".join(url_list)
- # now add in new record
- cur.execute(
- "INSERT INTO AltCovers (source_name, issue_id, url_list) VALUES(?, ?, ?)",
- (source_name, issue_id, url_list_str),
- )
-
- def get_alt_covers(self, source_name: str, issue_id: int) -> list[str]:
-
- con = lite.connect(self.db_file)
- with con:
- cur = con.cursor()
- con.text_factory = str
-
- # purge stale issue info - probably issue data won't change
- # much....
- a_month_ago = datetime.datetime.today() - datetime.timedelta(days=30)
- cur.execute("DELETE FROM AltCovers WHERE timestamp < ?", [str(a_month_ago)])
-
- cur.execute("SELECT url_list FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name])
- row = cur.fetchone()
- if row is None:
- return []
-
- url_list_str = row[0]
- if not url_list_str:
- return []
- url_list = str(url_list_str).split(",")
- return url_list
-
- def add_volume_info(self, source_name: str, cv_volume_record: CVVolumeResults) -> None:
-
+ def add_volume_info(self, source_name: str, volume_record: ComicVolume) -> None:
con = lite.connect(self.db_file)
with con:
@@ -259,25 +200,21 @@ class ComicCacher:
timestamp = datetime.datetime.now()
- if cv_volume_record["publisher"] is None:
- pub_name = ""
- else:
- pub_name = cv_volume_record["publisher"]["name"]
-
data = {
- "id": cv_volume_record["id"],
+ "id": volume_record["id"],
"source_name": source_name,
- "name": cv_volume_record["name"],
- "publisher": pub_name,
- "count_of_issues": cv_volume_record["count_of_issues"],
- "start_year": cv_volume_record["start_year"],
+ "name": volume_record["name"],
+ "publisher": volume_record.get("publisher", ""),
+ "count_of_issues": volume_record.get("count_of_issues"),
+ "start_year": volume_record.get("start_year"),
+ "image_url": volume_record.get("image_url", ""),
+ "description": volume_record.get("description", ""),
"timestamp": timestamp,
- "aliases": cv_volume_record["aliases"],
+ "aliases": "\n".join(volume_record.get("aliases", [])),
}
self.upsert(cur, "volumes", data)
- def add_volume_issues_info(self, source_name: str, volume_id: int, cv_volume_issues: list[CVIssuesResults]) -> None:
-
+ def add_volume_issues_info(self, source_name: str, volume_issues: list[ComicIssue]) -> None:
con = lite.connect(self.db_file)
with con:
@@ -287,40 +224,46 @@ class ComicCacher:
# add in issues
- for issue in cv_volume_issues:
+ for issue in volume_issues:
data = {
"id": issue["id"],
- "volume_id": volume_id,
+ "volume_id": issue["volume"]["id"],
"source_name": source_name,
"name": issue["name"],
"issue_number": issue["issue_number"],
- "site_detail_url": issue["site_detail_url"],
- "cover_date": issue["cover_date"],
- "super_url": issue["image"]["super_url"],
- "thumb_url": issue["image"]["thumb_url"],
- "description": issue["description"],
+ "site_detail_url": issue.get("site_detail_url"),
+ "cover_date": issue.get("cover_date"),
+ "image_url": issue.get("image_url", ""),
+ "thumb_url": issue.get("image_thumb_url", ""),
+ "description": issue.get("description", ""),
"timestamp": timestamp,
- "aliases": issue["aliases"],
+ "aliases": "\n".join(issue.get("aliases", [])),
+ "alt_image_urls": "\n".join(issue.get("alt_image_urls", [])),
+ "characters": "\n".join(issue.get("characters", [])),
+ "locations": "\n".join(issue.get("locations", [])),
+ "teams": "\n".join(issue.get("teams", [])),
+ "story_arcs": "\n".join(issue.get("story_arcs", [])),
+ "credits": json.dumps(issue.get("credits")),
+ "complete": issue["complete"],
}
self.upsert(cur, "issues", data)
- def get_volume_info(self, volume_id: int, source_name: str) -> CVVolumeResults | None:
-
- result: CVVolumeResults | None = None
+ def get_volume_info(self, volume_id: int, source_name: str, purge: bool = True) -> ComicVolume | None:
+ result: ComicVolume | None = None
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
- # purge stale volume info
- a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7)
- cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)])
+ if purge:
+ # purge stale volume info
+ a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7)
+ cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)])
# fetch
cur.execute(
- "SELECT source_name,id,name,publisher,count_of_issues,start_year,aliases FROM Volumes"
- " WHERE id=? AND source_name=?",
+ "SELECT * FROM Volumes" " WHERE id=? AND source_name=?",
[volume_id, source_name],
)
@@ -330,19 +273,22 @@ class ComicCacher:
return result
# since ID is primary key, there is only one row
- result = CVVolumeResults(
- id=row[1],
- name=row[2],
- count_of_issues=row[4],
- start_year=row[5],
- publisher=CVPublisher(name=row[3]),
- aliases=row[6],
+ result = ComicVolume(
+ id=row[0],
+ name=row[1],
+ publisher=row[2],
+ count_of_issues=row[3],
+ start_year=row[4],
+ image_url=row[5],
+ aliases=row[6].strip().splitlines(),
+ description=row[7],
)
return result
- def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[CVIssuesResults]:
-
+ def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[ComicIssue]:
+ # get_volume_info should only fail if someone is doing something weird
+ volume = self.get_volume_info(volume_id, source_name, False) or ComicVolume(id=volume_id, name="")
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
@@ -354,11 +300,11 @@ class ComicCacher:
cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)])
# fetch
- results: list[CVIssuesResults] = []
+ results: list[ComicIssue] = []
cur.execute(
(
- "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,super_url,thumb_url,description,aliases"
+ "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,alt_image_urls,characters,locations,credits,teams,story_arcs,complete"
" FROM Issues WHERE volume_id=? AND source_name=?"
),
[volume_id, source_name],
@@ -367,75 +313,78 @@ class ComicCacher:
# now process the results
for row in rows:
- record = CVIssuesResults(
+ record = ComicIssue(
id=row[1],
name=row[2],
issue_number=row[3],
site_detail_url=row[4],
cover_date=row[5],
- image=CVImage(super_url=row[6], thumb_url=row[7]),
+ image_url=row[6],
description=row[8],
- aliases=row[9],
+ volume=volume,
+ aliases=row[9].strip().splitlines(),
+ alt_image_urls=row[10].strip().splitlines(),
+ characters=row[11].strip().splitlines(),
+ locations=row[12].strip().splitlines(),
+ credits=json.loads(row[13]),
+ teams=row[14].strip().splitlines(),
+ story_arcs=row[15].strip().splitlines(),
+ complete=bool(row[16]),
)
results.append(record)
return results
- def add_issue_select_details(
- self,
- source_name: str,
- issue_id: int,
- image_url: str,
- thumb_image_url: str,
- cover_date: str,
- site_detail_url: str,
- ) -> None:
-
- con = lite.connect(self.db_file)
-
- with con:
- cur = con.cursor()
- con.text_factory = str
- timestamp = datetime.datetime.now()
-
- data = {
- "id": issue_id,
- "source_name": source_name,
- "super_url": image_url,
- "thumb_url": thumb_image_url,
- "cover_date": cover_date,
- "site_detail_url": site_detail_url,
- "timestamp": timestamp,
- }
- self.upsert(cur, "issues", data)
-
- def get_issue_select_details(self, issue_id: int, source_name: str) -> SelectDetails:
-
+ def get_issue_info(self, issue_id: int, source_name: str) -> ComicIssue | None:
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
+ # purge stale issue info - probably issue data won't change
+ # much....
+ a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7)
+ cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)])
+
cur.execute(
- "SELECT super_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=? AND source_name=?",
+ (
+ "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,volume_id,alt_image_urls,characters,locations,credits,teams,story_arcs,complete"
+ " FROM Issues WHERE id=? AND source_name=?"
+ ),
[issue_id, source_name],
)
row = cur.fetchone()
- details = SelectDetails(
- image_url=None,
- thumb_image_url=None,
- cover_date=None,
- site_detail_url=None,
- )
- if row is not None and row[0] is not None:
- details["image_url"] = row[0]
- details["thumb_image_url"] = row[1]
- details["cover_date"] = row[2]
- details["site_detail_url"] = row[3]
+ record = None
- return details
+ if row:
+ # get_volume_info should only fail if someone is doing something weird
+ volume = self.get_volume_info(row[10], source_name, False) or ComicVolume(id=row[10], name="")
+
+ # now process the results
+
+ record = ComicIssue(
+ id=row[1],
+ name=row[2],
+ issue_number=row[3],
+ site_detail_url=row[4],
+ cover_date=row[5],
+ image_url=row[6],
+ image_thumb_url=row[7],
+ description=row[8],
+ volume=volume,
+ aliases=row[9].strip().splitlines(),
+ alt_image_urls=row[11].strip().splitlines(),
+ characters=row[12].strip().splitlines(),
+ locations=row[13].strip().splitlines(),
+ credits=json.loads(row[14]),
+ teams=row[15].strip().splitlines(),
+ story_arcs=row[16].strip().splitlines(),
+ complete=bool(row[17]),
+ )
+
+ return record
def upsert(self, cur: lite.Cursor, tablename: str, data: dict[str, Any]) -> None:
"""This does an insert if the given PK doesn't exist, and an
diff --git a/comictalker/comictalkerapi.py b/comictalker/comictalkerapi.py
new file mode 100644
index 0000000..e9c3bbe
--- /dev/null
+++ b/comictalker/comictalkerapi.py
@@ -0,0 +1,38 @@
+"""Handles collecting data from source talkers."""
+
+# Copyright 2012-2014 Anthony Beville
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import logging
+
+import comictalker.talkers.comicvine
+from comictalker.talkerbase import ComicTalker, TalkerError
+
+logger = logging.getLogger(__name__)
+
+
+def get_comic_talker(source_name: str) -> type[ComicTalker]:
+ """Retrieve the available sources modules"""
+ sources = get_talkers()
+ if source_name not in sources:
+ raise TalkerError(source=source_name, code=4, desc="The talker does not exist")
+
+ talker = sources[source_name]
+ return talker
+
+
+def get_talkers():
+ """Returns all comic talker modules NOT objects"""
+ return {"comicvine": comictalker.talkers.comicvine.ComicVineTalker}
diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py
new file mode 100644
index 0000000..ca2b013
--- /dev/null
+++ b/comictalker/resulttypes.py
@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing_extensions import Required, TypedDict
+
+
+class Credits(TypedDict):
+ name: str
+ role: str
+
+
+class ComicVolume(TypedDict, total=False):
+ aliases: list[str]
+ count_of_issues: int
+ description: str
+ id: Required[int]
+ image_url: str
+ name: Required[str]
+ publisher: str
+ start_year: int
+
+
+class ComicIssue(TypedDict, total=False):
+ aliases: list[str]
+ cover_date: str
+ description: str
+ id: int
+ image_url: str
+ image_thumb_url: str
+ issue_number: Required[str]
+ name: Required[str]
+ site_detail_url: str
+ volume: ComicVolume
+ alt_image_urls: list[str]
+ characters: list[str]
+ locations: list[str]
+ credits: list[Credits]
+ teams: list[str]
+ story_arcs: list[str]
+ complete: bool # Is this a complete ComicIssue? or is there more data to fetch
diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py
new file mode 100644
index 0000000..a039cfc
--- /dev/null
+++ b/comictalker/talker_utils.py
@@ -0,0 +1,199 @@
+"""Generic sources utils to format API data and the like.
+"""
+# Copyright 2012-2014 Anthony Beville
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import logging
+import re
+from datetime import datetime
+
+from bs4 import BeautifulSoup
+
+from comicapi import utils
+from comicapi.genericmetadata import GenericMetadata
+from comicapi.issuestring import IssueString
+from comictaggerlib import ctversion
+from comictalker.talkerbase import ComicIssue
+
+logger = logging.getLogger(__name__)
+
+
+def map_comic_issue_to_metadata(
+ issue_results: ComicIssue, source: str, remove_html_tables: bool = False, use_year_volume: bool = False
+) -> GenericMetadata:
+ """Maps ComicIssue to generic metadata"""
+ metadata = GenericMetadata()
+ metadata.is_empty = False
+
+ # Is this best way to go about checking?
+ if issue_results["volume"].get("name"):
+ metadata.series = utils.xlate(issue_results["volume"]["name"])
+ if issue_results.get("issue_number"):
+ metadata.issue = IssueString(issue_results["issue_number"]).as_string()
+ if issue_results.get("name"):
+ metadata.title = utils.xlate(issue_results["name"])
+ if issue_results.get("image_url"):
+ metadata.cover_image = issue_results["image_url"]
+
+ if issue_results["volume"].get("publisher"):
+ metadata.publisher = utils.xlate(issue_results["volume"]["publisher"])
+
+ if issue_results.get("cover_date"):
+ metadata.day, metadata.month, metadata.year = utils.parse_date_str(issue_results["cover_date"])
+ elif issue_results["volume"].get("start_year"):
+ metadata.year = utils.xlate(issue_results["volume"]["start_year"], True)
+
+ metadata.comments = cleanup_html(issue_results["description"], remove_html_tables)
+ if use_year_volume:
+ metadata.volume = issue_results["volume"]["start_year"]
+
+ metadata.tag_origin = source
+ metadata.issue_id = issue_results["id"]
+
+ metadata.notes = (
+ f"Tagged with ComicTagger {ctversion.version} using info from {source} on"
+ f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Issue ID {issue_results['id']}]"
+ )
+ metadata.web_link = issue_results["site_detail_url"]
+
+ for person in issue_results["credits"]:
+ if "role" in person:
+ roles = person["role"].split(",")
+ for role in roles:
+ # can we determine 'primary' from CV??
+ metadata.add_credit(person["name"], role.title().strip(), False)
+
+ if issue_results.get("characters"):
+ metadata.characters = ", ".join(issue_results["characters"])
+ if issue_results.get("teams"):
+ metadata.teams = ", ".join(issue_results["teams"])
+ if issue_results.get("locations"):
+ metadata.locations = ", ".join(issue_results["locations"])
+ if issue_results.get("story_arcs"):
+ metadata.story_arc = ", ".join(issue_results["story_arcs"])
+
+ return metadata
+
+
+def parse_date_str(date_str: str) -> tuple[int | None, int | None, int | None]:
+ day = None
+ month = None
+ year = None
+ if date_str:
+ parts = date_str.split("-")
+ year = utils.xlate(parts[0], True)
+ if len(parts) > 1:
+ month = utils.xlate(parts[1], True)
+ if len(parts) > 2:
+ day = utils.xlate(parts[2], True)
+ return day, month, year
+
+
+def cleanup_html(string: str, remove_html_tables: bool = False) -> str:
+ """Cleans HTML code from any text. Will remove any HTML tables with remove_html_tables"""
+ if string is None:
+ return ""
+ # find any tables
+ soup = BeautifulSoup(string, "html.parser")
+ tables = soup.findAll("table")
+
+ # remove all newlines first
+ string = string.replace("\n", "")
+
+ # put in our own
+ string = string.replace("
", "\n")
+ string = string.replace("", "\n")
+ string = string.replace("", "\n\n")
+ string = string.replace("", "*")
+ string = string.replace("
", "*\n")
+ string = string.replace("", "*")
+ string = string.replace("
", "*\n")
+ string = string.replace("", "*")
+ string = string.replace("
", "*\n")
+ string = string.replace("", "*")
+ string = string.replace("
", "*\n")
+ string = string.replace("", "*")
+ string = string.replace("
", "*\n")
+ string = string.replace("", "*")
+ string = string.replace("
", "*\n")
+
+ # remove the tables
+ p = re.compile(r"")
+ if remove_html_tables:
+ string = p.sub("", string)
+ string = string.replace("*List of covers and their creators:*", "")
+ else:
+ string = p.sub("{}", string)
+
+ # now strip all other tags
+ p = re.compile(r"<[^<]*?>")
+ newstring = p.sub("", string)
+
+ newstring = newstring.replace(" ", " ")
+ newstring = newstring.replace("&", "&")
+
+ newstring = newstring.strip()
+
+ if not remove_html_tables:
+ # now rebuild the tables into text from BSoup
+ try:
+ table_strings = []
+ for table in tables:
+ rows = []
+ hdrs = []
+ col_widths = []
+ for hdr in table.findAll("th"):
+ item = hdr.string.strip()
+ hdrs.append(item)
+ col_widths.append(len(item))
+ rows.append(hdrs)
+
+ for row in table.findAll("tr"):
+ cols = []
+ col = row.findAll("td")
+ i = 0
+ for c in col:
+ item = c.string.strip()
+ cols.append(item)
+ if len(item) > col_widths[i]:
+ col_widths[i] = len(item)
+ i += 1
+ if len(cols) != 0:
+ rows.append(cols)
+ # now we have the data, make it into text
+ fmtstr = "|"
+ for w in col_widths:
+ fmtstr += f" {{:{w + 1}}}|"
+ table_text = ""
+ counter = 0
+ for row in rows:
+ table_text += fmtstr.format(*row) + "\n"
+ if counter == 0 and len(hdrs) != 0:
+ table_text += "|"
+ for w in col_widths:
+ table_text += "-" * (w + 2) + "|"
+ table_text += "\n"
+ counter += 1
+
+ table_strings.append(table_text + "\n")
+
+ newstring = newstring.format(*table_strings)
+ except Exception:
+ # we caught an error rebuilding the table.
+ # just bail and remove the formatting
+ logger.exception("table parse error")
+ newstring.replace("{}", "")
+
+ return newstring
diff --git a/comictalker/talkerbase.py b/comictalker/talkerbase.py
new file mode 100644
index 0000000..c624374
--- /dev/null
+++ b/comictalker/talkerbase.py
@@ -0,0 +1,186 @@
+"""A template for an information source
+"""
+# Copyright 2012-2014 Anthony Beville
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import logging
+from typing import Callable
+
+from comicapi.genericmetadata import GenericMetadata
+from comictalker.resulttypes import ComicIssue, ComicVolume
+
+logger = logging.getLogger(__name__)
+
+# NOTE: Series and Volume are synonymous. Some sources (ComicVine) use "volume" and others (MangaUpdates) use "series".
+
+
+class SourceDetails:
+ def __init__(
+ self,
+ name: str = "",
+ ident: str = "",
+ logo: str = "", # Will be scaled if greater than 100px width and 250px height in comictalker/talkers/logos
+ ):
+ self.name = name
+ self.id = ident
+ self.logo = logo
+
+
+class SourceStaticOptions:
+ def __init__(
+ self,
+ website: str = "",
+ has_issues: bool = False,
+ has_alt_covers: bool = False,
+ requires_apikey: bool = False,
+ has_nsfw: bool = False,
+ has_censored_covers: bool = False,
+ ) -> None:
+ self.website = website
+ self.has_issues = has_issues
+ self.has_alt_covers = has_alt_covers
+ self.requires_apikey = requires_apikey
+ self.has_nsfw = has_nsfw
+ self.has_censored_covers = has_censored_covers
+
+
+class TalkerError(Exception):
+ """Base class exception for information sources.
+
+ Attributes:
+ code -- a numerical code
+ 1 - General
+ 2 - Network
+ 3 - Data
+ desc -- description of the error
+ source -- the name of the source producing the error
+ """
+
+ codes = {
+ 1: "General",
+ 2: "Network",
+ 3: "Data",
+ 4: "Other",
+ }
+
+ def __init__(self, source: str = "", code: int = 4, desc: str = "", sub_code: int = 0) -> None:
+ super().__init__()
+ if desc == "":
+ desc = "Unknown"
+ self.desc = desc
+ self.code = code
+ self.code_name = self.codes[code]
+ self.sub_code = sub_code
+ self.source = source
+
+ def __str__(self):
+ return f"{self.source} encountered a {self.code_name} error. {self.desc}"
+
+
+class TalkerNetworkError(TalkerError):
+ """Network class exception for information sources
+
+ Attributes:
+ sub_code -- numerical code for finer detail
+ 1 -- connected refused
+ 2 -- api key
+ 3 -- rate limit
+ 4 -- timeout
+ """
+
+ net_codes = {
+ 0: "General network error.",
+ 1: "The connection was refused.",
+ 2: "An API key error occurred.",
+ 3: "Rate limit exceeded. Please wait a bit or enter a personal key if using the default.",
+ 4: "The connection timed out.",
+ 5: "Number of retries exceeded.",
+ }
+
+ def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None:
+ if desc == "":
+ desc = self.net_codes[sub_code]
+
+ super().__init__(source, 2, desc, sub_code)
+
+
+class TalkerDataError(TalkerError):
+ """Data class exception for information sources
+
+ Attributes:
+ sub_code -- numerical code for finer detail
+ 1 -- unexpected data
+ 2 -- malformed data
+ 3 -- missing data
+ """
+
+ data_codes = {
+ 0: "General data error.",
+ 1: "Unexpected data encountered.",
+ 2: "Malformed data encountered.",
+ 3: "Missing data encountered.",
+ }
+
+ def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None:
+ if desc == "":
+ desc = self.data_codes[sub_code]
+
+ super().__init__(source, 3, desc, sub_code)
+
+
+# Class talkers instance
+class ComicTalker:
+ """The base class for all comic source talkers"""
+
+ def __init__(self) -> None:
+ # Identity name for the information source etc.
+ self.source_details: SourceDetails = (
+ SourceDetails()
+ ) # Can use this to test if custom talker has been configured
+ self.static_options: SourceStaticOptions = SourceStaticOptions()
+
+ def check_api_key(self, key: str, url: str) -> bool:
+ """If the talker has or requires an API key, this function should test its validity"""
+ raise NotImplementedError
+
+ def search_for_series(
+ self,
+ series_name: str,
+ callback: Callable[[int, int], None] | None = None,
+ refresh_cache: bool = False,
+ literal: bool = False,
+ ) -> list[ComicVolume]:
+ """Searches for the series/volumes with the given series_name
+ callback is used for...
+ refresh_cache signals if the data in the cache should be used
+ literal indicates that no articles (a, the, etc.) should be removed when searching"""
+ raise NotImplementedError
+
+ def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]:
+ """Expected to return a list of issues with a given series/volume ID"""
+ raise NotImplementedError
+
+ def fetch_comic_data(self, issue_id: int = 0, series_id: int = 0, issue_number: str = "") -> GenericMetadata:
+ """This function is expected to handle a few possibilities:
+ 1. Only series_id passed in: Retrieve the SERIES/VOLUME information only
+ 2. series_id and issue_number: Retrieve the ISSUE information
+ 3. Only issue_id: Retrieve the ISSUE information"""
+ raise NotImplementedError
+
+ def fetch_issues_by_volume_issue_num_and_year(
+ self, volume_id_list: list[int], issue_number: str, year: str | int | None
+ ) -> list[ComicIssue]:
+ """Searches for a list of issues within the given year. Used solely by issueidentifer"""
+ raise NotImplementedError
diff --git a/comictalker/talkers/__init__.py b/comictalker/talkers/__init__.py
new file mode 100644
index 0000000..9d48db4
--- /dev/null
+++ b/comictalker/talkers/__init__.py
@@ -0,0 +1 @@
+from __future__ import annotations
diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py
new file mode 100644
index 0000000..0f9a4e0
--- /dev/null
+++ b/comictalker/talkers/comicvine.py
@@ -0,0 +1,674 @@
+"""ComicVine information source
+"""
+# Copyright 2012-2014 Anthony Beville
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import annotations
+
+import json
+import logging
+import time
+from typing import Any, Callable, cast
+from urllib.parse import urljoin, urlsplit
+
+import requests
+from typing_extensions import Required, TypedDict
+
+import comictalker.talker_utils as talker_utils
+from comicapi import utils
+from comicapi.genericmetadata import GenericMetadata
+from comicapi.issuestring import IssueString
+from comictaggerlib import ctversion
+from comictalker.comiccacher import ComicCacher
+from comictalker.resulttypes import ComicIssue, ComicVolume, Credits
+from comictalker.talkerbase import ComicTalker, SourceDetails, SourceStaticOptions, TalkerDataError, TalkerNetworkError
+
+logger = logging.getLogger(__name__)
+
+
+class CVTypeID:
+ Volume = "4050"
+ Issue = "4000"
+
+
+class CVImage(TypedDict, total=False):
+ icon_url: str
+ medium_url: str
+ screen_url: str
+ screen_large_url: str
+ small_url: str
+ super_url: Required[str]
+ thumb_url: str
+ tiny_url: str
+ original_url: str
+ image_tags: str
+
+
+class CVAltImages(TypedDict):
+ original_url: str
+ id: int
+ caption: str
+ image_tags: str
+
+
+class CVPublisher(TypedDict, total=False):
+ api_detail_url: str
+ id: int
+ name: Required[str]
+
+
+class CVVolume(TypedDict):
+ api_detail_url: str
+ id: int
+ name: str
+ site_detail_url: str
+
+
+class CVCredits(TypedDict):
+ api_detail_url: str
+ id: int
+ name: str
+ site_detail_url: str
+
+
+class CVPersonCredits(TypedDict):
+ api_detail_url: str
+ id: int
+ name: str
+ site_detail_url: str
+ role: str
+
+
+class CVVolumeResults(TypedDict):
+ aliases: str
+ count_of_issues: int
+ description: str
+ id: int
+ image: CVImage
+ name: str
+ publisher: CVPublisher
+ start_year: str
+ resource_type: str
+
+
+class CVResult(TypedDict):
+ error: str
+ limit: int
+ offset: int
+ number_of_page_results: int
+ number_of_total_results: int
+ status_code: int
+ results: (CVIssueDetailResults | CVVolumeResults | list[CVVolumeResults] | list[CVIssueDetailResults])
+ version: str
+
+
+class CVVolumeFullResult(TypedDict):
+ characters: list[CVCredits]
+ locations: list[CVCredits]
+ people: list[CVPersonCredits]
+ site_detail_url: str
+ count_of_issues: int
+ description: str
+ id: int
+ name: str
+ publisher: CVPublisher
+ start_year: str
+ resource_type: str
+
+
+class CVIssueDetailResults(TypedDict, total=False):
+ aliases: str
+ api_detail_url: str
+ associated_images: list[CVAltImages]
+ character_credits: list[CVCredits]
+ character_died_in: None
+ concept_credits: list[CVCredits]
+ cover_date: str
+ date_added: str
+ date_last_updated: str
+ deck: None
+ description: str
+ first_appearance_characters: None
+ first_appearance_concepts: None
+ first_appearance_locations: None
+ first_appearance_objects: None
+ first_appearance_storyarcs: None
+ first_appearance_teams: None
+ has_staff_review: bool
+ id: int
+ image: CVImage
+ issue_number: str
+ location_credits: list[CVCredits]
+ name: str
+ object_credits: list[CVCredits]
+ person_credits: list[CVPersonCredits]
+ site_detail_url: str
+ store_date: str
+ story_arc_credits: list[CVCredits]
+ team_credits: list[CVCredits]
+ team_disbanded_in: None
+ volume: CVVolume
+
+
+CV_RATE_LIMIT_STATUS = 107
+
+
+class ComicVineTalker(ComicTalker):
+ def __init__(
+ self, api_url, api_key, series_match_thresh, remove_html_tables, use_series_start_as_volume, wait_on_ratelimit
+ ):
+ super().__init__()
+ self.source_details = SourceDetails(
+ name="Comic Vine",
+ ident="comicvine",
+ )
+ self.static_options = SourceStaticOptions(
+ website="https://comicvine.gamespot.com/",
+ has_issues=True,
+ has_alt_covers=True,
+ requires_apikey=True,
+ has_nsfw=False,
+ has_censored_covers=False,
+ )
+
+ # Identity name for the information source
+ self.source_name = self.source_details.id
+ self.source_name_friendly = self.source_details.name
+
+ self.wait_for_rate_limit = wait_on_ratelimit
+ # NOTE: This was hardcoded before which is why it isn't passed in
+ self.wait_for_rate_limit_time = 20
+
+ self.issue_id: int | None = None
+
+ self.api_key = api_key if api_key else "27431e6787042105bd3e47e169a624521f89f3a4"
+ self.api_base_url = api_url if api_url else "https://comicvine.gamespot.com/api"
+
+ self.remove_html_tables = remove_html_tables
+ self.use_series_start_as_volume = use_series_start_as_volume
+
+ tmp_url = urlsplit(self.api_base_url)
+
+ # joinurl only works properly if there is a trailing slash
+ if tmp_url.path and tmp_url.path[-1] != "/":
+ tmp_url = tmp_url._replace(path=tmp_url.path + "/")
+
+ self.api_base_url = tmp_url.geturl()
+
+ self.series_match_thresh = series_match_thresh
+
+ def check_api_key(self, key: str, url: str) -> bool:
+ if not url:
+ url = self.api_base_url
+ try:
+ tmp_url = urlsplit(url)
+ if tmp_url.path and tmp_url.path[-1] != "/":
+ tmp_url = tmp_url._replace(path=tmp_url.path + "/")
+ url = tmp_url.geturl()
+ test_url = urljoin(url, "issue/1/")
+
+ cv_response: CVResult = requests.get(
+ test_url,
+ headers={"user-agent": "comictagger/" + ctversion.version},
+ params={
+ "api_key": key,
+ "format": "json",
+ "field_list": "name",
+ },
+ ).json()
+
+ # Bogus request, but if the key is wrong, you get error 100: "Invalid API Key"
+ return cv_response["status_code"] != 100
+ except Exception:
+ return False
+
+ def get_cv_content(self, url: str, params: dict[str, Any]) -> CVResult:
+ """
+ Get the content from the CV server. If we're in "wait mode" and status code is a rate limit error
+ sleep for a bit and retry.
+ """
+ total_time_waited = 0
+ limit_wait_time = 1
+ counter = 0
+ wait_times = [1, 2, 3, 4]
+ while True:
+ cv_response: CVResult = self.get_url_content(url, params)
+ if self.wait_for_rate_limit and cv_response["status_code"] == CV_RATE_LIMIT_STATUS:
+ logger.info(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n")
+ time.sleep(limit_wait_time * 60)
+ total_time_waited += limit_wait_time
+ limit_wait_time = wait_times[counter]
+ if counter < 3:
+ counter += 1
+ # don't wait much more than 20 minutes
+ if total_time_waited < self.wait_for_rate_limit_time:
+ continue
+ if cv_response["status_code"] != 1:
+ logger.debug(
+ f"{self.source_name_friendly} query failed with error #{cv_response['status_code']}: [{cv_response['error']}]."
+ )
+ raise TalkerNetworkError(
+ self.source_name_friendly, 0, f"{cv_response['status_code']}: {cv_response['error']}"
+ )
+
+ # it's all good
+ break
+ return cv_response
+
+ def get_url_content(self, url: str, params: dict[str, Any]) -> Any:
+ # connect to server:
+ # if there is a 500 error, try a few more times before giving up
+ # any other error, just bail
+ for tries in range(3):
+ try:
+ resp = requests.get(url, params=params, headers={"user-agent": "comictagger/" + ctversion.version})
+ if resp.status_code == 200:
+ return resp.json()
+ if resp.status_code == 500:
+ logger.debug(f"Try #{tries + 1}: ")
+ time.sleep(1)
+ logger.debug(str(resp.status_code))
+ else:
+ break
+
+ except requests.exceptions.Timeout:
+ logger.debug(f"Connection to {self.source_name_friendly} timed out.")
+ raise TalkerNetworkError(self.source_name_friendly, 4)
+ except requests.exceptions.RequestException as e:
+ logger.debug(f"Request exception: {e}")
+ raise TalkerNetworkError(self.source_name_friendly, 0, str(e)) from e
+ except json.JSONDecodeError as e:
+ logger.debug(f"JSON decode error: {e}")
+ raise TalkerDataError(self.source_name_friendly, 2, "ComicVine did not provide json")
+
+ raise TalkerNetworkError(self.source_name_friendly, 5)
+
+ def format_search_results(self, search_results: list[CVVolumeResults]) -> list[ComicVolume]:
+ formatted_results = []
+ for record in search_results:
+ # Flatten publisher to name only
+ if record.get("publisher") is None:
+ pub_name = ""
+ else:
+ pub_name = record["publisher"].get("name", "")
+
+ if record.get("image") is None:
+ image_url = ""
+ else:
+ image_url = record["image"].get("super_url", "")
+
+ if record.get("start_year") is None:
+ start_year = 0
+ else:
+ start_year = utils.xlate(record["start_year"], True)
+
+ formatted_results.append(
+ ComicVolume(
+ aliases=record["aliases"].split("\n") if record["aliases"] else [], # CV returns a null because...?
+ count_of_issues=record.get("count_of_issues", 0),
+ description=record.get("description", ""),
+ id=record["id"],
+ image_url=image_url,
+ name=record["name"],
+ publisher=pub_name,
+ start_year=start_year,
+ )
+ )
+
+ return formatted_results
+
+ def format_issue_results(
+ self, issue_results: list[CVIssueDetailResults], complete: bool = False
+ ) -> list[ComicIssue]:
+ formatted_results = []
+ for record in issue_results:
+ # Extract image super and thumb to name only
+ if record.get("image") is None:
+ image_url = ""
+ image_thumb_url = ""
+ else:
+ image_url = record["image"].get("super_url", "")
+ image_thumb_url = record["image"].get("thumb_url", "")
+
+ alt_images_list = []
+ for alt in record["associated_images"]:
+ alt_images_list.append(alt["original_url"])
+
+ character_list = []
+ if record.get("character_credits"):
+ for char in record["character_credits"]:
+ character_list.append(char["name"])
+
+ location_list = []
+ if record.get("location_credits"):
+ for loc in record["location_credits"]:
+ location_list.append(loc["name"])
+
+ teams_list = []
+ if record.get("team_credits"):
+ for loc in record["team_credits"]:
+ teams_list.append(loc["name"])
+
+ story_list = []
+ if record.get("story_arc_credits"):
+ for loc in record["story_arc_credits"]:
+ story_list.append(loc["name"])
+
+ persons_list = []
+ if record.get("person_credits"):
+ for person in record["person_credits"]:
+ persons_list.append(Credits(name=person["name"], role=person["role"]))
+
+ formatted_results.append(
+ ComicIssue(
+ aliases=record["aliases"].split("\n") if record["aliases"] else [],
+ cover_date=record.get("cover_date", ""),
+ description=record.get("description", ""),
+ id=record["id"],
+ image_url=image_url,
+ image_thumb_url=image_thumb_url,
+ issue_number=record["issue_number"],
+ name=record["name"],
+ site_detail_url=record.get("site_detail_url", ""),
+ volume=cast(ComicVolume, record["volume"]),
+ alt_image_urls=alt_images_list,
+ characters=character_list,
+ locations=location_list,
+ teams=teams_list,
+ story_arcs=story_list,
+ credits=persons_list,
+ complete=complete,
+ )
+ )
+
+ return formatted_results
+
+ def search_for_series(
+ self,
+ series_name: str,
+ callback: Callable[[int, int], None] | None = None,
+ refresh_cache: bool = False,
+ literal: bool = False,
+ ) -> list[ComicVolume]:
+ # Sanitize the series name for comicvine searching, comicvine search ignore symbols
+ search_series_name = utils.sanitize_title(series_name, literal)
+ logger.info(f"{self.source_name_friendly} searching: {search_series_name}")
+
+ # Before we search online, look in our cache, since we might have done this same search recently
+ # For literal searches always retrieve from online
+ cvc = ComicCacher()
+ if not refresh_cache and not literal:
+ cached_search_results = cvc.get_search_results(self.source_name, series_name)
+
+ if len(cached_search_results) > 0:
+ return cached_search_results
+
+ params = {
+ "api_key": self.api_key,
+ "format": "json",
+ "resources": "volume",
+ "query": search_series_name,
+ "field_list": "volume,name,id,start_year,publisher,image,description,count_of_issues,aliases",
+ "page": 1,
+ "limit": 100,
+ }
+
+ cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
+
+ search_results: list[CVVolumeResults] = []
+
+ # see http://api.comicvine.com/documentation/#handling_responses
+
+ current_result_count = cv_response["number_of_page_results"]
+ total_result_count = cv_response["number_of_total_results"]
+
+ # 8 Dec 2018 - Comic Vine changed query results again. Terms are now
+ # ORed together, and we get thousands of results. Good news is the
+ # results are sorted by relevance, so we can be smart about halting the search.
+ # 1. Don't fetch more than some sane amount of pages.
+ # 2. Halt when any result on the current page is less than or equal to a set ratio using thefuzz
+ max_results = 500 # 5 pages
+
+ total_result_count = min(total_result_count, max_results)
+
+ if callback is None:
+ logger.debug(
+ f"Found {cv_response['number_of_page_results']} of {cv_response['number_of_total_results']} results"
+ )
+ search_results.extend(cast(list[CVVolumeResults], cv_response["results"]))
+ page = 1
+
+ if callback is not None:
+ callback(current_result_count, total_result_count)
+
+ # see if we need to keep asking for more pages...
+ while current_result_count < total_result_count:
+
+ if not literal:
+ # Stop searching once any entry falls below the threshold
+ stop_searching = any(
+ not utils.titles_match(search_series_name, volume["name"], self.series_match_thresh)
+ for volume in cast(list[CVVolumeResults], cv_response["results"])
+ )
+
+ if stop_searching:
+ break
+
+ if callback is None:
+ logger.debug(f"getting another page of results {current_result_count} of {total_result_count}...")
+ page += 1
+
+ params["page"] = page
+ cv_response = self.get_cv_content(urljoin(self.api_base_url, "search"), params)
+
+ search_results.extend(cast(list[CVVolumeResults], cv_response["results"]))
+ current_result_count += cv_response["number_of_page_results"]
+
+ if callback is not None:
+ callback(current_result_count, total_result_count)
+
+ # Format result to ComicIssue
+ formatted_search_results = self.format_search_results(search_results)
+
+ # Cache these search results, even if it's literal we cache the results
+ # The most it will cause is extra processing time
+ cvc.add_search_results(self.source_name, series_name, formatted_search_results)
+
+ return formatted_search_results
+
+ # Get issue or volume information
+ def fetch_comic_data(self, issue_id: int = 0, series_id: int = 0, issue_number: str = "") -> GenericMetadata:
+ comic_data = GenericMetadata()
+ if issue_number and series_id:
+ comic_data = self.fetch_issue_data(series_id, issue_number)
+ elif issue_id:
+ comic_data = self.fetch_issue_data_by_issue_id(issue_id)
+
+ return comic_data
+
+ def fetch_partial_volume_data(self, series_id: int) -> ComicVolume:
+ # before we search online, look in our cache, since we might already have this info
+ cvc = ComicCacher()
+ cached_volume_result = cvc.get_volume_info(series_id, self.source_name)
+
+ if cached_volume_result is not None:
+ return cached_volume_result
+
+ volume_url = urljoin(self.api_base_url, f"volume/{CVTypeID.Volume}-{series_id}")
+
+ params = {
+ "api_key": self.api_key,
+ "format": "json",
+ "field_list": "name,id,start_year,publisher,count_of_issues,aliases",
+ }
+ cv_response = self.get_cv_content(volume_url, params)
+
+ volume_results = cast(CVVolumeResults, cv_response["results"])
+ formatted_volume_results = self.format_search_results([volume_results])
+
+ if volume_results:
+ cvc.add_volume_info(self.source_name, formatted_volume_results[0])
+
+ return formatted_volume_results[0]
+
+ def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]:
+ # before we search online, look in our cache, since we might already have this info
+ cvc = ComicCacher()
+ cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name)
+
+ volume_data = self.fetch_partial_volume_data(series_id)
+
+ if len(cached_volume_issues_result) == volume_data["count_of_issues"]:
+ return cached_volume_issues_result
+
+ params = {
+ "api_key": self.api_key,
+ "filter": f"volume:{series_id}",
+ "format": "json",
+ "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases,associated_images",
+ "offset": 0,
+ }
+ cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
+
+ current_result_count = cv_response["number_of_page_results"]
+ total_result_count = cv_response["number_of_total_results"]
+
+ volume_issues_result = cast(list[CVIssueDetailResults], cv_response["results"])
+ page = 1
+ offset = 0
+
+ # see if we need to keep asking for more pages...
+ while current_result_count < total_result_count:
+ page += 1
+ offset += cv_response["number_of_page_results"]
+
+ params["offset"] = offset
+ cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
+
+ volume_issues_result.extend(cast(list[CVIssueDetailResults], cv_response["results"]))
+ current_result_count += cv_response["number_of_page_results"]
+
+ # Format to expected output !! issues/ volume does NOT return publisher!!
+ formatted_volume_issues_result = self.format_issue_results(volume_issues_result)
+
+ cvc.add_volume_issues_info(self.source_name, formatted_volume_issues_result)
+
+ return formatted_volume_issues_result
+
+ def fetch_issues_by_volume_issue_num_and_year(
+ self, volume_id_list: list[int], issue_number: str, year: str | int | None
+ ) -> list[ComicIssue]:
+ volume_filter = ""
+ for vid in volume_id_list:
+ volume_filter += str(vid) + "|"
+ flt = f"volume:{volume_filter},issue_number:{issue_number}"
+
+ int_year = utils.xlate(year, True)
+ if int_year is not None:
+ flt += f",cover_date:{int_year}-1-1|{int_year + 1}-1-1"
+
+ params: dict[str, str | int] = {
+ "api_key": self.api_key,
+ "format": "json",
+ "field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases,associated_images",
+ "filter": flt,
+ }
+
+ cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
+
+ current_result_count = cv_response["number_of_page_results"]
+ total_result_count = cv_response["number_of_total_results"]
+
+ filtered_issues_result = cast(list[CVIssueDetailResults], cv_response["results"])
+ page = 1
+ offset = 0
+
+ # see if we need to keep asking for more pages...
+ while current_result_count < total_result_count:
+ page += 1
+ offset += cv_response["number_of_page_results"]
+
+ params["offset"] = offset
+ cv_response = self.get_cv_content(urljoin(self.api_base_url, "issues/"), params)
+
+ filtered_issues_result.extend(cast(list[CVIssueDetailResults], cv_response["results"]))
+ current_result_count += cv_response["number_of_page_results"]
+
+ formatted_filtered_issues_result = self.format_issue_results(filtered_issues_result)
+
+ return formatted_filtered_issues_result
+
+ def fetch_issue_data(self, series_id: int, issue_number: str) -> GenericMetadata:
+ issues_list_results = self.fetch_issues_by_volume(series_id)
+
+ # Loop through issue list to find the required issue info
+ f_record = None
+ for record in issues_list_results:
+ if not IssueString(issue_number).as_string():
+ issue_number = "1"
+ if (
+ IssueString(record["issue_number"]).as_string().casefold()
+ == IssueString(issue_number).as_string().casefold()
+ ):
+ f_record = record
+ break
+
+ if f_record and f_record["complete"]:
+ # Cache had full record
+ return talker_utils.map_comic_issue_to_metadata(
+ f_record,
+ self.source_name_friendly,
+ self.remove_html_tables,
+ self.use_series_start_as_volume,
+ )
+
+ if f_record is not None:
+ return self.fetch_issue_data_by_issue_id(f_record["id"])
+ else:
+ return GenericMetadata()
+
+ def fetch_issue_data_by_issue_id(self, issue_id: int) -> GenericMetadata:
+ # before we search online, look in our cache, since we might already have this info
+ cvc = ComicCacher()
+ cached_issues_result = cvc.get_issue_info(issue_id, self.source_name)
+
+ if cached_issues_result and cached_issues_result["complete"]:
+ return talker_utils.map_comic_issue_to_metadata(
+ cached_issues_result,
+ self.source_name_friendly,
+ self.remove_html_tables,
+ self.use_series_start_as_volume,
+ )
+
+ issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
+ params = {"api_key": self.api_key, "format": "json"}
+ cv_response = self.get_cv_content(issue_url, params)
+
+ issue_results = cast(CVIssueDetailResults, cv_response["results"])
+
+ # Format to expected output
+ formatted_issues_result = self.format_issue_results([issue_results], True)
+
+ # Due to issue/ not returning volume publisher, get it.
+ volume_info = self.fetch_partial_volume_data(formatted_issues_result[0]["volume"]["id"])
+ formatted_issues_result[0]["volume"]["publisher"] = volume_info["publisher"]
+
+ cvc.add_volume_issues_info(self.source_name, formatted_issues_result)
+
+ # Now, map the ComicIssue data to generic metadata
+ return talker_utils.map_comic_issue_to_metadata(
+ formatted_issues_result[0],
+ self.source_name_friendly,
+ self.remove_html_tables,
+ self.use_series_start_as_volume,
+ )
diff --git a/testing/comicdata.py b/testing/comicdata.py
index d2533fe..4340ba7 100644
--- a/testing/comicdata.py
+++ b/testing/comicdata.py
@@ -1,29 +1,29 @@
from __future__ import annotations
import comicapi.genericmetadata
-import comictaggerlib.resulttypes
+import comictalker.resulttypes
from comicapi import utils
search_results = [
- comictaggerlib.resulttypes.CVVolumeResults(
+ comictalker.resulttypes.ComicVolume(
count_of_issues=1,
description="this is a description",
id=1,
- image={"super_url": "https://test.org/image/1"},
+ image_url="https://test.org/image/1",
name="test",
- publisher=comictaggerlib.resulttypes.CVPublisher(name="test"),
- start_year="", # This is currently submitted as a string and returned as an int
- aliases=None,
+ publisher="test",
+ start_year=0,
+ aliases=[],
),
- comictaggerlib.resulttypes.CVVolumeResults(
+ comictalker.resulttypes.ComicVolume(
count_of_issues=1,
description="this is a description",
- id=1,
- image={"super_url": "https://test.org/image/2"},
+ id=2,
+ image_url="https://test.org/image/2",
name="test 2",
- publisher=comictaggerlib.resulttypes.CVPublisher(name="test"),
- start_year="", # This is currently submitted as a string and returned as an int
- aliases=None,
+ publisher="test",
+ start_year=0,
+ aliases=[],
),
]
diff --git a/testing/comicvine.py b/testing/comicvine.py
index 82dfca2..9c15978 100644
--- a/testing/comicvine.py
+++ b/testing/comicvine.py
@@ -3,7 +3,8 @@ from __future__ import annotations
from typing import Any
import comicapi.genericmetadata
-import comictaggerlib.comicvinetalker
+from comicapi import utils
+from comictalker.talker_utils import cleanup_html
def filter_field_list(cv_result, kwargs):
@@ -21,7 +22,7 @@ cv_issue_result: dict[str, Any] = {
"number_of_total_results": 1,
"status_code": 1,
"results": {
- "aliases": None,
+ "aliases": [],
"api_detail_url": "https://comicvine.gamespot.com/api/issue/4000-140529/",
"associated_images": [],
"character_credits": [],
@@ -116,7 +117,7 @@ cv_volume_result: dict[str, Any] = {
"number_of_total_results": 1,
"status_code": 1,
"results": {
- "aliases": None,
+ "aliases": [],
"api_detail_url": "https://comicvine.gamespot.com/api/volume/4050-23437/",
"count_of_issues": 6,
"date_added": "2008-10-16 05:25:47",
@@ -156,7 +157,24 @@ cv_not_found = {
"status_code": 101,
"results": [],
}
-date = comictaggerlib.comicvinetalker.ComicVineTalker().parse_date_str(cv_issue_result["results"]["cover_date"])
+comic_issue_result: dict[str, Any] = {
+ "aliases": cv_issue_result["results"]["aliases"],
+ "cover_date": cv_issue_result["results"]["cover_date"],
+ "description": cv_issue_result["results"]["description"],
+ "id": cv_issue_result["results"]["id"],
+ "image_url": cv_issue_result["results"]["image"]["super_url"],
+ "image_thumb_url": cv_issue_result["results"]["image"]["thumb_url"],
+ "issue_number": cv_issue_result["results"]["issue_number"],
+ "name": cv_issue_result["results"]["name"],
+ "site_detail_url": cv_issue_result["results"]["site_detail_url"],
+ "volume": {
+ "api_detail_url": cv_issue_result["results"]["volume"]["api_detail_url"],
+ "id": cv_issue_result["results"]["volume"]["id"],
+ "name": cv_issue_result["results"]["volume"]["name"],
+ "site_detail_url": cv_issue_result["results"]["volume"]["site_detail_url"],
+ },
+}
+date = utils.parse_date_str(cv_issue_result["results"]["cover_date"])
cv_md = comicapi.genericmetadata.GenericMetadata(
is_empty=False,
@@ -173,9 +191,7 @@ cv_md = comicapi.genericmetadata.GenericMetadata(
volume=None,
genre=None,
language=None,
- comments=comictaggerlib.comicvinetalker.ComicVineTalker().cleanup_html(
- cv_issue_result["results"]["description"], False
- ),
+ comments=cleanup_html(cv_issue_result["results"]["description"], False),
volume_count=None,
critical_rating=None,
country=None,
@@ -193,9 +209,9 @@ cv_md = comicapi.genericmetadata.GenericMetadata(
story_arc=None,
series_group=None,
scan_info=None,
- characters="",
- teams="",
- locations="",
+ characters=None,
+ teams=None,
+ locations=None,
credits=[
comicapi.genericmetadata.CreditMetadata(person=x["name"], role=x["role"].title(), primary=False)
for x in cv_issue_result["results"]["person_credits"]
@@ -207,7 +223,7 @@ cv_md = comicapi.genericmetadata.GenericMetadata(
rights=None,
identifier=None,
last_mark=None,
- cover_image=None,
+ cover_image=cv_issue_result["results"]["image"]["super_url"],
)
diff --git a/tests/comicarchive_test.py b/tests/comicarchive_test.py
index 004758d..b675431 100644
--- a/tests/comicarchive_test.py
+++ b/tests/comicarchive_test.py
@@ -16,7 +16,7 @@ def test_getPageNameList():
pageNameList = c.get_page_name_list()
assert pageNameList == [
- "!cover.jpg",
+ "!cover.jpg", # Depending on locale punctuation or numbers might come first (Linux)
"00.jpg",
"page0.jpg",
"Page1.jpeg",
diff --git a/tests/comiccacher_test.py b/tests/comiccacher_test.py
index 5ae7cd1..91b9291 100644
--- a/tests/comiccacher_test.py
+++ b/tests/comiccacher_test.py
@@ -2,13 +2,12 @@ from __future__ import annotations
import pytest
-import comictaggerlib.comiccacher
-import comictaggerlib.resulttypes
-from testing.comicdata import alt_covers, search_results, select_details
+import comictalker.comiccacher
+from testing.comicdata import search_results
def test_create_cache(settings):
- comictaggerlib.comiccacher.ComicCacher()
+ comictalker.comiccacher.ComicCacher()
assert (settings.get_settings_folder() / "settings").exists()
@@ -21,24 +20,13 @@ def test_search_results(comic_cache):
assert search_results == comic_cache.get_search_results("test", "test search")
-@pytest.mark.parametrize("alt_cover", alt_covers)
-def test_alt_covers(comic_cache, alt_cover):
- comic_cache.add_alt_covers(**alt_cover, source_name="test")
- assert alt_cover["url_list"] == comic_cache.get_alt_covers(issue_id=alt_cover["issue_id"], source_name="test")
-
-
@pytest.mark.parametrize("volume_info", search_results)
def test_volume_info(comic_cache, volume_info):
- comic_cache.add_volume_info(cv_volume_record=volume_info, source_name="test")
+ comic_cache.add_volume_info(volume_record=volume_info, source_name="test")
vi = volume_info.copy()
del vi["description"]
- del vi["image"]
- assert vi == comic_cache.get_volume_info(volume_id=volume_info["id"], source_name="test")
-
-
-@pytest.mark.parametrize("details", select_details)
-def test_issue_select_details(comic_cache, details):
- comic_cache.add_issue_select_details(**details, source_name="test")
- det = details.copy()
- del det["issue_id"]
- assert det == comic_cache.get_issue_select_details(details["issue_id"], "test")
+ del vi["image_url"]
+ cache_result = comic_cache.get_volume_info(volume_id=volume_info["id"], source_name="test")
+ del cache_result["description"]
+ del cache_result["image_url"]
+ assert vi == cache_result
diff --git a/tests/comicvinetalker_test.py b/tests/comicvinetalker_test.py
index 1d4a43c..ae8372f 100644
--- a/tests/comicvinetalker_test.py
+++ b/tests/comicvinetalker_test.py
@@ -3,57 +3,74 @@ from __future__ import annotations
import pytest
import comicapi.genericmetadata
-import comictaggerlib.comicvinetalker
+import comictalker.talkers.comicvine
import testing.comicvine
-from testing.comicdata import select_details
def test_search_for_series(comicvine_api, comic_cache):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
+ ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.search_for_series("cory doctorows futuristic tales of the here and now")
- for r in results:
- r["image"] = {"super_url": r["image"]["super_url"]}
- r["start_year"] = int(r["start_year"])
- del r["publisher"]["id"]
- del r["publisher"]["api_detail_url"]
cache_issues = comic_cache.get_search_results(ct.source_name, "cory doctorows futuristic tales of the here and now")
assert results == cache_issues
def test_fetch_volume_data(comicvine_api, comic_cache):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
- result = ct.fetch_volume_data(23437)
- result["start_year"] = int(result["start_year"])
- del result["publisher"]["id"]
- del result["publisher"]["api_detail_url"]
- assert result == comic_cache.get_volume_info(23437, ct.source_name)
+ ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
+ result = ct.fetch_partial_volume_data(23437)
+ del result["description"]
+ del result["image_url"]
+ cache_result = comic_cache.get_volume_info(23437, ct.source_name)
+ del cache_result["description"]
+ del cache_result["image_url"]
+ assert result == cache_result
def test_fetch_issues_by_volume(comicvine_api, comic_cache):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
+ ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.fetch_issues_by_volume(23437)
cache_issues = comic_cache.get_volume_issues_info(23437, ct.source_name)
for r in results:
- r["image"] = {"super_url": r["image"]["super_url"], "thumb_url": r["image"]["thumb_url"]}
del r["volume"]
+ del r["image_thumb_url"]
+ del r["characters"]
+ del r["locations"]
+ del r["story_arcs"]
+ del r["teams"]
+ for c in cache_issues:
+ del c["volume"]
+ del c["characters"]
+ del c["locations"]
+ del c["story_arcs"]
+ del c["teams"]
assert results == cache_issues
def test_fetch_issue_data_by_issue_id(comicvine_api, settings, mock_version):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
- result = ct.fetch_issue_data_by_issue_id(140529, settings)
+ ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
+ result = ct.fetch_comic_data(140529)
+ result.notes = None
assert result == testing.comicvine.cv_md
def test_fetch_issues_by_volume_issue_num_and_year(comicvine_api):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
+ ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
results = ct.fetch_issues_by_volume_issue_num_and_year([23437], "1", None)
- cv_expected = testing.comicvine.cv_issue_result["results"].copy()
+ cv_expected = testing.comicvine.comic_issue_result.copy()
testing.comicvine.filter_field_list(
cv_expected,
{"params": {"field_list": "id,volume,issue_number,name,image,cover_date,site_detail_url,description,aliases"}},
)
for r, e in zip(results, [cv_expected]):
+ del r["image_thumb_url"]
+ del r["image_url"]
+ del r["alt_image_urls"]
+ del r["characters"]
+ del r["credits"]
+ del r["locations"]
+ del r["story_arcs"]
+ del r["teams"]
+ del r["complete"]
+ del r["volume"]["publisher"]
assert r == e
@@ -66,36 +83,7 @@ cv_issue = [
@pytest.mark.parametrize("volume_id, issue_number, expected", cv_issue)
def test_fetch_issue_data(comicvine_api, settings, mock_version, volume_id, issue_number, expected):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
- results = ct.fetch_issue_data(volume_id, issue_number, settings)
+ ct = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
+ results = ct.fetch_issue_data(volume_id, issue_number)
+ results.notes = None
assert results == expected
-
-
-def test_fetch_issue_select_details(comicvine_api, mock_version):
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
- result = ct.fetch_issue_select_details(140529)
- expected = {
- "cover_date": testing.comicvine.cv_issue_result["results"]["cover_date"],
- "site_detail_url": testing.comicvine.cv_issue_result["results"]["site_detail_url"],
- "image_url": testing.comicvine.cv_issue_result["results"]["image"]["super_url"],
- "thumb_image_url": testing.comicvine.cv_issue_result["results"]["image"]["thumb_url"],
- }
- assert result == expected
-
-
-@pytest.mark.parametrize("details", select_details)
-def test_issue_select_details(comic_cache, details):
- expected = details.copy()
- del expected["issue_id"]
-
- ct = comictaggerlib.comicvinetalker.ComicVineTalker()
- ct.cache_issue_select_details(
- issue_id=details["issue_id"],
- image_url=details["image_url"],
- thumb_url=details["thumb_image_url"],
- cover_date=details["cover_date"],
- page_url=details["site_detail_url"],
- )
- result = comic_cache.get_issue_select_details(details["issue_id"], ct.source_name)
-
- assert result == expected
diff --git a/tests/conftest.py b/tests/conftest.py
index 1fac14f..7cccda8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -13,9 +13,9 @@ from PIL import Image
import comicapi.comicarchive
import comicapi.genericmetadata
-import comictaggerlib.comiccacher
-import comictaggerlib.comicvinetalker
import comictaggerlib.settings
+import comictalker.comiccacher
+import comictalker.talkers.comicvine
from comicapi import utils
from testing import comicvine, filenames
from testing.comicdata import all_seed_imprints, seed_imprints
@@ -54,7 +54,7 @@ def no_requests(monkeypatch) -> None:
@pytest.fixture
-def comicvine_api(monkeypatch, cbz, comic_cache) -> unittest.mock.Mock:
+def comicvine_api(monkeypatch, cbz, comic_cache) -> comictalker.talkers.comicvine.ComicVineTalker:
# Any arguments may be passed and mock_get() will always return our
# mocked object, which only has the .json() method or None for invalid urls.
@@ -111,7 +111,9 @@ def comicvine_api(monkeypatch, cbz, comic_cache) -> unittest.mock.Mock:
# apply the monkeypatch for requests.get to mock_get
monkeypatch.setattr(requests, "get", m_get)
- return m_get
+
+ cv = comictalker.talkers.comicvine.ComicVineTalker("", "", 90, False, False, False)
+ return cv
@pytest.fixture
@@ -153,5 +155,5 @@ def settings(tmp_path):
@pytest.fixture
-def comic_cache(settings) -> Generator[comictaggerlib.comiccacher.ComicCacher, Any, None]:
- yield comictaggerlib.comiccacher.ComicCacher()
+def comic_cache(settings) -> Generator[comictalker.comiccacher.ComicCacher, Any, None]:
+ yield comictalker.comiccacher.ComicCacher()
diff --git a/tests/issueidentifier_test.py b/tests/issueidentifier_test.py
index 2111839..a4e3498 100644
--- a/tests/issueidentifier_test.py
+++ b/tests/issueidentifier_test.py
@@ -4,14 +4,13 @@ import pytest
import comicapi.comicarchive
import comicapi.issuestring
-import comictaggerlib.comicvinetalker
import comictaggerlib.issueidentifier
import testing.comicdata
import testing.comicvine
-def test_crop(cbz_double_cover, settings, tmp_path):
- ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz_double_cover, settings)
+def test_crop(cbz_double_cover, settings, tmp_path, comicvine_api):
+ ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz_double_cover, settings, comicvine_api)
cropped = ii.crop_cover(cbz_double_cover.archiver.read_file("double_cover.jpg"))
original_cover = cbz_double_cover.get_page(0)
@@ -22,17 +21,16 @@ def test_crop(cbz_double_cover, settings, tmp_path):
@pytest.mark.parametrize("additional_md, expected", testing.comicdata.metadata_keys)
-def test_get_search_keys(cbz, settings, additional_md, expected):
- ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings)
+def test_get_search_keys(cbz, settings, additional_md, expected, comicvine_api):
+ ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api)
ii.set_additional_metadata(additional_md)
assert expected == ii.get_search_keys()
def test_get_issue_cover_match_score(cbz, settings, comicvine_api):
- ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings)
+ ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api)
score = ii.get_issue_cover_match_score(
- comictaggerlib.comicvinetalker.ComicVineTalker(),
int(
comicapi.issuestring.IssueString(
cbz.read_metadata(comicapi.comicarchive.MetaDataStyle.CIX).issue
@@ -52,12 +50,13 @@ def test_get_issue_cover_match_score(cbz, settings, comicvine_api):
def test_search(cbz, settings, comicvine_api):
- ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings)
+ ii = comictaggerlib.issueidentifier.IssueIdentifier(cbz, settings, comicvine_api)
results = ii.search()
cv_expected = {
"series": f"{testing.comicvine.cv_volume_result['results']['name']} ({testing.comicvine.cv_volume_result['results']['start_year']})",
"distance": 0,
"issue_number": testing.comicvine.cv_issue_result["results"]["issue_number"],
+ "alt_image_urls": [],
"cv_issue_count": testing.comicvine.cv_volume_result["results"]["count_of_issues"],
"issue_title": testing.comicvine.cv_issue_result["results"]["name"],
"issue_id": testing.comicvine.cv_issue_result["results"]["id"],
@@ -67,7 +66,6 @@ def test_search(cbz, settings, comicvine_api):
"publisher": testing.comicvine.cv_volume_result["results"]["publisher"]["name"],
"image_url": testing.comicvine.cv_issue_result["results"]["image"]["super_url"],
"thumb_url": testing.comicvine.cv_issue_result["results"]["image"]["thumb_url"],
- "page_url": testing.comicvine.cv_issue_result["results"]["site_detail_url"],
"description": testing.comicvine.cv_issue_result["results"]["description"],
}
for r, e in zip(results, [cv_expected]):