diff --git a/comictaggerlib/autotagmatchwindow.py b/comictaggerlib/autotagmatchwindow.py index 26ba997..34ebc72 100644 --- a/comictaggerlib/autotagmatchwindow.py +++ b/comictaggerlib/autotagmatchwindow.py @@ -27,6 +27,7 @@ from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.resulttypes import IssueResult, MultipleMatch from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) @@ -41,6 +42,7 @@ class AutoTagMatchWindow(QtWidgets.QDialog): style: int, fetch_func: Callable[[IssueResult], GenericMetadata], settings: ComicTaggerSettings, + talker_api: ComicTalker, ) -> None: super().__init__(parent) @@ -50,12 +52,12 @@ class AutoTagMatchWindow(QtWidgets.QDialog): self.current_match_set: MultipleMatch = match_set_list[0] - self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode) + self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode) gridlayout = QtWidgets.QGridLayout(self.altCoverContainer) gridlayout.addWidget(self.altCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) - self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode) + self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer) gridlayout.addWidget(self.archiveCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -241,15 +243,13 @@ class AutoTagMatchWindow(QtWidgets.QDialog): ) # now get the particular issue data - cv_md = self.fetch_func(match) - if cv_md is None: - QtWidgets.QMessageBox.critical( - self, "Network Issue", "Could not connect to Comic Vine to get issue details!" - ) + ct_md = self.fetch_func(match) + if ct_md is None: + QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not retrieve issue details!") return QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor)) - md.overlay(cv_md) + md.overlay(ct_md) success = ca.write_metadata(md, self._style) ca.load_cache([MetaDataStyle.CBI, MetaDataStyle.CIX]) diff --git a/comictaggerlib/autotagprogresswindow.py b/comictaggerlib/autotagprogresswindow.py index f8decb3..beeb585 100644 --- a/comictaggerlib/autotagprogresswindow.py +++ b/comictaggerlib/autotagprogresswindow.py @@ -22,22 +22,25 @@ from PyQt5 import QtCore, QtWidgets, uic from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) class AutoTagProgressWindow(QtWidgets.QDialog): - def __init__(self, parent: QtWidgets.QWidget) -> None: + def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("autotagprogresswindow.ui"), self) - self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.DataMode, False) + self.archiveCoverWidget = CoverImageWidget( + self.archiveCoverContainer, talker_api, CoverImageWidget.DataMode, False + ) gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer) gridlayout.addWidget(self.archiveCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) - self.testCoverWidget = CoverImageWidget(self.testCoverContainer, CoverImageWidget.DataMode, False) + self.testCoverWidget = CoverImageWidget(self.testCoverContainer, talker_api, CoverImageWidget.DataMode, False) gridlayout = QtWidgets.QGridLayout(self.testCoverContainer) gridlayout.addWidget(self.testCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/autotagstartwindow.py b/comictaggerlib/autotagstartwindow.py index 492187b..a48d418 100644 --- a/comictaggerlib/autotagstartwindow.py +++ b/comictaggerlib/autotagstartwindow.py @@ -47,7 +47,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.cbxAssumeIssueOne.setChecked(self.settings.assume_1_if_no_issue_num) self.cbxIgnoreLeadingDigitsInFilename.setChecked(self.settings.ignore_leading_numbers_in_filename) self.cbxRemoveAfterSuccess.setChecked(self.settings.remove_archive_after_successful_match) - self.cbxWaitForRateLimit.setChecked(self.settings.wait_and_retry_on_rate_limit) self.cbxAutoImprint.setChecked(self.settings.auto_imprint) nlmt_tip = """ The Name Length Match Tolerance is for eliminating automatic @@ -76,7 +75,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.assume_issue_one = False self.ignore_leading_digits_in_filename = False self.remove_after_success = False - self.wait_and_retry_on_rate_limit = False self.search_string = "" self.name_length_match_tolerance = self.settings.id_length_delta_thresh self.split_words = self.cbxSplitWords.isChecked() @@ -94,7 +92,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.ignore_leading_digits_in_filename = self.cbxIgnoreLeadingDigitsInFilename.isChecked() self.remove_after_success = self.cbxRemoveAfterSuccess.isChecked() self.name_length_match_tolerance = int(self.leNameLengthMatchTolerance.text()) - self.wait_and_retry_on_rate_limit = self.cbxWaitForRateLimit.isChecked() self.split_words = self.cbxSplitWords.isChecked() # persist some settings @@ -103,7 +100,6 @@ class AutoTagStartWindow(QtWidgets.QDialog): self.settings.assume_1_if_no_issue_num = self.assume_issue_one self.settings.ignore_leading_numbers_in_filename = self.ignore_leading_digits_in_filename self.settings.remove_archive_after_successful_match = self.remove_after_success - self.settings.wait_and_retry_on_rate_limit = self.wait_and_retry_on_rate_limit if self.cbxSpecifySearchString.isChecked(): self.search_string = self.leSearchString.text() diff --git a/comictaggerlib/cli.py b/comictaggerlib/cli.py index c09ed42..84681b0 100644 --- a/comictaggerlib/cli.py +++ b/comictaggerlib/cli.py @@ -28,31 +28,30 @@ from comicapi import utils from comicapi.comicarchive import ComicArchive, MetaDataStyle from comicapi.genericmetadata import GenericMetadata from comictaggerlib.cbltransformer import CBLTransformer -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.filerenamer import FileRenamer from comictaggerlib.issueidentifier import IssueIdentifier from comictaggerlib.resulttypes import IssueResult, MultipleMatch, OnlineMatchResults from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comictalker import ComicTalker +from comictalker.talkerbase import TalkerError logger = logging.getLogger(__name__) def actual_issue_data_fetch( - match: IssueResult, settings: ComicTaggerSettings, opts: argparse.Namespace + match: IssueResult, settings: ComicTaggerSettings, opts: argparse.Namespace, talker_api: ComicTalker ) -> GenericMetadata: # now get the particular issue data try: - comic_vine = ComicVineTalker() - comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit - cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], settings) - except ComicVineTalkerException: - logger.exception("Network error while getting issue details. Save aborted") + ct_md = talker_api.fetch_issue_data(match["volume_id"], match["issue_number"]) + except TalkerError as e: + logger.exception(f"Error retrieving issue details. Save aborted.\n{e}") return GenericMetadata() - if settings.apply_cbl_transform_on_cv_import: - cv_md = CBLTransformer(cv_md, settings).apply() + if settings.apply_cbl_transform_on_ct_import: + ct_md = CBLTransformer(ct_md, settings).apply() - return cv_md + return ct_md def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: GenericMetadata) -> bool: @@ -77,7 +76,11 @@ def actual_metadata_save(ca: ComicArchive, opts: argparse.Namespace, md: Generic def display_match_set_for_choice( - label: str, match_set: MultipleMatch, opts: argparse.Namespace, settings: ComicTaggerSettings + label: str, + match_set: MultipleMatch, + opts: argparse.Namespace, + settings: ComicTaggerSettings, + talker_api: ComicTalker, ) -> None: print(f"{match_set.ca.path} -- {label}:") @@ -107,11 +110,11 @@ def display_match_set_for_choice( # we know at this point, that the file is all good to go ca = match_set.ca md = create_local_metadata(opts, ca, settings) - cv_md = actual_issue_data_fetch(match_set.matches[int(i) - 1], settings, opts) + ct_md = actual_issue_data_fetch(match_set.matches[int(i) - 1], settings, opts, talker_api) if opts.overwrite: - md = cv_md + md = ct_md else: - md.overlay(cv_md) + md.overlay(ct_md) if opts.auto_imprint: md.fix_publisher() @@ -120,7 +123,7 @@ def display_match_set_for_choice( def post_process_matches( - match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings + match_results: OnlineMatchResults, opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker ) -> None: # now go through the match results if opts.show_save_summary: @@ -151,7 +154,7 @@ def post_process_matches( if len(match_results.multiple_matches) > 0: print("\nArchives with multiple high-confidence matches:\n------------------") for match_set in match_results.multiple_matches: - display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings) + display_match_set_for_choice("Multiple high-confidence matches", match_set, opts, settings, talker_api) if len(match_results.low_confidence_matches) > 0: print("\nArchives with low-confidence matches:\n------------------") @@ -161,10 +164,10 @@ def post_process_matches( else: label = "Multiple low-confidence matches" - display_match_set_for_choice(label, match_set, opts, settings) + display_match_set_for_choice(label, match_set, opts, settings, talker_api) -def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None: +def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None: if len(opts.files) < 1: logger.error("You must specify at least one filename. Use the -h option for more info") return @@ -172,10 +175,10 @@ def cli_mode(opts: argparse.Namespace, settings: ComicTaggerSettings) -> None: match_results = OnlineMatchResults() for f in opts.files: - process_file_cli(f, opts, settings, match_results) + process_file_cli(f, opts, settings, talker_api, match_results) sys.stdout.flush() - post_process_matches(match_results, opts, settings) + post_process_matches(match_results, opts, settings, talker_api) def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings: ComicTaggerSettings) -> GenericMetadata: @@ -210,7 +213,11 @@ def create_local_metadata(opts: argparse.Namespace, ca: ComicArchive, settings: def process_file_cli( - filename: str, opts: argparse.Namespace, settings: ComicTaggerSettings, match_results: OnlineMatchResults + filename: str, + opts: argparse.Namespace, + settings: ComicTaggerSettings, + talker_api: ComicTalker, + match_results: OnlineMatchResults, ) -> None: batch_mode = len(opts.files) > 1 @@ -373,31 +380,29 @@ def process_file_cli( # now, search online if opts.online: if opts.issue_id is not None: - # we were given the actual ID to search with + # we were given the actual issue ID to search with. try: - comic_vine = ComicVineTalker() - comic_vine.wait_for_rate_limit = opts.wait_on_cv_rate_limit - cv_md = comic_vine.fetch_issue_data_by_issue_id(opts.issue_id, settings) - except ComicVineTalkerException: - logger.exception("Network error while getting issue details. Save aborted") + ct_md = talker_api.fetch_issue_data_by_issue_id(opts.issue_id) + except TalkerError as e: + logger.exception(f"Error retrieving issue details. Save aborted.\n{e}") match_results.fetch_data_failures.append(str(ca.path.absolute())) return - if cv_md is None: + if ct_md is None: logger.error("No match for ID %s was found.", opts.issue_id) match_results.no_matches.append(str(ca.path.absolute())) return - if settings.apply_cbl_transform_on_cv_import: - cv_md = CBLTransformer(cv_md, settings).apply() + if settings.apply_cbl_transform_on_ct_import: + ct_md = CBLTransformer(ct_md, settings).apply() else: - ii = IssueIdentifier(ca, settings) - if md is None or md.is_empty: logger.error("No metadata given to search online with!") match_results.no_matches.append(str(ca.path.absolute())) return + ii = IssueIdentifier(ca, settings, talker_api) + def myoutput(text: str) -> None: if opts.verbose: IssueIdentifier.default_write_output(text) @@ -405,7 +410,6 @@ def process_file_cli( # use our overlaid MD struct to search ii.set_additional_metadata(md) ii.only_use_additional_meta_data = True - ii.wait_and_retry_on_rate_limit = opts.wait_on_cv_rate_limit ii.set_output_function(myoutput) ii.cover_page_index = md.get_cover_page_index_list()[0] matches = ii.search() @@ -452,15 +456,15 @@ def process_file_cli( # we got here, so we have a single match # now get the particular issue data - cv_md = actual_issue_data_fetch(matches[0], settings, opts) - if cv_md.is_empty: + ct_md = actual_issue_data_fetch(matches[0], settings, opts, talker_api) + if ct_md.is_empty: match_results.fetch_data_failures.append(str(ca.path.absolute())) return if opts.overwrite: - md = cv_md + md = ct_md else: - md.overlay(cv_md) + md.overlay(ct_md) if opts.auto_imprint: md.fix_publisher() diff --git a/comictaggerlib/coverimagewidget.py b/comictaggerlib/coverimagewidget.py index 80b6ba2..3300ed2 100644 --- a/comictaggerlib/coverimagewidget.py +++ b/comictaggerlib/coverimagewidget.py @@ -26,12 +26,12 @@ from PyQt5 import QtCore, QtGui, QtWidgets, uic from comicapi import utils from comicapi.comicarchive import ComicArchive -from comictaggerlib.comicvinetalker import ComicVineTalker from comictaggerlib.imagefetcher import ImageFetcher from comictaggerlib.imagepopup import ImagePopup from comictaggerlib.pageloader import PageLoader from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui.qtutils import get_qimage_from_data, reduce_widget_font_size +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) @@ -87,7 +87,9 @@ class CoverImageWidget(QtWidgets.QWidget): URLMode = 1 DataMode = 3 - def __init__(self, parent: QtWidgets.QWidget, mode: int, expand_on_click: bool = True) -> None: + def __init__( + self, parent: QtWidgets.QWidget, talker_api: ComicTalker, mode: int, expand_on_click: bool = True + ) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("coverimagewidget.ui"), self) @@ -98,6 +100,7 @@ class CoverImageWidget(QtWidgets.QWidget): self.alt_cover_url_list_fetch_complete, self.primary_url_fetch_complete, self.cover_remote_fetch_complete ) + self.talker_api = talker_api self.mode: int = mode self.page_loader: PageLoader | None = None self.showControls = True @@ -177,9 +180,8 @@ class CoverImageWidget(QtWidgets.QWidget): self.update_content() self.issue_id = issue_id - comic_vine = ComicVineTalker() - ComicVineTalker.url_fetch_complete = self.sig.emit_url - comic_vine.async_fetch_issue_cover_urls(self.issue_id) + ComicTalker.url_fetch_complete = self.sig.emit_url + self.talker_api.async_fetch_issue_cover_urls(self.issue_id) def set_image_data(self, image_data: bytes) -> None: if self.mode == CoverImageWidget.DataMode: @@ -209,10 +211,9 @@ class CoverImageWidget(QtWidgets.QWidget): self.label.setText("Searching for alt. covers...") # page URL should already be cached, so no need to defer - comic_vine = ComicVineTalker() - issue_page_url = comic_vine.fetch_issue_page_url(self.issue_id) - ComicVineTalker.alt_url_list_fetch_complete = self.sig.emit_list - comic_vine.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), cast(str, issue_page_url)) + issue_page_url = self.talker_api.fetch_issue_page_url(self.issue_id) + ComicTalker.alt_url_list_fetch_complete = self.sig.emit_list + self.talker_api.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), cast(str, issue_page_url)) def alt_cover_url_list_fetch_complete(self, url_list: list[str]) -> None: if url_list: @@ -269,6 +270,8 @@ class CoverImageWidget(QtWidgets.QWidget): self.cover_fetcher = ImageFetcher() ImageFetcher.image_fetch_complete = self.sig.emit_image self.cover_fetcher.fetch(self.url_list[self.imageIndex]) + # TODO: Figure out async + self.sig.emit_image(self.cover_fetcher.fetch(self.url_list[self.imageIndex])) # called when the image is done loading from internet def cover_remote_fetch_complete(self, image_data: bytes) -> None: diff --git a/comictaggerlib/imagefetcher.py b/comictaggerlib/imagefetcher.py index 076cb2d..f88d66a 100644 --- a/comictaggerlib/imagefetcher.py +++ b/comictaggerlib/imagefetcher.py @@ -81,7 +81,8 @@ class ImageFetcher: # first look in the DB image_data = self.get_image_from_cache(url) - if blocking or not qt_available: + # TODO: figure out async + if True: # if blocking or not qt_available: if not image_data: try: image_data = requests.get(url, headers={"user-agent": "comictagger/" + ctversion.version}).content diff --git a/comictaggerlib/issueidentifier.py b/comictaggerlib/issueidentifier.py index 1fce66b..35704a4 100644 --- a/comictaggerlib/issueidentifier.py +++ b/comictaggerlib/issueidentifier.py @@ -26,11 +26,13 @@ from comicapi import utils from comicapi.comicarchive import ComicArchive from comicapi.genericmetadata import GenericMetadata from comicapi.issuestring import IssueString -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.imagefetcher import ImageFetcher, ImageFetcherException from comictaggerlib.imagehasher import ImageHasher from comictaggerlib.resulttypes import IssueResult from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comictalker import ComicTalker +from comictalker.talkerbase import TalkerError +from comictalker.utils import parse_date_str logger = logging.getLogger(__name__) @@ -72,8 +74,9 @@ class IssueIdentifier: result_one_good_match = 4 result_multiple_good_matches = 5 - def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings) -> None: + def __init__(self, comic_archive: ComicArchive, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None: self.settings = settings + self.talker_api = talker_api self.comic_archive: ComicArchive = comic_archive self.image_hasher = 1 @@ -107,7 +110,6 @@ class IssueIdentifier: self.search_result = self.result_no_matches self.cover_page_index = 0 self.cancel = False - self.wait_and_retry_on_rate_limit = False self.match_list: list[IssueResult] = [] @@ -268,7 +270,6 @@ class IssueIdentifier: def get_issue_cover_match_score( self, - comic_vine: ComicVineTalker, issue_id: int, primary_img_url: str, primary_thumb_url: str, @@ -301,7 +302,7 @@ class IssueIdentifier: raise IssueIdentifierCancelled if use_remote_alternates: - alt_img_url_list = comic_vine.fetch_alternate_cover_urls(issue_id, page_url) + alt_img_url_list = self.talker_api.fetch_alternate_cover_urls(issue_id, page_url) for alt_url in alt_img_url_list: try: alt_url_image_data = ImageFetcher().fetch(alt_url, blocking=True) @@ -398,27 +399,24 @@ class IssueIdentifier: if keys["month"] is not None: self.log_msg("\tMonth: " + str(keys["month"])) - comic_vine = ComicVineTalker() - comic_vine.wait_for_rate_limit = self.wait_and_retry_on_rate_limit - - comic_vine.set_log_func(self.output_function) + self.talker_api.set_log_func(self.output_function) self.log_msg(f"Searching for {keys['series']} #{keys['issue_number']} ...") try: - cv_search_results = comic_vine.search_for_series(keys["series"]) - except ComicVineTalkerException: - self.log_msg("Network issue while searching for series. Aborting...") + ct_search_results = self.talker_api.search_for_series(keys["series"]) + except TalkerError as e: + self.log_msg(f"Error searching for series.\n{e}") return [] if self.cancel: return [] - if cv_search_results is None: + if ct_search_results is None: return [] series_second_round_list = [] - for item in cv_search_results: + for item in ct_search_results: length_approved = False publisher_approved = True date_approved = True @@ -444,7 +442,7 @@ class IssueIdentifier: # remove any series from publishers on the filter if item["publisher"] is not None: - publisher = item["publisher"]["name"] + publisher = item["publisher"] if publisher is not None and publisher.casefold() in self.publisher_filter: publisher_approved = False @@ -459,104 +457,150 @@ class IssueIdentifier: # now sort the list by name length series_second_round_list.sort(key=lambda x: len(x["name"]), reverse=False) - # build a list of volume IDs - volume_id_list = [] - for series in series_second_round_list: - volume_id_list.append(series["id"]) - - issue_list = None - try: - if len(volume_id_list) > 0: - issue_list = comic_vine.fetch_issues_by_volume_issue_num_and_year( - volume_id_list, keys["issue_number"], keys["year"] - ) - - except ComicVineTalkerException: - self.log_msg("Network issue while searching for series details. Aborting...") - return [] - - if issue_list is None: - return [] - - shortlist = [] - # now re-associate the issues and volumes - for issue in issue_list: + # If the sources lacks issue level data, don't look for it. + if not self.talker_api.static_options.has_issues: for series in series_second_round_list: - if series["id"] == issue["volume"]["id"]: - shortlist.append((series, issue)) - break + hash_list = [cover_hash] + if narrow_cover_hash is not None: + hash_list.append(narrow_cover_hash) - if keys["year"] is None: - self.log_msg(f"Found {len(shortlist)} series that have an issue #{keys['issue_number']}") + try: + image_url = series["image"] + thumb_url = series["image"] + page_url = "" + + score_item = self.get_issue_cover_match_score( + series["id"], + image_url, + thumb_url, + page_url, # Only required for alt covers + hash_list, + use_remote_alternates=False, + ) + except Exception: + self.match_list = [] + return self.match_list + + match: IssueResult = { + "series": f"{series['name']} ({series['start_year']})", + "distance": score_item["score"], + "issue_number": "", + "cv_issue_count": series["count_of_issues"], + "url_image_hash": score_item["hash"], + "issue_title": series["name"], + "issue_id": 0, + "volume_id": series["id"], + "month": 0, + "year": int(series["start_year"]), + "publisher": series["publisher"], + "image_url": image_url, + "thumb_url": thumb_url, + "page_url": page_url, + "description": series["description"], + } + + self.match_list.append(match) + + self.log_msg(f" --> {match['distance']}", newline=False) + + self.log_msg("") else: - self.log_msg( - f"Found {len(shortlist)} series that have an issue #{keys['issue_number']} from {keys['year']}" - ) - - # now we have a shortlist of volumes with the desired issue number - # Do first round of cover matching - counter = len(shortlist) - for series, issue in shortlist: - if self.callback is not None: - self.callback(counter, len(shortlist) * 3) - counter += 1 - - self.log_msg( - f"Examining covers for ID: {series['id']} {series['name']} ({series['start_year']}) ...", - newline=False, - ) - - # parse out the cover date - _, month, year = comic_vine.parse_date_str(issue["cover_date"]) - - # Now check the cover match against the primary image - hash_list = [cover_hash] - if narrow_cover_hash is not None: - hash_list.append(narrow_cover_hash) + # build a list of volume IDs + volume_id_list = [] + for series in series_second_round_list: + volume_id_list.append(series["id"]) + issue_list = None try: - image_url = issue["image"]["super_url"] - thumb_url = issue["image"]["thumb_url"] - page_url = issue["site_detail_url"] + if len(volume_id_list) > 0: + issue_list = self.talker_api.fetch_issues_by_volume_issue_num_and_year( + volume_id_list, keys["issue_number"], keys["year"] + ) + except TalkerError as e: + self.log_msg(f"Issue with while searching for series details. Aborting...\n{e}") + return [] - score_item = self.get_issue_cover_match_score( - comic_vine, - issue["id"], - image_url, - thumb_url, - page_url, - hash_list, - use_remote_alternates=False, + if issue_list is None: + return [] + + shortlist = [] + # now re-associate the issues and volumes + for issue in issue_list: + for series in series_second_round_list: + if series["id"] == issue["volume"]["id"]: + shortlist.append((series, issue)) + break + + if keys["year"] is None: + self.log_msg(f"Found {len(shortlist)} series that have an issue #{keys['issue_number']}") + else: + self.log_msg( + f"Found {len(shortlist)} series that have an issue #{keys['issue_number']} from {keys['year']}" ) - except Exception: - self.match_list = [] - return self.match_list - match: IssueResult = { - "series": f"{series['name']} ({series['start_year']})", - "distance": score_item["score"], - "issue_number": keys["issue_number"], - "cv_issue_count": series["count_of_issues"], - "url_image_hash": score_item["hash"], - "issue_title": issue["name"], - "issue_id": issue["id"], - "volume_id": series["id"], - "month": month, - "year": year, - "publisher": None, - "image_url": image_url, - "thumb_url": thumb_url, - "page_url": page_url, - "description": issue["description"], - } - if series["publisher"] is not None: - match["publisher"] = series["publisher"]["name"] + # now we have a shortlist of volumes with the desired issue number + # Do first round of cover matching + counter = len(shortlist) + for series, issue in shortlist: + if self.callback is not None: + self.callback(counter, len(shortlist) * 3) + counter += 1 - self.match_list.append(match) + self.log_msg( + f"Examining covers for ID: {series['id']} {series['name']} ({series['start_year']}) ...", + newline=False, + ) - self.log_msg(f" --> {match['distance']}", newline=False) + # parse out the cover date + _, month, year = parse_date_str(issue["cover_date"]) - self.log_msg("") + # Now check the cover match against the primary image + hash_list = [cover_hash] + if narrow_cover_hash is not None: + hash_list.append(narrow_cover_hash) + + try: + image_url = issue["image"] + thumb_url = issue["image_thumb"] + page_url = issue["site_detail_url"] + + score_item = self.get_issue_cover_match_score( + issue["id"], + image_url, + thumb_url, + page_url, + hash_list, + use_remote_alternates=False, + ) + except Exception: + self.match_list = [] + return self.match_list + + match: IssueResult = { + "series": f"{series['name']} ({series['start_year']})", + "distance": score_item["score"], + "issue_number": keys["issue_number"], + "cv_issue_count": series["count_of_issues"], + "url_image_hash": score_item["hash"], + "issue_title": issue["name"], + "issue_id": issue["id"], + "volume_id": series["id"], + "month": month, + "year": year, + "publisher": None, + "image_url": image_url, + "thumb_url": thumb_url, + "page_url": page_url, + "description": issue["description"], + } + if series["publisher"] is not None: + match["publisher"] = series["publisher"] + + self.match_list.append(match) + + self.log_msg(f" --> {match['distance']}", newline=False) + + self.log_msg("") if len(self.match_list) == 0: self.log_msg(":-(no matches!") @@ -608,7 +652,6 @@ class IssueIdentifier: self.log_msg(f"Examining alternate covers for ID: {m['volume_id']} {m['series']} ...", newline=False) try: score_item = self.get_issue_cover_match_score( - comic_vine, m["issue_id"], m["image_url"], m["thumb_url"], diff --git a/comictaggerlib/issueselectionwindow.py b/comictaggerlib/issueselectionwindow.py index 1e773b9..1ec92ce 100644 --- a/comictaggerlib/issueselectionwindow.py +++ b/comictaggerlib/issueselectionwindow.py @@ -20,11 +20,12 @@ import logging from PyQt5 import QtCore, QtGui, QtWidgets, uic from comicapi.issuestring import IssueString -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.coverimagewidget import CoverImageWidget -from comictaggerlib.resulttypes import CVIssuesResults from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.comictalker import ComicTalker +from comictalker.resulttypes import ComicIssue +from comictalker.talkerbase import TalkerError logger = logging.getLogger(__name__) @@ -41,13 +42,18 @@ class IssueSelectionWindow(QtWidgets.QDialog): volume_id = 0 def __init__( - self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, series_id: int, issue_number: str + self, + parent: QtWidgets.QWidget, + settings: ComicTaggerSettings, + talker_api: ComicTalker, + series_id: int, + issue_number: str, ) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("issueselectionwindow.ui"), self) - self.coverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.AltCoverMode) + self.coverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.AltCoverMode) gridlayout = QtWidgets.QGridLayout(self.coverImageContainer) gridlayout.addWidget(self.coverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -66,8 +72,9 @@ class IssueSelectionWindow(QtWidgets.QDialog): self.series_id = series_id self.issue_id: int | None = None self.settings = settings + self.talker_api = talker_api self.url_fetch_thread = None - self.issue_list: list[CVIssuesResults] = [] + self.issue_list: list[ComicIssue] = [] if issue_number is None or issue_number == "": self.issue_number = "1" @@ -97,15 +104,14 @@ class IssueSelectionWindow(QtWidgets.QDialog): QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor)) try: - comic_vine = ComicVineTalker() - comic_vine.fetch_volume_data(self.series_id) - self.issue_list = comic_vine.fetch_issues_by_volume(self.series_id) - except ComicVineTalkerException as e: + self.issue_list = self.talker_api.fetch_issues_by_volume(self.series_id) + except TalkerError as e: QtWidgets.QApplication.restoreOverrideCursor() - if e.code == ComicVineTalkerException.RateLimit: - QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message()) - else: - QtWidgets.QMessageBox.critical(self, "Network Issue", "Could not connect to Comic Vine to list issues!") + QtWidgets.QMessageBox.critical( + self, + f"{e.source} {e.code_name} Error", + f"{e}", + ) return self.twList.setRowCount(0) diff --git a/comictaggerlib/main.py b/comictaggerlib/main.py index d35fed1..97e0bac 100755 --- a/comictaggerlib/main.py +++ b/comictaggerlib/main.py @@ -27,10 +27,10 @@ import types from comicapi import utils from comictaggerlib import cli -from comictaggerlib.comicvinetalker import ComicVineTalker from comictaggerlib.ctversion import version from comictaggerlib.options import parse_cmd_line from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comictalker import ComicTalker if sys.version_info < (3, 10): import importlib_metadata @@ -40,6 +40,7 @@ else: logger = logging.getLogger("comictagger") logging.getLogger("comicapi").setLevel(logging.DEBUG) logging.getLogger("comictaggerlib").setLevel(logging.DEBUG) +logging.getLogger("sourcesapi").setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG) try: @@ -135,19 +136,6 @@ def ctmain() -> None: ) # Need to load setting before anything else - # manage the CV API key - # None comparison is used so that the empty string can unset the value - if opts.cv_api_key is not None or opts.cv_url is not None: - SETTINGS.cv_api_key = opts.cv_api_key if opts.cv_api_key is not None else SETTINGS.cv_api_key - SETTINGS.cv_url = opts.cv_url if opts.cv_url is not None else SETTINGS.cv_url - SETTINGS.save() - if opts.only_set_cv_key: - print("Key set") # noqa: T201 - return - - ComicVineTalker.api_key = SETTINGS.cv_api_key - ComicVineTalker.api_base_url = SETTINGS.cv_url - signal.signal(signal.SIGINT, signal.SIG_DFL) logger.info( @@ -161,6 +149,8 @@ def ctmain() -> None: for pkg in sorted(importlib_metadata.distributions(), key=lambda x: x.name): logger.debug("%s\t%s", pkg.name, pkg.version) + talker_api = ComicTalker(SETTINGS.comic_info_source) + utils.load_publishers() update_publishers() @@ -170,7 +160,7 @@ def ctmain() -> None: if opts.no_gui: try: - cli.cli_mode(opts, SETTINGS) + cli.cli_mode(opts, SETTINGS, talker_api) except Exception: logger.exception("CLI mode failed") else: @@ -206,7 +196,7 @@ def ctmain() -> None: QtWidgets.QApplication.processEvents() try: - tagger_window = TaggerWindow(opts.files, SETTINGS, opts=opts) + tagger_window = TaggerWindow(opts.files, SETTINGS, talker_api, opts=opts) tagger_window.setWindowIcon(QtGui.QIcon(ComicTaggerSettings.get_graphic("app.png"))) tagger_window.show() diff --git a/comictaggerlib/matchselectionwindow.py b/comictaggerlib/matchselectionwindow.py index c60ecb7..1861b34 100644 --- a/comictaggerlib/matchselectionwindow.py +++ b/comictaggerlib/matchselectionwindow.py @@ -25,6 +25,7 @@ from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.resulttypes import IssueResult from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) @@ -32,17 +33,23 @@ logger = logging.getLogger(__name__) class MatchSelectionWindow(QtWidgets.QDialog): volume_id = 0 - def __init__(self, parent: QtWidgets.QWidget, matches: list[IssueResult], comic_archive: ComicArchive) -> None: + def __init__( + self, + parent: QtWidgets.QWidget, + matches: list[IssueResult], + comic_archive: ComicArchive, + talker_api: ComicTalker, + ) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("matchselectionwindow.ui"), self) - self.altCoverWidget = CoverImageWidget(self.altCoverContainer, CoverImageWidget.AltCoverMode) + self.altCoverWidget = CoverImageWidget(self.altCoverContainer, talker_api, CoverImageWidget.AltCoverMode) gridlayout = QtWidgets.QGridLayout(self.altCoverContainer) gridlayout.addWidget(self.altCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) - self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, CoverImageWidget.ArchiveMode) + self.archiveCoverWidget = CoverImageWidget(self.archiveCoverContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.archiveCoverContainer) gridlayout.addWidget(self.archiveCoverWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/options.py b/comictaggerlib/options.py index 03f52b4..5281fa8 100644 --- a/comictaggerlib/options.py +++ b/comictaggerlib/options.py @@ -80,6 +80,7 @@ def define_args() -> argparse.ArgumentParser: action="store_true", help="Export RAR archive to Zip format.", ) + # TODO: update for new api commands.add_argument( "--only-set-cv-key", action="store_true", @@ -107,10 +108,12 @@ def define_args() -> argparse.ArgumentParser: dest="config_path", help="""Config directory defaults to ~/.ComicTagger\non Linux/Mac and %%APPDATA%% on Windows\n""", ) + # TODO: update for new api parser.add_argument( "--cv-api-key", help="Use the given Comic Vine API Key (persisted in settings).", ) + # TODO: update for new api parser.add_argument( "--cv-url", help="Use the given Comic Vine URL (persisted in settings).", @@ -214,6 +217,7 @@ def define_args() -> argparse.ArgumentParser: action="store_true", help="Be noisy when doing what it does.", ) + # TODO: update for new api parser.add_argument( "-w", "--wait-on-cv-rate-limit", @@ -369,7 +373,7 @@ def parse_cmd_line() -> argparse.Namespace: opts.copy, opts.rename, opts.export_to_zip, - opts.only_set_cv_key, + opts.only_set_cv_key, # TODO: update for new api ] ) @@ -385,9 +389,11 @@ def parse_cmd_line() -> argparse.Namespace: for item in globs: opts.files.extend(glob.glob(item)) + # TODO: update for new api if opts.only_set_cv_key and opts.cv_api_key is None and opts.cv_url is None: parser.exit(message="Key not given!\n", status=1) + # TODO: update for new api if not opts.only_set_cv_key and opts.no_gui and not opts.files: parser.exit(message="Command requires at least one filename!\n", status=1) diff --git a/comictaggerlib/pagebrowser.py b/comictaggerlib/pagebrowser.py index 1071206..7913336 100644 --- a/comictaggerlib/pagebrowser.py +++ b/comictaggerlib/pagebrowser.py @@ -24,17 +24,18 @@ from comicapi.comicarchive import ComicArchive from comicapi.genericmetadata import GenericMetadata from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) class PageBrowserWindow(QtWidgets.QDialog): - def __init__(self, parent: QtWidgets.QWidget, metadata: GenericMetadata) -> None: + def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker, metadata: GenericMetadata) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("pagebrowser.ui"), self) - self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode) + self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.pageContainer) gridlayout.addWidget(self.pageWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/pagelisteditor.py b/comictaggerlib/pagelisteditor.py index 9a68a52..ca93d4f 100644 --- a/comictaggerlib/pagelisteditor.py +++ b/comictaggerlib/pagelisteditor.py @@ -23,6 +23,7 @@ from comicapi.comicarchive import ComicArchive, MetaDataStyle from comicapi.genericmetadata import ImageMetadata, PageType from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) @@ -67,12 +68,12 @@ class PageListEditor(QtWidgets.QWidget): PageType.Deleted: "Deleted", } - def __init__(self, parent: QtWidgets.QWidget) -> None: + def __init__(self, parent: QtWidgets.QWidget, talker_api: ComicTalker) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("pagelisteditor.ui"), self) - self.pageWidget = CoverImageWidget(self.pageContainer, CoverImageWidget.ArchiveMode) + self.pageWidget = CoverImageWidget(self.pageContainer, talker_api, CoverImageWidget.ArchiveMode) gridlayout = QtWidgets.QGridLayout(self.pageContainer) gridlayout.addWidget(self.pageWidget) gridlayout.setContentsMargins(0, 0, 0, 0) diff --git a/comictaggerlib/renamewindow.py b/comictaggerlib/renamewindow.py index 6bf04b5..7f5af28 100644 --- a/comictaggerlib/renamewindow.py +++ b/comictaggerlib/renamewindow.py @@ -28,6 +28,7 @@ from comictaggerlib.filerenamer import FileRenamer from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.settingswindow import SettingsWindow from comictaggerlib.ui.qtutils import center_window_on_parent +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) @@ -44,6 +45,7 @@ class RenameWindow(QtWidgets.QDialog): comic_archive_list: list[ComicArchive], data_style: int, settings: ComicTaggerSettings, + talker_api: ComicTalker, ) -> None: super().__init__(parent) @@ -59,6 +61,7 @@ class RenameWindow(QtWidgets.QDialog): ) self.settings = settings + self.talker_api = talker_api self.comic_archive_list = comic_archive_list self.data_style = data_style self.rename_list: list[RenameItem] = [] @@ -157,7 +160,7 @@ class RenameWindow(QtWidgets.QDialog): self.twList.setSortingEnabled(True) def modify_settings(self) -> None: - settingswin = SettingsWindow(self, self.settings) + settingswin = SettingsWindow(self, self.settings, self.talker_api) settingswin.setModal(True) settingswin.show_rename_tab() settingswin.exec() diff --git a/comictaggerlib/resulttypes.py b/comictaggerlib/resulttypes.py index 31062d4..1982dd7 100644 --- a/comictaggerlib/resulttypes.py +++ b/comictaggerlib/resulttypes.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing_extensions import NotRequired, Required, TypedDict +from typing_extensions import TypedDict from comicapi.comicarchive import ComicArchive @@ -44,117 +44,3 @@ class SelectDetails(TypedDict): thumb_image_url: str | None cover_date: str | None site_detail_url: str | None - - -class CVResult(TypedDict): - error: str - limit: int - offset: int - number_of_page_results: int - number_of_total_results: int - status_code: int - results: ( - CVIssuesResults - | CVIssueDetailResults - | CVVolumeResults - | list[CVIssuesResults] - | list[CVVolumeResults] - | list[CVIssueDetailResults] - ) - version: str - - -class CVImage(TypedDict, total=False): - icon_url: str - medium_url: str - screen_url: str - screen_large_url: str - small_url: str - super_url: Required[str] - thumb_url: str - tiny_url: str - original_url: str - image_tags: str - - -class CVVolume(TypedDict): - api_detail_url: str - id: int - name: str - site_detail_url: str - - -class CVIssuesResults(TypedDict): - cover_date: str - description: str - id: int - image: CVImage - issue_number: str - name: str - site_detail_url: str - volume: NotRequired[CVVolume] - - -class CVPublisher(TypedDict, total=False): - api_detail_url: str - id: int - name: Required[str] - - -class CVVolumeResults(TypedDict): - count_of_issues: int - description: NotRequired[str] - id: int - image: NotRequired[CVImage] - name: str - publisher: CVPublisher - start_year: str - resource_type: NotRequired[str] - - -class CVCredits(TypedDict): - api_detail_url: str - id: int - name: str - site_detail_url: str - - -class CVPersonCredits(TypedDict): - api_detail_url: str - id: int - name: str - site_detail_url: str - role: str - - -class CVIssueDetailResults(TypedDict): - aliases: None - api_detail_url: str - character_credits: list[CVCredits] - character_died_in: None - concept_credits: list[CVCredits] - cover_date: str - date_added: str - date_last_updated: str - deck: None - description: str - first_appearance_characters: None - first_appearance_concepts: None - first_appearance_locations: None - first_appearance_objects: None - first_appearance_storyarcs: None - first_appearance_teams: None - has_staff_review: bool - id: int - image: CVImage - issue_number: str - location_credits: list[CVCredits] - name: str - object_credits: list[CVCredits] - person_credits: list[CVPersonCredits] - site_detail_url: str - store_date: str - story_arc_credits: list[CVCredits] - team_credits: list[CVCredits] - team_disbanded_in: None - volume: CVVolume diff --git a/comictaggerlib/settings.py b/comictaggerlib/settings.py index 26a75d1..39ed6d7 100644 --- a/comictaggerlib/settings.py +++ b/comictaggerlib/settings.py @@ -56,6 +56,30 @@ class ComicTaggerSettings: def get_ui_file(filename: str | pathlib.Path) -> pathlib.Path: return ComicTaggerSettings.base_dir() / "ui" / filename + @staticmethod + def get_source_settings(source_name: str, source_settings: dict): + def get_type(section: str, setting: dict): + if setting["type"] is str: + return config.get(section, setting["name"]) + elif setting["type"] is int: + return config.getint(section, setting["name"]) + elif setting["type"] is float: + return config.getfloat(section, setting["name"]) + elif setting["type"] is bool: + return config.getboolean(section, setting["name"]) + + settings_dir = ComicTaggerSettings.get_settings_folder() + settings_file = os.path.join(settings_dir, "settings") + config = configparser.RawConfigParser() + try: + with open(settings_file, encoding="utf-8") as f: + config.read_file(f) + for setting in source_settings.values(): + setting["value"] = get_type(source_name, setting) + return True + except Exception: + return False + def __init__(self, folder: str | pathlib.Path | None) -> None: # General Settings self.rar_exe_path = "" @@ -80,6 +104,7 @@ class ComicTaggerSettings: # identifier settings self.id_length_delta_thresh = 5 self.id_publisher_filter = "Panini Comics, Abril, Planeta DeAgostini, Editorial Televisa, Dino Comics" + self.comic_info_source = "comicvine" # Default to CV as should always be present # Show/ask dialog flags self.ask_about_cbi_in_rar = True @@ -93,12 +118,8 @@ class ComicTaggerSettings: self.remove_fcbd = False self.remove_publisher = False - # Comic Vine settings - self.use_series_start_as_volume = False - self.clear_form_before_populating_from_cv = False - self.remove_html_tables = False - self.cv_api_key = "" - self.cv_url = "" + # Comic source general settings + self.clear_form_before_populating = False self.auto_imprint = False self.sort_series_by_year = True @@ -114,7 +135,7 @@ class ComicTaggerSettings: self.copy_storyarcs_to_tags = False self.copy_notes_to_comments = False self.copy_weblink_to_comments = False - self.apply_cbl_transform_on_cv_import = False + self.apply_cbl_transform_on_ct_import = False self.apply_cbl_transform_on_bulk_operation = False # Rename settings @@ -219,6 +240,8 @@ class ComicTaggerSettings: self.id_length_delta_thresh = self.config.getint("identifier", "id_length_delta_thresh") if self.config.has_option("identifier", "id_publisher_filter"): self.id_publisher_filter = self.config.get("identifier", "id_publisher_filter") + if self.config.has_option("identifier", "always_use_publisher_filter"): + self.always_use_publisher_filter = self.config.getboolean("identifier", "always_use_publisher_filter") if self.config.has_option("filenameparser", "complicated_parser"): self.complicated_parser = self.config.getboolean("filenameparser", "complicated_parser") @@ -238,27 +261,18 @@ class ComicTaggerSettings: if self.config.has_option("dialogflags", "ask_about_usage_stats"): self.ask_about_usage_stats = self.config.getboolean("dialogflags", "ask_about_usage_stats") - if self.config.has_option("comicvine", "use_series_start_as_volume"): - self.use_series_start_as_volume = self.config.getboolean("comicvine", "use_series_start_as_volume") - if self.config.has_option("comicvine", "clear_form_before_populating_from_cv"): - self.clear_form_before_populating_from_cv = self.config.getboolean( - "comicvine", "clear_form_before_populating_from_cv" + if self.config.has_option("comic_source_general", "clear_form_before_populating"): + self.clear_form_before_populating = self.config.getboolean( + "comic_source_general", "clear_form_before_populating" ) - if self.config.has_option("comicvine", "remove_html_tables"): - self.remove_html_tables = self.config.getboolean("comicvine", "remove_html_tables") - - if self.config.has_option("comicvine", "sort_series_by_year"): - self.sort_series_by_year = self.config.getboolean("comicvine", "sort_series_by_year") - if self.config.has_option("comicvine", "exact_series_matches_first"): - self.exact_series_matches_first = self.config.getboolean("comicvine", "exact_series_matches_first") - if self.config.has_option("comicvine", "always_use_publisher_filter"): - self.always_use_publisher_filter = self.config.getboolean("comicvine", "always_use_publisher_filter") - - if self.config.has_option("comicvine", "cv_api_key"): - self.cv_api_key = self.config.get("comicvine", "cv_api_key") - - if self.config.has_option("comicvine", "cv_url"): - self.cv_url = self.config.get("comicvine", "cv_url") + if self.config.has_option("comic_source_general", "sort_series_by_year"): + self.sort_series_by_year = self.config.getboolean("comic_source_general", "sort_series_by_year") + if self.config.has_option("comic_source_general", "exact_series_matches_first"): + self.exact_series_matches_first = self.config.getboolean( + "comic_source_general", "exact_series_matches_first" + ) + if self.config.has_option("comic_source_general", "comic_info_source"): + self.comic_info_source = self.config.get("identifier", "comic_info_source") if self.config.has_option("cbl_transform", "assume_lone_credit_is_primary"): self.assume_lone_credit_is_primary = self.config.getboolean( @@ -276,9 +290,9 @@ class ComicTaggerSettings: self.copy_storyarcs_to_tags = self.config.getboolean("cbl_transform", "copy_storyarcs_to_tags") if self.config.has_option("cbl_transform", "copy_weblink_to_comments"): self.copy_weblink_to_comments = self.config.getboolean("cbl_transform", "copy_weblink_to_comments") - if self.config.has_option("cbl_transform", "apply_cbl_transform_on_cv_import"): - self.apply_cbl_transform_on_cv_import = self.config.getboolean( - "cbl_transform", "apply_cbl_transform_on_cv_import" + if self.config.has_option("cbl_transform", "apply_cbl_transform_on_ct_import"): + self.apply_cbl_transform_on_ct_import = self.config.getboolean( + "cbl_transform", "apply_cbl_transform_on_ct_import" ) if self.config.has_option("cbl_transform", "apply_cbl_transform_on_bulk_operation"): self.apply_cbl_transform_on_bulk_operation = self.config.getboolean( @@ -352,6 +366,7 @@ class ComicTaggerSettings: self.config.set("identifier", "id_length_delta_thresh", self.id_length_delta_thresh) self.config.set("identifier", "id_publisher_filter", self.id_publisher_filter) + self.config.set("identifier", "always_use_publisher_filter", self.always_use_publisher_filter) if not self.config.has_section("dialogflags"): self.config.add_section("dialogflags") @@ -369,19 +384,13 @@ class ComicTaggerSettings: self.config.set("filenameparser", "remove_fcbd", self.remove_fcbd) self.config.set("filenameparser", "remove_publisher", self.remove_publisher) - if not self.config.has_section("comicvine"): - self.config.add_section("comicvine") + if not self.config.has_section("comic_source_general"): + self.config.add_section("comic_source_general") - self.config.set("comicvine", "use_series_start_as_volume", self.use_series_start_as_volume) - self.config.set("comicvine", "clear_form_before_populating_from_cv", self.clear_form_before_populating_from_cv) - self.config.set("comicvine", "remove_html_tables", self.remove_html_tables) - - self.config.set("comicvine", "sort_series_by_year", self.sort_series_by_year) - self.config.set("comicvine", "exact_series_matches_first", self.exact_series_matches_first) - self.config.set("comicvine", "always_use_publisher_filter", self.always_use_publisher_filter) - - self.config.set("comicvine", "cv_api_key", self.cv_api_key) - self.config.set("comicvine", "cv_url", self.cv_url) + self.config.set("comic_source_general", "clear_form_before_populating", self.clear_form_before_populating) + self.config.set("comic_source_general", "sort_series_by_year", self.sort_series_by_year) + self.config.set("comic_source_general", "exact_series_matches_first", self.exact_series_matches_first) + self.config.set("comic_source_general", "comic_info_source", self.comic_info_source) if not self.config.has_section("cbl_transform"): self.config.add_section("cbl_transform") @@ -393,7 +402,7 @@ class ComicTaggerSettings: self.config.set("cbl_transform", "copy_storyarcs_to_tags", self.copy_storyarcs_to_tags) self.config.set("cbl_transform", "copy_notes_to_comments", self.copy_notes_to_comments) self.config.set("cbl_transform", "copy_weblink_to_comments", self.copy_weblink_to_comments) - self.config.set("cbl_transform", "apply_cbl_transform_on_cv_import", self.apply_cbl_transform_on_cv_import) + self.config.set("cbl_transform", "apply_cbl_transform_on_ct_import", self.apply_cbl_transform_on_ct_import) self.config.set( "cbl_transform", "apply_cbl_transform_on_bulk_operation", @@ -421,5 +430,7 @@ class ComicTaggerSettings: self.config.set("autotag", "wait_and_retry_on_rate_limit", self.wait_and_retry_on_rate_limit) self.config.set("autotag", "auto_imprint", self.auto_imprint) + # NOTE: Source settings are added in settingswindow module + with open(self.settings_file, "w", encoding="utf-8") as configfile: self.config.write(configfile) diff --git a/comictaggerlib/settingswindow.py b/comictaggerlib/settingswindow.py index ab40957..43ecfdf 100644 --- a/comictaggerlib/settingswindow.py +++ b/comictaggerlib/settingswindow.py @@ -24,11 +24,11 @@ from PyQt5 import QtCore, QtGui, QtWidgets, uic from comicapi import utils from comicapi.genericmetadata import md_test -from comictaggerlib.comicvinecacher import ComicVineCacher -from comictaggerlib.comicvinetalker import ComicVineTalker from comictaggerlib.filerenamer import FileRenamer from comictaggerlib.imagefetcher import ImageFetcher from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comiccacher import ComicCacher +from comictalker.comictalker import ComicTalker logger = logging.getLogger(__name__) @@ -126,7 +126,7 @@ Spider-Geddon #1 - New Players; Check In class SettingsWindow(QtWidgets.QDialog): - def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings) -> None: + def __init__(self, parent: QtWidgets.QWidget, settings: ComicTaggerSettings, talker_api: ComicTalker) -> None: super().__init__(parent) uic.loadUi(ComicTaggerSettings.get_ui_file("settingswindow.ui"), self) @@ -136,6 +136,7 @@ class SettingsWindow(QtWidgets.QDialog): ) self.settings = settings + self.talker_api = talker_api self.name = "Settings" if platform.system() == "Windows": @@ -183,7 +184,6 @@ class SettingsWindow(QtWidgets.QDialog): self.btnBrowseRar.clicked.connect(self.select_rar) self.btnClearCache.clicked.connect(self.clear_cache) self.btnResetSettings.clicked.connect(self.reset_settings) - self.btnTestKey.clicked.connect(self.test_api_key) self.btnTemplateHelp.clicked.connect(self.show_template_help) self.leRenameTemplate.textEdited.connect(self.rename__test) self.cbxMoveFiles.clicked.connect(self.rename_test) @@ -191,6 +191,96 @@ class SettingsWindow(QtWidgets.QDialog): self.leDirectory.textEdited.connect(self.rename_test) self.cbxComplicatedParser.clicked.connect(self.switch_parser) + self.sources: dict = {} + self.generate_source_option_tabs() + + def generate_source_option_tabs(self) -> None: + # Add source sub tabs to Comic Sources tab + for source in self.talker_api.sources.values(): + # Add source to general tab dropdown list + self.cobxInfoSource.addItem(source.source_details.name, source.source_details.id) + # Use a dict to make a var name from var + source_info = {} + tab_name = source.source_details.id + source_info[tab_name] = {"tab": QtWidgets.QWidget(), "widgets": {}} + layout_grid = QtWidgets.QGridLayout() + row = 0 + for option in source.source_details.settings_options.values(): + if not option["hidden"]: + current_widget = None + if option["type"] is bool: + # bool equals a checkbox (QCheckBox) + current_widget = QtWidgets.QCheckBox(option["text"]) + # Set widget status + # This works because when a talker class is initialised it loads its settings from disk + if option["value"]: + current_widget.setChecked(option["value"]) + # Add widget and span all columns + layout_grid.addWidget(current_widget, row, 0, 1, -1) + if option["type"] is int: + # int equals a spinbox (QSpinBox) + lbl = QtWidgets.QLabel(option["text"]) + # Create a label + layout_grid.addWidget(lbl, row, 0) + current_widget = QtWidgets.QSpinBox() + current_widget.setRange(0, 9999) + if option["value"]: + current_widget.setValue(option["value"]) + layout_grid.addWidget(current_widget, row, 1, alignment=QtCore.Qt.AlignLeft) + if option["type"] is float: + # float equals a spinbox (QDoubleSpinBox) + lbl = QtWidgets.QLabel(option["text"]) + # Create a label + layout_grid.addWidget(lbl, row, 0) + current_widget = QtWidgets.QDoubleSpinBox() + current_widget.setRange(0, 9999.99) + if option["value"]: + current_widget.setValue(option["value"]) + layout_grid.addWidget(current_widget, row, 1, alignment=QtCore.Qt.AlignLeft) + if option["type"] is str: + # str equals a text field (QLineEdit) + lbl = QtWidgets.QLabel(option["text"]) + # Create a label + layout_grid.addWidget(lbl, row, 0) + current_widget = QtWidgets.QLineEdit() + # Set widget status + if option["value"]: + current_widget.setText(option["value"]) + layout_grid.addWidget(current_widget, row, 1) + # Special case for api_key, make a test button + if option["name"] == "api_key": + btn = QtWidgets.QPushButton("Test Key") + layout_grid.addWidget(btn, row, 2) + btn.clicked.connect(lambda: self.test_api_key(source.source_details.id)) + row += 1 + + if current_widget: + # Add tooltip text + current_widget.setToolTip(option["help_text"]) + + source_info[tab_name]["widgets"][option["name"]] = current_widget + else: + # An empty current_widget implies an unsupported type + logger.info( + "Unsupported talker option found. Name: " + + str(option["name"]) + + " Type: " + + str(option["type"]) + ) + + # Add vertical spacer + vspacer = QtWidgets.QSpacerItem(20, 40, QtWidgets.QSizePolicy.Minimum, QtWidgets.QSizePolicy.Expanding) + layout_grid.addItem(vspacer, row, 0) + # Display the new widgets + source_info[tab_name]["tab"].setLayout(layout_grid) + + # Add new sub tab to Comic Source tab + self.tComicSourcesOptions.addTab(source_info[tab_name]["tab"], source.source_details.name) + self.sources.update(source_info) + + # Select active source in dropdown + self.cobxInfoSource.setCurrentIndex(self.cobxInfoSource.findData(self.settings.comic_info_source)) + def rename_test(self) -> None: self.rename__test(self.leRenameTemplate.text()) @@ -228,17 +318,12 @@ class SettingsWindow(QtWidgets.QDialog): self.cbxRemovePublisher.setChecked(self.settings.remove_publisher) self.switch_parser() - self.cbxUseSeriesStartAsVolume.setChecked(self.settings.use_series_start_as_volume) - self.cbxClearFormBeforePopulating.setChecked(self.settings.clear_form_before_populating_from_cv) - self.cbxRemoveHtmlTables.setChecked(self.settings.remove_html_tables) + self.cbxClearFormBeforePopulating.setChecked(self.settings.clear_form_before_populating) self.cbxUseFilter.setChecked(self.settings.always_use_publisher_filter) self.cbxSortByYear.setChecked(self.settings.sort_series_by_year) self.cbxExactMatches.setChecked(self.settings.exact_series_matches_first) - self.leKey.setText(self.settings.cv_api_key) - self.leURL.setText(self.settings.cv_url) - self.cbxAssumeLoneCreditIsPrimary.setChecked(self.settings.assume_lone_credit_is_primary) self.cbxCopyCharactersToTags.setChecked(self.settings.copy_characters_to_tags) self.cbxCopyTeamsToTags.setChecked(self.settings.copy_teams_to_tags) @@ -246,7 +331,7 @@ class SettingsWindow(QtWidgets.QDialog): self.cbxCopyStoryArcsToTags.setChecked(self.settings.copy_storyarcs_to_tags) self.cbxCopyNotesToComments.setChecked(self.settings.copy_notes_to_comments) self.cbxCopyWebLinkToComments.setChecked(self.settings.copy_weblink_to_comments) - self.cbxApplyCBLTransformOnCVIMport.setChecked(self.settings.apply_cbl_transform_on_cv_import) + self.cbxApplyCBLTransformOnCVIMport.setChecked(self.settings.apply_cbl_transform_on_ct_import) self.cbxApplyCBLTransformOnBatchOperation.setChecked(self.settings.apply_cbl_transform_on_bulk_operation) self.leRenameTemplate.setText(self.settings.rename_template) @@ -289,24 +374,20 @@ class SettingsWindow(QtWidgets.QDialog): self.settings.id_length_delta_thresh = int(self.leNameLengthDeltaThresh.text()) self.settings.id_publisher_filter = str(self.tePublisherFilter.toPlainText()) + self.settings.comic_info_source = str(self.cobxInfoSource.itemData(self.cobxInfoSource.currentIndex())) + # Also change current talker_api object + self.talker_api.source = self.settings.comic_info_source self.settings.complicated_parser = self.cbxComplicatedParser.isChecked() self.settings.remove_c2c = self.cbxRemoveC2C.isChecked() self.settings.remove_fcbd = self.cbxRemoveFCBD.isChecked() self.settings.remove_publisher = self.cbxRemovePublisher.isChecked() - self.settings.use_series_start_as_volume = self.cbxUseSeriesStartAsVolume.isChecked() - self.settings.clear_form_before_populating_from_cv = self.cbxClearFormBeforePopulating.isChecked() - self.settings.remove_html_tables = self.cbxRemoveHtmlTables.isChecked() - + self.settings.clear_form_before_populating = self.cbxClearFormBeforePopulating.isChecked() self.settings.always_use_publisher_filter = self.cbxUseFilter.isChecked() self.settings.sort_series_by_year = self.cbxSortByYear.isChecked() self.settings.exact_series_matches_first = self.cbxExactMatches.isChecked() - self.settings.cv_api_key = self.leKey.text().strip() - ComicVineTalker.api_key = self.settings.cv_api_key - self.settings.cv_url = self.leURL.text().strip() - ComicVineTalker.api_base_url = self.settings.cv_url self.settings.assume_lone_credit_is_primary = self.cbxAssumeLoneCreditIsPrimary.isChecked() self.settings.copy_characters_to_tags = self.cbxCopyCharactersToTags.isChecked() self.settings.copy_teams_to_tags = self.cbxCopyTeamsToTags.isChecked() @@ -314,7 +395,7 @@ class SettingsWindow(QtWidgets.QDialog): self.settings.copy_storyarcs_to_tags = self.cbxCopyStoryArcsToTags.isChecked() self.settings.copy_notes_to_comments = self.cbxCopyNotesToComments.isChecked() self.settings.copy_weblink_to_comments = self.cbxCopyWebLinkToComments.isChecked() - self.settings.apply_cbl_transform_on_cv_import = self.cbxApplyCBLTransformOnCVIMport.isChecked() + self.settings.apply_cbl_transform_on_ct_import = self.cbxApplyCBLTransformOnCVIMport.isChecked() self.settings.apply_cbl_transform_on_bulk_operation = self.cbxApplyCBLTransformOnBatchOperation.isChecked() self.settings.rename_template = str(self.leRenameTemplate.text()) @@ -326,6 +407,51 @@ class SettingsWindow(QtWidgets.QDialog): self.settings.rename_strict = self.cbxRenameStrict.isChecked() + # Read settings from sources tabs and generate self.settings.config data + for source in self.talker_api.sources.values(): + source_info = self.sources[source.source_details.id] + if not self.settings.config.has_section(source.source_details.id): + self.settings.config.add_section(source.source_details.id) + # Iterate over sources options and get the tab setting + for option in source.source_details.settings_options.values(): + # Only save visible here + if option["name"] in source_info["widgets"]: + # Set the tab setting for the talker class var + if option["type"] is bool: + current_widget: QtWidgets.QCheckBox = source_info["widgets"][option["name"]] + option["value"] = current_widget.isChecked() + if option["type"] is int: + current_widget: QtWidgets.QSpinBox = source_info["widgets"][option["name"]] + option["value"] = current_widget.value() + if option["type"] is float: + current_widget: QtWidgets.QDoubleSpinBox = source_info["widgets"][option["name"]] + option["value"] = current_widget.value() + if option["type"] is str: + current_widget: QtWidgets.QLineEdit = source_info["widgets"][option["name"]] + option["value"] = current_widget.text().strip() + + else: + # Handle hidden, assume changed programmatically + if option["name"] == "enabled": + # Set to disabled if is not the selected talker + if source.source_details.id != self.settings.comic_info_source: + source.source_details.settings_options["enabled"]["value"] = False + else: + source.source_details.settings_options["enabled"]["value"] = True + else: + # Ensure correct type + if option["type"] is bool: + option["value"] = bool(option["value"]) + if option["type"] is int: + option["value"] = int(option["value"]) + if option["type"] is float: + option["value"] = float(option["value"]) + if option["type"] is str: + option["value"] = str(option["value"]).strip() + + # Save out option + self.settings.config.set(source.source_details.id, option["name"], option["value"]) + self.settings.save() QtWidgets.QDialog.accept(self) @@ -334,11 +460,13 @@ class SettingsWindow(QtWidgets.QDialog): def clear_cache(self) -> None: ImageFetcher().clear_cache() - ComicVineCacher().clear_cache() + ComicCacher().clear_cache() QtWidgets.QMessageBox.information(self, self.name, "Cache has been cleared.") - def test_api_key(self) -> None: - if ComicVineTalker().test_key(self.leKey.text().strip(), self.leURL.text().strip()): + def test_api_key(self, source_id) -> None: + key = self.sources[source_id]["widgets"]["api_key"].text().strip() + url = self.sources[source_id]["widgets"]["url_root"].text().strip() + if self.talker_api.check_api_key(key, url, source_id): QtWidgets.QMessageBox.information(self, "API Key Test", "Key is valid!") else: QtWidgets.QMessageBox.warning(self, "API Key Test", "Key is NOT valid.") diff --git a/comictaggerlib/taggerwindow.py b/comictaggerlib/taggerwindow.py index 8aa69db..53bfb9c 100644 --- a/comictaggerlib/taggerwindow.py +++ b/comictaggerlib/taggerwindow.py @@ -44,7 +44,6 @@ from comictaggerlib.autotagmatchwindow import AutoTagMatchWindow from comictaggerlib.autotagprogresswindow import AutoTagProgressWindow from comictaggerlib.autotagstartwindow import AutoTagStartWindow from comictaggerlib.cbltransformer import CBLTransformer -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.crediteditorwindow import CreditEditorWindow from comictaggerlib.exportwindow import ExportConflictOpts, ExportWindow @@ -61,6 +60,8 @@ from comictaggerlib.settingswindow import SettingsWindow from comictaggerlib.ui.qtutils import center_window_on_parent, reduce_widget_font_size from comictaggerlib.versionchecker import VersionChecker from comictaggerlib.volumeselectionwindow import VolumeSelectionWindow +from comictalker.comictalker import ComicTalker +from comictalker.talkerbase import TalkerError logger = logging.getLogger(__name__) @@ -77,6 +78,7 @@ class TaggerWindow(QtWidgets.QMainWindow): self, file_list: list[str], settings: ComicTaggerSettings, + talker_api: ComicTalker, parent: QtWidgets.QWidget | None = None, opts: argparse.Namespace | None = None, ) -> None: @@ -84,6 +86,7 @@ class TaggerWindow(QtWidgets.QMainWindow): uic.loadUi(ComicTaggerSettings.get_ui_file("taggerwindow.ui"), self) self.settings = settings + self.talker_api = talker_api self.log_window = self.setup_logger() # prevent multiple instances @@ -117,12 +120,12 @@ class TaggerWindow(QtWidgets.QMainWindow): ) sys.exit() - self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, CoverImageWidget.ArchiveMode) + self.archiveCoverWidget = CoverImageWidget(self.coverImageContainer, talker_api, CoverImageWidget.ArchiveMode) grid_layout = QtWidgets.QGridLayout(self.coverImageContainer) grid_layout.addWidget(self.archiveCoverWidget) grid_layout.setContentsMargins(0, 0, 0, 0) - self.page_list_editor = PageListEditor(self.tabPages) + self.page_list_editor = PageListEditor(self.tabPages, self.talker_api) grid_layout = QtWidgets.QGridLayout(self.tabPages) grid_layout.addWidget(self.page_list_editor) @@ -1021,7 +1024,8 @@ Have fun! issue_number = str(self.leIssueNum.text()).strip() - if autoselect and issue_number == "": + # Only need this check is the source has issue level data. + if autoselect and issue_number == "" and self.talker_api.static_options.has_issues: QtWidgets.QMessageBox.information( self, "Automatic Identify Search", "Can't auto-identify without an issue number (yet!)" ) @@ -1047,6 +1051,7 @@ Have fun! cover_index_list, cast(ComicArchive, self.comic_archive), self.settings, + self.talker_api, autoselect, literal, ) @@ -1064,23 +1069,25 @@ Have fun! self.form_to_metadata() try: - comic_vine = ComicVineTalker() - new_metadata = comic_vine.fetch_issue_data(selector.volume_id, selector.issue_number, self.settings) - except ComicVineTalkerException as e: - QtWidgets.QApplication.restoreOverrideCursor() - if e.code == ComicVineTalkerException.RateLimit: - QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message()) + # Does the source support issue level data? + if self.talker_api.static_options.has_issues: + new_metadata = self.talker_api.fetch_issue_data(selector.volume_id, selector.issue_number) else: - QtWidgets.QMessageBox.critical( - self, "Network Issue", "Could not connect to Comic Vine to get issue details.!" - ) + new_metadata = self.talker_api.fetch_volume_data(selector.volume_id) + except TalkerError as e: + QtWidgets.QApplication.restoreOverrideCursor() + QtWidgets.QMessageBox.critical( + self, + f"{e.source} {e.code_name} Error", + f"{e}", + ) else: QtWidgets.QApplication.restoreOverrideCursor() if new_metadata is not None: - if self.settings.apply_cbl_transform_on_cv_import: + if self.settings.apply_cbl_transform_on_ct_import: new_metadata = CBLTransformer(new_metadata, self.settings).apply() - if self.settings.clear_form_before_populating_from_cv: + if self.settings.clear_form_before_populating: self.clear_form() self.metadata.overlay(new_metadata) @@ -1364,7 +1371,7 @@ Have fun! def show_settings(self) -> None: - settingswin = SettingsWindow(self, self.settings) + settingswin = SettingsWindow(self, self.settings, self.talker_api) settingswin.setModal(True) settingswin.exec() if settingswin.result(): @@ -1669,24 +1676,25 @@ Have fun! def actual_issue_data_fetch(self, match: IssueResult) -> GenericMetadata: - # now get the particular issue data - cv_md = GenericMetadata() + # now get the particular issue data OR series data + ct_md = GenericMetadata() QtWidgets.QApplication.setOverrideCursor(QtGui.QCursor(QtCore.Qt.CursorShape.WaitCursor)) try: - comic_vine = ComicVineTalker() - comic_vine.wait_for_rate_limit = self.settings.wait_and_retry_on_rate_limit - cv_md = comic_vine.fetch_issue_data(match["volume_id"], match["issue_number"], self.settings) - except ComicVineTalkerException: - logger.exception("Network error while getting issue details. Save aborted") + if self.talker_api.static_options.has_issues: + ct_md = self.talker_api.fetch_issue_data(match["volume_id"], match["issue_number"]) + else: + ct_md = self.talker_api.fetch_volume_data(match["volume_id"]) + except TalkerError as e: + logger.exception(f"Save aborted.\n{e}") - if not cv_md.is_empty: - if self.settings.apply_cbl_transform_on_cv_import: - cv_md = CBLTransformer(cv_md, self.settings).apply() + if not ct_md.is_empty: + if self.settings.apply_cbl_transform_on_ct_import: + ct_md = CBLTransformer(ct_md, self.settings).apply() QtWidgets.QApplication.restoreOverrideCursor() - return cv_md + return ct_md def auto_tag_log(self, text: str) -> None: IssueIdentifier.default_write_output(text) @@ -1701,7 +1709,7 @@ Have fun! self, ca: ComicArchive, match_results: OnlineMatchResults, dlg: AutoTagStartWindow ) -> tuple[bool, OnlineMatchResults]: success = False - ii = IssueIdentifier(ca, self.settings) + ii = IssueIdentifier(ca, self.settings, self.talker_api) # read in metadata, and parse file name if not there try: @@ -1738,7 +1746,6 @@ Have fun! md.issue = utils.xlate(md.volume) ii.set_additional_metadata(md) ii.only_use_additional_meta_data = True - ii.wait_and_retry_on_rate_limit = dlg.wait_and_retry_on_rate_limit ii.set_output_function(self.auto_tag_log) ii.cover_page_index = md.get_cover_page_index_list()[0] if self.atprogdialog is not None: @@ -1787,15 +1794,15 @@ Have fun! self.auto_tag_log("Online search: Low confidence match, but saving anyways, as indicated...\n") # now get the particular issue data - cv_md = self.actual_issue_data_fetch(matches[0]) - if cv_md is None: + ct_md = self.actual_issue_data_fetch(matches[0]) + if ct_md is None: match_results.fetch_data_failures.append(str(ca.path.absolute())) - if cv_md is not None: + if ct_md is not None: if dlg.cbxRemoveMetadata.isChecked(): - md = cv_md + md = ct_md else: - md.overlay(cv_md) + md.overlay(ct_md) if self.settings.auto_imprint: md.fix_publisher() @@ -1838,7 +1845,7 @@ Have fun! if not atstartdlg.exec(): return - self.atprogdialog = AutoTagProgressWindow(self) + self.atprogdialog = AutoTagProgressWindow(self, self.talker_api) self.atprogdialog.setModal(True) self.atprogdialog.show() self.atprogdialog.progressBar.setMaximum(len(ca_list)) @@ -1925,7 +1932,12 @@ Have fun! match_results.multiple_matches.extend(match_results.low_confidence_matches) if reply == QtWidgets.QMessageBox.StandardButton.Yes: matchdlg = AutoTagMatchWindow( - self, match_results.multiple_matches, style, self.actual_issue_data_fetch, self.settings + self, + match_results.multiple_matches, + style, + self.actual_issue_data_fetch, + self.settings, + self.talker_api, ) matchdlg.setModal(True) matchdlg.exec() @@ -1988,7 +2000,7 @@ Have fun! def show_page_browser(self) -> None: if self.page_browser is None: - self.page_browser = PageBrowserWindow(self, self.metadata) + self.page_browser = PageBrowserWindow(self, self.talker_api, self.metadata) if self.comic_archive is not None: self.page_browser.set_comic_archive(self.comic_archive) self.page_browser.finished.connect(self.page_browser_closed) @@ -2055,7 +2067,7 @@ Have fun! "File Rename", "If you rename files now, unsaved data in the form will be lost. Are you sure?" ): - dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings) + dlg = RenameWindow(self, ca_list, self.load_data_style, self.settings, self.talker_api) dlg.setModal(True) if dlg.exec() and self.comic_archive is not None: self.fileSelectionList.update_selected_rows() diff --git a/comictaggerlib/ui/autotagstartwindow.ui b/comictaggerlib/ui/autotagstartwindow.ui index 09d8450..bb3d543 100644 --- a/comictaggerlib/ui/autotagstartwindow.ui +++ b/comictaggerlib/ui/autotagstartwindow.ui @@ -10,7 +10,7 @@ 0 0 519 - 440 + 452 @@ -44,8 +44,28 @@ - - + + + + Removes existing metadata before applying retrieved metadata + + + Overwrite metadata + + + + + + + Checks the publisher against a list of imprints. + + + Auto Imprint + + + + + 0 @@ -53,24 +73,11 @@ - Specify series search string for all selected archives: + Split words in filenames (e.g. 'judgedredd' to 'judge dredd') (Experimental) - - - - - 0 - 0 - - - - Save on low confidence match - - - - + @@ -83,26 +90,6 @@ - - - - - 0 - 0 - - - - Remove archives from list after successful tagging - - - - - - - Wait and retry when Comic Vine rate limit is exceeded (experimental) - - - @@ -129,28 +116,8 @@ - - - - Checks the publisher against a list of imprints. - - - Auto Imprint - - - - - - - Removes existing metadata before applying retrieved metadata - - - Overwrite metadata - - - - - + + 0 @@ -158,11 +125,47 @@ - Split words in filenames (e.g. 'judgedredd' to 'judge dredd') (Experimental) + Save on low confidence match - + + + + + 0 + 0 + + + + Remove archives from list after successful tagging + + + + + + + + 0 + 0 + + + + Specify series search string for all selected archives: + + + + + + + + 0 + 0 + + + + + @@ -179,16 +182,6 @@ - - - - 0 - 0 - - - - - diff --git a/comictaggerlib/ui/settingswindow.ui b/comictaggerlib/ui/settingswindow.ui index 2f61133..55aaa80 100644 --- a/comictaggerlib/ui/settingswindow.ui +++ b/comictaggerlib/ui/settingswindow.ui @@ -28,7 +28,7 @@ - 0 + 4 @@ -279,53 +279,56 @@ - + - Comic Vine + Comic Sources - - - - 0 - 0 - + + + 0 - - + + + General + + + + + 0 + 0 + 641 + 341 + + + + 6 + + + 6 + + + 6 + + + 6 + - + - Use Series Start Date as Volume + Select Information Source: + + + - Clear Form Before Importing Comic Vine data - - - - - - - Remove HTML tables from CV summary field - - - - - - - - 0 - 0 - - - - Qt::Horizontal + Clear Form Before Importing Comic Data @@ -343,132 +346,24 @@ + + + + Qt::Vertical + + + + 20 + 40 + + + + - - + + - - - - - 0 - 0 - - - - Qt::Horizontal - - - - - - - - 0 - 0 - - - - - - - - 0 - 0 - - - - <html><head/><body><p>A personal API key from <a href="http://www.comicvine.com/api/"><span style=" text-decoration: underline; color:#0000ff;">Comic Vine</span></a> is recommended in order to search for tag data. Login (or create a new account) there to get your key, and enter it below.</p></body></html> - - - Qt::RichText - - - Qt::AlignBottom|Qt::AlignLeading|Qt::AlignLeft - - - true - - - true - - - Qt::LinksAccessibleByKeyboard|Qt::LinksAccessibleByMouse - - - - - - - Test Key - - - - - - - - 0 - 0 - - - - false - - - - - - - - 0 - 0 - - - - - 120 - 0 - - - - - 200 - 16777215 - - - - Comic Vine API Key - - - - - - - Comic Vine URL - - - - - - - - - - - - - Qt::Vertical - - - - 20 - 40 - - - - @@ -479,7 +374,7 @@ - Apply CBL Transforms on ComicVine Import + Apply CBL Transforms on Comic Import @@ -507,7 +402,7 @@ 11 21 251 - 199 + 206 @@ -830,8 +725,8 @@ By default only removes restricted characters and filenames for the current Oper accept() - 248 - 254 + 258 + 477 157 @@ -846,8 +741,8 @@ By default only removes restricted characters and filenames for the current Oper reject() - 316 - 260 + 326 + 477 286 diff --git a/comictaggerlib/volumeselectionwindow.py b/comictaggerlib/volumeselectionwindow.py index 328e2d0..b8c58c2 100644 --- a/comictaggerlib/volumeselectionwindow.py +++ b/comictaggerlib/volumeselectionwindow.py @@ -25,15 +25,16 @@ from PyQt5.QtCore import pyqtSignal from comicapi import utils from comicapi.comicarchive import ComicArchive from comicapi.genericmetadata import GenericMetadata -from comictaggerlib.comicvinetalker import ComicVineTalker, ComicVineTalkerException from comictaggerlib.coverimagewidget import CoverImageWidget from comictaggerlib.issueidentifier import IssueIdentifier from comictaggerlib.issueselectionwindow import IssueSelectionWindow from comictaggerlib.matchselectionwindow import MatchSelectionWindow from comictaggerlib.progresswindow import IDProgressWindow -from comictaggerlib.resulttypes import CVVolumeResults from comictaggerlib.settings import ComicTaggerSettings from comictaggerlib.ui.qtutils import reduce_widget_font_size +from comictalker.comictalker import ComicTalker +from comictalker.resulttypes import ComicVolume +from comictalker.talkerbase import TalkerError logger = logging.getLogger(__name__) @@ -42,26 +43,26 @@ class SearchThread(QtCore.QThread): searchComplete = pyqtSignal() progressUpdate = pyqtSignal(int, int) - def __init__(self, series_name: str, refresh: bool, literal: bool = False) -> None: + def __init__(self, talker_api: ComicTalker, series_name: str, refresh: bool, literal: bool = False) -> None: QtCore.QThread.__init__(self) + self.talker_api = talker_api self.series_name = series_name self.refresh: bool = refresh - self.error_code: int | None = None - self.cv_error = False - self.cv_search_results: list[CVVolumeResults] = [] + self.error_e: TalkerError + self.ct_error = False + self.ct_search_results: list[ComicVolume] = [] self.literal = literal def run(self) -> None: - comic_vine = ComicVineTalker() try: - self.cv_error = False - self.cv_search_results = comic_vine.search_for_series( + self.ct_error = False + self.ct_search_results = self.talker_api.search_for_series( self.series_name, self.prog_callback, self.refresh, self.literal ) - except ComicVineTalkerException as e: - self.cv_search_results = [] - self.cv_error = True - self.error_code = e.code + except TalkerError as e: + self.ct_search_results = [] + self.ct_error = True + self.error_e = e finally: self.searchComplete.emit() @@ -103,6 +104,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): cover_index_list: list[int], comic_archive: ComicArchive, settings: ComicTaggerSettings, + talker_api: ComicTalker, autoselect: bool = False, literal: bool = False, ) -> None: @@ -110,7 +112,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): uic.loadUi(ComicTaggerSettings.get_ui_file("volumeselectionwindow.ui"), self) - self.imageWidget = CoverImageWidget(self.imageContainer, CoverImageWidget.URLMode) + self.imageWidget = CoverImageWidget(self.imageContainer, talker_api, CoverImageWidget.URLMode) gridlayout = QtWidgets.QGridLayout(self.imageContainer) gridlayout.addWidget(self.imageWidget) gridlayout.setContentsMargins(0, 0, 0, 0) @@ -135,7 +137,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.comic_archive = comic_archive self.immediate_autoselect = autoselect self.cover_index_list = cover_index_list - self.cv_search_results: list[CVVolumeResults] = [] + self.ct_search_results: list[ComicVolume] = [] self.literal = literal self.ii: IssueIdentifier | None = None self.iddialog: IDProgressWindow | None = None @@ -145,6 +147,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.use_filter = self.settings.always_use_publisher_filter + # Load to retrieve settings + self.talker_api = talker_api + self.twList.resizeColumnsToContents() self.twList.currentItemChanged.connect(self.current_item_changed) self.twList.cellDoubleClicked.connect(self.cell_double_clicked) @@ -160,11 +165,19 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.twList.selectRow(0) def update_buttons(self) -> None: - enabled = bool(self.cv_search_results) + enabled = bool(self.ct_search_results) self.btnRequery.setEnabled(enabled) - self.btnIssues.setEnabled(enabled) - self.btnAutoSelect.setEnabled(enabled) + + if self.talker_api.static_options.has_issues: + self.btnIssues.setEnabled(enabled) + self.btnAutoSelect.setEnabled(enabled) + else: + self.btnIssues.setEnabled(False) + self.btnIssues.setToolTip("Unsupported by " + self.talker_api.talker.source_details.name) + self.btnAutoSelect.setEnabled(False) + self.btnAutoSelect.setToolTip("Unsupported by " + self.talker_api.talker.source_details.name) + self.buttonBox.button(QtWidgets.QDialogButtonBox.StandardButton.Ok).setEnabled(enabled) def requery(self) -> None: @@ -177,40 +190,43 @@ class VolumeSelectionWindow(QtWidgets.QDialog): def auto_select(self) -> None: - if self.comic_archive is None: - QtWidgets.QMessageBox.information(self, "Auto-Select", "You need to load a comic first!") - return + if self.talker_api.static_options.has_issues: + if self.comic_archive is None: + QtWidgets.QMessageBox.information(self, "Auto-Select", "You need to load a comic first!") + return - if self.issue_number is None or self.issue_number == "": - QtWidgets.QMessageBox.information(self, "Auto-Select", "Can't auto-select without an issue number (yet!)") - return + if self.issue_number is None or self.issue_number == "": + QtWidgets.QMessageBox.information( + self, "Auto-Select", "Can't auto-select without an issue number (yet!)" + ) + return - self.iddialog = IDProgressWindow(self) - self.iddialog.setModal(True) - self.iddialog.rejected.connect(self.identify_cancel) - self.iddialog.show() + self.iddialog = IDProgressWindow(self) + self.iddialog.setModal(True) + self.iddialog.rejected.connect(self.identify_cancel) + self.iddialog.show() - self.ii = IssueIdentifier(self.comic_archive, self.settings) + self.ii = IssueIdentifier(self.comic_archive, self.settings, self.talker_api) - md = GenericMetadata() - md.series = self.series_name - md.issue = self.issue_number - md.year = self.year - md.issue_count = self.issue_count + md = GenericMetadata() + md.series = self.series_name + md.issue = self.issue_number + md.year = self.year + md.issue_count = self.issue_count - self.ii.set_additional_metadata(md) - self.ii.only_use_additional_meta_data = True + self.ii.set_additional_metadata(md) + self.ii.only_use_additional_meta_data = True - self.ii.cover_page_index = int(self.cover_index_list[0]) + self.ii.cover_page_index = int(self.cover_index_list[0]) - self.id_thread = IdentifyThread(self.ii) - self.id_thread.identifyComplete.connect(self.identify_complete) - self.id_thread.identifyLogMsg.connect(self.log_id_output) - self.id_thread.identifyProgress.connect(self.identify_progress) + self.id_thread = IdentifyThread(self.ii) + self.id_thread.identifyComplete.connect(self.identify_complete) + self.id_thread.identifyLogMsg.connect(self.log_id_output) + self.id_thread.identifyProgress.connect(self.identify_progress) - self.id_thread.start() + self.id_thread.start() - self.iddialog.exec() + self.iddialog.exec() def log_id_output(self, text: str) -> None: if self.iddialog is not None: @@ -263,7 +279,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): choices = True if choices: - selector = MatchSelectionWindow(self, matches, self.comic_archive) + selector = MatchSelectionWindow(self, matches, self.comic_archive, self.talker_api) selector.setModal(True) selector.exec() if selector.result(): @@ -279,9 +295,9 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.show_issues() def show_issues(self) -> None: - selector = IssueSelectionWindow(self, self.settings, self.volume_id, self.issue_number) + selector = IssueSelectionWindow(self, self.settings, self.talker_api, self.volume_id, self.issue_number) title = "" - for record in self.cv_search_results: + for record in self.ct_search_results: if record["id"] == self.volume_id: title = record["name"] title += " (" + str(record["start_year"]) + ")" @@ -310,7 +326,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.progdialog.canceled.connect(self.search_canceled) self.progdialog.setModal(True) self.progdialog.setMinimumDuration(300) - self.search_thread = SearchThread(self.series_name, refresh, self.literal) + self.search_thread = SearchThread(self.talker_api, self.series_name, refresh, self.literal) self.search_thread.searchComplete.connect(self.search_complete) self.search_thread.progressUpdate.connect(self.search_progress_update) self.search_thread.start() @@ -338,26 +354,26 @@ class VolumeSelectionWindow(QtWidgets.QDialog): if self.progdialog is not None: self.progdialog.accept() del self.progdialog - if self.search_thread is not None and self.search_thread.cv_error: - if self.search_thread.error_code == ComicVineTalkerException.RateLimit: - QtWidgets.QMessageBox.critical(self, "Comic Vine Error", ComicVineTalker.get_rate_limit_message()) - else: - QtWidgets.QMessageBox.critical( - self, "Network Issue", "Could not connect to Comic Vine to search for series!" - ) + if self.search_thread is not None and self.search_thread.ct_error: + # TODO Currently still opens the window + QtWidgets.QMessageBox.critical( + self, + f"{self.search_thread.error_e.source} {self.search_thread.error_e.code_name} Error", + f"{self.search_thread.error_e}", + ) return - self.cv_search_results = self.search_thread.cv_search_results if self.search_thread is not None else [] + self.ct_search_results = self.search_thread.ct_search_results if self.search_thread is not None else [] # filter the publishers if enabled set if self.use_filter: try: publisher_filter = {s.strip().casefold() for s in self.settings.id_publisher_filter.split(",")} # use '' as publisher name if None - self.cv_search_results = list( + self.ct_search_results = list( filter( - lambda d: ("" if d["publisher"] is None else str(d["publisher"]["name"]).casefold()) + lambda d: ("" if d["publisher"] is None else str(d["publisher"]).casefold()) not in publisher_filter, - self.cv_search_results, + self.ct_search_results, ) ) except Exception: @@ -369,8 +385,8 @@ class VolumeSelectionWindow(QtWidgets.QDialog): # sort by start_year if set if self.settings.sort_series_by_year: try: - self.cv_search_results = sorted( - self.cv_search_results, + self.ct_search_results = sorted( + self.ct_search_results, key=lambda i: (str(i["start_year"]), str(i["count_of_issues"])), reverse=True, ) @@ -378,8 +394,8 @@ class VolumeSelectionWindow(QtWidgets.QDialog): logger.exception("bad data error sorting results by start_year,count_of_issues") else: try: - self.cv_search_results = sorted( - self.cv_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True + self.ct_search_results = sorted( + self.ct_search_results, key=lambda i: str(i["count_of_issues"]), reverse=True ) except Exception: logger.exception("bad data error sorting results by count_of_issues") @@ -390,7 +406,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): sanitized = utils.sanitize_title(self.series_name, False).casefold() sanitized_no_articles = utils.sanitize_title(self.series_name, True).casefold() - deques: list[deque[CVVolumeResults]] = [deque(), deque(), deque()] + deques: list[deque[ComicVolume]] = [deque(), deque(), deque()] def categorize(result): # We don't remove anything on this one so that we only get exact matches @@ -402,10 +418,10 @@ class VolumeSelectionWindow(QtWidgets.QDialog): return 1 return 2 - for comic in self.cv_search_results: + for comic in self.ct_search_results: deques[categorize(comic)].append(comic) logger.info("Length: %d, %d, %d", len(deques[0]), len(deques[1]), len(deques[2])) - self.cv_search_results = list(itertools.chain.from_iterable(deques)) + self.ct_search_results = list(itertools.chain.from_iterable(deques)) except Exception: logger.exception("bad data error filtering exact/near matches") @@ -416,7 +432,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.twList.setRowCount(0) row = 0 - for record in self.cv_search_results: + for record in self.ct_search_results: self.twList.insertRow(row) item_text = record["name"] @@ -440,7 +456,7 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.twList.setItem(row, 2, item) if record["publisher"] is not None: - item_text = record["publisher"]["name"] + item_text = record["publisher"] item.setData(QtCore.Qt.ItemDataRole.ToolTipRole, item_text) item = QtWidgets.QTableWidgetItem(item_text) item.setFlags(QtCore.Qt.ItemFlag.ItemIsSelectable | QtCore.Qt.ItemFlag.ItemIsEnabled) @@ -452,12 +468,12 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.twList.selectRow(0) self.twList.resizeColumnsToContents() - if not self.cv_search_results: + if not self.ct_search_results: QtCore.QCoreApplication.processEvents() QtWidgets.QMessageBox.information(self, "Search Result", "No matches found!") QtCore.QTimer.singleShot(200, self.close_me) - if self.immediate_autoselect and self.cv_search_results: + if self.immediate_autoselect and self.ct_search_results: # defer the immediate autoselect so this dialog has time to pop up QtCore.QCoreApplication.processEvents() QtCore.QTimer.singleShot(10, self.do_immediate_autoselect) @@ -467,7 +483,11 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.auto_select() def cell_double_clicked(self, r: int, c: int) -> None: - self.show_issues() + if self.talker_api.static_options.has_issues: + self.show_issues() + else: + # Pass back to have taggerwindow get full series data + self.accept() def current_item_changed(self, curr: QtCore.QModelIndex | None, prev: QtCore.QModelIndex | None) -> None: @@ -479,11 +499,11 @@ class VolumeSelectionWindow(QtWidgets.QDialog): self.volume_id = self.twList.item(curr.row(), 0).data(QtCore.Qt.ItemDataRole.UserRole) # list selection was changed, update the info on the volume - for record in self.cv_search_results: + for record in self.ct_search_results: if record["id"] == self.volume_id: if record["description"] is None: self.teDetails.setText("") else: self.teDetails.setText(record["description"]) - self.imageWidget.set_url(record["image"]["super_url"]) + self.imageWidget.set_url(record["image"]) break diff --git a/comictalker/__init__.py b/comictalker/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/comictalker/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/comictaggerlib/comicvinecacher.py b/comictalker/comiccacher.py similarity index 64% rename from comictaggerlib/comicvinecacher.py rename to comictalker/comiccacher.py index e602697..cfc9d39 100644 --- a/comictaggerlib/comicvinecacher.py +++ b/comictalker/comiccacher.py @@ -22,16 +22,16 @@ import sqlite3 as lite from typing import Any from comictaggerlib import ctversion -from comictaggerlib.resulttypes import CVIssuesResults, CVVolumeResults, SelectDetails from comictaggerlib.settings import ComicTaggerSettings +from comictalker.resulttypes import ComicIssue, ComicVolume, SelectDetails logger = logging.getLogger(__name__) -class ComicVineCacher: +class ComicCacher: def __init__(self) -> None: self.settings_folder = ComicTaggerSettings.get_settings_folder() - self.db_file = os.path.join(self.settings_folder, "cv_cache.db") + self.db_file = os.path.join(self.settings_folder, "comic_cache.db") self.version_file = os.path.join(self.settings_folder, "cache_version.txt") # verify that cache is from same version as this one @@ -72,42 +72,45 @@ class ComicVineCacher: # create tables with con: cur = con.cursor() - # name,id,start_year,publisher,image,description,count_of_issues + # source_name,name,id,start_year,publisher,image,description,count_of_issues cur.execute( "CREATE TABLE VolumeSearchCache(" + "search_term TEXT," - + "id INT," + + "id INT NOT NULL," + "name TEXT," + "start_year INT," + "publisher TEXT," + "count_of_issues INT," + "image_url TEXT," + "description TEXT," - + "timestamp DATE DEFAULT (datetime('now','localtime'))) " + + "timestamp DATE DEFAULT (datetime('now','localtime'))," + + "source_name TEXT NOT NULL)" ) cur.execute( "CREATE TABLE Volumes(" - + "id INT," + + "id INT NOT NULL," + "name TEXT," + "publisher TEXT," + "count_of_issues INT," + "start_year INT," + "timestamp DATE DEFAULT (datetime('now','localtime')), " - + "PRIMARY KEY (id))" + + "source_name TEXT NOT NULL," + + "PRIMARY KEY (id, source_name))" ) cur.execute( "CREATE TABLE AltCovers(" - + "issue_id INT," + + "issue_id INT NOT NULL," + "url_list TEXT," + "timestamp DATE DEFAULT (datetime('now','localtime')), " - + "PRIMARY KEY (issue_id))" + + "source_name TEXT NOT NULL," + + "PRIMARY KEY (issue_id, source_name))" ) cur.execute( "CREATE TABLE Issues(" - + "id INT," + + "id INT NOT NULL," + "volume_id INT," + "name TEXT," + "issue_number TEXT," @@ -117,10 +120,11 @@ class ComicVineCacher: + "site_detail_url TEXT," + "description TEXT," + "timestamp DATE DEFAULT (datetime('now','localtime')), " - + "PRIMARY KEY (id))" + + "source_name TEXT NOT NULL," + + "PRIMARY KEY (id, source_name))" ) - def add_search_results(self, search_term: str, cv_search_results: list[CVVolumeResults]) -> None: + def add_search_results(self, source_name: str, search_term: str, ct_search_results: list[ComicVolume]) -> None: con = lite.connect(self.db_file) @@ -132,35 +136,27 @@ class ComicVineCacher: cur.execute("DELETE FROM VolumeSearchCache WHERE search_term = ?", [search_term.casefold()]) # now add in new results - for record in cv_search_results: - - if record["publisher"] is None: - pub_name = "" - else: - pub_name = record["publisher"]["name"] - - if record["image"] is None: - url = "" - else: - url = record["image"]["super_url"] + for record in ct_search_results: cur.execute( "INSERT INTO VolumeSearchCache " - + "(search_term, id, name, start_year, publisher, count_of_issues, image_url, description) " - + "VALUES(?, ?, ?, ?, ?, ?, ?, ?)", + + "(source_name, search_term, id, name, start_year, publisher, count_of_issues, image_url, " + + "description) " + + "VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)", ( + source_name, search_term.casefold(), record["id"], record["name"], record["start_year"], - pub_name, + record["publisher"], record["count_of_issues"], - url, + record["image"], record["description"], ), ) - def get_search_results(self, search_term: str) -> list[CVVolumeResults]: + def get_search_results(self, source_name: str, search_term: str) -> list[ComicVolume]: results = [] con = lite.connect(self.db_file) @@ -173,27 +169,29 @@ class ComicVineCacher: cur.execute("DELETE FROM VolumeSearchCache WHERE timestamp < ?", [str(a_day_ago)]) # fetch - cur.execute("SELECT * FROM VolumeSearchCache WHERE search_term=?", [search_term.casefold()]) + cur.execute( + "SELECT * FROM VolumeSearchCache WHERE search_term=? AND source_name=?", + [search_term.casefold(), source_name], + ) rows = cur.fetchall() # now process the results for record in rows: - result = CVVolumeResults( - { - "id": record[1], - "name": record[2], - "start_year": record[3], - "count_of_issues": record[5], - "description": record[7], - "publisher": {"name": record[4]}, - "image": {"super_url": record[6]}, - } + result = ComicVolume( + id=record[1], + name=record[2], + start_year=record[3], + count_of_issues=record[5], + description=record[7], + publisher=record[4], + image=record[6], + # "source": record[9], # Not needed? ) results.append(result) return results - def add_alt_covers(self, issue_id: int, url_list: list[str]) -> None: + def add_alt_covers(self, source_name: str, issue_id: int, url_list: list[str]) -> None: con = lite.connect(self.db_file) @@ -202,13 +200,16 @@ class ComicVineCacher: cur = con.cursor() # remove all previous entries with this search term - cur.execute("DELETE FROM AltCovers WHERE issue_id = ?", [issue_id]) + cur.execute("DELETE FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name]) url_list_str = ", ".join(url_list) # now add in new record - cur.execute("INSERT INTO AltCovers (issue_id, url_list) VALUES(?, ?)", (issue_id, url_list_str)) + cur.execute( + "INSERT INTO AltCovers (source_name, issue_id, url_list) VALUES(?, ?, ?)", + (source_name, issue_id, url_list_str), + ) - def get_alt_covers(self, issue_id: int) -> list[str]: + def get_alt_covers(self, source_name: str, issue_id: int) -> list[str]: con = lite.connect(self.db_file) with con: @@ -220,7 +221,7 @@ class ComicVineCacher: a_month_ago = datetime.datetime.today() - datetime.timedelta(days=30) cur.execute("DELETE FROM AltCovers WHERE timestamp < ?", [str(a_month_ago)]) - cur.execute("SELECT url_list FROM AltCovers WHERE issue_id=?", [issue_id]) + cur.execute("SELECT url_list FROM AltCovers WHERE issue_id=? AND source_name=?", [issue_id, source_name]) row = cur.fetchone() if row is None: return [] @@ -234,7 +235,7 @@ class ComicVineCacher: url_list.append(str(item).strip()) return url_list - def add_volume_info(self, cv_volume_record: CVVolumeResults) -> None: + def add_volume_info(self, source_name: str, volume_record: ComicVolume) -> None: con = lite.connect(self.db_file) @@ -244,21 +245,18 @@ class ComicVineCacher: timestamp = datetime.datetime.now() - if cv_volume_record["publisher"] is None: - pub_name = "" - else: - pub_name = cv_volume_record["publisher"]["name"] - data = { - "name": cv_volume_record["name"], - "publisher": pub_name, - "count_of_issues": cv_volume_record["count_of_issues"], - "start_year": cv_volume_record["start_year"], + "id": volume_record["id"], + "source_name": source_name, + "name": volume_record["name"], + "publisher": volume_record["publisher"], + "count_of_issues": volume_record["count_of_issues"], + "start_year": volume_record["start_year"], "timestamp": timestamp, } - self.upsert(cur, "volumes", "id", cv_volume_record["id"], data) + self.upsert(cur, "volumes", data) - def add_volume_issues_info(self, volume_id: int, cv_volume_issues: list[CVIssuesResults]) -> None: + def add_volume_issues_info(self, source_name: str, volume_id: int, volume_issues: list[ComicIssue]) -> None: con = lite.connect(self.db_file) @@ -269,23 +267,25 @@ class ComicVineCacher: # add in issues - for issue in cv_volume_issues: + for issue in volume_issues: data = { + "id": issue["id"], "volume_id": volume_id, + "source_name": source_name, "name": issue["name"], "issue_number": issue["issue_number"], "site_detail_url": issue["site_detail_url"], "cover_date": issue["cover_date"], - "super_url": issue["image"]["super_url"], - "thumb_url": issue["image"]["thumb_url"], + "super_url": issue["image"], + "thumb_url": issue["image_thumb"], "description": issue["description"], "timestamp": timestamp, } - self.upsert(cur, "issues", "id", issue["id"], data) + self.upsert(cur, "issues", data) - def get_volume_info(self, volume_id: int) -> CVVolumeResults | None: + def get_volume_info(self, volume_id: int, source_name: str) -> ComicVolume | None: - result: CVVolumeResults | None = None + result: ComicVolume | None = None con = lite.connect(self.db_file) with con: @@ -297,7 +297,10 @@ class ComicVineCacher: cur.execute("DELETE FROM Volumes WHERE timestamp < ?", [str(a_week_ago)]) # fetch - cur.execute("SELECT id,name,publisher,count_of_issues,start_year FROM Volumes WHERE id = ?", [volume_id]) + cur.execute( + "SELECT source_name,id,name,publisher,count_of_issues,start_year FROM Volumes WHERE id=? AND source_name=?", + [volume_id, source_name], + ) row = cur.fetchone() @@ -305,19 +308,18 @@ class ComicVineCacher: return result # since ID is primary key, there is only one row - result = CVVolumeResults( - { - "id": row[0], - "name": row[1], - "count_of_issues": row[3], - "start_year": row[4], - "publisher": {"name": row[2]}, - } + result = ComicVolume( + # source_name: row[0], + id=row[1], + name=row[2], + count_of_issues=row[4], + start_year=row[5], + publisher=row[3], ) return result - def get_volume_issues_info(self, volume_id: int) -> list[CVIssuesResults]: + def get_volume_issues_info(self, volume_id: int, source_name: str) -> list[ComicIssue]: con = lite.connect(self.db_file) with con: @@ -330,29 +332,29 @@ class ComicVineCacher: cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)]) # fetch - results: list[CVIssuesResults] = [] + results: list[ComicIssue] = [] cur.execute( ( - "SELECT id,name,issue_number,site_detail_url,cover_date,super_url,thumb_url,description" - " FROM Issues WHERE volume_id = ?" + "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,super_url,thumb_url,description" + + " FROM Issues WHERE volume_id=? AND source_name=?" ), - [volume_id], + [volume_id, source_name], ) rows = cur.fetchall() # now process the results for row in rows: - record = CVIssuesResults( - { - "id": row[0], - "name": row[1], - "issue_number": row[2], - "site_detail_url": row[3], - "cover_date": row[4], - "image": {"super_url": row[5], "thumb_url": row[6]}, - "description": row[7], - } + record = ComicIssue( + id=row[1], + name=row[2], + issue_number=row[3], + site_detail_url=row[4], + cover_date=row[5], + image=row[6], + image_thumb=row[7], + description=row[8], + volume={"id": volume_id, "name": row[2]}, ) results.append(record) @@ -371,31 +373,33 @@ class ComicVineCacher: timestamp = datetime.datetime.now() data = { + "id": issue_id, "super_url": image_url, "thumb_url": thumb_image_url, "cover_date": cover_date, "site_detail_url": site_detail_url, "timestamp": timestamp, } - self.upsert(cur, "issues", "id", issue_id, data) + self.upsert(cur, "issues", data) - def get_issue_select_details(self, issue_id: int) -> SelectDetails: + def get_issue_select_details(self, issue_id: int, source_name: str) -> SelectDetails: con = lite.connect(self.db_file) with con: cur = con.cursor() con.text_factory = str - cur.execute("SELECT super_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=?", [issue_id]) + cur.execute( + "SELECT super_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=? " + "AND source_name=?", + [issue_id, source_name], + ) row = cur.fetchone() details = SelectDetails( - { - "image_url": None, - "thumb_image_url": None, - "cover_date": None, - "site_detail_url": None, - } + image_url=None, + thumb_image_url=None, + cover_date=None, + site_detail_url=None, ) if row is not None and row[0] is not None: details["image_url"] = row[0] @@ -405,11 +409,10 @@ class ComicVineCacher: return details - def upsert(self, cur: lite.Cursor, tablename: str, pkname: str, pkval: Any, data: dict[str, Any]) -> None: + def upsert(self, cur: lite.Cursor, tablename: str, data: dict[str, Any]) -> None: """This does an insert if the given PK doesn't exist, and an update it if does - TODO: look into checking if UPDATE is needed TODO: should the cursor be created here, and not up the stack? """ @@ -432,13 +435,5 @@ class ComicVineCacher: ins_slots += "?" set_slots += key + " = ?" - keys += ", " + pkname - vals.append(pkval) - ins_slots += ", ?" - condition = pkname + " = ?" - - sql_ins = f"INSERT OR IGNORE INTO {tablename} ({keys}) VALUES ({ins_slots})" + sql_ins = f"INSERT OR REPLACE INTO {tablename} ({keys}) VALUES ({ins_slots})" cur.execute(sql_ins, vals) - - sql_upd = f"UPDATE {tablename} SET {set_slots} WHERE {condition}" - cur.execute(sql_upd, vals) diff --git a/comictalker/comictalker.py b/comictalker/comictalker.py new file mode 100644 index 0000000..0a98894 --- /dev/null +++ b/comictalker/comictalker.py @@ -0,0 +1,222 @@ +"""Handles collecting data from source talkers. +""" +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging +from typing import Callable + +from comicapi.genericmetadata import GenericMetadata +from comictalker.resulttypes import ComicIssue, ComicVolume +from comictalker.talkerbase import SourceStaticOptions, TalkerError + +logger = logging.getLogger(__name__) + + +# To signal image loaded etc. +def list_fetch_complete(url_list: list[str]) -> None: + ... + + +def url_fetch_complete(image_url: str, thumb_url: str | None) -> None: + ... + + +class ComicTalker: + alt_url_list_fetch_complete = list_fetch_complete + url_fetch_complete = url_fetch_complete + + def __init__(self, source_name) -> None: + # ID of the source to use e.g. comicvine + self.source = source_name + # Retrieve the available sources modules + self.sources = self.get_talkers() + # Set the active talker + self.talker = self.get_active_talker() + self.static_options = self.get_static_options() + + def get_active_talker(self): + # This should always work because it will have errored at get_talkers if there are none + if not self.sources[self.source] is None: + return self.sources[self.source] + + @staticmethod + def get_talkers(): + # Hardcode import for now. Placed here to prevent circular import + import comictalker.talkers.comicvine + + return {"comicvine": comictalker.talkers.comicvine.ComicVineTalker()} + + # For issueidentifier + def set_log_func(self, log_func: Callable[[str], None]) -> None: + self.talker.log_func = log_func + + def get_static_options(self) -> SourceStaticOptions: + return self.talker.source_details.static_options + + def check_api_key(self, key: str, url: str, source_id: str): + for source in self.sources.values(): + if source.source_details.id == source_id: + return source.check_api_key(key, url) + # Return false as back up or error? + return False + + # Master function to search for series/volumes + def search_for_series( + self, + series_name: str, + callback: Callable[[int, int], None] | None = None, + refresh_cache: bool = False, + literal: bool = False, + ) -> list[ComicVolume]: + try: + series_result = self.talker.search_for_series(series_name, callback, refresh_cache, literal) + return series_result + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'search_for_series'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'search_for_series'", + ) + + # Master function to fetch series data (i.e for sources without issue level details) + def fetch_volume_data(self, series_id: int) -> GenericMetadata: + try: + series_result = self.talker.fetch_volume_data(series_id) + return series_result + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_volume_data'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_volume_data'", + ) + + # Master function to get issues in a series/volume + def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]: + try: + issues_result = self.talker.fetch_issues_by_volume(series_id) + return issues_result + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issues_by_volume'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_issues_by_volume'", + ) + + # Master function to get issue information + def fetch_issue_data(self, series_id: int, issue_number: str) -> GenericMetadata: + try: + issue_result = self.talker.fetch_issue_data(series_id, issue_number) + return issue_result + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issue_data'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_issue_data'", + ) + + # For CLI + def fetch_issue_data_by_issue_id(self, issue_id: int) -> GenericMetadata: + try: + issue_result = self.talker.fetch_issue_data_by_issue_id(issue_id) + return issue_result + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issue_data_by_issue_id'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_issue_data_by_issue_id'", + ) + + # For issueidentifer + def fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> list[str]: + try: + alt_covers = self.talker.fetch_alternate_cover_urls(issue_id, issue_page_url) + return alt_covers + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_alternate_cover_urls'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_alternate_cover_urls'", + ) + + # For issueidentifier + def fetch_issues_by_volume_issue_num_and_year( + self, volume_id_list: list[int], issue_number: str, year: str | int | None + ) -> list[ComicIssue]: + try: + issue_results = self.talker.fetch_issues_by_volume_issue_num_and_year(volume_id_list, issue_number, year) + return issue_results + except NotImplementedError: + logger.warning( + f"{self.talker.source_details.name} has not implemented: 'fetch_issues_by_volume_issue_num_and_year'" + ) + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_issues_by_volume_issue_num_and_year'", + ) + + def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]: + try: + cover_urls = self.talker.fetch_issue_cover_urls(issue_id) + return cover_urls + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issue_cover_urls'") + raise TalkerError( + self.talker.source_details.name, + 4, + "The source has not implemented: 'fetch_issue_cover_urls'", + ) + + # Master function to get issue cover. Used by coverimagewidget + def async_fetch_issue_cover_urls(self, issue_id: int) -> None: + try: + # TODO: Figure out async + image_url, thumb_url = self.fetch_issue_cover_urls(issue_id) + ComicTalker.url_fetch_complete(image_url or "", thumb_url) + logger.info("Should be downloading image: %s thumb: %s", image_url, thumb_url) + return + + # Should be all that's needed? CV functions will trigger everything. + self.talker.async_fetch_issue_cover_urls(issue_id) + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'async_fetch_issue_cover_urls'") + + # Used by coverimagewidget.start_alt_cover_search + def fetch_issue_page_url(self, issue_id: int) -> str | None: + try: + page_url = self.talker.fetch_issue_page_url(issue_id) + return page_url + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issue_page_url'") + return None + + def async_fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> None: + try: + # TODO: Figure out async + url_list = self.fetch_alternate_cover_urls(issue_id, issue_page_url) + ComicTalker.alt_url_list_fetch_complete(url_list) + logger.info("Should be downloading alt image list: %s", url_list) + return + + self.talker.async_fetch_alternate_cover_urls(issue_id, issue_page_url) + except NotImplementedError: + logger.warning(f"{self.talker.source_details.name} has not implemented: 'async_fetch_alternate_cover_urls'") diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py new file mode 100644 index 0000000..967e0e2 --- /dev/null +++ b/comictalker/resulttypes.py @@ -0,0 +1,32 @@ +from __future__ import annotations + +from typing_extensions import Required, TypedDict + + +class SelectDetails(TypedDict): + image_url: str | None + thumb_image_url: str | None + cover_date: str | None + site_detail_url: str | None + + +class ComicVolume(TypedDict, total=False): + count_of_issues: int + description: str + id: Required[int] + image: str + name: Required[str] + publisher: str + start_year: str + + +class ComicIssue(TypedDict, total=False): + cover_date: str + description: str + id: int + image: str + image_thumb: str + issue_number: Required[str] + name: Required[str] + site_detail_url: str + volume: ComicVolume diff --git a/comictalker/talkerbase.py b/comictalker/talkerbase.py new file mode 100644 index 0000000..22a71d4 --- /dev/null +++ b/comictalker/talkerbase.py @@ -0,0 +1,226 @@ +"""A template for an information source +""" +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging +from typing import Any, Callable + +from typing_extensions import Required, Type, TypedDict + +from comicapi.genericmetadata import GenericMetadata +from comictalker.resulttypes import ComicIssue, ComicVolume + +logger = logging.getLogger(__name__) + +# NOTE: Series and Volume are synonymous. Some sources (ComicVine) use "volume" and others (MangaUpdates) use "series". + + +class SourceDetails: + def __init__( + self, + name: str, + ident: str, + static_options: SourceStaticOptions, + settings_options: dict[str, SourceSettingsOptions], + ): + self.name = name + self.id = ident + self.static_options = static_options + self.settings_options = settings_options + + +class SourceStaticOptions: + def __init__( + self, + logo_url: str = "", + has_issues: bool = False, + has_alt_covers: bool = False, + requires_apikey: bool = False, + has_nsfw: bool = False, + has_censored_covers: bool = False, + ) -> None: + self.logo_url = logo_url + self.has_issues = has_issues + self.has_alt_covers = has_alt_covers + self.requires_apikey = requires_apikey + self.has_nsfw = has_nsfw + self.has_censored_covers = has_censored_covers + + +class SourceSettingsOptions(TypedDict): + # Source settings options and used to generate settings options in panel + name: Required[str] # Internal name for setting i.e "remove_html_tables" + text: Required[str] # Display text i.e "Remove HTML tables" + help_text: str # Tooltip text i.e "Enabling this will remove HTML tables from the description." + hidden: Required[bool] # To hide an option from the settings menu. + type: Required[Type] + value: Any + + +class TalkerError(Exception): + """Base class exception for information sources. + + Attributes: + code -- a numerical code + 1 - General + 2 - Network + 3 - Data + desc -- description of the error + source -- the name of the source producing the error + """ + + codes = { + 1: "General", + 2: "Network", + 3: "Data", + 4: "Other", + } + + def __init__(self, source: str = "", code: int = 4, desc: str = "", sub_code: int = 0) -> None: + super().__init__() + if desc == "": + desc = "Unknown" + self.desc = desc + self.code = code + self.code_name = self.codes[code] + self.sub_code = sub_code + self.source = source + + def __str__(self): + return f"{self.source} encountered a {self.code_name} error. {self.desc}" + + +class TalkerNetworkError(TalkerError): + """Network class exception for information sources + + Attributes: + sub_code -- numerical code for finer detail + 1 -- connected refused + 2 -- api key + 3 -- rate limit + 4 -- timeout + """ + + net_codes = { + 0: "General network error.", + 1: "The connection was refused.", + 2: "An API key error occurred.", + 3: "Rate limit exceeded. Please wait a bit or enter a personal key if using the default.", + 4: "The connection timed out.", + 5: "Number of retries exceeded.", + } + + def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None: + if desc == "": + desc = self.net_codes[sub_code] + + super().__init__(source, 2, desc, sub_code) + + +class TalkerDataError(TalkerError): + """Data class exception for information sources + + Attributes: + sub_code -- numerical code for finer detail + 1 -- unexpected data + 2 -- malformed data + 3 -- missing data + """ + + data_codes = { + 0: "General data error.", + 1: "Unexpected data encountered.", + 2: "Malformed data encountered.", + 3: "Missing data encountered.", + } + + def __init__(self, source: str = "", sub_code: int = 0, desc: str = "") -> None: + if desc == "": + desc = self.data_codes[sub_code] + + super().__init__(source, 3, desc, sub_code) + + +# Class talkers instance +class TalkerBase: + """This is the class for mysource.""" + + def __init__(self) -> None: + # Identity name for the information source etc. + self.source_details: SourceDetails | None = None # Can use this to test if custom talker has been configured + + self.log_func: Callable[[str], None] | None = None + + def set_log_func(self, log_func: Callable[[str], None]) -> None: + self.log_func = log_func + + # issueidentifier will set_log_func to print any write_log to console otherwise logger.info is not printed + def write_log(self, text: str) -> None: + if self.log_func is None: + logger.info(text) + else: + self.log_func(text) + + def check_api_key(self, key: str, url: str): + raise NotImplementedError + + # Search for series/volumes + def search_for_series( + self, + series_name: str, + callback: Callable[[int, int], None] | None = None, + refresh_cache: bool = False, + literal: bool = False, + ) -> list[ComicVolume]: + raise NotImplementedError + + # Get volume/series information + def fetch_volume_data(self, series_id: int) -> GenericMetadata: + raise NotImplementedError + + # Get issues in a series/volume + def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]: + raise NotImplementedError + + # Get issue information + def fetch_issue_data(self, series_id: int, issue_number: str) -> GenericMetadata: + raise NotImplementedError + + # For CLI + def fetch_issue_data_by_issue_id(self, issue_id: int) -> GenericMetadata: + raise NotImplementedError + + def fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> list[str]: + raise NotImplementedError + + def fetch_issues_by_volume_issue_num_and_year( + self, volume_id_list: list[int], issue_number: str, year: str | int | None + ) -> list[ComicIssue]: + raise NotImplementedError + + def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]: + raise NotImplementedError + + # Used by coverimagewidget to load cover image + def async_fetch_issue_cover_urls(self, issue_id: int) -> None: + raise NotImplementedError + + # Used by coverimagewidget.start_alt_cover_search + def fetch_issue_page_url(self, issue_id: int) -> str | None: + raise NotImplementedError + + def async_fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> None: + raise NotImplementedError diff --git a/comictalker/talkers/__init__.py b/comictalker/talkers/__init__.py new file mode 100644 index 0000000..9d48db4 --- /dev/null +++ b/comictalker/talkers/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/comictaggerlib/comicvinetalker.py b/comictalker/talkers/comicvine.py similarity index 62% rename from comictaggerlib/comicvinetalker.py rename to comictalker/talkers/comicvine.py index 83e397a..82406b5 100644 --- a/comictaggerlib/comicvinetalker.py +++ b/comictalker/talkers/comicvine.py @@ -1,5 +1,5 @@ -"""A python class to manage communication with Comic Vine's REST API""" -# +"""ComicVine information source +""" # Copyright 2012-2014 Anthony Beville # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -24,14 +24,23 @@ from typing import Any, Callable, cast import requests from bs4 import BeautifulSoup +from typing_extensions import Required, TypedDict from comicapi import utils from comicapi.genericmetadata import GenericMetadata from comicapi.issuestring import IssueString from comictaggerlib import ctversion -from comictaggerlib.comicvinecacher import ComicVineCacher -from comictaggerlib.resulttypes import CVIssueDetailResults, CVIssuesResults, CVResult, CVVolumeResults, SelectDetails from comictaggerlib.settings import ComicTaggerSettings +from comictalker.comiccacher import ComicCacher +from comictalker.comictalker import ComicTalker +from comictalker.resulttypes import ComicIssue, ComicVolume, SelectDetails +from comictalker.talkerbase import ( + SourceDetails, + SourceSettingsOptions, + SourceStaticOptions, + TalkerBase, + TalkerNetworkError, +) logger = logging.getLogger(__name__) @@ -42,81 +51,236 @@ try: except ImportError: qt_available = False -logger = logging.getLogger(__name__) - class CVTypeID: Volume = "4050" Issue = "4000" -class ComicVineTalkerException(Exception): - Unknown = -1 - Network = -2 - InvalidKey = 100 - RateLimit = 107 - - def __init__(self, code: int = -1, desc: str = "") -> None: - super().__init__() - self.desc = desc - self.code = code - - def __str__(self) -> str: - if self.code in (ComicVineTalkerException.Unknown, ComicVineTalkerException.Network): - return self.desc - - return f"CV error #{self.code}: [{self.desc}]. \n" +class CVImage(TypedDict, total=False): + icon_url: str + medium_url: str + screen_url: str + screen_large_url: str + small_url: str + super_url: Required[str] + thumb_url: str + tiny_url: str + original_url: str + image_tags: str -def list_fetch_complete(url_list: list[str]) -> None: - ... +class CVPublisher(TypedDict, total=False): + api_detail_url: str + id: int + name: Required[str] -def url_fetch_complete(image_url: str, thumb_url: str | None) -> None: - ... +class CVVolume(TypedDict): + api_detail_url: str + id: int + name: str + site_detail_url: str -class ComicVineTalker: - logo_url = "http://static.comicvine.com/bundles/comicvinesite/images/logo.png" - api_key = "" - api_base_url = "" +class CVCredits(TypedDict): + api_detail_url: str + id: int + name: str + site_detail_url: str - alt_url_list_fetch_complete = list_fetch_complete - url_fetch_complete = url_fetch_complete - @staticmethod - def get_rate_limit_message() -> str: - if ComicVineTalker.api_key == "": - return "Comic Vine rate limit exceeded. You should configure your own Comic Vine API key." +class CVPersonCredits(TypedDict): + api_detail_url: str + id: int + name: str + site_detail_url: str + role: str - return "Comic Vine rate limit exceeded. Please wait a bit." +class CVVolumeResults(TypedDict): + count_of_issues: int + description: str + id: int + image: CVImage + name: str + publisher: CVPublisher + start_year: str + resource_type: str + + +class CVResult(TypedDict): + error: str + limit: int + offset: int + number_of_page_results: int + number_of_total_results: int + status_code: int + results: ( + CVIssuesResults + | CVIssueDetailResults + | CVVolumeResults + | list[CVIssuesResults] + | list[CVVolumeResults] + | list[CVIssueDetailResults] + ) + version: str + + +class CVIssuesResults(TypedDict): + cover_date: str + description: str + id: int + image: CVImage + issue_number: str + name: str + site_detail_url: str + volume: CVVolume + + +class CVVolumeFullResult(TypedDict): + characters: list[CVCredits] + locations: list[CVCredits] + people: list[CVPersonCredits] + site_detail_url: str + count_of_issues: int + description: str + id: int + name: str + publisher: CVPublisher + start_year: str + resource_type: str + + +class CVIssueDetailResults(TypedDict): + aliases: None + api_detail_url: str + character_credits: list[CVCredits] + character_died_in: None + concept_credits: list[CVCredits] + cover_date: str + date_added: str + date_last_updated: str + deck: None + description: str + first_appearance_characters: None + first_appearance_concepts: None + first_appearance_locations: None + first_appearance_objects: None + first_appearance_storyarcs: None + first_appearance_teams: None + has_staff_review: bool + id: int + image: CVImage + issue_number: str + location_credits: list[CVCredits] + name: str + object_credits: list[CVCredits] + person_credits: list[CVPersonCredits] + site_detail_url: str + store_date: str + story_arc_credits: list[CVCredits] + team_credits: list[CVCredits] + team_disbanded_in: None + volume: CVVolume + + +class ComicVineTalker(TalkerBase): def __init__(self) -> None: - self.wait_for_rate_limit = False + super().__init__() + self.source_details = source_details = SourceDetails( + name="Comic Vine", + ident="comicvine", + static_options=SourceStaticOptions( + logo_url="http://static.comicvine.com/bundles/comicvinesite/images/logo.png", + has_issues=True, + has_alt_covers=True, + requires_apikey=True, + has_nsfw=False, + has_censored_covers=False, + ), + settings_options={ + "enabled": SourceSettingsOptions( + name="enabled", text="Enabled", help_text="", hidden=True, type=bool, value=True + ), + "order": SourceSettingsOptions( + name="order", text="Order", help_text="", hidden=True, type=int, value=1 + ), + "remove_html_tables": SourceSettingsOptions( + name="remove_html_tables", + text="Remove HTML tables", + help_text="Remove tables in description", + hidden=False, + type=bool, + value=False, + ), + "use_series_start_as_volume": SourceSettingsOptions( + name="use_series_start_as_volume", + text="Use series start year as volume number", + help_text="Use the series start year as the volume number", + hidden=False, + type=bool, + value=False, + ), + "wait_on_ratelimit": SourceSettingsOptions( + name="wait_on_ratelimit", + text="Retry on API limit", + help_text="If the Comic Vine API limit is reached, wait and retry", + hidden=False, + type=bool, + value=False, + ), + "ratelimit_waittime": SourceSettingsOptions( + name="ratelimit_waittime", + text="API maximum wait time (minutes)", + help_text="Maximum time to wait before abandoning retries", + hidden=False, + type=int, + value=20, + ), + "url_root": SourceSettingsOptions( + name="url_root", + text="Comic Vine API address", + help_text="Example: https://api.comicsource.net", + hidden=False, + type=str, + value="https://comicvine.gamespot.com/api", + ), + "api_key": SourceSettingsOptions( + name="api_key", + text="API key", + help_text="Comic Vine API key", + hidden=False, + type=str, + value="27431e6787042105bd3e47e169a624521f89f3a4", + ), + }, + ) - # key that is registered to comictagger - default_api_key = "27431e6787042105bd3e47e169a624521f89f3a4" - default_url = "https://comicvine.gamespot.com/api" + # Identity name for the information source + self.source_name = self.source_details.id + self.source_name_friendly = self.source_details.name + + # Overwrite any source_details.options that have saved settings + source_settings = ComicTaggerSettings.get_source_settings( + self.source_name, self.source_details.settings_options + ) + if not source_settings: + # No saved settings, do something? + ... + + self.wait_for_rate_limit = source_details.settings_options["wait_on_ratelimit"]["value"] + self.wait_for_rate_limit_time = source_details.settings_options["ratelimit_waittime"]["value"] self.issue_id: int | None = None - self.api_key = ComicVineTalker.api_key or default_api_key - self.api_base_url = ComicVineTalker.api_base_url or default_url - - self.log_func: Callable[[str], None] | None = None + self.api_key = source_details.settings_options["api_key"]["value"] + self.api_base_url = source_details.settings_options["url_root"]["value"] + # Used for async cover loading etc. if qt_available: self.nam = QtNetwork.QNetworkAccessManager() - def set_log_func(self, log_func: Callable[[str], None]) -> None: - self.log_func = log_func - - def write_log(self, text: str) -> None: - if self.log_func is None: - logger.info(text) - else: - self.log_func(text) - def parse_date_str(self, date_str: str) -> tuple[int | None, int | None, int | None]: day = None month = None @@ -130,7 +294,7 @@ class ComicVineTalker: day = utils.xlate(parts[2], True) return day, month, year - def test_key(self, key: str, url: str) -> bool: + def check_api_key(self, key: str, url: str) -> bool: if not url: url = self.api_base_url try: @@ -156,7 +320,7 @@ class ComicVineTalker: wait_times = [1, 2, 3, 4] while True: cv_response: CVResult = self.get_url_content(url, params) - if self.wait_for_rate_limit and cv_response["status_code"] == ComicVineTalkerException.RateLimit: + if self.wait_for_rate_limit and cv_response["status_code"] == 107: self.write_log(f"Rate limit encountered. Waiting for {limit_wait_time} minutes\n") time.sleep(limit_wait_time * 60) total_time_waited += limit_wait_time @@ -164,13 +328,15 @@ class ComicVineTalker: if counter < 3: counter += 1 # don't wait much more than 20 minutes - if total_time_waited < 20: + if total_time_waited < self.wait_for_rate_limit_time: continue if cv_response["status_code"] != 1: self.write_log( f"Comic Vine query failed with error #{cv_response['status_code']}: [{cv_response['error']}]. \n" ) - raise ComicVineTalkerException(cv_response["status_code"], cv_response["error"]) + raise TalkerNetworkError( + self.source_name_friendly, 0, str(cv_response["status_code"]) + ":" + str(cv_response["error"]) + ) # it's all good break @@ -192,11 +358,73 @@ class ComicVineTalker: else: break + except requests.exceptions.Timeout: + self.write_log("Connection to " + self.source_name_friendly + " timed out.\n") + raise TalkerNetworkError(self.source_name_friendly, 4) except requests.exceptions.RequestException as e: self.write_log(str(e) + "\n") - raise ComicVineTalkerException(ComicVineTalkerException.Network, "Network Error!") from e + raise TalkerNetworkError(self.source_name_friendly, 0, str(e)) from e - raise ComicVineTalkerException(ComicVineTalkerException.Unknown, "Error on Comic Vine server") + raise TalkerNetworkError(self.source_name_friendly, 5) + + def format_search_results(self, search_results: list[CVVolumeResults]) -> list[ComicVolume]: + + formatted_results = [] + for record in search_results: + # Flatten publisher to name only + if record.get("publisher") is None: + pub_name = "" + else: + pub_name = record["publisher"]["name"] + + if record.get("image") is None: + image = "" + else: + if record["image"].get("super_url") is None: + image = "" + else: + image = record["image"]["super_url"] + + formatted_results.append( + ComicVolume( + count_of_issues=record.get("count_of_issues", 0), + description=record.get("description", ""), + id=record["id"], + image=image, + name=record["name"], + publisher=pub_name, + start_year=record.get("start_year", ""), + ) + ) + + return formatted_results + + def format_issue_results(self, issue_results: list[CVIssuesResults]) -> list[ComicIssue]: + formatted_results = [] + for record in issue_results: + # Extract image super and thumb to name only + if record.get("image") is None: + image = "" + image_thumb = "" + else: + image = record["image"].get("super_url", "") + image_thumb = record["image"].get("thumb_url", "") + + formatted_results.append( + ComicIssue( + cover_date=record.get("cover_date", ""), + description=record.get("description", ""), + id=record["id"], + image=image, + image_thumb=image_thumb, + issue_number=record["issue_number"], + name=record["name"], + site_detail_url=record.get("site_detail_url", ""), + volume=cast(ComicVolume, record["volume"]), + ) + ) + + return formatted_results def search_for_series( self, @@ -204,17 +432,17 @@ class ComicVineTalker: callback: Callable[[int, int], None] | None = None, refresh_cache: bool = False, literal: bool = False, - ) -> list[CVVolumeResults]: + ) -> list[ComicVolume]: # Sanitize the series name for comicvine searching, comicvine search ignore symbols search_series_name = utils.sanitize_title(series_name, literal) - logger.info("Searching: %s", search_series_name) + logger.info("Comic Vine searching: %s", search_series_name) # Before we search online, look in our cache, since we might have done this same search recently # For literal searches always retrieve from online - cvc = ComicVineCacher() + cvc = ComicCacher() if not refresh_cache and not literal: - cached_search_results = cvc.get_search_results(series_name) + cached_search_results = cvc.get_search_results(self.source_name, series_name) if len(cached_search_results) > 0: return cached_search_results @@ -305,17 +533,37 @@ class ComicVineTalker: search_results.remove(record) break + # Format result to ComicSearchResult + formatted_search_results = self.format_search_results(search_results) + # Cache these search results, even if it's literal we cache the results # The most it will cause is extra processing time - cvc.add_search_results(series_name, search_results) + cvc.add_search_results(self.source_name, series_name, formatted_search_results) - return search_results + return formatted_search_results - def fetch_volume_data(self, series_id: int) -> CVVolumeResults: + # To support volume only searching + def fetch_volume_data(self, series_id: int) -> GenericMetadata: + # TODO New cache table or expand current? Makes sense to cache as multiple chapters will want the same data + + volume_url = self.api_base_url + "/volume/" + CVTypeID.Volume + "-" + str(series_id) + + params = { + "api_key": self.api_key, + "format": "json", + "field_list": "name,id,start_year,publisher,count_of_issues,site_detail_url,description,characters,people,locations", + } + cv_response = self.get_cv_content(volume_url, params) + + volume_results = cast(CVVolumeFullResult, cv_response["results"]) + + return self.map_cv_volume_data_to_metadata(volume_results) + + def fetch_partial_volume_data(self, series_id: int) -> ComicVolume: # before we search online, look in our cache, since we might already have this info - cvc = ComicVineCacher() - cached_volume_result = cvc.get_volume_info(series_id) + cvc = ComicCacher() + cached_volume_result = cvc.get_volume_info(series_id, self.source_name) if cached_volume_result is not None: return cached_volume_result @@ -330,16 +578,17 @@ class ComicVineTalker: cv_response = self.get_cv_content(volume_url, params) volume_results = cast(CVVolumeResults, cv_response["results"]) + formatted_volume_results = self.format_search_results([volume_results]) if volume_results: - cvc.add_volume_info(volume_results) + cvc.add_volume_info(self.source_name, formatted_volume_results[0]) - return volume_results + return formatted_volume_results[0] - def fetch_issues_by_volume(self, series_id: int) -> list[CVIssuesResults]: + def fetch_issues_by_volume(self, series_id: int) -> list[ComicIssue]: # before we search online, look in our cache, since we might already have this info - cvc = ComicVineCacher() - cached_volume_issues_result = cvc.get_volume_issues_info(series_id) + cvc = ComicCacher() + cached_volume_issues_result = cvc.get_volume_issues_info(series_id, self.source_name) if cached_volume_issues_result: return cached_volume_issues_result @@ -373,13 +622,16 @@ class ComicVineTalker: self.repair_urls(volume_issues_result) - cvc.add_volume_issues_info(series_id, volume_issues_result) + # Format to expected output + formatted_volume_issues_result = self.format_issue_results(volume_issues_result) - return volume_issues_result + cvc.add_volume_issues_info(self.source_name, series_id, formatted_volume_issues_result) + + return formatted_volume_issues_result def fetch_issues_by_volume_issue_num_and_year( self, volume_id_list: list[int], issue_number: str, year: str | int | None - ) -> list[CVIssuesResults]: + ) -> list[ComicIssue]: volume_filter = "" for vid in volume_id_list: volume_filter += str(vid) + "|" @@ -387,7 +639,7 @@ class ComicVineTalker: int_year = utils.xlate(year, True) if int_year is not None: - flt += f",cover_date:{int_year}-1-1|{int_year+1}-1-1" + flt += f",cover_date:{int_year}-1-1|{int_year + 1}-1-1" params: dict[str, str | int] = { "api_key": self.api_key, @@ -418,10 +670,12 @@ class ComicVineTalker: self.repair_urls(filtered_issues_result) - return filtered_issues_result + formatted_filtered_issues_result = self.format_issue_results(filtered_issues_result) - def fetch_issue_data(self, series_id: int, issue_number: str, settings: ComicTaggerSettings) -> GenericMetadata: - volume_results = self.fetch_volume_data(series_id) + return formatted_filtered_issues_result + + def fetch_issue_data(self, series_id: int, issue_number: str) -> GenericMetadata: + volume_results = self.fetch_partial_volume_data(series_id) issues_list_results = self.fetch_issues_by_volume(series_id) f_record = None @@ -444,10 +698,9 @@ class ComicVineTalker: else: return GenericMetadata() - # Now, map the Comic Vine data to generic metadata - return self.map_cv_data_to_metadata(volume_results, issue_results, settings) + return self.map_cv_data_to_metadata(volume_results, issue_results) - def fetch_issue_data_by_issue_id(self, issue_id: int, settings: ComicTaggerSettings) -> GenericMetadata: + def fetch_issue_data_by_issue_id(self, issue_id: int) -> GenericMetadata: issue_url = self.api_base_url + "/issue/" + CVTypeID.Issue + "-" + str(issue_id) params = {"api_key": self.api_key, "format": "json"} @@ -455,17 +708,70 @@ class ComicVineTalker: issue_results = cast(CVIssueDetailResults, cv_response["results"]) - volume_results = self.fetch_volume_data(issue_results["volume"]["id"]) + volume_results = self.fetch_partial_volume_data(issue_results["volume"]["id"]) - # Now, map the Comic Vine data to generic metadata - md = self.map_cv_data_to_metadata(volume_results, issue_results, settings) + md = self.map_cv_data_to_metadata(volume_results, issue_results) md.is_empty = False return md + # To support volume only searching + def map_cv_volume_data_to_metadata(self, volume_results: CVVolumeFullResult) -> GenericMetadata: + + own_settings = self.source_details.settings_options + # Now, map the Comic Vine data to generic metadata + metadata = GenericMetadata() + metadata.is_empty = False + + metadata.series = utils.xlate(volume_results["name"]) + + if volume_results["publisher"] is not None: + metadata.publisher = utils.xlate(volume_results["publisher"]["name"]) + metadata.year = int(volume_results["start_year"]) + + metadata.comments = self.cleanup_html( + volume_results["description"], own_settings["remove_html_tables"]["value"] + ) + if own_settings["use_series_start_as_volume"]["value"]: + metadata.volume = int(volume_results["start_year"]) + + # TODO How to handle multiple sources? Leave this to sourcesapi to write? + metadata.notes = ( + f"Tagged with ComicTagger {ctversion.version} using info from Comic Vine on" + f" {datetime.now():%Y-%m-%d %H:%M:%S}. [Volume ID {volume_results['id']}]" + ) + metadata.web_link = volume_results["site_detail_url"] + + # TODO No role is given. No unknown role, ignore or presume writer/artist? + if "people" in volume_results: + person_credits = volume_results["people"] + for person in person_credits: + if "role" in person: + roles = person["role"].split(",") + for role in roles: + # can we determine 'primary' from CV?? + metadata.add_credit(person["name"], role.title().strip(), False) + + if "characters" in volume_results: + character_credits = volume_results["characters"] + character_list = [] + for character in character_credits: + character_list.append(character["name"]) + metadata.characters = ", ".join(character_list) + + if "locations" in volume_results: + location_credits = volume_results["locations"] + location_list = [] + for location in location_credits: + location_list.append(location["name"]) + metadata.locations = ", ".join(location_list) + + return metadata + def map_cv_data_to_metadata( - self, volume_results: CVVolumeResults, issue_results: CVIssueDetailResults, settings: ComicTaggerSettings + self, volume_results: ComicVolume, issue_results: CVIssueDetailResults ) -> GenericMetadata: + own_settings = self.source_details.settings_options # Now, map the Comic Vine data to generic metadata metadata = GenericMetadata() metadata.is_empty = False @@ -475,11 +781,11 @@ class ComicVineTalker: metadata.title = utils.xlate(issue_results["name"]) if volume_results["publisher"] is not None: - metadata.publisher = utils.xlate(volume_results["publisher"]["name"]) + metadata.publisher = utils.xlate(volume_results["publisher"]) metadata.day, metadata.month, metadata.year = self.parse_date_str(issue_results["cover_date"]) - metadata.comments = self.cleanup_html(issue_results["description"], settings.remove_html_tables) - if settings.use_series_start_as_volume: + metadata.comments = self.cleanup_html(issue_results["description"], own_settings["remove_html_tables"]["value"]) + if own_settings["use_series_start_as_volume"]["value"]: metadata.volume = int(volume_results["start_year"]) metadata.notes = ( @@ -523,6 +829,7 @@ class ComicVineTalker: return metadata + # TODO Move to utils? def cleanup_html(self, string: str, remove_html_tables: bool) -> str: if string is None: return "" @@ -667,18 +974,18 @@ class ComicVineTalker: def fetch_cached_issue_select_details(self, issue_id: int) -> SelectDetails: # before we search online, look in our cache, since we might already have this info - cvc = ComicVineCacher() - return cvc.get_issue_select_details(issue_id) + cvc = ComicCacher() + return cvc.get_issue_select_details(issue_id, self.source_name) def cache_issue_select_details( self, issue_id: int, image_url: str, thumb_url: str, cover_date: str, page_url: str ) -> None: - cvc = ComicVineCacher() + cvc = ComicCacher() cvc.add_issue_select_details(issue_id, image_url, thumb_url, cover_date, page_url) def fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> list[str]: url_list = self.fetch_cached_alternate_cover_urls(issue_id) - if url_list is not None: + if url_list: return url_list # scrape the CV issue page URL to get the alternate cover URLs @@ -717,22 +1024,21 @@ class ComicVineTalker: def fetch_cached_alternate_cover_urls(self, issue_id: int) -> list[str]: # before we search online, look in our cache, since we might already have this info - cvc = ComicVineCacher() - url_list = cvc.get_alt_covers(issue_id) + cvc = ComicCacher() + url_list = cvc.get_alt_covers(self.source_name, issue_id) return url_list def cache_alternate_cover_urls(self, issue_id: int, url_list: list[str]) -> None: - cvc = ComicVineCacher() - cvc.add_alt_covers(issue_id, url_list) + cvc = ComicCacher() + cvc.add_alt_covers(self.source_name, issue_id, url_list) def async_fetch_issue_cover_urls(self, issue_id: int) -> None: self.issue_id = issue_id details = self.fetch_cached_issue_select_details(issue_id) if details["image_url"] is not None: - ComicVineTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"]) - return + ComicTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"]) issue_url = ( self.api_base_url @@ -771,15 +1077,19 @@ class ComicVineTalker: self.cache_issue_select_details(cast(int, self.issue_id), image_url, thumb_url, cover_date, page_url) - ComicVineTalker.url_fetch_complete(image_url, thumb_url) + ComicTalker.url_fetch_complete(image_url, thumb_url) def async_fetch_alternate_cover_urls(self, issue_id: int, issue_page_url: str) -> None: + # bypass async for now + url_list = self.fetch_alternate_cover_urls(issue_id, issue_page_url) + ComicTalker.alt_url_list_fetch_complete(url_list) + return + # This async version requires the issue page url to be provided! self.issue_id = issue_id url_list = self.fetch_cached_alternate_cover_urls(issue_id) if url_list: - ComicVineTalker.alt_url_list_fetch_complete(url_list) - return + ComicTalker.alt_url_list_fetch_complete(url_list) self.nam.finished.connect(self.async_fetch_alternate_cover_urls_complete) self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_page_url)))) @@ -792,15 +1102,13 @@ class ComicVineTalker: # cache this alt cover URL list self.cache_alternate_cover_urls(cast(int, self.issue_id), alt_cover_url_list) - ComicVineTalker.alt_url_list_fetch_complete(alt_cover_url_list) + ComicTalker.alt_url_list_fetch_complete(alt_cover_url_list) - def repair_urls( - self, issue_list: list[CVIssuesResults] | list[CVVolumeResults] | list[CVIssueDetailResults] - ) -> None: + def repair_urls(self, issue_list: list[CVIssuesResults] | list[ComicIssue] | list[CVIssueDetailResults]) -> None: # make sure there are URLs for the image fields for issue in issue_list: if issue["image"] is None: issue["image"] = { - "super_url": ComicVineTalker.logo_url, - "thumb_url": ComicVineTalker.logo_url, + "super_url": self.source_details.static_options["logo_url"], + "thumb_url": self.source_details.static_options["logo_url"], } diff --git a/comictalker/utils.py b/comictalker/utils.py new file mode 100644 index 0000000..67cc7ad --- /dev/null +++ b/comictalker/utils.py @@ -0,0 +1,134 @@ +"""Generic sources utils to format API data and the like. +""" +# Copyright 2012-2014 Anthony Beville +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from __future__ import annotations + +import logging +import re + +from bs4 import BeautifulSoup + +from comicapi import utils + +logger = logging.getLogger(__name__) + + +def parse_date_str(date_str: str) -> tuple[int | None, int | None, int | None]: + day = None + month = None + year = None + if date_str: + parts = date_str.split("-") + year = utils.xlate(parts[0], True) + if len(parts) > 1: + month = utils.xlate(parts[1], True) + if len(parts) > 2: + day = utils.xlate(parts[2], True) + return day, month, year + + +def cleanup_html(string: str, remove_html_tables: bool) -> str: + if string is None: + return "" + # find any tables + soup = BeautifulSoup(string, "html.parser") + tables = soup.findAll("table") + + # remove all newlines first + string = string.replace("\n", "") + + # put in our own + string = string.replace("
", "\n") + string = string.replace("", "\n") + string = string.replace("

", "\n\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("

", "*") + string = string.replace("

", "*\n") + string = string.replace("
", "*") + string = string.replace("
", "*\n") + string = string.replace("
", "*") + string = string.replace("
", "*\n") + + # remove the tables + p = re.compile(r".*?") + if remove_html_tables: + string = p.sub("", string) + string = string.replace("*List of covers and their creators:*", "") + else: + string = p.sub("{}", string) + + # now strip all other tags + p = re.compile(r"<[^<]*?>") + newstring = p.sub("", string) + + newstring = newstring.replace(" ", " ") + newstring = newstring.replace("&", "&") + + newstring = newstring.strip() + + if not remove_html_tables: + # now rebuild the tables into text from BSoup + try: + table_strings = [] + for table in tables: + rows = [] + hdrs = [] + col_widths = [] + for hdr in table.findAll("th"): + item = hdr.string.strip() + hdrs.append(item) + col_widths.append(len(item)) + rows.append(hdrs) + + for row in table.findAll("tr"): + cols = [] + col = row.findAll("td") + i = 0 + for c in col: + item = c.string.strip() + cols.append(item) + if len(item) > col_widths[i]: + col_widths[i] = len(item) + i += 1 + if len(cols) != 0: + rows.append(cols) + # now we have the data, make it into text + fmtstr = "" + for w in col_widths: + fmtstr += f" {{:{w + 1}}}|" + width = sum(col_widths) + len(col_widths) * 2 + table_text = "" + counter = 0 + for row in rows: + table_text += fmtstr.format(*row) + "\n" + if counter == 0 and len(hdrs) != 0: + table_text += "-" * width + "\n" + counter += 1 + + table_strings.append(table_text) + + newstring = newstring.format(*table_strings) + except Exception: + # we caught an error rebuilding the table. + # just bail and remove the formatting + logger.exception("table parse error") + newstring.replace("{}", "") + + return newstring