From c9cd58fecb14e72e77b6394cac86d2ebe6a60222 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Sat, 22 Oct 2022 01:43:56 +0100 Subject: [PATCH] Remove fetch_issue_cover_urls and async_fetch_issue_cover_urls. Reduce API calls by using data already available with coverimagewidget. --- comictaggerlib/autotagmatchwindow.py | 4 +- comictaggerlib/coverimagewidget.py | 14 ++-- comictaggerlib/issueidentifier.py | 2 +- comictaggerlib/issueselectionwindow.py | 2 +- comictaggerlib/matchselectionwindow.py | 2 +- comictaggerlib/resulttypes.py | 7 -- comictalker/comiccacher.py | 57 +------------ comictalker/comictalker.py | 36 ++------ comictalker/resulttypes.py | 7 -- comictalker/talkerbase.py | 11 +-- comictalker/talkers/comicvine.py | 112 +++---------------------- 11 files changed, 35 insertions(+), 219 deletions(-) diff --git a/comictaggerlib/autotagmatchwindow.py b/comictaggerlib/autotagmatchwindow.py index 34ebc72..9f37a98 100644 --- a/comictaggerlib/autotagmatchwindow.py +++ b/comictaggerlib/autotagmatchwindow.py @@ -179,7 +179,9 @@ class AutoTagMatchWindow(QtWidgets.QDialog): if prev is not None and prev.row() == curr.row(): return None - self.altCoverWidget.set_issue_id(self.current_match()["issue_id"]) + self.altCoverWidget.set_issue_details( + self.current_match()["issue_id"], self.current_match()["page_url"], self.current_match()["image_url"] + ) if self.current_match()["description"] is None: self.teDescription.setText("") else: diff --git a/comictaggerlib/coverimagewidget.py b/comictaggerlib/coverimagewidget.py index b1fdb86..dded97c 100644 --- a/comictaggerlib/coverimagewidget.py +++ b/comictaggerlib/coverimagewidget.py @@ -110,6 +110,7 @@ class CoverImageWidget(QtWidgets.QWidget): self.comic_archive: ComicArchive | None = None self.issue_id: int | None = None + self.issue_url: str | None = None self.url_list: list[str] = [] if self.page_loader is not None: self.page_loader.abandoned = True @@ -133,6 +134,7 @@ class CoverImageWidget(QtWidgets.QWidget): def reset_widget(self) -> None: self.comic_archive = None self.issue_id = None + self.issue_url = None self.url_list = [] if self.page_loader is not None: self.page_loader.abandoned = True @@ -175,14 +177,15 @@ class CoverImageWidget(QtWidgets.QWidget): self.imageCount = 1 self.update_content() - def set_issue_id(self, issue_id: int) -> None: + def set_issue_details(self, issue_id: int, issue_url: str, image_url: str) -> None: if self.mode == CoverImageWidget.AltCoverMode: self.reset_widget() self.update_content() self.issue_id = issue_id + self.issue_url = issue_url ComicTalker.url_fetch_complete = self.sig.emit_url - self.talker_api.async_fetch_issue_cover_urls(self.issue_id) + ComicTalker.url_fetch_complete(image_url, None) def set_image_data(self, image_data: bytes) -> None: if self.mode == CoverImageWidget.DataMode: @@ -203,17 +206,18 @@ class CoverImageWidget(QtWidgets.QWidget): self.update_content() # defer the alt cover search - QtCore.QTimer.singleShot(1, self.start_alt_cover_search) + if self.talker_api.static_options.has_alt_covers: + QtCore.QTimer.singleShot(1, self.start_alt_cover_search) def start_alt_cover_search(self) -> None: - if self.issue_id is not None: + if self.issue_url is not None and self.issue_id is not None: # now we need to get the list of alt cover URLs self.label.setText("Searching for alt. covers...") # page URL should already be cached, so no need to defer ComicTalker.alt_url_list_fetch_complete = self.sig.emit_list - self.talker_api.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id)) + self.talker_api.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), self.issue_url) def alt_cover_url_list_fetch_complete(self, url_list: list[str]) -> None: if url_list: diff --git a/comictaggerlib/issueidentifier.py b/comictaggerlib/issueidentifier.py index faa7bed..c93cbe7 100644 --- a/comictaggerlib/issueidentifier.py +++ b/comictaggerlib/issueidentifier.py @@ -271,7 +271,7 @@ class IssueIdentifier: raise IssueIdentifierCancelled if use_remote_alternates: - alt_img_url_list = self.talker_api.fetch_alternate_cover_urls(issue_id) + alt_img_url_list = self.talker_api.fetch_alternate_cover_urls(issue_id, page_url) for alt_url in alt_img_url_list: try: alt_url_image_data = ImageFetcher().fetch(alt_url, blocking=True) diff --git a/comictaggerlib/issueselectionwindow.py b/comictaggerlib/issueselectionwindow.py index 2cb3b60..6efa7aa 100644 --- a/comictaggerlib/issueselectionwindow.py +++ b/comictaggerlib/issueselectionwindow.py @@ -180,7 +180,7 @@ class IssueSelectionWindow(QtWidgets.QDialog): for record in self.issue_list: if record["id"] == self.issue_id: self.issue_number = record["issue_number"] - self.coverWidget.set_issue_id(self.issue_id) + self.coverWidget.set_issue_details(self.issue_id, record["site_detail_url"], record["image"]) if record["description"] is None: self.teDescription.setText("") else: diff --git a/comictaggerlib/matchselectionwindow.py b/comictaggerlib/matchselectionwindow.py index 1861b34..b6f013b 100644 --- a/comictaggerlib/matchselectionwindow.py +++ b/comictaggerlib/matchselectionwindow.py @@ -149,7 +149,7 @@ class MatchSelectionWindow(QtWidgets.QDialog): if prev is not None and prev.row() == curr.row(): return - self.altCoverWidget.set_issue_id(self.current_match()["issue_id"]) + self.altCoverWidget.set_issue_id(self.current_match()["page_url"], self.current_match()["image_url"]) if self.current_match()["description"] is None: self.teDescription.setText("") else: diff --git a/comictaggerlib/resulttypes.py b/comictaggerlib/resulttypes.py index 1982dd7..8fe1d33 100644 --- a/comictaggerlib/resulttypes.py +++ b/comictaggerlib/resulttypes.py @@ -37,10 +37,3 @@ class MultipleMatch: def __init__(self, ca: ComicArchive, match_list: list[IssueResult]) -> None: self.ca: ComicArchive = ca self.matches: list[IssueResult] = match_list - - -class SelectDetails(TypedDict): - image_url: str | None - thumb_image_url: str | None - cover_date: str | None - site_detail_url: str | None diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index f1a173b..1e033ec 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -23,7 +23,7 @@ from typing import Any from comictaggerlib import ctversion from comictaggerlib.settings import ComicTaggerSettings -from comictalker.resulttypes import ComicIssue, ComicVolume, SelectDetails +from comictalker.resulttypes import ComicIssue, ComicVolume logger = logging.getLogger(__name__) @@ -392,61 +392,6 @@ class ComicCacher: return results - def add_issue_select_details( - self, - source_name: str, - issue_id: int, - image_url: str, - thumb_image_url: str, - cover_date: str, - site_detail_url: str, - ) -> None: - - con = lite.connect(self.db_file) - - with con: - cur = con.cursor() - con.text_factory = str - timestamp = datetime.datetime.now() - - data = { - "id": issue_id, - "source_name": source_name, - "image_url": image_url, - "thumb_url": thumb_image_url, - "cover_date": cover_date, - "site_detail_url": site_detail_url, - "timestamp": timestamp, - } - self.upsert(cur, "issues", data) - - def get_issue_select_details(self, issue_id: int, source_name: str) -> SelectDetails: - - con = lite.connect(self.db_file) - with con: - cur = con.cursor() - con.text_factory = str - - cur.execute( - "SELECT image_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=? AND source_name=?", - [issue_id, source_name], - ) - row = cur.fetchone() - - details = SelectDetails( - image_url=None, - thumb_image_url=None, - cover_date=None, - site_detail_url=None, - ) - if row is not None and row[0] is not None: - details["image_url"] = row[0] - details["thumb_image_url"] = row[1] - details["cover_date"] = row[2] - details["site_detail_url"] = row[3] - - return details - def upsert(self, cur: lite.Cursor, tablename: str, data: dict[str, Any]) -> None: """This does an insert if the given PK doesn't exist, and an update it if does diff --git a/comictalker/comictalker.py b/comictalker/comictalker.py index 77aa692..f4c13ad 100644 --- a/comictalker/comictalker.py +++ b/comictalker/comictalker.py @@ -123,9 +123,9 @@ class ComicTalker: ) # For issueidentifer - def fetch_alternate_cover_urls(self, issue_id: int) -> list[str]: + def fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> list[str]: try: - alt_covers = self.talker.fetch_alternate_cover_urls(issue_id) + alt_covers = self.talker.fetch_alternate_cover_urls(issue_id, issue_url) return alt_covers except NotImplementedError: logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_alternate_cover_urls'") @@ -152,40 +152,14 @@ class ComicTalker: "The source has not implemented: 'fetch_issues_by_volume_issue_num_and_year'", ) - def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]: - try: - cover_urls = self.talker.fetch_issue_cover_urls(issue_id) - return cover_urls - except NotImplementedError: - logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issue_cover_urls'") - raise TalkerError( - self.talker.source_details.name, - 4, - "The source has not implemented: 'fetch_issue_cover_urls'", - ) - - # Master function to get issue cover. Used by coverimagewidget - def async_fetch_issue_cover_urls(self, issue_id: int) -> None: + def async_fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> None: try: # TODO: Figure out async - image_url, thumb_url = self.fetch_issue_cover_urls(issue_id) - ComicTalker.url_fetch_complete(image_url or "", thumb_url) - logger.info("Should be downloading image: %s thumb: %s", image_url, thumb_url) - return - - # Should be all that's needed? CV functions will trigger everything. - self.talker.async_fetch_issue_cover_urls(issue_id) - except NotImplementedError: - logger.warning(f"{self.talker.source_details.name} has not implemented: 'async_fetch_issue_cover_urls'") - - def async_fetch_alternate_cover_urls(self, issue_id: int) -> None: - try: - # TODO: Figure out async - url_list = self.fetch_alternate_cover_urls(issue_id) + url_list = self.fetch_alternate_cover_urls(issue_id, issue_url) ComicTalker.alt_url_list_fetch_complete(url_list) logger.info("Should be downloading alt image list: %s", url_list) return - self.talker.async_fetch_alternate_cover_urls(issue_id) + self.talker.async_fetch_alternate_cover_urls(issue_id, issue_url) except NotImplementedError: logger.warning(f"{self.talker.source_details.name} has not implemented: 'async_fetch_alternate_cover_urls'") diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py index 8f090d2..a21bbba 100644 --- a/comictalker/resulttypes.py +++ b/comictalker/resulttypes.py @@ -3,13 +3,6 @@ from __future__ import annotations from typing_extensions import Required, TypedDict -class SelectDetails(TypedDict): - image_url: str | None - thumb_image_url: str | None - cover_date: str | None - site_detail_url: str | None - - class ComicVolume(TypedDict, total=False): aliases: str # Newline separated count_of_issues: int diff --git a/comictalker/talkerbase.py b/comictalker/talkerbase.py index 049ff3a..d18a035 100644 --- a/comictalker/talkerbase.py +++ b/comictalker/talkerbase.py @@ -195,7 +195,7 @@ class TalkerBase: def fetch_comic_data(self, series_id: int, issue_number: str = "") -> GenericMetadata: raise NotImplementedError - def fetch_alternate_cover_urls(self, issue_id: int) -> list[str]: + def fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> list[str]: raise NotImplementedError def fetch_issues_by_volume_issue_num_and_year( @@ -203,12 +203,5 @@ class TalkerBase: ) -> list[ComicIssue]: raise NotImplementedError - def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]: - raise NotImplementedError - - # Used by coverimagewidget to load cover image - def async_fetch_issue_cover_urls(self, issue_id: int) -> None: - raise NotImplementedError - - def async_fetch_alternate_cover_urls(self, issue_id: int) -> None: + def async_fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> None: raise NotImplementedError diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py index 65a9234..300b518 100644 --- a/comictalker/talkers/comicvine.py +++ b/comictalker/talkers/comicvine.py @@ -21,7 +21,7 @@ import re import time from datetime import datetime from typing import Any, Callable, cast -from urllib.parse import urlencode, urljoin, urlsplit +from urllib.parse import urljoin, urlsplit import requests from bs4 import BeautifulSoup @@ -34,7 +34,7 @@ from comictaggerlib import ctversion from comictaggerlib.settings import ComicTaggerSettings from comictalker.comiccacher import ComicCacher from comictalker.comictalker import ComicTalker -from comictalker.resulttypes import ComicIssue, ComicVolume, SelectDetails +from comictalker.resulttypes import ComicIssue, ComicVolume from comictalker.talkerbase import ( SourceDetails, SourceSettingsOptions, @@ -791,6 +791,7 @@ class ComicVineTalker(TalkerBase): metadata.series = utils.xlate(issue_results["volume"]["name"]) metadata.issue = IssueString(issue_results["issue_number"]).as_string() metadata.title = utils.xlate(issue_results["name"]) + metadata.cover_image = issue_results["image"]["super_url"] if volume_results["publisher"] is not None: metadata.publisher = utils.xlate(volume_results["publisher"]) @@ -936,74 +937,13 @@ class ComicVineTalker(TalkerBase): return newstring - def fetch_issue_date(self, issue_id: int) -> tuple[int | None, int | None]: - details = self.fetch_issue_select_details(issue_id) - _, month, year = utils.parse_date_str(details["cover_date"] or "") - return month, year - - def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]: - details = self.fetch_issue_select_details(issue_id) - return details["image_url"], details["thumb_image_url"] - - def fetch_issue_page_url(self, issue_id: int) -> str | None: - details = self.fetch_issue_select_details(issue_id) - return details["site_detail_url"] - - def fetch_issue_select_details(self, issue_id: int) -> SelectDetails: - cached_details = self.fetch_cached_issue_select_details(issue_id) - if cached_details["image_url"] is not None: - return cached_details - - issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}") - logger.error("%s, %s", self.api_base_url, issue_url) - - params = {"api_key": self.api_key, "format": "json", "field_list": "image,cover_date,site_detail_url"} - - cv_response = self.get_cv_content(issue_url, params) - results = cast(CVIssueDetailResults, cv_response["results"]) - - details = SelectDetails( - image_url=results["image"]["super_url"], - thumb_image_url=results["image"]["thumb_url"], - cover_date=results["cover_date"], - site_detail_url=results["site_detail_url"], - ) - - if ( - details["image_url"] is not None - and details["thumb_image_url"] is not None - and details["cover_date"] is not None - and details["site_detail_url"] is not None - ): - self.cache_issue_select_details( - issue_id, - details["image_url"], - details["thumb_image_url"], - details["cover_date"], - details["site_detail_url"], - ) - return details - - def fetch_cached_issue_select_details(self, issue_id: int) -> SelectDetails: - - # before we search online, look in our cache, since we might already have this info - cvc = ComicCacher() - return cvc.get_issue_select_details(issue_id, self.source_name) - - def cache_issue_select_details( - self, issue_id: int, image_url: str, thumb_url: str, cover_date: str, page_url: str - ) -> None: - cvc = ComicCacher() - cvc.add_issue_select_details(self.source_name, issue_id, image_url, thumb_url, cover_date, page_url) - - def fetch_alternate_cover_urls(self, issue_id: int) -> list[str]: + def fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> list[str]: url_list = self.fetch_cached_alternate_cover_urls(issue_id) if url_list: return url_list - issue_page_url = self.fetch_issue_page_url(issue_id) # scrape the CV issue page URL to get the alternate cover URLs - content = requests.get(issue_page_url, headers={"user-agent": "comictagger/" + ctversion.version}).text + content = requests.get(issue_url, headers={"user-agent": "comictagger/" + ctversion.version}).text alt_cover_url_list = self.parse_out_alt_cover_urls(content) # cache this alt cover URL list @@ -1047,28 +987,6 @@ class ComicVineTalker(TalkerBase): cvc = ComicCacher() cvc.add_alt_covers(self.source_name, issue_id, url_list) - def async_fetch_issue_cover_urls(self, issue_id: int) -> None: - - self.issue_id = issue_id - details = self.fetch_cached_issue_select_details(issue_id) - if details["image_url"] is not None: - ComicTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"]) - - issue_url = urlsplit(self.api_base_url) - issue_url = issue_url._replace( - query=urlencode( - { - "api_key": self.api_key, - "format": "json", - "field_list": "image,cover_date,site_detail_url", - } - ), - path=f"issue/{CVTypeID.Issue}-{issue_id}", - ) - - self.nam.finished.connect(self.async_fetch_issue_cover_url_complete) - self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(issue_url.geturl()))) - def async_fetch_issue_cover_url_complete(self, reply: QtNetwork.QNetworkReply) -> None: # read in the response data = reply.readAll() @@ -1087,16 +1005,12 @@ class ComicVineTalker(TalkerBase): image_url = result["image"]["super_url"] thumb_url = result["image"]["thumb_url"] - cover_date = result["cover_date"] - page_url = result["site_detail_url"] - - self.cache_issue_select_details(cast(int, self.issue_id), image_url, thumb_url, cover_date, page_url) ComicTalker.url_fetch_complete(image_url, thumb_url) - def async_fetch_alternate_cover_urls(self, issue_id: int) -> None: + def async_fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> None: # bypass async for now - url_list = self.fetch_alternate_cover_urls(issue_id) + url_list = self.fetch_alternate_cover_urls(issue_id, issue_url) ComicTalker.alt_url_list_fetch_complete(url_list) return @@ -1106,10 +1020,8 @@ class ComicVineTalker(TalkerBase): if url_list: ComicTalker.alt_url_list_fetch_complete(url_list) - issue_page_url = self.fetch_issue_page_url(issue_id) - self.nam.finished.connect(self.async_fetch_alternate_cover_urls_complete) - self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_page_url)))) + self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_url)))) def async_fetch_alternate_cover_urls_complete(self, reply: QtNetwork.QNetworkReply) -> None: # read in the response @@ -1125,7 +1037,7 @@ class ComicVineTalker(TalkerBase): # make sure there are URLs for the image fields for issue in issue_list: if issue["image"] is None: - issue["image"] = { - "super_url": self.source_details.static_options["logo_url"], - "thumb_url": self.source_details.static_options["logo_url"], - } + issue["image"] = CVImage( + super_url=self.source_details.static_options.logo_url, + thumb_url=self.source_details.static_options.logo_url, + )