lordwelch/comictagger
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove fetch_issue_cover_urls and async_fetch_issue_cover_urls. Reduce API calls by using data already available with coverimagewidget.

Browse Source
This commit is contained in:
Mizaki 2022-10-22 01:43:56 +01:00
parent fb1616aaa1
commit c9cd58fecb
11 changed files with 35 additions and 219 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
comictaggerlib/autotagmatchwindow.py
View File

@ -179,7 +179,9 @@ class AutoTagMatchWindow(QtWidgets.QDialog):
if prev is not None and prev.row() == curr.row():
return None
self.altCoverWidget.set_issue_id(self.current_match()["issue_id"])
self.altCoverWidget.set_issue_details(
self.current_match()["issue_id"], self.current_match()["page_url"], self.current_match()["image_url"]
)
if self.current_match()["description"] is None:
self.teDescription.setText("")
else:

14
comictaggerlib/coverimagewidget.py
View File

@ -110,6 +110,7 @@ class CoverImageWidget(QtWidgets.QWidget):
self.comic_archive: ComicArchive | None = None
self.issue_id: int | None = None
self.issue_url: str | None = None
self.url_list: list[str] = []
if self.page_loader is not None:
self.page_loader.abandoned = True
@ -133,6 +134,7 @@ class CoverImageWidget(QtWidgets.QWidget):
def reset_widget(self) -> None:
self.comic_archive = None
self.issue_id = None
self.issue_url = None
self.url_list = []
if self.page_loader is not None:
self.page_loader.abandoned = True
@ -175,14 +177,15 @@ class CoverImageWidget(QtWidgets.QWidget):
self.imageCount = 1
self.update_content()
def set_issue_id(self, issue_id: int) -> None:
def set_issue_details(self, issue_id: int, issue_url: str, image_url: str) -> None:
if self.mode == CoverImageWidget.AltCoverMode:
self.reset_widget()
self.update_content()
self.issue_id = issue_id
self.issue_url = issue_url
ComicTalker.url_fetch_complete = self.sig.emit_url
self.talker_api.async_fetch_issue_cover_urls(self.issue_id)
ComicTalker.url_fetch_complete(image_url, None)
def set_image_data(self, image_data: bytes) -> None:
if self.mode == CoverImageWidget.DataMode:
@ -203,17 +206,18 @@ class CoverImageWidget(QtWidgets.QWidget):
self.update_content()
# defer the alt cover search
QtCore.QTimer.singleShot(1, self.start_alt_cover_search)
if self.talker_api.static_options.has_alt_covers:
QtCore.QTimer.singleShot(1, self.start_alt_cover_search)
def start_alt_cover_search(self) -> None:
if self.issue_id is not None:
if self.issue_url is not None and self.issue_id is not None:
# now we need to get the list of alt cover URLs
self.label.setText("Searching for alt. covers...")
# page URL should already be cached, so no need to defer
ComicTalker.alt_url_list_fetch_complete = self.sig.emit_list
self.talker_api.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id))
self.talker_api.async_fetch_alternate_cover_urls(utils.xlate(self.issue_id), self.issue_url)
def alt_cover_url_list_fetch_complete(self, url_list: list[str]) -> None:
if url_list:

2
comictaggerlib/issueidentifier.py
View File

@ -271,7 +271,7 @@ class IssueIdentifier:
raise IssueIdentifierCancelled
if use_remote_alternates:
alt_img_url_list = self.talker_api.fetch_alternate_cover_urls(issue_id)
alt_img_url_list = self.talker_api.fetch_alternate_cover_urls(issue_id, page_url)
for alt_url in alt_img_url_list:
try:
alt_url_image_data = ImageFetcher().fetch(alt_url, blocking=True)

2
comictaggerlib/issueselectionwindow.py
View File

@ -180,7 +180,7 @@ class IssueSelectionWindow(QtWidgets.QDialog):
for record in self.issue_list:
if record["id"] == self.issue_id:
self.issue_number = record["issue_number"]
self.coverWidget.set_issue_id(self.issue_id)
self.coverWidget.set_issue_details(self.issue_id, record["site_detail_url"], record["image"])
if record["description"] is None:
self.teDescription.setText("")
else:

2
comictaggerlib/matchselectionwindow.py
View File

@ -149,7 +149,7 @@ class MatchSelectionWindow(QtWidgets.QDialog):
if prev is not None and prev.row() == curr.row():
return
self.altCoverWidget.set_issue_id(self.current_match()["issue_id"])
self.altCoverWidget.set_issue_id(self.current_match()["page_url"], self.current_match()["image_url"])
if self.current_match()["description"] is None:
self.teDescription.setText("")
else:

7
comictaggerlib/resulttypes.py
View File

@ -37,10 +37,3 @@ class MultipleMatch:
def __init__(self, ca: ComicArchive, match_list: list[IssueResult]) -> None:
self.ca: ComicArchive = ca
self.matches: list[IssueResult] = match_list
class SelectDetails(TypedDict):
image_url: str | None
thumb_image_url: str | None
cover_date: str | None
site_detail_url: str | None

57
comictalker/comiccacher.py
View File

@ -23,7 +23,7 @@ from typing import Any
from comictaggerlib import ctversion
from comictaggerlib.settings import ComicTaggerSettings
from comictalker.resulttypes import ComicIssue, ComicVolume, SelectDetails
from comictalker.resulttypes import ComicIssue, ComicVolume
logger = logging.getLogger(__name__)
@ -392,61 +392,6 @@ class ComicCacher:
return results
def add_issue_select_details(
self,
source_name: str,
issue_id: int,
image_url: str,
thumb_image_url: str,
cover_date: str,
site_detail_url: str,
) -> None:
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
timestamp = datetime.datetime.now()
data = {
"id": issue_id,
"source_name": source_name,
"image_url": image_url,
"thumb_url": thumb_image_url,
"cover_date": cover_date,
"site_detail_url": site_detail_url,
"timestamp": timestamp,
}
self.upsert(cur, "issues", data)
def get_issue_select_details(self, issue_id: int, source_name: str) -> SelectDetails:
con = lite.connect(self.db_file)
with con:
cur = con.cursor()
con.text_factory = str
cur.execute(
"SELECT image_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=? AND source_name=?",
[issue_id, source_name],
)
row = cur.fetchone()
details = SelectDetails(
image_url=None,
thumb_image_url=None,
cover_date=None,
site_detail_url=None,
)
if row is not None and row[0] is not None:
details["image_url"] = row[0]
details["thumb_image_url"] = row[1]
details["cover_date"] = row[2]
details["site_detail_url"] = row[3]
return details
def upsert(self, cur: lite.Cursor, tablename: str, data: dict[str, Any]) -> None:
"""This does an insert if the given PK doesn't exist, and an
update it if does

36
comictalker/comictalker.py
View File

@ -123,9 +123,9 @@ class ComicTalker:
)
# For issueidentifer
def fetch_alternate_cover_urls(self, issue_id: int) -> list[str]:
def fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> list[str]:
try:
alt_covers = self.talker.fetch_alternate_cover_urls(issue_id)
alt_covers = self.talker.fetch_alternate_cover_urls(issue_id, issue_url)
return alt_covers
except NotImplementedError:
logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_alternate_cover_urls'")
@ -152,40 +152,14 @@ class ComicTalker:
"The source has not implemented: 'fetch_issues_by_volume_issue_num_and_year'",
)
def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]:
try:
cover_urls = self.talker.fetch_issue_cover_urls(issue_id)
return cover_urls
except NotImplementedError:
logger.warning(f"{self.talker.source_details.name} has not implemented: 'fetch_issue_cover_urls'")
raise TalkerError(
self.talker.source_details.name,
4,
"The source has not implemented: 'fetch_issue_cover_urls'",
)
# Master function to get issue cover. Used by coverimagewidget
def async_fetch_issue_cover_urls(self, issue_id: int) -> None:
def async_fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> None:
try:
# TODO: Figure out async
image_url, thumb_url = self.fetch_issue_cover_urls(issue_id)
ComicTalker.url_fetch_complete(image_url or "", thumb_url)
logger.info("Should be downloading image: %s thumb: %s", image_url, thumb_url)
return
# Should be all that's needed? CV functions will trigger everything.
self.talker.async_fetch_issue_cover_urls(issue_id)
except NotImplementedError:
logger.warning(f"{self.talker.source_details.name} has not implemented: 'async_fetch_issue_cover_urls'")
def async_fetch_alternate_cover_urls(self, issue_id: int) -> None:
try:
# TODO: Figure out async
url_list = self.fetch_alternate_cover_urls(issue_id)
url_list = self.fetch_alternate_cover_urls(issue_id, issue_url)
ComicTalker.alt_url_list_fetch_complete(url_list)
logger.info("Should be downloading alt image list: %s", url_list)
return
self.talker.async_fetch_alternate_cover_urls(issue_id)
self.talker.async_fetch_alternate_cover_urls(issue_id, issue_url)
except NotImplementedError:
logger.warning(f"{self.talker.source_details.name} has not implemented: 'async_fetch_alternate_cover_urls'")

7
comictalker/resulttypes.py
View File

@ -3,13 +3,6 @@ from __future__ import annotations
from typing_extensions import Required, TypedDict
class SelectDetails(TypedDict):
image_url: str | None
thumb_image_url: str | None
cover_date: str | None
site_detail_url: str | None
class ComicVolume(TypedDict, total=False):
aliases: str # Newline separated
count_of_issues: int

11
comictalker/talkerbase.py
View File

@ -195,7 +195,7 @@ class TalkerBase:
def fetch_comic_data(self, series_id: int, issue_number: str = "") -> GenericMetadata:
raise NotImplementedError
def fetch_alternate_cover_urls(self, issue_id: int) -> list[str]:
def fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> list[str]:
raise NotImplementedError
def fetch_issues_by_volume_issue_num_and_year(
@ -203,12 +203,5 @@ class TalkerBase:
) -> list[ComicIssue]:
raise NotImplementedError
def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]:
raise NotImplementedError
# Used by coverimagewidget to load cover image
def async_fetch_issue_cover_urls(self, issue_id: int) -> None:
raise NotImplementedError
def async_fetch_alternate_cover_urls(self, issue_id: int) -> None:
def async_fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> None:
raise NotImplementedError

112
comictalker/talkers/comicvine.py
View File

@ -21,7 +21,7 @@ import re
import time
from datetime import datetime
from typing import Any, Callable, cast
from urllib.parse import urlencode, urljoin, urlsplit
from urllib.parse import urljoin, urlsplit
import requests
from bs4 import BeautifulSoup
@ -34,7 +34,7 @@ from comictaggerlib import ctversion
from comictaggerlib.settings import ComicTaggerSettings
from comictalker.comiccacher import ComicCacher
from comictalker.comictalker import ComicTalker
from comictalker.resulttypes import ComicIssue, ComicVolume, SelectDetails
from comictalker.resulttypes import ComicIssue, ComicVolume
from comictalker.talkerbase import (
SourceDetails,
SourceSettingsOptions,
@ -791,6 +791,7 @@ class ComicVineTalker(TalkerBase):
metadata.series = utils.xlate(issue_results["volume"]["name"])
metadata.issue = IssueString(issue_results["issue_number"]).as_string()
metadata.title = utils.xlate(issue_results["name"])
metadata.cover_image = issue_results["image"]["super_url"]
if volume_results["publisher"] is not None:
metadata.publisher = utils.xlate(volume_results["publisher"])
@ -936,74 +937,13 @@ class ComicVineTalker(TalkerBase):
return newstring
def fetch_issue_date(self, issue_id: int) -> tuple[int | None, int | None]:
details = self.fetch_issue_select_details(issue_id)
_, month, year = utils.parse_date_str(details["cover_date"] or "")
return month, year
def fetch_issue_cover_urls(self, issue_id: int) -> tuple[str | None, str | None]:
details = self.fetch_issue_select_details(issue_id)
return details["image_url"], details["thumb_image_url"]
def fetch_issue_page_url(self, issue_id: int) -> str | None:
details = self.fetch_issue_select_details(issue_id)
return details["site_detail_url"]
def fetch_issue_select_details(self, issue_id: int) -> SelectDetails:
cached_details = self.fetch_cached_issue_select_details(issue_id)
if cached_details["image_url"] is not None:
return cached_details
issue_url = urljoin(self.api_base_url, f"issue/{CVTypeID.Issue}-{issue_id}")
logger.error("%s, %s", self.api_base_url, issue_url)
params = {"api_key": self.api_key, "format": "json", "field_list": "image,cover_date,site_detail_url"}
cv_response = self.get_cv_content(issue_url, params)
results = cast(CVIssueDetailResults, cv_response["results"])
details = SelectDetails(
image_url=results["image"]["super_url"],
thumb_image_url=results["image"]["thumb_url"],
cover_date=results["cover_date"],
site_detail_url=results["site_detail_url"],
)
if (
details["image_url"] is not None
and details["thumb_image_url"] is not None
and details["cover_date"] is not None
and details["site_detail_url"] is not None
):
self.cache_issue_select_details(
issue_id,
details["image_url"],
details["thumb_image_url"],
details["cover_date"],
details["site_detail_url"],
)
return details
def fetch_cached_issue_select_details(self, issue_id: int) -> SelectDetails:
# before we search online, look in our cache, since we might already have this info
cvc = ComicCacher()
return cvc.get_issue_select_details(issue_id, self.source_name)
def cache_issue_select_details(
self, issue_id: int, image_url: str, thumb_url: str, cover_date: str, page_url: str
) -> None:
cvc = ComicCacher()
cvc.add_issue_select_details(self.source_name, issue_id, image_url, thumb_url, cover_date, page_url)
def fetch_alternate_cover_urls(self, issue_id: int) -> list[str]:
def fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> list[str]:
url_list = self.fetch_cached_alternate_cover_urls(issue_id)
if url_list:
return url_list
issue_page_url = self.fetch_issue_page_url(issue_id)
# scrape the CV issue page URL to get the alternate cover URLs
content = requests.get(issue_page_url, headers={"user-agent": "comictagger/" + ctversion.version}).text
content = requests.get(issue_url, headers={"user-agent": "comictagger/" + ctversion.version}).text
alt_cover_url_list = self.parse_out_alt_cover_urls(content)
# cache this alt cover URL list
@ -1047,28 +987,6 @@ class ComicVineTalker(TalkerBase):
cvc = ComicCacher()
cvc.add_alt_covers(self.source_name, issue_id, url_list)
def async_fetch_issue_cover_urls(self, issue_id: int) -> None:
self.issue_id = issue_id
details = self.fetch_cached_issue_select_details(issue_id)
if details["image_url"] is not None:
ComicTalker.url_fetch_complete(details["image_url"], details["thumb_image_url"])
issue_url = urlsplit(self.api_base_url)
issue_url = issue_url._replace(
query=urlencode(
{
"api_key": self.api_key,
"format": "json",
"field_list": "image,cover_date,site_detail_url",
}
),
path=f"issue/{CVTypeID.Issue}-{issue_id}",
)
self.nam.finished.connect(self.async_fetch_issue_cover_url_complete)
self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(issue_url.geturl())))
def async_fetch_issue_cover_url_complete(self, reply: QtNetwork.QNetworkReply) -> None:
# read in the response
data = reply.readAll()
@ -1087,16 +1005,12 @@ class ComicVineTalker(TalkerBase):
image_url = result["image"]["super_url"]
thumb_url = result["image"]["thumb_url"]
cover_date = result["cover_date"]
page_url = result["site_detail_url"]
self.cache_issue_select_details(cast(int, self.issue_id), image_url, thumb_url, cover_date, page_url)
ComicTalker.url_fetch_complete(image_url, thumb_url)
def async_fetch_alternate_cover_urls(self, issue_id: int) -> None:
def async_fetch_alternate_cover_urls(self, issue_id: int, issue_url: str) -> None:
# bypass async for now
url_list = self.fetch_alternate_cover_urls(issue_id)
url_list = self.fetch_alternate_cover_urls(issue_id, issue_url)
ComicTalker.alt_url_list_fetch_complete(url_list)
return
@ -1106,10 +1020,8 @@ class ComicVineTalker(TalkerBase):
if url_list:
ComicTalker.alt_url_list_fetch_complete(url_list)
issue_page_url = self.fetch_issue_page_url(issue_id)
self.nam.finished.connect(self.async_fetch_alternate_cover_urls_complete)
self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_page_url))))
self.nam.get(QtNetwork.QNetworkRequest(QtCore.QUrl(str(issue_url))))
def async_fetch_alternate_cover_urls_complete(self, reply: QtNetwork.QNetworkReply) -> None:
# read in the response
@ -1125,7 +1037,7 @@ class ComicVineTalker(TalkerBase):
# make sure there are URLs for the image fields
for issue in issue_list:
if issue["image"] is None:
issue["image"] = {
"super_url": self.source_details.static_options["logo_url"],
"thumb_url": self.source_details.static_options["logo_url"],
}
issue["image"] = CVImage(
super_url=self.source_details.static_options.logo_url,
thumb_url=self.source_details.static_options.logo_url,
)