From 73738010b8769855b3764df83e620869acd63a5d Mon Sep 17 00:00:00 2001 From: Mizaki Date: Wed, 15 Feb 2023 16:48:07 +0000 Subject: [PATCH 01/13] Add additional fields to ComicIssue and add a genre field to ComicSeries to allow for filtering of search results from the cache. --- comictalker/comiccacher.py | 33 +++++++++++++++++++++++++++----- comictalker/resulttypes.py | 5 +++++ comictalker/talker_utils.py | 8 ++++++++ comictalker/talkers/comicvine.py | 5 +++++ 4 files changed, 46 insertions(+), 5 deletions(-) diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index bd3ea8c..733a594 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -2,7 +2,7 @@ # # Copyright 2012-2014 Anthony Beville # -# Licensed under the Apache License, Version 2.0 (the "License; +# Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # @@ -92,6 +92,7 @@ class ComicCacher: + "image_url TEXT," + "aliases TEXT," # Newline separated + "description TEXT," + + "genres TEXT," # Newline separated. For filtering etc. + "timestamp DATE DEFAULT (datetime('now','localtime')), " + "source_name TEXT NOT NULL," + "PRIMARY KEY (id, source_name))" @@ -117,6 +118,10 @@ class ComicCacher: + "credits TEXT," # JSON: "{"name": "Bob Shakespeare", "role": "Writer"}" + "teams TEXT," # Newline separated + "story_arcs TEXT," # Newline separated + + "genres TEXT," # Newline separated + + "tags TEXT," # Newline separated + + "rating FLOAT," + + "manga TEXT," # Yes/Yes (Right to Left)/No + "complete BOOL," # Is the data complete? Includes characters, locations, credits. + "PRIMARY KEY (id, source_name))" ) @@ -150,6 +155,7 @@ class ComicCacher: "start_year": record.start_year, "image_url": record.image_url, "description": record.description, + "genres": "\n".join(record.genres), "timestamp": datetime.datetime.now(), "aliases": "\n".join(record.aliases), } @@ -181,6 +187,7 @@ class ComicCacher: image_url=record[9], aliases=record[10].strip().splitlines(), description=record[11], + genres=record[12].strip().splitlines(), ) results.append(result) @@ -204,6 +211,7 @@ class ComicCacher: "start_year": series_record.start_year, "image_url": series_record.image_url, "description": series_record.description, + "genres": "\n".join(series_record.genres), "timestamp": timestamp, "aliases": "\n".join(series_record.aliases), } @@ -237,6 +245,10 @@ class ComicCacher: "locations": "\n".join(issue.locations), "teams": "\n".join(issue.teams), "story_arcs": "\n".join(issue.story_arcs), + "genres": "\n".join(issue.genres), + "tags": "\n".join(issue.tags), + "rating": issue.rating, + "manga": issue.manga, "credits": json.dumps([dataclasses.asdict(x) for x in issue.credits]), "complete": issue.complete, } @@ -273,6 +285,7 @@ class ComicCacher: image_url=row[5], aliases=row[6].strip().splitlines(), description=row[7], + genres=row[8].strip().splitlines(), ) return result @@ -283,6 +296,7 @@ class ComicCacher: id=series_id, name="", description="", + genres=[], image_url="", publisher="", start_year=None, @@ -304,7 +318,7 @@ class ComicCacher: cur.execute( ( - "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,alt_image_urls,characters,locations,credits,teams,story_arcs,complete" + "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,alt_image_urls,characters,locations,credits,teams,story_arcs,genres,tags,rating,manga,complete" " FROM Issues WHERE series_id=? AND source_name=?" ), [series_id, source_name], @@ -335,7 +349,11 @@ class ComicCacher: credits=credits, teams=row[14].strip().splitlines(), story_arcs=row[15].strip().splitlines(), - complete=bool(row[16]), + genres=row[16].strip().splitlines(), + tags=row[17].strip().splitlines(), + rating=row[18], + manga=row[19], + complete=bool(row[20]), ) results.append(record) @@ -355,7 +373,7 @@ class ComicCacher: cur.execute( ( - "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,description,aliases,series_id,alt_image_urls,characters,locations,credits,teams,story_arcs,complete" + "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,description,aliases,series_id,alt_image_urls,characters,locations,credits,teams,story_arcs,genres,tags,rating,manga,complete" " FROM Issues WHERE id=? AND source_name=?" ), [issue_id, source_name], @@ -370,6 +388,7 @@ class ComicCacher: id=row[10], name="", description="", + genres=[], image_url="", publisher="", start_year=None, @@ -395,7 +414,11 @@ class ComicCacher: credits=json.loads(row[13]), teams=row[14].strip().splitlines(), story_arcs=row[15].strip().splitlines(), - complete=bool(row[16]), + genres=row[16].strip().splitlines(), + tags=row[17].strip().splitlines(), + rating=row[18], + manga=row[19], + complete=bool(row[20]), ) return record diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py index 42f4fce..f4156dd 100644 --- a/comictalker/resulttypes.py +++ b/comictalker/resulttypes.py @@ -20,6 +20,7 @@ class ComicSeries: name: str publisher: str start_year: int | None + genres: list[str] def copy(self) -> ComicSeries: return copy.deepcopy(self) @@ -33,6 +34,10 @@ class ComicIssue: id: str image_url: str issue_number: str + rating: float + manga: str + genres: list[str] + tags: list[str] name: str site_detail_url: str series: ComicSeries diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index bd605ca..b69479d 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -72,6 +72,14 @@ def map_comic_issue_to_metadata( metadata.locations = ", ".join(issue_results.locations) if issue_results.story_arcs: metadata.story_arc = ", ".join(issue_results.story_arcs) + if issue_results.genres: + metadata.genre = ", ".join(issue_results.genres) + if issue_results.tags: + metadata.tags = set(issue_results.tags) + if issue_results.manga: + metadata.manga = issue_results.manga + if issue_results.rating: + metadata.critical_rating = issue_results.rating return metadata diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py index f3d4d29..56f8f52 100644 --- a/comictalker/talkers/comicvine.py +++ b/comictalker/talkers/comicvine.py @@ -507,6 +507,7 @@ class ComicVineTalker(ComicTalker): name=record["name"], publisher=pub_name, start_year=start_year, + genres=[], ) ) @@ -568,6 +569,10 @@ class ComicVineTalker(ComicTalker): locations=location_list, teams=teams_list, story_arcs=story_list, + rating=0, + manga="", + genres=[], + tags=[], credits=persons_list, complete=complete, ) From c9638ba0d9227a88e99934c6d2cad95614808693 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Thu, 2 Mar 2023 00:10:52 +0000 Subject: [PATCH 02/13] Format manga and rating --- comictalker/talker_utils.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index b69479d..5f4f8fa 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -77,9 +77,12 @@ def map_comic_issue_to_metadata( if issue_results.tags: metadata.tags = set(issue_results.tags) if issue_results.manga: - metadata.manga = issue_results.manga + if issue_results.manga in ["Yes", "yes", "No", "no", "YesAndRightToLeft", "yesandrighttoleft"]: + metadata.manga = issue_results.manga + else: + metadata.manga = "Unknown" if issue_results.rating: - metadata.critical_rating = issue_results.rating + metadata.critical_rating = utils.xlate(issue_results.rating, is_float=True) return metadata From 9e86b5e331484d2373d6a44252f2b6a288e4e988 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Thu, 2 Mar 2023 00:23:56 +0000 Subject: [PATCH 03/13] Fix tests --- testing/comicdata.py | 2 ++ testing/comicvine.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/testing/comicdata.py b/testing/comicdata.py index 5528154..5e8d9fa 100644 --- a/testing/comicdata.py +++ b/testing/comicdata.py @@ -14,6 +14,7 @@ search_results = [ publisher="test", start_year=0, aliases=[], + genres=[], ), comictalker.resulttypes.ComicSeries( count_of_issues=1, @@ -24,6 +25,7 @@ search_results = [ publisher="test", start_year=0, aliases=[], + genres=[], ), ] diff --git a/testing/comicvine.py b/testing/comicvine.py index 716abc1..ccffd12 100644 --- a/testing/comicvine.py +++ b/testing/comicvine.py @@ -176,6 +176,7 @@ comic_issue_result = ComicIssue( image_url=cv_volume_result["results"]["image"]["super_url"], publisher=cv_volume_result["results"]["publisher"]["name"], start_year=int(cv_volume_result["results"]["start_year"]), + genres=[], ), characters=[], alt_image_urls=[], @@ -183,6 +184,10 @@ comic_issue_result = ComicIssue( credits=[], locations=[], story_arcs=[], + rating=0, + manga="", + genres=[], + tags=[], teams=[], ) date = utils.parse_date_str(cv_issue_result["results"]["cover_date"]) From 6cf259191e52e1778ebcf844325300cb7795524c Mon Sep 17 00:00:00 2001 From: Mizaki Date: Fri, 12 May 2023 21:48:45 +0100 Subject: [PATCH 04/13] Add volume and count_of_volumes to ComicSeries --- comictalker/comiccacher.py | 34 ++++++++++++++++++++++---------- comictalker/resulttypes.py | 2 ++ comictalker/talker_utils.py | 2 ++ comictalker/talkers/comicvine.py | 2 ++ testing/comicdata.py | 4 ++++ testing/comicvine.py | 2 ++ 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index 733a594..80bb49d 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -88,6 +88,8 @@ class ComicCacher: + "name TEXT," + "publisher TEXT," + "count_of_issues INT," + + "count_of_volumes INT," + + "volume TEXT," + "start_year INT," + "image_url TEXT," + "aliases TEXT," # Newline separated @@ -152,6 +154,8 @@ class ComicCacher: "name": record.name, "publisher": record.publisher, "count_of_issues": record.count_of_issues, + "count_of_volumes": record.count_of_volumes, + "volume": record.volume, "start_year": record.start_year, "image_url": record.image_url, "description": record.description, @@ -183,11 +187,13 @@ class ComicCacher: name=record[5], publisher=record[6], count_of_issues=record[7], - start_year=record[8], - image_url=record[9], - aliases=record[10].strip().splitlines(), - description=record[11], - genres=record[12].strip().splitlines(), + count_of_volumes=record[8], + volume=record[9], + start_year=record[10], + image_url=record[11], + aliases=record[12].strip().splitlines(), + description=record[13], + genres=record[14].strip().splitlines(), ) results.append(result) @@ -208,6 +214,8 @@ class ComicCacher: "name": series_record.name, "publisher": series_record.publisher, "count_of_issues": series_record.count_of_issues, + "count_of_volumes": series_record.count_of_volumes, + "volume": series_record.volume, "start_year": series_record.start_year, "image_url": series_record.image_url, "description": series_record.description, @@ -281,11 +289,13 @@ class ComicCacher: name=row[1], publisher=row[2], count_of_issues=row[3], - start_year=row[4], - image_url=row[5], - aliases=row[6].strip().splitlines(), - description=row[7], - genres=row[8].strip().splitlines(), + count_of_volumes=row[4], + volume=row[5], + start_year=row[6], + image_url=row[7], + aliases=row[8].strip().splitlines(), + description=row[9], + genres=row[10].strip().splitlines(), ) return result @@ -302,6 +312,8 @@ class ComicCacher: start_year=None, aliases=[], count_of_issues=None, + count_of_volumes=None, + volume=None, ) con = lite.connect(self.db_file) with con: @@ -394,6 +406,8 @@ class ComicCacher: start_year=None, aliases=[], count_of_issues=None, + count_of_volumes=None, + volume=None, ) # now process the results diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py index f4156dd..45a38e0 100644 --- a/comictalker/resulttypes.py +++ b/comictalker/resulttypes.py @@ -14,6 +14,8 @@ class Credit: class ComicSeries: aliases: list[str] count_of_issues: int | None + count_of_volumes: int | None + volume: str | None description: str id: str image_url: str diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index 5f4f8fa..19485c2 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -35,6 +35,8 @@ def map_comic_issue_to_metadata( metadata.series = utils.xlate(issue_results.series.name) metadata.issue = IssueString(issue_results.issue_number).as_string() + metadata.volume = utils.xlate(issue_results.series.volume, True) + metadata.volume_count = utils.xlate(issue_results.series.count_of_volumes, True) if issue_results.name: metadata.title = utils.xlate(issue_results.name) diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py index 56f8f52..3f7d11e 100644 --- a/comictalker/talkers/comicvine.py +++ b/comictalker/talkers/comicvine.py @@ -501,6 +501,8 @@ class ComicVineTalker(ComicTalker): ComicSeries( aliases=aliases.splitlines(), count_of_issues=record.get("count_of_issues", 0), + count_of_volumes=None, + volume=None, description=record.get("description", ""), id=str(record["id"]), image_url=image_url, diff --git a/testing/comicdata.py b/testing/comicdata.py index 5e8d9fa..7698437 100644 --- a/testing/comicdata.py +++ b/testing/comicdata.py @@ -7,6 +7,8 @@ from comicapi import utils search_results = [ comictalker.resulttypes.ComicSeries( count_of_issues=1, + count_of_volumes=1, + volume="1", description="this is a description", id="1", image_url="https://test.org/image/1", @@ -18,6 +20,8 @@ search_results = [ ), comictalker.resulttypes.ComicSeries( count_of_issues=1, + count_of_volumes=1, + volume="1", description="this is a description", id="2", image_url="https://test.org/image/2", diff --git a/testing/comicvine.py b/testing/comicvine.py index ccffd12..44d169b 100644 --- a/testing/comicvine.py +++ b/testing/comicvine.py @@ -172,6 +172,8 @@ comic_issue_result = ComicIssue( name=cv_issue_result["results"]["volume"]["name"], aliases=[], count_of_issues=cv_volume_result["results"]["count_of_issues"], + count_of_volumes=None, + volume=None, description=cv_volume_result["results"]["description"], image_url=cv_volume_result["results"]["image"]["super_url"], publisher=cv_volume_result["results"]["publisher"]["name"], From 2598fc546aeffd6fa21b984155838043c03cf029 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Fri, 12 May 2023 22:47:36 +0100 Subject: [PATCH 05/13] Use new xlate_int and xlate_float --- comictalker/talker_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index e81b37c..5da2d23 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -44,8 +44,8 @@ def map_comic_issue_to_metadata( metadata.series = utils.xlate(issue_results.series.name) metadata.issue = IssueString(issue_results.issue_number).as_string() - metadata.volume = utils.xlate(issue_results.series.volume, True) - metadata.volume_count = utils.xlate(issue_results.series.count_of_volumes, True) + metadata.volume = utils.xlate_int(issue_results.series.volume) + metadata.volume_count = utils.xlate_int(issue_results.series.count_of_volumes) if issue_results.name: metadata.title = utils.xlate(issue_results.name) @@ -93,7 +93,7 @@ def map_comic_issue_to_metadata( else: metadata.manga = "Unknown" if issue_results.rating: - metadata.critical_rating = utils.xlate(issue_results.rating, is_float=True) + metadata.critical_rating = utils.xlate_float(issue_results.rating) return metadata From 0b77078a93b026956963f8e41692424c712aa0de Mon Sep 17 00:00:00 2001 From: Mizaki Date: Fri, 12 May 2023 23:46:34 +0100 Subject: [PATCH 06/13] Retrieve all fields instead of by (many) names --- comictalker/comiccacher.py | 86 ++++++++++++++++---------------------- 1 file changed, 37 insertions(+), 49 deletions(-) diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index e61f914..aef76d4 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -328,44 +328,38 @@ class ComicCacher: # fetch results: list[ComicIssue] = [] - cur.execute( - ( - "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,thumb_url,description,aliases,alt_image_urls,characters,locations,credits,teams,story_arcs,genres,tags,rating,manga,complete" - " FROM Issues WHERE series_id=? AND source_name=?" - ), - [series_id, source_name], - ) + cur.execute("SELECT * FROM Issues WHERE series_id=? AND source_name=?", [series_id, source_name]) rows = cur.fetchall() # now process the results for row in rows: credits = [] try: - for credit in json.loads(row[13]): + for credit in json.loads(row[15]): credits.append(Credit(**credit)) except Exception: logger.exception("credits failed") record = ComicIssue( - id=row[1], + id=row[0], name=row[2], issue_number=row[3], - site_detail_url=row[4], - cover_date=row[5], - image_url=row[6], + site_detail_url=row[7], + cover_date=row[6], + image_url=row[4], description=row[8], series=series, - aliases=row[9].strip().splitlines(), - alt_image_urls=row[10].strip().splitlines(), - characters=row[11].strip().splitlines(), - locations=row[12].strip().splitlines(), + aliases=row[11].strip().splitlines(), + alt_image_urls=row[12].strip().splitlines(), + characters=row[13].strip().splitlines(), + locations=row[14].strip().splitlines(), credits=credits, - teams=row[14].strip().splitlines(), - story_arcs=row[15].strip().splitlines(), - genres=row[16].strip().splitlines(), - tags=row[17].strip().splitlines(), - rating=row[18], - manga=row[19], - complete=bool(row[20]), + teams=row[16].strip().splitlines(), + story_arcs=row[17].strip().splitlines(), + genres=row[18].strip().splitlines(), + tags=row[19].strip().splitlines(), + rating=row[20], + manga=row[21], + complete=bool(row[22]), ) results.append(record) @@ -383,21 +377,15 @@ class ComicCacher: a_week_ago = datetime.datetime.today() - datetime.timedelta(days=7) cur.execute("DELETE FROM Issues WHERE timestamp < ?", [str(a_week_ago)]) - cur.execute( - ( - "SELECT source_name,id,name,issue_number,site_detail_url,cover_date,image_url,description,aliases,series_id,alt_image_urls,characters,locations,credits,teams,story_arcs,genres,tags,rating,manga,complete" - " FROM Issues WHERE id=? AND source_name=?" - ), - [issue_id, source_name], - ) + cur.execute("SELECT * FROM Issues WHERE id=? AND source_name=?", [issue_id, source_name]) row = cur.fetchone() record = None if row: # get_series_info should only fail if someone is doing something weird - series = self.get_series_info(row[10], source_name, False) or ComicSeries( - id=row[10], + series = self.get_series_info(row[1], source_name, False) or ComicSeries( + id=row[1], name="", description="", genres=[], @@ -413,31 +401,31 @@ class ComicCacher: # now process the results credits = [] try: - for credit in json.loads(row[13]): + for credit in json.loads(row[15]): credits.append(Credit(**credit)) except Exception: logger.exception("credits failed") record = ComicIssue( - id=row[1], + id=row[0], name=row[2], issue_number=row[3], - site_detail_url=row[4], - cover_date=row[5], - image_url=row[6], - description=row[7], + site_detail_url=row[7], + cover_date=row[6], + image_url=row[4], + description=row[8], series=series, - aliases=row[8].strip().splitlines(), - alt_image_urls=row[10].strip().splitlines(), - characters=row[11].strip().splitlines(), - locations=row[12].strip().splitlines(), + aliases=row[11].strip().splitlines(), + alt_image_urls=row[12].strip().splitlines(), + characters=row[13].strip().splitlines(), + locations=row[14].strip().splitlines(), credits=credits, - teams=row[14].strip().splitlines(), - story_arcs=row[15].strip().splitlines(), - genres=row[16].strip().splitlines(), - tags=row[17].strip().splitlines(), - rating=row[18], - manga=row[19], - complete=bool(row[20]), + teams=row[16].strip().splitlines(), + story_arcs=row[17].strip().splitlines(), + genres=row[18].strip().splitlines(), + tags=row[19].strip().splitlines(), + rating=row[20], + manga=row[21], + complete=bool(row[22]), ) return record From d65a120eb5b00c594a43e4f7738e30af9ad89ad0 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Sun, 14 May 2023 00:50:37 +0100 Subject: [PATCH 07/13] Add issue_count --- comictalker/talker_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index 5da2d23..ecb97a4 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -44,6 +44,10 @@ def map_comic_issue_to_metadata( metadata.series = utils.xlate(issue_results.series.name) metadata.issue = IssueString(issue_results.issue_number).as_string() + + # Rely on comic talker to valid this number + metadata.issue_count = utils.xlate_int(issue_results.series.volume) + metadata.volume = utils.xlate_int(issue_results.series.volume) metadata.volume_count = utils.xlate_int(issue_results.series.count_of_volumes) From 97e64fa918a26f7b492b511284861f133ad5ee34 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Thu, 18 May 2023 02:02:21 +0100 Subject: [PATCH 08/13] Add maturity_rating, language and country to ComicIssue and pass to metadata. --- comictalker/comiccacher.py | 26 +++++++++++++++++++------- comictalker/resulttypes.py | 5 ++++- comictalker/talker_utils.py | 12 ++++++++++-- comictalker/talkers/comicvine.py | 8 +++++++- testing/comicvine.py | 5 ++++- 5 files changed, 44 insertions(+), 12 deletions(-) diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index aef76d4..1d9cfd8 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -122,8 +122,11 @@ class ComicCacher: + "story_arcs TEXT," # Newline separated + "genres TEXT," # Newline separated + "tags TEXT," # Newline separated - + "rating FLOAT," - + "manga TEXT," # Yes/Yes (Right to Left)/No + + "critical_rating FLOAT," + + "manga TEXT," # Yes/YesAndRightToLeft/No + + "maturity_rating TEXT," + + "language TEXT," # 2-letter iso code + + "country TEXT," # 2-letter iso code + "complete BOOL," # Is the data complete? Includes characters, locations, credits. + "PRIMARY KEY (id, source_name))" ) @@ -255,8 +258,11 @@ class ComicCacher: "story_arcs": "\n".join(issue.story_arcs), "genres": "\n".join(issue.genres), "tags": "\n".join(issue.tags), - "rating": issue.rating, + "critical_rating": issue.critical_rating, "manga": issue.manga, + "maturity_rating": issue.maturity_rating, + "language": issue.language, + "country": issue.country, "credits": json.dumps([dataclasses.asdict(x) for x in issue.credits]), "complete": issue.complete, } @@ -357,9 +363,12 @@ class ComicCacher: story_arcs=row[17].strip().splitlines(), genres=row[18].strip().splitlines(), tags=row[19].strip().splitlines(), - rating=row[20], + critical_rating=row[20], manga=row[21], - complete=bool(row[22]), + maturity_rating=row[22], + language=row[23], + country=row[24], + complete=bool(row[25]), ) results.append(record) @@ -423,9 +432,12 @@ class ComicCacher: story_arcs=row[17].strip().splitlines(), genres=row[18].strip().splitlines(), tags=row[19].strip().splitlines(), - rating=row[20], + critical_rating=row[20], manga=row[21], - complete=bool(row[22]), + maturity_rating=row[22], + language=row[23], + country=row[24], + complete=bool(row[25]), ) return record diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py index 45a38e0..c7fa18c 100644 --- a/comictalker/resulttypes.py +++ b/comictalker/resulttypes.py @@ -36,11 +36,14 @@ class ComicIssue: id: str image_url: str issue_number: str - rating: float + critical_rating: float + maturity_rating: str manga: str genres: list[str] tags: list[str] name: str + language: str # 2-letter iso code + country: str # 2-letter iso code site_detail_url: str series: ComicSeries alt_image_urls: list[str] diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index ecb97a4..b5c3094 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -96,8 +96,16 @@ def map_comic_issue_to_metadata( metadata.manga = issue_results.manga else: metadata.manga = "Unknown" - if issue_results.rating: - metadata.critical_rating = utils.xlate_float(issue_results.rating) + if issue_results.critical_rating: + metadata.critical_rating = utils.xlate_float(issue_results.critical_rating) + + if issue_results.language: + # 2-letter code # TODO Run check against pycountry? + metadata.language = issue_results.language + + if issue_results.country: + # 2-letter code # TODO Run check against pycountry? + metadata.language = issue_results.country return metadata diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py index 3dc77ef..9476c0d 100644 --- a/comictalker/talkers/comicvine.py +++ b/comictalker/talkers/comicvine.py @@ -563,6 +563,9 @@ class ComicVineTalker(ComicTalker): series = self._fetch_series_data(record["volume"]["id"]) + # Make below an option? + # TODO Infer country by publisher (and language)? Check series desc for "Translated into English"? + formatted_results.append( ComicIssue( aliases=record["aliases"].split("\n") if record["aliases"] else [], @@ -579,8 +582,11 @@ class ComicVineTalker(ComicTalker): locations=location_list, teams=teams_list, story_arcs=story_list, - rating=0, + critical_rating=0, + maturity_rating="", manga="", + language="", + country="", genres=[], tags=[], credits=persons_list, diff --git a/testing/comicvine.py b/testing/comicvine.py index 44d169b..7cb9df5 100644 --- a/testing/comicvine.py +++ b/testing/comicvine.py @@ -186,8 +186,11 @@ comic_issue_result = ComicIssue( credits=[], locations=[], story_arcs=[], - rating=0, + critical_rating=0, + maturity_rating="", manga="", + language="", + country="", genres=[], tags=[], teams=[], From 8e2411a086725a9bedeb0d304c5cd1453481ad49 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Tue, 23 May 2023 00:02:56 +0100 Subject: [PATCH 09/13] Add country functions to utils and try to convert a country name to ISO country name --- comicapi/utils.py | 34 ++++++++++++++++++++++++++++++++++ comictalker/talker_utils.py | 20 ++++++++++++++++---- 2 files changed, 50 insertions(+), 4 deletions(-) diff --git a/comicapi/utils.py b/comicapi/utils.py index 9cb579c..9b2c2f5 100644 --- a/comicapi/utils.py +++ b/comicapi/utils.py @@ -239,6 +239,7 @@ def get_language_from_iso(iso: str | None) -> str | None: def get_language_iso(string: str | None) -> str | None: if string is None: return None + # Why return this if ISO failed? lang = string.casefold() try: @@ -248,6 +249,39 @@ def get_language_iso(string: str | None) -> str | None: return lang +def get_country_from_iso(iso: str | None) -> str | None: + return countries[iso] + + +def get_country_iso(string: str | None) -> str | None: + if string is None: + return None + + try: + country = pycountry.countries.lookup(string) + if country: + # Assume first entry is correct + return country[0].alpha_2 + except LookupError: + pass + return None + + +def get_country_iso_name(string: str | None) -> str | None: + """Return country's ISO name from "other" name""" + if string is None: + return None + + try: + country = pycountry.countries.search_fuzzy(string) + if country: + # Assume first entry is correct + return country[0].name + except LookupError: + pass + return None + + def get_publisher(publisher: str) -> tuple[str, str]: imprint = "" diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index b5c3094..4767e1c 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -99,13 +99,25 @@ def map_comic_issue_to_metadata( if issue_results.critical_rating: metadata.critical_rating = utils.xlate_float(issue_results.critical_rating) + # 2-letter country code if issue_results.language: - # 2-letter code # TODO Run check against pycountry? - metadata.language = issue_results.language + if utils.get_language_from_iso(issue_results.language): + metadata.language = issue_results.language + else: + code = utils.get_language_iso(issue_results.language) + if code: + metadata.language = code + # ISO country name if issue_results.country: - # 2-letter code # TODO Run check against pycountry? - metadata.language = issue_results.country + # Only CBI supports country currently in CT and is a string name + # TODO Use https://codeberg.org/plotski/countryguess as pycountry seems stale? + # Return current entry if conversion fails + metadata.country = issue_results.country + if not utils.get_country_iso(issue_results.country): + code = utils.get_country_iso_name(issue_results.country) + if code: + metadata.country = code return metadata From e72347656bc2810acef6602e5f71a5809d1692c3 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Tue, 23 May 2023 00:27:58 +0100 Subject: [PATCH 10/13] Add format (1-shot, limited series, etc.) --- comictalker/comiccacher.py | 7 +++++++ comictalker/resulttypes.py | 1 + comictalker/talker_utils.py | 6 +++++- comictalker/talkers/comicvine.py | 1 + testing/comicdata.py | 2 ++ testing/comicvine.py | 1 + 6 files changed, 17 insertions(+), 1 deletion(-) diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index 1d9cfd8..90e0931 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -95,6 +95,7 @@ class ComicCacher: + "aliases TEXT," # Newline separated + "description TEXT," + "genres TEXT," # Newline separated. For filtering etc. + + "format TEXT," + "timestamp DATE DEFAULT (datetime('now','localtime')), " + "source_name TEXT NOT NULL," + "PRIMARY KEY (id, source_name))" @@ -163,6 +164,7 @@ class ComicCacher: "image_url": record.image_url, "description": record.description, "genres": "\n".join(record.genres), + "format": record.format, "timestamp": datetime.datetime.now(), "aliases": "\n".join(record.aliases), } @@ -197,6 +199,7 @@ class ComicCacher: aliases=record[12].strip().splitlines(), description=record[13], genres=record[14].strip().splitlines(), + format=record[15], ) results.append(result) @@ -223,6 +226,7 @@ class ComicCacher: "image_url": series_record.image_url, "description": series_record.description, "genres": "\n".join(series_record.genres), + "format": series_record.format, "timestamp": timestamp, "aliases": "\n".join(series_record.aliases), } @@ -302,6 +306,7 @@ class ComicCacher: aliases=row[8].strip().splitlines(), description=row[9], genres=row[10].strip().splitlines(), + format=row[11], ) return result @@ -320,6 +325,7 @@ class ComicCacher: count_of_issues=None, count_of_volumes=None, volume=None, + format=None, ) con = lite.connect(self.db_file) with con: @@ -405,6 +411,7 @@ class ComicCacher: count_of_issues=None, count_of_volumes=None, volume=None, + format=None, ) # now process the results diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py index c7fa18c..debaa2e 100644 --- a/comictalker/resulttypes.py +++ b/comictalker/resulttypes.py @@ -23,6 +23,7 @@ class ComicSeries: publisher: str start_year: int | None genres: list[str] + format: str | None def copy(self) -> ComicSeries: return copy.deepcopy(self) diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index 4767e1c..180397d 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -45,9 +45,13 @@ def map_comic_issue_to_metadata( metadata.series = utils.xlate(issue_results.series.name) metadata.issue = IssueString(issue_results.issue_number).as_string() - # Rely on comic talker to valid this number + # Rely on comic talker to validate this number metadata.issue_count = utils.xlate_int(issue_results.series.volume) + # TODO Generate list in utils (same as language and countries) and check against that in separate PR + if issue_results.series.format: + metadata.format = issue_results.series.format + metadata.volume = utils.xlate_int(issue_results.series.volume) metadata.volume_count = utils.xlate_int(issue_results.series.count_of_volumes) diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py index 9476c0d..3225306 100644 --- a/comictalker/talkers/comicvine.py +++ b/comictalker/talkers/comicvine.py @@ -518,6 +518,7 @@ class ComicVineTalker(ComicTalker): publisher=pub_name, start_year=start_year, genres=[], + format=None, ) ) diff --git a/testing/comicdata.py b/testing/comicdata.py index 7698437..4baf139 100644 --- a/testing/comicdata.py +++ b/testing/comicdata.py @@ -17,6 +17,7 @@ search_results = [ start_year=0, aliases=[], genres=[], + format=None, ), comictalker.resulttypes.ComicSeries( count_of_issues=1, @@ -30,6 +31,7 @@ search_results = [ start_year=0, aliases=[], genres=[], + format=None, ), ] diff --git a/testing/comicvine.py b/testing/comicvine.py index 7cb9df5..a4efb53 100644 --- a/testing/comicvine.py +++ b/testing/comicvine.py @@ -179,6 +179,7 @@ comic_issue_result = ComicIssue( publisher=cv_volume_result["results"]["publisher"]["name"], start_year=int(cv_volume_result["results"]["start_year"]), genres=[], + format=None, ), characters=[], alt_image_urls=[], From d1917502314bbf2c90453707b72cc450035f0cf6 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Thu, 25 May 2023 01:32:52 +0100 Subject: [PATCH 11/13] Remove attempted validation of language and country plus minor changes --- comicapi/utils.py | 2 +- comictalker/comiccacher.py | 4 ++-- comictalker/talker_utils.py | 24 +++++------------------- comictalker/talkers/comicvine.py | 3 --- 4 files changed, 8 insertions(+), 25 deletions(-) diff --git a/comicapi/utils.py b/comicapi/utils.py index 9b2c2f5..650edce 100644 --- a/comicapi/utils.py +++ b/comicapi/utils.py @@ -239,7 +239,7 @@ def get_language_from_iso(iso: str | None) -> str | None: def get_language_iso(string: str | None) -> str | None: if string is None: return None - # Why return this if ISO failed? + # Return current string if all else fails lang = string.casefold() try: diff --git a/comictalker/comiccacher.py b/comictalker/comiccacher.py index 90e0931..c7c8a60 100644 --- a/comictalker/comiccacher.py +++ b/comictalker/comiccacher.py @@ -126,8 +126,8 @@ class ComicCacher: + "critical_rating FLOAT," + "manga TEXT," # Yes/YesAndRightToLeft/No + "maturity_rating TEXT," - + "language TEXT," # 2-letter iso code - + "country TEXT," # 2-letter iso code + + "language TEXT," + + "country TEXT," + "complete BOOL," # Is the data complete? Includes characters, locations, credits. + "PRIMARY KEY (id, source_name))" ) diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index 180397d..7da85bd 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -93,35 +93,21 @@ def map_comic_issue_to_metadata( metadata.story_arc = ", ".join(issue_results.story_arcs) if issue_results.genres: metadata.genre = ", ".join(issue_results.genres) + if issue_results.tags: metadata.tags = set(issue_results.tags) + if issue_results.manga: - if issue_results.manga in ["Yes", "yes", "No", "no", "YesAndRightToLeft", "yesandrighttoleft"]: - metadata.manga = issue_results.manga - else: - metadata.manga = "Unknown" + metadata.manga = issue_results.manga + if issue_results.critical_rating: metadata.critical_rating = utils.xlate_float(issue_results.critical_rating) - # 2-letter country code if issue_results.language: - if utils.get_language_from_iso(issue_results.language): - metadata.language = issue_results.language - else: - code = utils.get_language_iso(issue_results.language) - if code: - metadata.language = code + metadata.language = issue_results.language - # ISO country name if issue_results.country: - # Only CBI supports country currently in CT and is a string name - # TODO Use https://codeberg.org/plotski/countryguess as pycountry seems stale? - # Return current entry if conversion fails metadata.country = issue_results.country - if not utils.get_country_iso(issue_results.country): - code = utils.get_country_iso_name(issue_results.country) - if code: - metadata.country = code return metadata diff --git a/comictalker/talkers/comicvine.py b/comictalker/talkers/comicvine.py index 3225306..337828c 100644 --- a/comictalker/talkers/comicvine.py +++ b/comictalker/talkers/comicvine.py @@ -564,9 +564,6 @@ class ComicVineTalker(ComicTalker): series = self._fetch_series_data(record["volume"]["id"]) - # Make below an option? - # TODO Infer country by publisher (and language)? Check series desc for "Translated into English"? - formatted_results.append( ComicIssue( aliases=record["aliases"].split("\n") if record["aliases"] else [], From a395e5541fc828065ec10934f84b06549fed562f Mon Sep 17 00:00:00 2001 From: Mizaki Date: Thu, 25 May 2023 15:00:53 +0100 Subject: [PATCH 12/13] Remove invalid comments --- comictalker/resulttypes.py | 4 ++-- comictalker/talker_utils.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/comictalker/resulttypes.py b/comictalker/resulttypes.py index debaa2e..d7a1076 100644 --- a/comictalker/resulttypes.py +++ b/comictalker/resulttypes.py @@ -43,8 +43,8 @@ class ComicIssue: genres: list[str] tags: list[str] name: str - language: str # 2-letter iso code - country: str # 2-letter iso code + language: str + country: str site_detail_url: str series: ComicSeries alt_image_urls: list[str] diff --git a/comictalker/talker_utils.py b/comictalker/talker_utils.py index 7da85bd..92b32d6 100644 --- a/comictalker/talker_utils.py +++ b/comictalker/talker_utils.py @@ -48,7 +48,6 @@ def map_comic_issue_to_metadata( # Rely on comic talker to validate this number metadata.issue_count = utils.xlate_int(issue_results.series.volume) - # TODO Generate list in utils (same as language and countries) and check against that in separate PR if issue_results.series.format: metadata.format = issue_results.series.format From 3b92993ef6fb40e3d956056ea778a91dacfdd399 Mon Sep 17 00:00:00 2001 From: Mizaki Date: Sat, 3 Jun 2023 00:11:40 +0100 Subject: [PATCH 13/13] Remove country name code --- comicapi/utils.py | 29 ----------------------------- 1 file changed, 29 deletions(-) diff --git a/comicapi/utils.py b/comicapi/utils.py index 650edce..7499c9a 100644 --- a/comicapi/utils.py +++ b/comicapi/utils.py @@ -253,35 +253,6 @@ def get_country_from_iso(iso: str | None) -> str | None: return countries[iso] -def get_country_iso(string: str | None) -> str | None: - if string is None: - return None - - try: - country = pycountry.countries.lookup(string) - if country: - # Assume first entry is correct - return country[0].alpha_2 - except LookupError: - pass - return None - - -def get_country_iso_name(string: str | None) -> str | None: - """Return country's ISO name from "other" name""" - if string is None: - return None - - try: - country = pycountry.countries.search_fuzzy(string) - if country: - # Assume first entry is correct - return country[0].name - except LookupError: - pass - return None - - def get_publisher(publisher: str) -> tuple[str, str]: imprint = ""