diff --git a/comictaggerlib/comicvinecacher.py b/comictaggerlib/comicvinecacher.py index 91b41bd..bb05f3c 100644 --- a/comictaggerlib/comicvinecacher.py +++ b/comictaggerlib/comicvinecacher.py @@ -110,10 +110,8 @@ class ComicVineCacher: "volume_id INT," + "name TEXT," + "issue_number TEXT," + - "image_url TEXT," + - "image_hash TEXT," + - "thumb_image_url TEXT," + - "thumb_image_hash TEXT," + + "super_url TEXT," + + "thumb_url TEXT," + "cover_date TEXT," + "site_detail_url TEXT," + "timestamp DATE DEFAULT (datetime('now','localtime')), " + @@ -262,18 +260,33 @@ class ComicVineCacher: } self.upsert( cur, "volumes", "id", cv_volume_record['id'], data) - # now add in issues - for issue in cv_volume_record['issues']: + def add_volume_issues_info( self, volume_id, cv_volume_issues ): + + con = lite.connect( self.db_file ) + + with con: + + cur = con.cursor() + + timestamp = datetime.datetime.now() + + # add in issues + + for issue in cv_volume_issues: data = { - "volume_id": cv_volume_record['id'], - "name": issue['name'], - "issue_number": issue['issue_number'], - "timestamp": timestamp + "volume_id": volume_id, + "name": issue['name'], + "issue_number": issue['issue_number'], + "site_detail_url": issue['site_detail_url'], + "cover_date": issue['cover_date'], + "super_url": issue['image']['super_url'], + "thumb_url": issue['image']['thumb_url'], + "timestamp": timestamp } self.upsert( cur, "issues" , "id", issue['id'], data) - + def get_volume_info( self, volume_id ): @@ -287,10 +300,6 @@ class ComicVineCacher: # purge stale volume info a_week_ago = datetime.datetime.today()-datetime.timedelta(days=7) cur.execute( "DELETE FROM Volumes WHERE timestamp < ?", [ str(a_week_ago) ] ) - - # purge stale issue info - probably issue data won't change much.... - a_month_ago = datetime.datetime.today()-datetime.timedelta(days=30) - cur.execute( "DELETE FROM Issues WHERE timestamp < ?", [ str(a_month_ago) ] ) # fetch cur.execute("SELECT id,name,publisher,count_of_issues,start_year FROM Volumes WHERE id = ?", [ volume_id ] ) @@ -310,23 +319,48 @@ class ComicVineCacher: result['count_of_issues'] = row[3] result['start_year'] = row[4] result['issues'] = list() + + return result - cur.execute("SELECT id,name,issue_number,image_url,image_hash FROM Issues WHERE volume_id = ?", [ volume_id ] ) + def get_volume_issues_info( self, volume_id ): + + result = None + + con = lite.connect( self.db_file ) + with con: + cur = con.cursor() + con.text_factory = unicode + + # purge stale issue info - probably issue data won't change much.... + a_week_ago = datetime.datetime.today()-datetime.timedelta(days=7) + cur.execute( "DELETE FROM Issues WHERE timestamp < ?", [ str(a_week_ago) ] ) + + # fetch + results = list() + + cur.execute("SELECT id,name,issue_number,site_detail_url,cover_date,super_url,thumb_url FROM Issues WHERE volume_id = ?", [ volume_id ] ) rows = cur.fetchall() # now process the results for row in rows: record = dict() - record['id'] = row[0] - record['name'] = row[1] - record['issue_number'] = row[2] - record['image_url'] = row[3] - record['image_hash'] = row[4] - result['issues'].append(record) + record['id'] = row[0] + record['name'] = row[1] + record['issue_number'] = row[2] + record['site_detail_url'] = row[3] + record['cover_date'] = row[4] + record['image'] = dict() + record['image']['super_url'] = row[5] + record['image']['thumb_url'] = row[6] + + results.append(record) - return result - + if len(results) == 0: + return None + + return results + def add_issue_select_details( self, issue_id, image_url, thumb_image_url, cover_date, site_detail_url ): @@ -338,8 +372,8 @@ class ComicVineCacher: timestamp = datetime.datetime.now() data = { - "image_url": image_url, - "thumb_image_url": thumb_image_url, + "super_url": image_url, + "thumb_url": thumb_image_url, "cover_date": cover_date, "site_detail_url": site_detail_url, "timestamp": timestamp @@ -355,7 +389,7 @@ class ComicVineCacher: cur = con.cursor() con.text_factory = unicode - cur.execute("SELECT image_url,thumb_image_url,cover_date,site_detail_url FROM Issues WHERE id=?", [ issue_id ]) + cur.execute("SELECT super_url,thumb_url,cover_date,site_detail_url FROM Issues WHERE id=?", [ issue_id ]) row = cur.fetchone() details = dict() diff --git a/comictaggerlib/comicvinetalker.py b/comictaggerlib/comicvinetalker.py index 3bb6058..27139a2 100644 --- a/comictaggerlib/comicvinetalker.py +++ b/comictaggerlib/comicvinetalker.py @@ -198,14 +198,66 @@ class ComicVineTalker(QObject): cvc.add_volume_info( volume_results ) return volume_results + + def fetchIssuesByVolume( self, series_id ): + + # before we search online, look in our cache, since we might already + # have this info + cvc = ComicVineCacher( ) + cached_volume_issues_result = cvc.get_volume_issues_info( series_id ) + + if cached_volume_issues_result is not None: + return cached_volume_issues_result + + #--------------------------------- + issues_url = self.api_base_url + "/issues/" + "?api_key=" + self.api_key + "&filter=volume:" + str(series_id) + "&field_list=id,volume_id,issue_number,name,image,cover_date,site_detail_url&format=json" + content = self.getUrlContent(issues_url) + cv_response = json.loads(content) + + if cv_response[ 'status_code' ] != 1: + print >> sys.stderr, "Comic Vine query failed with error: [{0}]. ".format( cv_response[ 'error' ] ) + return None + #------------------------------------ + + limit = cv_response['limit'] + current_result_count = cv_response['number_of_page_results'] + total_result_count = cv_response['number_of_total_results'] + #print "ATB total_result_count", total_result_count + + #print "ATB Found {0} of {1} results\n".format( cv_response['number_of_page_results'], cv_response['number_of_total_results']) + volume_issues_result = cv_response['results'] + page = 1 + offset = 0 + # see if we need to keep asking for more pages... + while ( current_result_count < total_result_count ): + #print "ATB getting another page of issue results {0} of {1}...\n".format( current_result_count, total_result_count) + page += 1 + offset += cv_response['number_of_page_results'] + + #print issues_url+ "&offset="+str(offset) + content = self.getUrlContent(issues_url + "&offset="+str(offset)) + cv_response = json.loads(content) + + if cv_response[ 'status_code' ] != 1: + self.writeLog( "Comic Vine query failed with error: [{0}]. \n".format( cv_response[ 'error' ] )) + return None + volume_issues_result.extend( cv_response['results']) + current_result_count += cv_response['number_of_page_results'] + + + cvc.add_volume_issues_info( series_id, volume_issues_result ) + + return volume_issues_result + def fetchIssueData( self, series_id, issue_number, settings ): volume_results = self.fetchVolumeData( series_id ) + issues_list_results = self.fetchIssuesByVolume( series_id ) found = False - for record in volume_results['issues']: + for record in issues_list_results: if IssueString(issue_number).asFloat() is None: issue_number = 1 if IssueString(record['issue_number']).asString().lower() == IssueString(issue_number).asString().lower(): diff --git a/comictaggerlib/issueidentifier.py b/comictaggerlib/issueidentifier.py index 03fd6c6..9de70a5 100644 --- a/comictaggerlib/issueidentifier.py +++ b/comictaggerlib/issueidentifier.py @@ -441,11 +441,12 @@ class IssueIdentifier: try: cv_series_results = comicVine.fetchVolumeData( series['id'] ) + issue_list = comicVine.fetchIssuesByVolume( series['id'] ) + except ComicVineTalkerException: self.log_msg( "Network issue while searching for series details. Aborting...") return [] - issue_list = cv_series_results['issues'] for issue in issue_list: num_s = IssueString(issue['issue_number']).asString() diff --git a/comictaggerlib/issueselectionwindow.py b/comictaggerlib/issueselectionwindow.py index 8ccf403..a4c18d4 100644 --- a/comictaggerlib/issueselectionwindow.py +++ b/comictaggerlib/issueselectionwindow.py @@ -20,6 +20,7 @@ limitations under the License. import sys import os +import re from PyQt4 import QtCore, QtGui, uic from PyQt4.QtCore import QUrl, pyqtSignal, QByteArray @@ -92,6 +93,7 @@ class IssueSelectionWindow(QtGui.QDialog): try: comicVine = ComicVineTalker( ) volume_data = comicVine.fetchVolumeData( self.series_id ) + self.issue_list = comicVine.fetchIssuesByVolume( self.series_id ) except ComicVineTalkerException: QtGui.QApplication.restoreOverrideCursor() QtGui.QMessageBox.critical(self, self.tr("Network Issue"), self.tr("Could not connect to ComicVine to list issues!")) @@ -99,8 +101,6 @@ class IssueSelectionWindow(QtGui.QDialog): while self.twList.rowCount() > 0: self.twList.removeRow(0) - - self.issue_list = volume_data['issues'] self.twList.setSortingEnabled(False) @@ -117,6 +117,10 @@ class IssueSelectionWindow(QtGui.QDialog): self.twList.setItem(row, 0, item) item_text = record['name'] + # ComicVine gives redundant info in the title. Strip out the + # volume name and issue number + item_text = re.sub( ".* #.+ - ", "", item_text ) + item = QtGui.QTableWidgetItem(item_text) item.setData( QtCore.Qt.ToolTipRole, item_text ) item.setFlags(QtCore.Qt.ItemIsSelectable| QtCore.Qt.ItemIsEnabled)