Compare commits

..

5 Commits

Author SHA1 Message Date
1d3d6e2741 bump version 1.1.32-rc1 2019-09-22 12:47:19 +01:00
c9724527b5 Fixed TLS version for the Comic Vine (#135)
* Fixed TLS version for the comicvine

* Fixed TLS version for the Comic Vine - Auto-Identify and Auto-Tag functions
2019-09-22 12:40:59 +01:00
2891209b4e bump version 2019-02-04 20:27:37 +01:00
5b87e19d3e Limit Comic Vine search result queries (#119)
* Tweaked search string based on new comic vine search behavior
Placated Beaufitul Soup by passing the parser

* Limit search results fetching after recent Comic Vine changes.
Also, minor debug comment tweaks.
2019-02-04 20:16:44 +01:00
tlc
674e24fc41 Enable Zip64 (#96) 2018-09-20 00:09:24 +02:00
5 changed files with 60 additions and 15 deletions

View File

@ -153,6 +153,7 @@ class ZipArchiver:
zf = zipfile.ZipFile(
self.path,
mode='a',
allowZip64=True,
compression=zipfile.ZIP_DEFLATED)
zf.writestr(archive_file, data)
zf.close()
@ -185,7 +186,7 @@ class ZipArchiver:
os.close(tmp_fd)
zin = zipfile.ZipFile(self.path, 'r')
zout = zipfile.ZipFile(tmp_name, 'w')
zout = zipfile.ZipFile(tmp_name, 'w', allowZip64=True)
for item in zin.infolist():
buffer = zin.read(item.filename)
if (item.filename not in exclude_list):
@ -270,7 +271,7 @@ class ZipArchiver:
"""Replace the current zip with one copied from another archive"""
try:
zout = zipfile.ZipFile(self.path, 'w')
zout = zipfile.ZipFile(self.path, 'w', allowZip64=True)
for fname in otherArchive.getArchiveFilenameList():
data = otherArchive.readArchiveFile(fname)
if data is not None:

View File

@ -123,7 +123,7 @@ def which(program):
def removearticles(text):
text = text.lower()
articles = ['and', 'a', '&', 'issue']
articles = ['and', 'a', '&', 'issue', 'the']
newText = ''
for word in text.split(' '):
if word not in articles:

View File

@ -105,7 +105,7 @@ class ComicVineTalker(QObject):
self.log_func = None
# always use a tls context for urlopen
self.ssl = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
self.ssl = ssl.SSLContext(ssl.PROTOCOL_TLS)
def setLogFunc(self, log_func):
self.log_func = log_func
@ -188,7 +188,7 @@ class ComicVineTalker(QObject):
# connect to server:
# if there is a 500 error, try a few more times before giving up
# any other error, just bail
# print "ATB---", url
#print("---", url)
for tries in range(3):
try:
resp = urllib.request.urlopen(url, context=self.ssl)
@ -226,10 +226,10 @@ class ComicVineTalker(QObject):
original_series_name = series_name
# Split and rejoin to remove extra internal spaces
# Split and rejoin to remove extra internal spaces
query_word_list = series_name.split()
query_string = " ".join( query_word_list ).strip()
#print "Query string = ", query_string
#print ("Query string = ", query_string)
query_string = urllib.parse.quote_plus(query_string.encode("utf-8"))
@ -246,6 +246,19 @@ class ComicVineTalker(QObject):
current_result_count = cv_response['number_of_page_results']
total_result_count = cv_response['number_of_total_results']
# 8 Dec 2018 - Comic Vine changed query results again. Terms are now
# ORed together, and we get thousands of results. Good news is the
# results are sorted by relevance, so we can be smart about halting
# the search.
# 1. Don't fetch more than some sane amount of pages.
max_results = 500
# 2. Halt when not all of our search terms are present in a result
# 3. Halt when the results contain more (plus threshold) words than
# our search
result_word_count_max = len(query_word_list) + 3
total_result_count = min(total_result_count, max_results)
if callback is None:
self.writeLog(
"Found {0} of {1} results\n".format(
@ -258,7 +271,29 @@ class ComicVineTalker(QObject):
callback(current_result_count, total_result_count)
# see if we need to keep asking for more pages...
stop_searching = False
while (current_result_count < total_result_count):
last_result = search_results[-1]['name']
# See if the last result's name has all the of the search terms.
# if not, break out of this, loop, we're done.
#print("Searching for {} in '{}'".format(query_word_list, last_result))
for term in query_word_list:
if term not in last_result.lower():
#print("Term '{}' not in last result. Halting search result fetching".format(term))
stop_searching = True
break
# Also, stop searching when the word count of last results is too much longer
# than our search terms list
if len(utils.removearticles(last_result).split()) > result_word_count_max:
#print("Last result '{}' is too long. Halting search result fetching".format(last_result))
stop_searching = True
if stop_searching:
break
if callback is None:
self.writeLog(
"getting another page of results {0} of {1}...\n".format(
@ -274,6 +309,15 @@ class ComicVineTalker(QObject):
if callback is not None:
callback(current_result_count, total_result_count)
# Remove any search results that don't contain all the search terms
# (iterate backwards for easy removal)
for i in range(len(search_results) - 1, -1, -1):
record = search_results[i]
for term in query_word_list:
if term not in record['name'].lower():
del search_results[i]
break
# for record in search_results:
#print(u"{0}: {1} ({2})".format(record['id'], record['name'] , record['start_year']))
# print(record)
@ -328,16 +372,16 @@ class ComicVineTalker(QObject):
limit = cv_response['limit']
current_result_count = cv_response['number_of_page_results']
total_result_count = cv_response['number_of_total_results']
# print "ATB total_result_count", total_result_count
#print("total_result_count", total_result_count)
#print("ATB Found {0} of {1} results".format(cv_response['number_of_page_results'], cv_response['number_of_total_results']))
#print("Found {0} of {1} results".format(cv_response['number_of_page_results'], cv_response['number_of_total_results']))
volume_issues_result = cv_response['results']
page = 1
offset = 0
# see if we need to keep asking for more pages...
while (current_result_count < total_result_count):
#print("ATB getting another page of issue results {0} of {1}...".format(current_result_count, total_result_count))
#print("getting another page of issue results {0} of {1}...".format(current_result_count, total_result_count))
page += 1
offset += cv_response['number_of_page_results']
@ -380,16 +424,16 @@ class ComicVineTalker(QObject):
limit = cv_response['limit']
current_result_count = cv_response['number_of_page_results']
total_result_count = cv_response['number_of_total_results']
# print "ATB total_result_count", total_result_count
#print("total_result_count", total_result_count)
#print("ATB Found {0} of {1} results\n".format(cv_response['number_of_page_results'], cv_response['number_of_total_results']))
#print("Found {0} of {1} results\n".format(cv_response['number_of_page_results'], cv_response['number_of_total_results']))
filtered_issues_result = cv_response['results']
page = 1
offset = 0
# see if we need to keep asking for more pages...
while (current_result_count < total_result_count):
#print("ATB getting another page of issue results {0} of {1}...\n".format(current_result_count, total_result_count))
#print("getting another page of issue results {0} of {1}...\n".format(current_result_count, total_result_count))
page += 1
offset += cv_response['number_of_page_results']

View File

@ -1,3 +1,3 @@
# This file should contain only these comments, and the line below.
# Used by packaging makefiles and app
version = "1.1.30-rc1"
version = "1.1.32-rc1"

View File

@ -67,7 +67,7 @@ class ImageFetcher(QObject):
self.create_image_db()
# always use a tls context for urlopen
self.ssl = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
self.ssl = ssl.SSLContext(ssl.PROTOCOL_TLS)
def clearCache(self):
os.unlink(self.db_file)