From 0041ad4b3b63a5033923abd003e74e3703d9ef6d Mon Sep 17 00:00:00 2001 From: Matthew Welch Date: Thu, 22 Aug 2019 10:43:14 -0700 Subject: [PATCH] Changed the database to use models. The database is now used through models instead of direct SQL queries. --- app.py | 9 +- comics/comics.py | 11 +- scripts/create_user.py | 1 + scripts/database.py | 439 ++++++++++++++++++----------- scripts/func.py | 24 +- scripts/tmdb.py | 119 ++++---- test.py | 65 +---- tv/templates/tv/episodeViewer.html | 24 +- tv/tv.py | 15 +- update_tables.sql | 28 ++ 10 files changed, 429 insertions(+), 306 deletions(-) create mode 100644 scripts/create_user.py create mode 100644 update_tables.sql diff --git a/app.py b/app.py index 72628ba..eacdc91 100644 --- a/app.py +++ b/app.py @@ -6,6 +6,7 @@ from flask_log import Logging import threading import logging import inotify.adapters, inotify.constants +import inspect import scripts.func as func from scripts import database @@ -95,14 +96,14 @@ def update_comic_db(sender, **kw): try: database.add_comics(kw["meta"], kw["thumbnails"]) except Exception as e: - print(e) + print(inspect.stack()[0][3], type(e), e) def update_movie_db(sender, **kw): try: database.add_movies(kw["movies"]) except Exception as e: - print(e) + print(inspect.stack()[0][3], type(e), e) def update_tv_show_db(sender, **kw): @@ -111,14 +112,14 @@ def update_tv_show_db(sender, **kw): if kw["tv_episodes"]: database.add_tv_episodes(kw["tv_episodes"]) except Exception as e: - print(e) + print(inspect.stack()[0][3], type(e), e) def update_tv_episodes_db(sender, **kw): try: database.add_tv_episodes(kw["tv_episodes"]) except Exception as e: - print(e) + print(inspect.stack()[0][3], type(e), e) func.comic_loaded.connect(update_comic_db) diff --git a/comics/comics.py b/comics/comics.py index 0cba804..3185653 100644 --- a/comics/comics.py +++ b/comics/comics.py @@ -5,6 +5,7 @@ from urllib import parse from io import BytesIO from wand.image import Image import os, pytz, datetime +import inspect from scripts import database, func @@ -24,7 +25,7 @@ def index(): end = len(publishers) if len(publishers) < max_items*page else max_items*page return render_template("comics/index.html", title="Comics", publishers=publishers, page=page, max_items=max_items, start=start, end=end, item_count=len(publishers)) except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return str(type(e)) + " " + str(e) @@ -67,7 +68,7 @@ def search(): publisher_start=publisher_start, series_start=series_start, comics_start=comics_start, publisher_end=publisher_end, series_end=series_end, comics_end=comics_end) except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return str(type(e))+" "+str(e) @@ -98,7 +99,7 @@ def comics_publisher(publisher): return render_template("comics/publisherSeriesView.html", title="Comics", publisher_series=publisher_series, start=start, end=end, page=page, max_items=max_items, item_count=len(publisher_series)) except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return str(type(e)) + " " + str(e) @@ -124,7 +125,7 @@ def comic_viewer(publisher, series, series_year, issue): title = "Comics: "+meta["series"]+": #"+meta["issueText"]+" "+(meta["title"] or "") return render_template("comics/comicView.html", title=title, on_mobile=on_mobile, prev_url=prev_url, next_url=next_url, comic=meta, page_number=page_number) except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return str(type(e)) + " " + str(e) @@ -137,7 +138,7 @@ def comic_gallery(publisher, series, series_year, issue): end = meta["pageCount"] if meta["pageCount"] < max_items*page else max_items*page return render_template("comics/comicGallery.html", title="Comics", comic=meta, start=start, end=end, page=page, max_items=max_items, item_count=meta["pageCount"]) except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return str(type(e)) + " " + str(e) diff --git a/scripts/create_user.py b/scripts/create_user.py new file mode 100644 index 0000000..98fd184 --- /dev/null +++ b/scripts/create_user.py @@ -0,0 +1 @@ +import click diff --git a/scripts/database.py b/scripts/database.py index 0f29371..0401781 100644 --- a/scripts/database.py +++ b/scripts/database.py @@ -5,8 +5,13 @@ from flask_login import UserMixin from werkzeug.security import generate_password_hash, check_password_hash from io import BytesIO from wand.image import Image +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import create_engine +from sqlalchemy import orm +from sqlalchemy import Column, Integer, String, BLOB, Boolean +from sqlalchemy.orm import sessionmaker import sqlite3 -import os, time +import os, time, inspect from comicapi.issuestring import IssueString @@ -22,6 +27,183 @@ DATABASE = RPI_DATABASE if os.path.exists(RPI_DATABASE) else MC_DATABASE IMDB_DATABASE = RPI_IMDB_DATABASE if os.path.exists(RPI_IMDB_DATABASE) else MC_IMDB_DATABASE +engine = create_engine("sqlite:///"+DATABASE+"?check_same_thread=False") +Session =sessionmaker(bind=engine) +session = Session() +Base = declarative_base() + + +class Comic(Base): + __tablename__ = "comics" + + path = Column(String, unique=True) + tagOrigin = Column(String) + series = Column(String) + issue = Column(Integer) + issueText = Column(String) + title = Column(String) + publisher = Column(String) + month = Column(Integer) + year = Column(Integer) + day = Column(Integer) + seriesYear = Column(Integer) + issueCount = Column(Integer) + volume = Column(String) + genre = Column(String) + language = Column(String) + comments = Column(String) + volumeCount = Column(Integer) + criticalRating = Column(String) + country = Column(String) + alternateSeries = Column(String) + alternateNumber = Column(String) + alternateCount = Column(Integer) + imprint = Column(String) + notes = Column(String) + webLink = Column(String) + format = Column(String) + manga = Column(String) + blackAndWhite = Column(String) + pageCount = Column(Integer) + maturityRating = Column(String) + storyArc = Column(String) + seriesGroup = Column(String) + scanInfo = Column(String) + characters = Column(String) + teams = Column(String) + locations = Column(String) + id = Column(Integer, primary_key=True, autoincrement=True) + + def __init__(self, data): + i = 0 + try: + for column in self.__table__.columns: + if column.name == "id": + continue + setattr(self, column.name, data[i]) + i += 1 + except Exception as e: + print(inspect.stack()[0][3], type(e), e) + + def __repr__(self): + return "".format(series=self.series, issue=self.issueText) + + +class ComicThumbnail(Base): + __tablename__ = "comic_thumbnails" + + comic_id = Column(Integer) + pageNumber = Column(Integer) + image = Column(BLOB) + type = Column(String) + id = Column(Integer, primary_key=True, autoincrement=True) + + def __init__(self, data): + i = 0 + try: + for column in self.__table__.columns: + if column.name == "id": + continue + setattr(self, column.name, data[i]) + i += 1 + except Exception as e: + print(inspect.stack()[0][3], type(e), e) + + +class Movie(Base): + __tablename__ = "movies" + + path = Column(String) + imdb_id = Column(String, primary_key=True) + tmdb_id = Column(Integer) + title = Column(String) + year = Column(Integer) + length = Column(Integer) + description = Column(String) + extended = Column(Boolean) + directors_cut = Column(Boolean) + poster_path = Column(String) + backdrop_path = Column(String) + + def __init__(self, data): + i = 0 + try: + for column in self.__table__.columns: + setattr(self, column.name, data[i]) + i += 1 + except Exception as e: + print(inspect.stack()[0][3], type(e), e) + + +class TvShow(Base): + __tablename__ = "tv_shows" + + imdb_id = Column(String, primary_key=True) + tmdb_id = Column(Integer) + title = Column(String) + year = Column(Integer) + description = Column(String) + poster_path = Column(String) + path = Column(String) + + def __init__(self, data): + i = 0 + try: + for column in self.__table__.columns: + setattr(self, column.name, data[i]) + i += 1 + except Exception as e: + print(inspect.stack()[0][3], type(e), e) + + +class TvEpisode(Base): + __tablename__ = "tv_episodes" + + imdb_id = Column(String, primary_key=True) + parent_imdb_id = Column(String) + tmdb_id = Column(Integer) + title = Column(String) + season = Column(Integer) + episode = Column(Integer) + description = Column(String) + still_path = Column(String) + path = Column(String) + + def __init__(self, data): + i = 0 + try: + for column in self.__table__.columns: + setattr(self, column.name, data[i]) + i += 1 + except Exception as e: + print(inspect.stack()[0][3], type(e), e) + + +class User(Base, UserMixin): + __tablename__ = "users" + + username = Column(String, unique=True, primary_key=True) + passwordHash = Column(String(128)) + isAdmin = Column(Boolean, default=False) + + def __init__(self, data): + i = 0 + for column in self.__table__.columns: + setattr(self, column.name, data[i]) + i += 1 + + @orm.reconstructor + def get_user_data(self): + pass + + def get_id(self): + return self.username + + def check_password(self, password): + result = check_password_hash(self.passwordHash, password) + return result + + def get_db(): db = getattr(g, '_database', None) if db is None: @@ -87,10 +269,11 @@ def initialize_db(): "primary" INTEGER NOT NULL )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "comic_thumbnails" ( - "id" INTEGER, + "comic_id" INTEGER, "pageNumber" INTEGER, "image" BLOB, - "type" TEXT + "type" TEXT, + "id" INTEGER PRIMARY KEY AUTOINCREMENT )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "users" ( "username" TEXT UNIQUE PRIMARY KEY, @@ -99,7 +282,7 @@ def initialize_db(): )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "movies" ( "path" TEXT, - "imdb_id" INTEGER, + "imdb_id" INTEGER PRIMARY KEY, "tmdb_id" INTEGER, "title" TEXT, "year" INTEGER, @@ -111,7 +294,7 @@ def initialize_db(): "backdrop_path" TEXT )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "tv_shows" ( - "imdb_id" TEXT UNIQUE, + "imdb_id" TEXT PRIMARY KEY , "tmdb_id" INTEGER UNIQUE, "title" TEXT, "year" INTEGER, @@ -120,7 +303,7 @@ def initialize_db(): "path" TEXT )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "tv_episodes" ( - "imdb_id" INTEGER UNIQUE, + "imdb_id" INTEGER PRIMARY KEY, "parent_imdb_id" INTEGER, "tmdb_id" INTEGER UNIQUE, "title" TEXT, @@ -131,7 +314,7 @@ def initialize_db(): "path" TEXT )""") get_db().execute("CREATE INDEX IF NOT EXISTS path_index ON comics(path);") - get_db().execute("CREATE INDEX IF NOT EXISTS id_index ON comic_thumbnails(id);") + get_db().execute("CREATE INDEX IF NOT EXISTS comic_id_index ON comic_thumbnails(comic_id);") get_db().commit() get_imdb().execute("CREATE INDEX IF NOT EXISTS original_title_index ON title_basics(originalTitle)") @@ -140,36 +323,31 @@ def initialize_db(): get_imdb().commit() -def table_exists(table_name): - cursor = get_db().execute("SELECT name FROM sqlite_master WHERE type='table' AND name='{}'".format(table_name)) - get_db().commit() - if cursor.rowcount == 0: - return False - return True - - def add_movies(movies): - for movie in movies: - get_db().execute("INSERT INTO movies(path, imdb_id, tmdb_id, title, year, length, description, extended, directors_cut, poster_path, backdrop_path) VALUES(?,?,?,?,?,?,?,?,?,?,?)", - (movie[0], movie[1], movie[2], movie[3], movie[4], movie[5], movie[6], movie[7], movie[8], movie[9], movie[10])) - get_db().commit() + for movie_data in movies: + movie = Movie(movie_data) + session.add(movie) + session.commit() -def add_tv_shows(tv_show): +def add_tv_shows(tv_show_data): try: - get_db().execute("INSERT INTO tv_shows(imdb_id, tmdb_id, title, year, description, poster_path, path) VALUES(?,?,?,?,?,?,?)", tv_show) - get_db().commit() + tv_show = TvShow(tv_show_data) + session.add(tv_show) + session.commit() except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) def add_tv_episodes(episodes): - for episode in episodes: + for episode_data in episodes: try: - get_db().execute("INSERT INTO tv_episodes(imdb_id, parent_imdb_id, tmdb_id, title, season, episode, description, still_path, path) VALUES(?,?,?,?,?,?,?,?,?)", episode) - get_db().commit() + episode = TvEpisode(episode_data) + if not session.query(TvEpisode).filter(TvEpisode.tmdb_id == episode.tmdb_id).one_or_none(): + session.add(episode) + session.commit() except Exception as e: - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) def add_comics(meta, thumbnails): @@ -183,137 +361,107 @@ def add_comics(meta, thumbnails): info[1].notes, info[1].webLink, info[1].format, info[1].manga, info[1].blackAndWhite, info[1].pageCount, info[1].maturityRating, info[1].storyArc, info[1].seriesGroup, info[1].scanInfo, info[1].characters, info[1].teams, info[1].locations]) - for comic in data: - for i in range(len(comic)): - if comic[i] == "": - comic[i] = None - for comic, images in zip(data, thumbnails): - get_db().execute("""INSERT INTO comics(path, tagOrigin, series, issue, issueText, title, publisher, - month, year, day, seriesYear, issueCount, volume, genre, language, comments, volumeCount, - criticalRating, country, alternateSeries, alternateNumber, alternateCount, imprint, - notes, webLink, format, manga, blackAndWhite, pageCount, maturityRating, storyArc, - seriesGroup, scanInfo, characters, teams, locations) - VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)""", comic) - get_db().commit() - comic_id = get_db().execute("SELECT id from comics ORDER BY id DESC LIMIT 1").fetchone()[0] + for comic_data in data: + for i in range(len(comic_data)): + if comic_data[i] == "": + comic_data[i] = None + for comic_data, images in zip(data, thumbnails): + comic = Comic(comic_data) + session.add(comic) + session.commit() + comic_id = session.query(Comic.id).order_by(Comic.id.desc()).first()[0] for index in range(len(images)): - get_db().execute("INSERT INTO comic_thumbnails(id, pageNumber, image, type) VALUES (?,?,?,?)", (comic_id, index, images[index][0], images[index][1])) - get_db().commit() + thumbnail = ComicThumbnail([comic_id, index, images[index][0], images[index][1]]) + session.add(thumbnail) + session.commit() print("#"*18) print("# {} comic{} added #".format(len(meta), "s" if len(meta)>1 else "")) print("#"*18) -def add_comic_thumbnails(thumbnails): - comic_id = 1 - row = get_db().execute("SELECT id from comic_thumbnails ORDER BY id DESC LIMIT 1").fetchone() - if row: - comic_id = row[0]+1 - print("start comics added:", len(thumbnails)) - for images in thumbnails: - print("pages in comic:", len(images)) - for index in range(len(images)): - print("comic page added:", index) - get_db().execute("INSERT INTO comic_thumbnails(id, pageNumber, image) VALUES (?,?,?)", (comic_id, index, images[index])) - comic_id += 1 - get_db().commit() - - def db_get_all_comics(): - result = get_db().execute("SELECT * FROM comics ORDER BY series, issue").fetchall() - get_db().commit() + result = session.query(Comic).all() return result def db_get_series_by_publisher(publisher): - series = [] - rows = get_db().execute("SELECT publisher, series, seriesYear, id, min(issue) FROM comics WHERE publisher=? GROUP BY publisher, series, seriesYear ORDER BY series, seriesYear", [publisher]).fetchall() - for row in rows: - series.append(row) + result = session.query(Comic).filter(Comic.publisher == publisher).group_by(Comic.publisher, Comic.series, Comic.seriesYear).order_by(Comic.series, Comic.seriesYear).all() + series = [comic.__dict__ for comic in result] return series def db_get_comics_in_series(series, publisher, series_year): - comics = [] - rows = get_db().execute("SELECT * FROM comics WHERE series=? AND publisher=? AND seriesYear=? ORDER BY series, issue", [series, publisher, series_year]).fetchall() - get_db().commit() - for row in rows: - comics.append(row) + result = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesYear == series_year)\ + .order_by(Comic.issue).all() + comics = [comic.__dict__ for comic in result] return comics def get_publishers(): - publishers = [] - rows = get_db().execute("SELECT DISTINCT publisher FROM comics ORDER BY publisher").fetchall() - - for row in rows: - publishers.append(row[0]) + result = session.query(Comic.publisher).distinct(Comic.publisher).order_by(Comic.publisher).all() + publishers = [r for (r,) in result] return publishers def db_get_comic_by_id(comic_id): - row = get_db().execute("SELECT * FROM comics WHERE id=?", [comic_id]).fetchone() - return row + comic = session.query(Comic).filter(Comic.id == comic_id).one().__dict__ + return comic def db_get_thumbnail_by_id_page(comic_id, pageNumber): - row = get_db().execute("SELECT * FROM comic_thumbnails WHERE id=? AND pageNumber=?", [comic_id, pageNumber]).fetchone() - return row + thumbnail = session.query(ComicThumbnail).filter(ComicThumbnail.comic_id == comic_id, ComicThumbnail.pageNumber == pageNumber).one().__dict__ + return thumbnail def db_get_comic(publisher, series, series_year, issue): - row = get_db().execute("SELECT * FROM comics WHERE publisher=? AND series=? AND seriesYear=? AND issue=?", - [publisher, series, series_year, issue]).fetchone() - return row + comic = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesYear == series_year, Comic.issue == issue).one().__dict__ + return comic def comic_path_in_db(path): try: - result = get_db().execute("SELECT path FROM comics WHERE path=?", [path]) - get_db().commit() - if result.fetchone(): + result = session.query(Comic).filter(Comic.path == path).one_or_none() + if result: return True except Exception as e: print(path) - print(e) + print(inspect.stack()[0][3], type(e), e) return False def movie_path_in_db(path): try: - result = get_db().execute("SELECT path FROM movies WHERE path=?", [path]) - get_db().commit() - if result.fetchone(): + result = session.query(Movie).filter(Movie.path == path).one_or_none() + if result: return True except Exception as e: print(path) - print(e) + print(inspect.stack()[0][3], type(e), e) return False def tv_show_path_in_db(path): try: - result = get_db().execute("SELECT path FROM tv_shows WHERE path=?", [path]).fetchone() + result = session.query(TvShow).filter(TvShow.path == path).one_or_none() if result: return True except Exception as e: print(path) - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return False def tv_episode_path_in_db(path): try: - result = get_db().execute("SELECT path FROM tv_episodes WHERE path=?", [path]).fetchone() + result = session.query(TvEpisode).filter(TvEpisode.path == path).one_or_none() if result: return True except Exception as e: print(path) - print(type(e), e) + print(inspect.stack()[0][3], type(e), e) return False - def verify_paths(): rows = get_db().execute("SELECT path FROM comics").fetchall() get_db().commit() @@ -369,47 +517,36 @@ def tmdb_get_tv_episode_by_imdb_id(imdb_id): def db_get_all_movies(): - rows = get_db().execute("SELECT * FROM movies ORDER BY title, year;").fetchall() - return rows + movies = session.query(Movie).all() + return movies -def db_get_movie_by_imdb_id(imdb_id, extended=0, directors_cut=0): - row = get_db().execute("SELECT * FROM movies WHERE imdb_id=? AND extended=? AND directors_cut=?", [imdb_id, extended, directors_cut]).fetchone() - return row +def db_get_movie_by_imdb_id(imdb_id, extended=False, directors_cut=False): + result = session.query(Movie).filter(Movie.imdb_id == imdb_id, Movie.extended == extended, + Movie.directors_cut == directors_cut).one() + return result def get_all_tv_shows(): - rows = get_db().execute("SELECT * FROM tv_shows ORDER BY title, year;").fetchall() - return rows + result = session.query(TvShow).order_by(TvShow.title, TvShow.year).all() + return result def get_tv_show_episodes_by_imdb_id(imdb_id): - rows = get_db().execute("SELECT * FROM tv_episodes WHERE parent_imdb_id=? ORDER BY season, episode", [imdb_id]).fetchall() - return rows + result = session.query(TvEpisode).filter(TvEpisode.parent_imdb_id == imdb_id).order_by(TvEpisode.season, TvEpisode.episode).all() + return result def db_get_episode_by_imdb_id(imdb_id): - row = get_db().execute("SELECT * FROM tv_episodes WHERE imdb_id=?", [imdb_id]).fetchone() - return row + result = session.query(TvEpisode).filter(TvEpisode.imdb_id == imdb_id).one() + return result -def db_search_table_columns_by_query(query, table, columns, group="", order=""): +def db_search_table_columns_by_query(query, table, columns, group=[], order=[]): results = {} - final_query = "%"+query.replace(" ", "%")+"%" - sqlite_base_statement = "SELECT * FROM "+table+" WHERE {condition} {group} {order}" - if not group == "": - group = "GROUP BY "+group - if not order == "": - order = "ORDER BY "+order + final_query = "%" + query.replace(" ", "%") + "%" for column in columns: - sqlite_statement = sqlite_base_statement.format(condition=column+" LIKE '"+final_query+"'", group=group, order=order) - results[column] = get_db().execute(sqlite_statement).fetchall() - - # sqlite_condition = "" - # for column in columns: - # sqlite_condition += column+" LIKE '"+final_query+"'"+(" OR " if column != columns[-1] else "") - # sqlite_statement = "SELECT * FROM {table} WHERE {condition}".format(table=table, condition=sqlite_condition) - # rows = get_db().execute(sqlite_statement).fetchall() + results[column.name] = session.query(table).filter(column.like(final_query)).group_by(*group).order_by(*order).all() return results @@ -418,23 +555,23 @@ def db_search_comics(query): series = [] comics = [] - results = db_search_table_columns_by_query(query, "comics", ["publisher", "title", "series", "year"]) - series_results = db_search_table_columns_by_query(query, "comics", ["publisher", "title", "series", "year"], - group="series, seriesYear", order="issue") + results = db_search_table_columns_by_query(query, Comic, [Comic.publisher, Comic.title, Comic.series, Comic.year]) + series_results = db_search_table_columns_by_query(query, Comic, [Comic.publisher, Comic.title, Comic.series, Comic.year], + group=[Comic.series, Comic.seriesYear], order=[Comic.issue]) for row in results["publisher"]: if row["publisher"] not in publishers: - publishers.append(row["publisher"]) + publishers.append(row.publisher) for row in series_results["series"]: - dict = {"publisher": row["publisher"],"series": row["series"],"seriesYear": row["seriesYear"],"id": row["id"]} + dict = row.__dict__ if dict not in series: series.append(dict) for row in results["title"]: - dict = {"publisher": row["publisher"],"series": row["series"],"seriesYear": row["seriesYear"],"issue": row["issue"],"id": row["id"],"title": row["title"]} + dict = row.__dict__ if dict not in comics: comics.append(dict) for row in results["year"]: - dict = {"publisher": row["publisher"],"series": row["series"],"seriesYear": row["seriesYear"],"issue": row["issue"],"id": row["id"],"title": row["title"]} + dict = row.__dict__ if dict not in comics: comics.append(dict) @@ -442,32 +579,32 @@ def db_search_comics(query): def db_search_movies(query): - results = db_search_table_columns_by_query(query, "movies", ["title", "year", "description"], order="title") + results = db_search_table_columns_by_query(query, Movie, [Movie.title, Movie.year, Movie.description], order=[Movie.title]) movies = [] for movie in results["title"]: - if movie not in movies: - movies.append(movie) + if movie.__dict__ not in movies: + movies.append(movie.__dict__) for movie in results["description"]: - if movie not in movies: - movies.append(movie) + if movie.__dict__ not in movies: + movies.append(movie.__dict__) for movie in results["year"]: - if movie not in movies: - movies.append(movie) + if movie.__dict__ not in movies: + movies.append(movie.__dict__) return movies def db_search_tv_shows(query): - results = db_search_table_columns_by_query(query, "tv_shows", ["title", "year", "description"], order="title") + results = db_search_table_columns_by_query(query, TvShow, [TvShow.title, TvShow.year, TvShow.description], order=[TvShow.title]) tv_shows = [] for show in results["title"]: - if show not in tv_shows: - tv_shows.append(show) + if show.__dict__ not in tv_shows: + tv_shows.append(show.__dict__) for show in results["description"]: - if show not in tv_shows: - tv_shows.append(show) + if show.__dict__ not in tv_shows: + tv_shows.append(show.__dict__) for show in results["year"]: - if show not in tv_shows: - tv_shows.append(show) + if show.__dict__ not in tv_shows: + tv_shows.append(show.__dict__) return tv_shows @@ -496,28 +633,14 @@ def fix_thumbnails(): image = Image(file=BytesIO(row["image"])) if image.width > new_width or image.height > new_height: print("id:", row["id"], "pageNumber:", row["pageNumber"]) - get_db().execute("UPDATE comic_thumbnails SET image=? WHERE id=? AND pageNumber=?", + get_db().execute("UPDATE comic_thumbnails SET image=? WHERE comic_id=? AND pageNumber=?", [resize_image(image, new_width, new_height).make_blob(), row["id"], row["pageNumber"]]) get_db().commit() print("Finished fix thumbnail size") def get_user(username): - user_data = get_db().execute("SELECT * FROM users WHERE username=?", [username]).fetchone() - if user_data: - return User(user_data) - return None - - -class User(UserMixin): - def __init__(self, user): - self.username = user["username"] - self.password_hash = user["passwordHash"] - self.is_admin = user["isAdmin"] - - def get_id(self): - return self.username - - def check_password(self, password): - result = check_password_hash(self.password_hash, password) + result = session.query(User).filter(User.username == username).one() + if result: return result + return None diff --git a/scripts/func.py b/scripts/func.py index c511214..3ea7e63 100644 --- a/scripts/func.py +++ b/scripts/func.py @@ -142,14 +142,14 @@ def get_movies(): title = match.group(1) print("movie title:", title) year = int(match.group(2)) - extended = 0 - directors_cut = 0 + extended = False + directors_cut = False if match.group(3): - extended = 1 + extended = True imdb_data = database.imdb_get_movie(title.replace(match.group(3), ""), year) elif match.group(4): imdb_data = database.imdb_get_movie(title.replace(match.group(4), ""), year) - directors_cut = 1 + directors_cut = True else: imdb_data = database.imdb_get_movie(title, year) if not imdb_data: @@ -215,30 +215,30 @@ def get_tv_episodes(): rows = database.get_all_tv_shows() for tv_show in rows: episodes = [] - for video in sorted(os.listdir(tv_show["path"])): + for video in sorted(os.listdir(tv_show.path)): video_match = re.fullmatch(video_pattern, video) if video_match: - path = tv_show["path"] + video + path = tv_show.path + video if not database.tv_episode_path_in_db(path): season = int(video_match.group(1)) episode = int(video_match.group(2)) episode_name = video_match.group(3) - episode_imdb_data = database.imdb_get_tv_episode(tv_show["imdb_id"], season, episode) + episode_imdb_data = database.imdb_get_tv_episode(tv_show.imdb_id, season, episode) if not episode_imdb_data: - print("could not get imdb data for:", tv_show["title"], tv_show["year"], season, episode) + print("could not get imdb data for:", tv_show.title, tv_show.year, season, episode) continue episode_imdb_id = episode_imdb_data["tconst"] episode_tmdb_data = database.tmdb_get_tv_episode_by_imdb_id(episode_imdb_id) if not episode_tmdb_data: - print("could not get tmdb data for:", tv_show["title"], tv_show["year"], season, episode) + print("could not get tmdb data for:", tv_show.title, tv_show.year, season, episode) with open("/var/lib/rpiWebApp/log.txt", "a") as f: - f.write("could not get tmdb data for: " + episode_imdb_id + " " + tv_show["title"] + " " + str( - tv_show["year"]) + " " + str(season) + " " + str(episode) + "\n") + f.write("could not get tmdb data for: " + episode_imdb_id + " " + tv_show.title + " " + str( + tv_show.year) + " " + str(season) + " " + str(episode) + "\n") continue episode_tmdb_id = episode_tmdb_data[0] episode_description = episode_tmdb_data[1] episode_still_path = episode_tmdb_data[2] - episodes.append((episode_imdb_id, tv_show["imdb_id"], episode_tmdb_id, episode_name, season, episode, + episodes.append((episode_imdb_id, tv_show.imdb_id, episode_tmdb_id, episode_name, season, episode, episode_description, episode_still_path, path)) tv_episodes_loaded.send("anonymous", tv_episodes=episodes) print("finish load tv episodes.") diff --git a/scripts/tmdb.py b/scripts/tmdb.py index 0d5ffb8..a216740 100644 --- a/scripts/tmdb.py +++ b/scripts/tmdb.py @@ -1,5 +1,6 @@ import requests import logging +import inspect logging.getLogger("requests").setLevel(logging.WARNING) logging.getLogger("urllib3").setLevel(logging.WARNING) @@ -10,65 +11,77 @@ TMDB_IMG_URL = "https://image.tmdb.org/t/p/original" def get_movie_data(imdb_id): - data = { - "api_key": API_KEY, - "language": "en-US", - "external_source": "imdb_id" - } - r = requests.get(TMDB_FIND_URL+imdb_id, data=data) - info = dict(r.json()) - if "status_code" in info.keys(): - print("error getting tmdb data, status code:", info["status_code"]) - return None - print("tmdb movie title:", info["movie_results"][0]["title"]) - movie_id = info["movie_results"][0]["id"] - overview = info["movie_results"][0]["overview"] - poster_path = info["movie_results"][0]["poster_path"] - backdrop_path = info["movie_results"][0]["backdrop_path"] + try: + data = { + "api_key": API_KEY, + "language": "en-US", + "external_source": "imdb_id" + } + r = requests.get(TMDB_FIND_URL+imdb_id, data=data) + info = dict(r.json()) + if "status_code" in info.keys(): + print("error getting tmdb data, status code:", info["status_code"]) + return None + if info["movie_results"] == []: + print("no tmdb results for:", imdb_id) + return None + print("tmdb movie title:", info["movie_results"][0]["title"]) + movie_id = info["movie_results"][0]["id"] + overview = info["movie_results"][0]["overview"] + poster_path = info["movie_results"][0]["poster_path"] + backdrop_path = info["movie_results"][0]["backdrop_path"] - return movie_id, overview, poster_path, backdrop_path + return movie_id, overview, poster_path, backdrop_path + except Exception as e: + print(inspect.stack()[0][3], type(e), e) def get_tv_show_data(imdb_id): - data = { - "api_key": API_KEY, - "language": "en-US", - "external_source": "imdb_id" - } - r = requests.get(TMDB_FIND_URL+imdb_id, data=data) - info = dict(r.json()) - if "status_code" in info.keys(): - print("error getting tmdb data, status code:", info["status_code"]) - return None - if info["tv_results"] == []: - print("no tmdb results for:", imdb_id) - return None - print("tmdb movie title:", info["tv_results"][0]["name"]) - tv_show_id = info["tv_results"][0]["id"] - overview = info["tv_results"][0]["overview"] - poster_path = info["tv_results"][0]["poster_path"] + try: + data = { + "api_key": API_KEY, + "language": "en-US", + "external_source": "imdb_id" + } + r = requests.get(TMDB_FIND_URL+imdb_id, data=data) + info = dict(r.json()) + if "status_code" in info.keys(): + print("error getting tmdb data, status code:", info["status_code"]) + return None + if info["tv_results"] == []: + print("no tmdb results for:", imdb_id) + return None + print("tmdb tv show title:", info["tv_results"][0]["name"]) + tv_show_id = info["tv_results"][0]["id"] + overview = info["tv_results"][0]["overview"] + poster_path = info["tv_results"][0]["poster_path"] - return tv_show_id, overview, poster_path + return tv_show_id, overview, poster_path + except Exception as e: + print(inspect.stack()[0][3], type(e), e) def get_tv_episode_data(imdb_id): - data = { - "api_key": API_KEY, - "language": "en-US", - "external_source": "imdb_id" - } - r = requests.get(TMDB_FIND_URL+imdb_id, data=data) - info = dict(r.json()) - if "status_code" in info.keys(): - print("error getting tmdb data, status code:", info["status_code"]) - return None - if info["tv_episode_results"] == []: - print("no tmdb results for:", imdb_id) - return None - print("tmdb movie title:", info["tv_episode_results"][0]["name"]) - tv_episode_id = info["tv_episode_results"][0]["id"] - name = info["tv_episode_results"][0]["name"] - overview = info["tv_episode_results"][0]["overview"] - still_path = info["tv_episode_results"][0]["still_path"] + try: + data = { + "api_key": API_KEY, + "language": "en-US", + "external_source": "imdb_id" + } + r = requests.get(TMDB_FIND_URL+imdb_id, data=data) + info = dict(r.json()) + if "status_code" in info.keys(): + print("error getting tmdb data, status code:", info["status_code"]) + return None + if info["tv_episode_results"] == []: + print("no tmdb results for:", imdb_id) + return None + print("tmdb tv episode title:", info["tv_episode_results"][0]["name"]) + tv_episode_id = info["tv_episode_results"][0]["id"] + name = info["tv_episode_results"][0]["name"] + overview = info["tv_episode_results"][0]["overview"] + still_path = info["tv_episode_results"][0]["still_path"] - return tv_episode_id, overview, still_path + return tv_episode_id, overview, still_path + except Exception as e: + print(inspect.stack()[0][3], type(e), e) diff --git a/test.py b/test.py index fbd4aa0..25c0872 100644 --- a/test.py +++ b/test.py @@ -2,6 +2,7 @@ import threading import time import os import sqlite3 +import inspect from comicapi import comicarchive from comicapi.issuestring import IssueString from urllib import parse @@ -9,12 +10,21 @@ from io import BytesIO from PIL import Image import datetime, pytz from werkzeug.security import generate_password_hash +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy import create_engine +from sqlalchemy import Column, Integer, String, BLOB, Boolean +from sqlalchemy.orm import sessionmaker from scripts import database db = sqlite3.connect(database.DATABASE) db.row_factory = sqlite3.Row +engine = create_engine("sqlite:///"+database.DATABASE+"?check_same_thread=False") +Session =sessionmaker(bind=engine) +session = Session() +Base = declarative_base() + DC_Comics = ["DC", "Young Animal", "WildStorm", "Earth-M", "Vertigo", "Sandman Universe", "DC Black Label", "Wonder Comics", "DC Ink", "DC Zoom", "Mad", "All Star", "Amalgam Comics", "DC Focus", "Elseworlds", "First Wave", "Helix", "Impact Comics", "Johnny DC", "Minx", "Paradox Press", "Piranha Press", "Tangent Comics", "WildStorm Productions", "America's Best Comics", "Cliffhanger", "CMX Manga", "Homage Comics", "WildStorm Signature", "Zuda Comics"] @@ -23,58 +33,3 @@ Marvel = ["Aircel Comics", "alibu Comics‎", "Atlas Comics", "Atlas", "CrossGen def get_db(): return db - - -def db_search_table_columns_by_query(query, table, columns, group="", order=""): - results = {} - final_query = "%"+query.replace(" ", "%")+"%" - sqlite_base_statement = "SELECT * FROM "+table+" WHERE {condition} {group} {order}" - if not group == "": - group = "GROUP BY "+group - if not order == "": - order = "ORDER BY "+order - for column in columns: - sqlite_statement = sqlite_base_statement.format(condition=column+" LIKE '"+final_query+"'", group=group, order=order) - results[column] = get_db().execute(sqlite_statement).fetchall() - - # sqlite_condition = "" - # for column in columns: - # sqlite_condition += column+" LIKE '"+final_query+"'"+(" OR " if column != columns[-1] else "") - # sqlite_statement = "SELECT * FROM {table} WHERE {condition}".format(table=table, condition=sqlite_condition) - # rows = get_db().execute(sqlite_statement).fetchall() - return results - - -def db_search_comics(query): - publishers = [] - series = [] - comics = [] - - results = db_search_table_columns_by_query(query, "comics", ["publisher", "title", "series", "year"]) - series_results = db_search_table_columns_by_query(query, "comics", ["publisher", "title", "series", "year"], - group="series, seriesYear", order="issue") - - for row in results["publisher"]: - if row["publisher"] not in publishers: - publishers.append(row["publisher"]) - for row in series_results["series"]: - dict = {"publisher": row["publisher"],"series": row["series"],"seriesYear": row["seriesYear"],"id": row["id"]} - if dict not in series: - series.append(dict) - for row in results["title"]: - dict = {"publisher": row["publisher"],"series": row["series"],"seriesYear": row["seriesYear"],"issue": row["issue"],"id": row["id"],"title": row["title"]} - if dict not in comics: - comics.append(dict) - for row in results["year"]: - dict = {"publisher": row["publisher"],"series": row["series"],"seriesYear": row["seriesYear"],"issue": row["issue"],"id": row["id"],"title": row["title"]} - if dict not in comics: - comics.append(dict) - - return {"publishers": publishers, "series": series, "comics": comics} - - -results = db_search_comics("lord") - - -for key in results.keys(): - print(key, results[key]) diff --git a/tv/templates/tv/episodeViewer.html b/tv/templates/tv/episodeViewer.html index b63bc51..af0324e 100644 --- a/tv/templates/tv/episodeViewer.html +++ b/tv/templates/tv/episodeViewer.html @@ -2,18 +2,18 @@ {% block content %} {% for episode in episodes %} - {% if episode["season"] == season_num and episode["episode"] == episode_num %} + {% if episode.season == season_num and episode.episode == episode_num %}
-

{{ episode["title"] }}

-

{{ episode["description"] }}

+

{{ episode.title }}

+

{{ episode.description }}

{% endif %} {% endfor %} @@ -23,11 +23,11 @@
@@ -35,9 +35,9 @@