from flask import g, current_app from flask_login import UserMixin from werkzeug.security import check_password_hash from io import BytesIO from wand.image import Image from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import create_engine from sqlalchemy import orm from sqlalchemy import Column, Integer, String, BLOB, Boolean, Table, MetaData, DateTime from sqlalchemy.orm import sessionmaker import sqlite3 import os import inspect from comicapi.issuestring import IssueString from scripts import tmdb RPI_DATABASE = "/var/lib/rpiWebApp/database.db" RPI_IMDB_DATABASE = "/var/lib/rpiWebApp/imdb.db" MC_DATABASE = "***REMOVED***" MC_IMDB_DATABASE = "C:\\Users\\Matthew\\Documents\\MyPrograms\\Websites\\rpi_web_interface\\imdb.db" DATABASE = RPI_DATABASE if os.path.exists(RPI_DATABASE) else MC_DATABASE IMDB_DATABASE = RPI_IMDB_DATABASE if os.path.exists(RPI_IMDB_DATABASE) else MC_IMDB_DATABASE engine = create_engine("sqlite:///"+DATABASE+"?check_same_thread=False") Session = sessionmaker(bind=engine) session = Session() Base = declarative_base() class Comic(Base): __tablename__ = "comics" path = Column(String, unique=True) tagOrigin = Column(String) series = Column(String) issue = Column(Integer) issueText = Column(String) title = Column(String) publisher = Column(String) month = Column(Integer) year = Column(Integer) day = Column(Integer) seriesYear = Column(Integer) issueCount = Column(Integer) volume = Column(String) genre = Column(String) language = Column(String) comments = Column(String) volumeCount = Column(Integer) criticalRating = Column(String) country = Column(String) alternateSeries = Column(String) alternateNumber = Column(String) alternateCount = Column(Integer) imprint = Column(String) notes = Column(String) webLink = Column(String) format = Column(String) manga = Column(String) blackAndWhite = Column(String) pageCount = Column(Integer) maturityRating = Column(String) storyArc = Column(String) seriesGroup = Column(String) scanInfo = Column(String) characters = Column(String) teams = Column(String) locations = Column(String) id = Column(Integer, primary_key=True, autoincrement=True) def __init__(self, data): i = 0 try: for column in self.__table__.columns: if column.name == "id": continue setattr(self, column.name, data[i]) i += 1 except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) +" "+ str(e)) #print(inspect.stack()[0][3], type(e), e) def __repr__(self): return "".format(series=self.series, issue=self.issueText) class ComicThumbnail(Base): __tablename__ = "comic_thumbnails" comic_id = Column(Integer) pageNumber = Column(Integer) image = Column(BLOB) type = Column(String) id = Column(Integer, primary_key=True, autoincrement=True) def __init__(self, data): i = 0 try: for column in self.__table__.columns: if column.name == "id": continue setattr(self, column.name, data[i]) i += 1 except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) +" "+ str(e)) #print(inspect.stack()[0][3], type(e), e) class Movie(Base): __tablename__ = "movies" path = Column(String) imdb_id = Column(String, primary_key=True) tmdb_id = Column(Integer) title = Column(String) year = Column(Integer) length = Column(Integer) description = Column(String) extended = Column(Boolean) directors_cut = Column(Boolean) poster_path = Column(String) backdrop_path = Column(String) def __init__(self, data): i = 0 try: for column in self.__table__.columns: setattr(self, column.name, data[i]) i += 1 except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) +" "+ str(e)) #print(inspect.stack()[0][3], type(e), e) class TvShow(Base): __tablename__ = "tv_shows" imdb_id = Column(String, primary_key=True) tmdb_id = Column(Integer) title = Column(String) year = Column(Integer) description = Column(String) poster_path = Column(String) path = Column(String) def __init__(self, data): i = 0 try: for column in self.__table__.columns: setattr(self, column.name, data[i]) i += 1 except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) class TvEpisode(Base): __tablename__ = "tv_episodes" imdb_id = Column(String, primary_key=True) parent_imdb_id = Column(String) tmdb_id = Column(Integer) title = Column(String) season = Column(Integer) episode = Column(Integer) description = Column(String) still_path = Column(String) path = Column(String) def __init__(self, data): i = 0 try: for column in self.__table__.columns: setattr(self, column.name, data[i]) i += 1 except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) class User(Base, UserMixin): __tablename__ = "users" username = Column(String, unique=True, primary_key=True) passwordHash = Column(String(128)) isAdmin = Column(Boolean, default=False) def __init__(self, data): i = 0 for column in self.__table__.columns: setattr(self, column.name, data[i]) i += 1 self.init_user_data() @orm.reconstructor def init_user_data(self): meta = MetaData() self.comics = Table( "comics", meta, Column("comic_id", Integer, primary_key=True), Column("viewed", Boolean, default=False), Column("date", DateTime) ) self.movies = Table( "movies", meta, Column("imdb_id", Integer, primary_key=True), Column("viewed", Boolean, default=False), Column("date", DateTime) ) self.tv_episodes = Table( "tv_episodes", meta, Column("imdb_id", Integer, primary_key=True), Column("viewed", Boolean, default=False), Column("date", DateTime) ) self.tv_shows = Table( "tv_shows", meta, Column("imdb_id", Integer, primary_key=True), Column("viewed", Boolean, default=False), Column("date", DateTime) ) user_engine = create_engine("sqlite:///" + "user_" + self.username + ".db" + "?check_same_thread=False") self.conn = engine.connect() meta.create_all(user_engine) def set_comic_viewed(self, comic_id, val=True): self.conn.execute(self.comics.insert().values(comic_id=comic_id, viewed=val)) def is_comic_viewed(self, comic_id): q = self.comics.select().where(self.comics.c.comic_id == comic_id) result = self.conn.execute(q).fetchone() if result: return result["viewed"] def set_movie_viewed(self, imdb_id, val=True): self.conn.execute(self.movies.insert().values(comic_id=imdb_id, viewed=val)) def is_movie_viewed(self, imdb_id): q = self.movies.select().where(self.movies.c.imdb_id == imdb_id) result = self.conn.execute(q).fetchone() if result: return result["viewed"] def set_tv_episode_viewed(self, imdb_id, val=True): self.conn.execute(self.tv_episodes.insert().values(comic_id=imdb_id, viewed=val)) def is_tv_episode_viewed(self, imdb_id): q = self.tv_episodes.select().where(self.tv_episodes.c.imdb_id == imdb_id) result = self.conn.execute(q).fetchone() if result: return result["viewed"] def set_tv_show_viewed(self, imdb_id, val=True): self.conn.execute(self.tv_shows.insert().values(comic_id=imdb_id, viewed=val)) def is_tv_show_viewed(self, imdb_id): q = self.tv_shows.select().where(self.tv_shows.c.imdb_id == imdb_id) result = self.conn.execute(q).fetchone() if result: return result["viewed"] def get_id(self): return self.username def check_password(self, password): result = check_password_hash(self.passwordHash, password) return result def get_db(): db = getattr(g, '_database', None) if db is None: db = g._database = sqlite3.connect(DATABASE) db.row_factory = sqlite3.Row return db def get_imdb(): db = getattr(g, '_imdb_database', None) if db is None: db = g._imdb_database = sqlite3.connect(IMDB_DATABASE) db.row_factory = sqlite3.Row return db def initialize_db(): get_db().execute("""CREATE TABLE IF NOT EXISTS "comics" ( "path" TEXT UNIQUE, "tagOrigin" TEXT, "series" TEXT, "issue" REAL, "issueText" TEXT, "title" TEXT, "publisher" TEXT, "month" INTEGER, "year" INTEGER, "day" INTEGER, "seriesYear" INTEGER, "issueCount" INTEGER, "volume" TEXT, "genre" TEXT, "language" TEXT, "comments" TEXT, "volumeCount" INTEGER, "criticalRating" TEXT, "country" TEXT, "alternateSeries" TEXT, "alternateNumber" TEXT, "alternateCount" INTEGER, "imprint" TEXT, "notes" TEXT, "webLink" TEXT, "format" TEXT, "manga" TEXT, "blackAndWhite" TEXT, "pageCount" INTEGER, "maturityRating" TEXT, "storyArc" TEXT, "seriesGroup" TEXT, "scanInfo" TEXT, "characters" TEXT, "teams" TEXT, "locations" TEXT, "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "credits" ( "id" INTEGER NOT NULL, "role" TEXT, "name" TEXT, "primary" INTEGER NOT NULL )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "comic_thumbnails" ( "comic_id" INTEGER, "pageNumber" INTEGER, "image" BLOB, "type" TEXT, "id" INTEGER PRIMARY KEY AUTOINCREMENT )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "users" ( "username" TEXT UNIQUE PRIMARY KEY, "passwordHash" VARCHAR(128), "isAdmin" INTEGER NOT NULL DEFAULT 0 )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "movies" ( "path" TEXT, "imdb_id" INTEGER PRIMARY KEY, "tmdb_id" INTEGER, "title" TEXT, "year" INTEGER, "length" INTEGER, "description" TEXT, "extended" INTEGER, "directors_cut" INTEGER, "poster_path" TEXT, "backdrop_path" TEXT )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "tv_shows" ( "imdb_id" TEXT PRIMARY KEY , "tmdb_id" INTEGER UNIQUE, "title" TEXT, "year" INTEGER, "description" TEXT, "poster_path" TEXT, "path" TEXT )""") get_db().execute("""CREATE TABLE IF NOT EXISTS "tv_episodes" ( "imdb_id" INTEGER PRIMARY KEY, "parent_imdb_id" INTEGER, "tmdb_id" INTEGER UNIQUE, "title" TEXT, "season" INTEGER, "episode" INTEGER, "description" TEXT, "still_path" TEXT, "path" TEXT )""") get_db().execute("CREATE INDEX IF NOT EXISTS path_index ON comics(path);") get_db().execute("CREATE INDEX IF NOT EXISTS comic_id_index ON comic_thumbnails(comic_id);") get_db().commit() get_imdb().execute("CREATE INDEX IF NOT EXISTS original_title_index ON title_basics(originalTitle)") get_imdb().execute("CREATE INDEX IF NOT EXISTS primary_title_index ON title_basics(primaryTitle)") get_imdb().execute("CREATE INDEX IF NOT EXISTS parent_tconst_index ON title_episode(parentTconst)") get_imdb().commit() def add_movies(movies): for movie_data in movies: movie = Movie(movie_data) session.add(movie) session.commit() def add_tv_shows(tv_show_data): try: tv_show = TvShow(tv_show_data) session.add(tv_show) session.commit() except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) def add_tv_episodes(episodes): for episode_data in episodes: try: episode = TvEpisode(episode_data) if not session.query(TvEpisode).filter(TvEpisode.tmdb_id == episode.tmdb_id).one_or_none(): session.add(episode) session.commit() except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) def add_comics(meta, thumbnails): data = [] for info in meta: issue = IssueString(info[1].issue).asFloat() data.append([info[0], info[1].tagOrigin, info[1].series, (issue or -1), info[1].issue, info[1].title, info[1].publisher, info[1].month, info[1].year, info[1].day, info[1].seriesYear, info[1].issueCount, info[1].volume, info[1].genre, info[1].language, info[1].comments, info[1].volumeCount, info[1].criticalRating, info[1].country, info[1].alternateSeries, info[1].alternateNumber, info[1].alternateCount, info[1].imprint, info[1].notes, info[1].webLink, info[1].format, info[1].manga, info[1].blackAndWhite, info[1].pageCount, info[1].maturityRating, info[1].storyArc, info[1].seriesGroup, info[1].scanInfo, info[1].characters, info[1].teams, info[1].locations]) for comic_data in data: for i in range(len(comic_data)): if comic_data[i] == "": comic_data[i] = None for comic_data, images in zip(data, thumbnails): comic = Comic(comic_data) session.add(comic) session.commit() comic_id = session.query(Comic.id).order_by(Comic.id.desc()).first()[0] for index in range(len(images)): thumbnail = ComicThumbnail([comic_id, index, images[index][0], images[index][1]]) session.add(thumbnail) session.commit() current_app.logger.info("{} comic{} added".format(len(meta), "s" if len(meta)>1 else "")) print("#"*18) print("# {} comic{} added #".format(len(meta), "s" if len(meta)>1 else "")) print("#"*18) def db_get_all_comics(): result = session.query(Comic).all() return result def db_get_series_by_publisher(publisher): result = session.query(Comic).filter(Comic.publisher == publisher).group_by(Comic.publisher, Comic.series, Comic.seriesYear).order_by(Comic.series, Comic.seriesYear).all() series = [comic.__dict__ for comic in result] return series def db_get_comics_in_series(series, publisher, series_year): result = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesYear == series_year)\ .order_by(Comic.issue).all() comics = [comic.__dict__ for comic in result] return comics def get_publishers(): result = session.query(Comic.publisher).distinct(Comic.publisher).order_by(Comic.publisher).all() publishers = [r for (r,) in result] return publishers def db_get_comic_by_id(comic_id): comic = session.query(Comic).filter(Comic.id == comic_id).one().__dict__ return comic def db_get_thumbnail_by_id_page(comic_id, pageNumber): thumbnail = session.query(ComicThumbnail).filter(ComicThumbnail.comic_id == comic_id, ComicThumbnail.pageNumber == pageNumber).one().__dict__ return thumbnail def db_get_comic(publisher, series, series_year, issue): comic = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesYear == series_year, Comic.issue == issue).one().__dict__ return comic def comic_path_in_db(path): try: result = session.query(Comic).filter(Comic.path == path).one_or_none() if result: return True except Exception as e: # print(path) current_app.logger.info(path) current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) return False def movie_path_in_db(path): try: result = session.query(Movie).filter(Movie.path == path).one_or_none() if result: return True except Exception as e: # print(path) current_app.logger.info(path) current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) return False def tv_show_path_in_db(path): try: result = session.query(TvShow).filter(TvShow.path == path).one_or_none() if result: return True except Exception as e: # print(path) current_app.logger.info(path) current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) return False def tv_episode_path_in_db(path): try: result = session.query(TvEpisode).filter(TvEpisode.path == path).one_or_none() if result: return True except Exception as e: # print(path) current_app.logger.info(path) current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) # print(inspect.stack()[0][3], type(e), e) return False def verify_paths(): rows = get_db().execute("SELECT path FROM comics").fetchall() get_db().commit() for row in rows: if not os.path.exists(row["path"]): get_db().execute("DELETE FROM comic_thumbnails WHERE id IN (SELECT id FROM comics WHERE path=?)", [row["path"]]) get_db().execute("DELETE FROM comics WHERE path=?", [row["path"]]) get_db().commit() def verify_path(path): if not os.path.exists(path): row = get_db().execute("SELECT path FROM comics WHERE path=?", [path]).fetchone() get_db().commit() if row: get_db().execute("DELETE FROM comic_thumbnails WHERE id IN (SELECT id FROM comics WHERE path=?)", [path]) get_db().execute("DELETE FROM comics WHERE path=?", [path]) get_db().commit() def imdb_get_movie(title, year): row = get_imdb().execute("SELECT tconst, runtimeMinutes FROM title_basics WHERE (originalTitle LIKE ? OR primaryTitle LIKE ?) AND (titleType LIKE '%movie' OR titleType='video') AND startYear=?", (title, title, year)).fetchone() return row def imdb_get_tv_show(title, year): row = get_imdb().execute( "SELECT tconst FROM title_basics WHERE (originalTitle LIKE ? OR primaryTitle LIKE ?) AND (titleType LIKE 'tvSeries' OR titleType LIKE 'tvMiniSeries') AND startYear=?", (title, title, year)).fetchone() return row def imdb_get_tv_episode(imdb_id, season, episode): row = get_imdb().execute( "SELECT tconst FROM title_episode WHERE parentTconst=? AND seasonNumber=? AND episodeNumber=?", [imdb_id, season, episode]).fetchone() return row def tmdb_get_movie_by_imdb_id(imdb_id): data = tmdb.get_movie_data(imdb_id) return data def tmdb_get_tv_show_by_imdb_id(imdb_id): data = tmdb.get_tv_show_data(imdb_id) return data def tmdb_get_tv_episode_by_imdb_id(imdb_id): data = tmdb.get_tv_episode_data(imdb_id) return data def db_get_all_movies(): movies = session.query(Movie).order_by(Movie.title, Movie.year).all() return movies def db_get_movie_by_imdb_id(imdb_id, extended=False, directors_cut=False): result = session.query(Movie).filter(Movie.imdb_id == imdb_id, Movie.extended == extended, Movie.directors_cut == directors_cut).one() return result def get_all_tv_shows(): result = session.query(TvShow).order_by(TvShow.title, TvShow.year).all() return result def get_tv_show_episodes_by_imdb_id(imdb_id): result = session.query(TvEpisode).filter(TvEpisode.parent_imdb_id == imdb_id).order_by(TvEpisode.season, TvEpisode.episode).all() return result def db_get_episode_by_imdb_id(imdb_id): result = session.query(TvEpisode).filter(TvEpisode.imdb_id == imdb_id).one() return result def db_search_table_columns_by_query(query, table, columns, group=[], order=[]): results = {} final_query = "%" + query.replace(" ", "%") + "%" for column in columns: results[column.name] = session.query(table).filter(column.like(final_query)).group_by(*group).order_by(*order).all() return results def db_search_comics(query): publishers = [] series = [] comics = [] results = db_search_table_columns_by_query(query, Comic, [Comic.publisher, Comic.title, Comic.series, Comic.year]) series_results = db_search_table_columns_by_query(query, Comic, [Comic.publisher, Comic.title, Comic.series, Comic.year], group=[Comic.series, Comic.seriesYear], order=[Comic.issue]) for row in results["publisher"]: if row["publisher"] not in publishers: publishers.append(row.publisher) for row in series_results["series"]: dict = row.__dict__ if dict not in series: series.append(dict) for row in results["title"]: dict = row.__dict__ if dict not in comics: comics.append(dict) for row in results["year"]: dict = row.__dict__ if dict not in comics: comics.append(dict) return {"publisher": publishers, "series": series, "comics": comics} def db_search_movies(query): results = db_search_table_columns_by_query(query, Movie, [Movie.title, Movie.year, Movie.description], order=[Movie.title]) movies = [] for movie in results["title"]: if movie.__dict__ not in movies: movies.append(movie.__dict__) for movie in results["description"]: if movie.__dict__ not in movies: movies.append(movie.__dict__) for movie in results["year"]: if movie.__dict__ not in movies: movies.append(movie.__dict__) return movies def db_search_tv_shows(query): results = db_search_table_columns_by_query(query, TvShow, [TvShow.title, TvShow.year, TvShow.description], order=[TvShow.title]) tv_shows = [] for show in results["title"]: if show.__dict__ not in tv_shows: tv_shows.append(show.__dict__) for show in results["description"]: if show.__dict__ not in tv_shows: tv_shows.append(show.__dict__) for show in results["year"]: if show.__dict__ not in tv_shows: tv_shows.append(show.__dict__) return tv_shows def resize_image(image, new_width=256, new_height=256): new_image = image orig_height = new_image.height orig_width = new_image.width if orig_height >= orig_width: width = int((orig_width / orig_height) * new_height) height = new_height else: height = int((orig_height / orig_width) * new_width) width = new_width new_image.thumbnail(width, height) return new_image def fix_thumbnails(): new_height = 256 new_width = 256 print("Start fix thumbnail size") rows = get_db().execute("SELECT * FROM comic_thumbnails") print("got list of all thumbnails\n") for row in rows: image = Image(file=BytesIO(row["image"])) if image.width > new_width or image.height > new_height: print("id:", row["id"], "pageNumber:", row["pageNumber"]) get_db().execute("UPDATE comic_thumbnails SET image=? WHERE comic_id=? AND pageNumber=?", [resize_image(image, new_width, new_height).make_blob(), row["id"], row["pageNumber"]]) get_db().commit() print("Finished fix thumbnail size") def get_user(username): result = session.query(User).filter(User.username == username).one() if result: return result return None