From 1d2a28032b6a329bb37d6a4577d2171d35aae18c Mon Sep 17 00:00:00 2001 From: Matthew Welch Date: Fri, 27 Mar 2020 17:08:03 -0700 Subject: [PATCH] moved database from sqlite to postgresql --- movies/templates/movies/moviesList.html | 2 +- rpiWebApp.py | 11 +- scripts/database.py | 353 +++++++++++------------- scripts/fix_thumbnail_size.py | 2 +- scripts/func.py | 41 ++- scripts/imdb_import.py | 4 +- scripts/tmdb.py | 25 +- test.sql | 20 -- tv/templates/tv/episodeViewer.html | 4 +- tv/tv.py | 27 +- 10 files changed, 233 insertions(+), 256 deletions(-) delete mode 100644 test.sql diff --git a/movies/templates/movies/moviesList.html b/movies/templates/movies/moviesList.html index e387b69..cfba6b4 100644 --- a/movies/templates/movies/moviesList.html +++ b/movies/templates/movies/moviesList.html @@ -4,7 +4,7 @@
- {{ movies[i].title }} ({{ movies[i].year }}) + {{ movies[i].title }} ({{ movies[i].year }}) {% if movies[i].extended %}Extended Edition{% endif %}{% if movies[i].directors_cut %}Director's Cut{% endif %}
diff --git a/rpiWebApp.py b/rpiWebApp.py index f78a12e..79566bc 100644 --- a/rpiWebApp.py +++ b/rpiWebApp.py @@ -60,15 +60,12 @@ MOBILE_DEVICES = ["android", "blackberry", "ipad", "iphone"] def get_comics(): with app.app_context(): func.get_comics() - database.verify_paths() i = inotify.adapters.InotifyTree(func.COMICS_DIRECTORY) for event in i.event_gen(yield_nones=False): (header, type_names, path, filename) = event file_path = path+"/"+filename if "IN_CLOSE_WRITE" in type_names: func.get_comic(file_path) - if "IN_DELETE" in type_names: - database.verify_path(file_path) def get_movies(): @@ -85,11 +82,17 @@ def get_tv_shows(): with app.app_context(): func.get_tv_shows() func.get_tv_episodes() + i = inotify.adapters.InotifyTree(func.TV_SHOWS_DIRECTORY) + for event in i.event_gen(yield_nones=False): + (header, type_names, path, filename) = event + if "IN_CLOSE_WRITE" in type_names: + func.get_tv_shows() + func.get_tv_episodes() with app.app_context(): current_app.logger.info("server start") - database.initialize_db() + #database.initialize_db() thread = threading.Thread(target=get_comics, args=()) thread.daemon = True thread.start() diff --git a/scripts/database.py b/scripts/database.py index e4d97d7..e016075 100644 --- a/scripts/database.py +++ b/scripts/database.py @@ -6,30 +6,37 @@ from io import BytesIO from wand.image import Image from sqlalchemy.ext.declarative import declarative_base from sqlalchemy import create_engine -from sqlalchemy import orm -from sqlalchemy import Column, Integer, String, BLOB, Boolean, Table, MetaData, DateTime +from sqlalchemy.exc import IntegrityError +from sqlalchemy import Column, Integer, String, BLOB, Boolean, DateTime, Numeric, func, over from sqlalchemy.orm import sessionmaker, scoped_session -from sqlalchemy.pool import QueuePool, NullPool +from sqlalchemy.pool import NullPool +from sqlalchemy.sql.expression import cast +import sqlalchemy import sqlite3 +import psycopg2 import os import inspect import logging +import datetime from comicapi.issuestring import IssueString from scripts import tmdb -RPI_DATABASE = "/usb/storage/rpiWebApp/database.db" -RPI_IMDB_DATABASE = "/usb/storage/rpiWebApp/imdb.db" +RPI_DATABASE = "/var/lib/rpiWebApp/database.db" +RPI_IMDB_DATABASE = "/var/lib/rpiWebApp/imdb.db" +RPI_USER_DATABASE = "/var/lib/rpiWebApp/users.db" -MC_DATABASE = "***REMOVED***" -MC_IMDB_DATABASE = "C:\\Users\\Matthew\\Documents\\MyPrograms\\Websites\\rpi_web_interface\\imdb.db" +MC_DATABASE = "***REMOVED***database.db" +MC_IMDB_DATABASE = "***REMOVED***imdb.db" +MC_USER_DATABASE = "***REMOVED***users.db" DATABASE = RPI_DATABASE if os.path.exists(RPI_DATABASE) else MC_DATABASE IMDB_DATABASE = RPI_IMDB_DATABASE if os.path.exists(RPI_IMDB_DATABASE) else MC_IMDB_DATABASE +USER_DATABASE = RPI_USER_DATABASE if os.path.exists(RPI_USER_DATABASE) else MC_USER_DATABASE -engine = create_engine("sqlite:///"+DATABASE+"?check_same_thread=False", poolclass=NullPool) +engine = create_engine("***REMOVED***", poolclass=NullPool) logging.getLogger('sqlalchemy.engine').setLevel(logging.INFO) session_factory = sessionmaker(bind=engine) Session = scoped_session(session_factory) @@ -41,38 +48,38 @@ class Comic(Base): __tablename__ = "comics" path = Column(String, unique=True) - tagOrigin = Column(String) + tagorigin = Column(String) series = Column(String) issue = Column(Integer) - issueText = Column(String) + issuetext = Column(String) title = Column(String) publisher = Column(String) month = Column(Integer) year = Column(Integer) day = Column(Integer) - seriesYear = Column(Integer) - issueCount = Column(Integer) + seriesyear = Column(Integer) + issuecount = Column(Integer) volume = Column(String) genre = Column(String) language = Column(String) comments = Column(String) - volumeCount = Column(Integer) - criticalRating = Column(String) + volumecount = Column(Integer) + criticalrating = Column(String) country = Column(String) - alternateSeries = Column(String) - alternateNumber = Column(String) - alternateCount = Column(Integer) + alternateseries = Column(String) + alternatenumber = Column(String) + alternatecount = Column(Integer) imprint = Column(String) notes = Column(String) - webLink = Column(String) + weblink = Column(String) format = Column(String) manga = Column(String) - blackAndWhite = Column(String) - pageCount = Column(Integer) - maturityRating = Column(String) - storyArc = Column(String) - seriesGroup = Column(String) - scanInfo = Column(String) + blackandwhite = Column(String) + pagecount = Column(Integer) + maturityrating = Column(String) + storyarc = Column(String) + seriesgroup = Column(String) + scaninfo = Column(String) characters = Column(String) teams = Column(String) locations = Column(String) @@ -90,14 +97,14 @@ class Comic(Base): current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) +" "+ str(e)) def __repr__(self): - return "".format(series=self.series, issue=self.issueText) + return "".format(series=self.series, issue=self.issuetext) class ComicThumbnail(Base): __tablename__ = "comic_thumbnails" comic_id = Column(Integer) - pageNumber = Column(Integer) + pagenumber = Column(Integer) image = Column(BLOB) type = Column(String) id = Column(Integer, primary_key=True, autoincrement=True) @@ -117,8 +124,8 @@ class ComicThumbnail(Base): class Movie(Base): __tablename__ = "movies" - path = Column(String) - imdb_id = Column(String, primary_key=True) + path = Column(String, primary_key=True) + imdb_id = Column(String) tmdb_id = Column(Integer) title = Column(String) year = Column(Integer) @@ -183,10 +190,68 @@ class TvEpisode(Base): current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) +class DupComic(Comic): + __tablename__ = "duplicate_comics" + + path = Column(String, unique=True) + tagorigin = Column(String) + series = Column(String) + issue = Column(Integer) + issuetext = Column(String) + title = Column(String) + publisher = Column(String) + month = Column(Integer) + year = Column(Integer) + day = Column(Integer) + seriesyear = Column(Integer) + issuecount = Column(Integer) + volume = Column(String) + genre = Column(String) + language = Column(String) + comments = Column(String) + volumecount = Column(Integer) + criticalrating = Column(String) + country = Column(String) + alternateseries = Column(String) + alternatenumber = Column(String) + alternatecount = Column(Integer) + imprint = Column(String) + notes = Column(String) + weblink = Column(String) + format = Column(String) + manga = Column(String) + blackandwhite = Column(String) + pagecount = Column(Integer) + maturityrating = Column(String) + storyarc = Column(String) + seriesgroup = Column(String) + scaninfo = Column(String) + characters = Column(String) + teams = Column(String) + locations = Column(String) + id = Column(Integer) + pk = Column(Integer, primary_key=True, autoincrement=True) + + __mapper_args__ = { + "concrete": True + } + + def __init__(self, data): + i = 0 + try: + for column in self.__table__.columns: + if column.name == "id" or column.name == "pk": + continue + setattr(self, column.name, data[i]) + i += 1 + except Exception as e: + current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) + + class User(Base, UserMixin): __tablename__ = "users" - id = Column(String, primary_key=True) + id = Column(Numeric, primary_key=True) username = Column(String) email = Column(String, unique=True) passwordHash = Column(String(128)) @@ -218,10 +283,11 @@ class UserTvMovieData(Base): id = Column(Integer, primary_key=True, autoincrement=True) user = Column(String) imdb_id = Column(String) - parent_imdb_id = Column(String) + parent_imdb = Column(String) time = Column(Integer) length = Column(Integer) finished = Column(Boolean, default=False) + time_stamp = Column(DateTime) def __init__(self, data): i = 0 @@ -256,9 +322,9 @@ class UserTvMovieData(Base): def get_db(): db = getattr(g, '_database', None) if db is None: - db = g._database = sqlite3.connect(DATABASE) + db = g._database = psycopg2.connect(host="bombur", database="rpiwebapp", user="rpiwebapp", password="hello").cursor() - db.row_factory = sqlite3.Row + #db.row_factory = sqlite3.Row return db @@ -271,118 +337,6 @@ def get_imdb(): return db -def initialize_db(): - get_db().execute("""CREATE TABLE IF NOT EXISTS "comics" ( - "path" TEXT UNIQUE, - "tagOrigin" TEXT, - "series" TEXT, - "issue" REAL, - "issueText" TEXT, - "title" TEXT, - "publisher" TEXT, - "month" INTEGER, - "year" INTEGER, - "day" INTEGER, - "seriesYear" INTEGER, - "issueCount" INTEGER, - "volume" TEXT, - "genre" TEXT, - "language" TEXT, - "comments" TEXT, - "volumeCount" INTEGER, - "criticalRating" TEXT, - "country" TEXT, - "alternateSeries" TEXT, - "alternateNumber" TEXT, - "alternateCount" INTEGER, - "imprint" TEXT, - "notes" TEXT, - "webLink" TEXT, - "format" TEXT, - "manga" TEXT, - "blackAndWhite" TEXT, - "pageCount" INTEGER, - "maturityRating" TEXT, - "storyArc" TEXT, - "seriesGroup" TEXT, - "scanInfo" TEXT, - "characters" TEXT, - "teams" TEXT, - "locations" TEXT, - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "credits" ( - "id" INTEGER NOT NULL, - "role" TEXT, - "name" TEXT, - "primary" INTEGER NOT NULL - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "comic_thumbnails" ( - "comic_id" INTEGER, - "pageNumber" INTEGER, - "image" BLOB, - "type" TEXT, - "id" INTEGER PRIMARY KEY AUTOINCREMENT - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "users" ( - "id" TEXT PRIMARY KEY, - "username" TEXT, - "email" TEXT UNIQUE, - "passwordHash" VARCHAR(128), - "isAdmin" INTEGER NOT NULL DEFAULT 0 - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "movies" ( - "path" TEXT, - "imdb_id" INTEGER PRIMARY KEY, - "tmdb_id" INTEGER, - "title" TEXT, - "year" INTEGER, - "length" INTEGER, - "description" TEXT, - "extended" INTEGER, - "directors_cut" INTEGER, - "poster_path" TEXT, - "backdrop_path" TEXT - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "tv_shows" ( - "imdb_id" TEXT PRIMARY KEY , - "tmdb_id" INTEGER UNIQUE, - "title" TEXT, - "year" INTEGER, - "description" TEXT, - "poster_path" TEXT, - "path" TEXT - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "tv_episodes" ( - "imdb_id" INTEGER PRIMARY KEY, - "parent_imdb_id" INTEGER, - "tmdb_id" INTEGER UNIQUE, - "title" TEXT, - "season" INTEGER, - "episode" INTEGER, - "description" TEXT, - "still_path" TEXT, - "path" TEXT - )""") - get_db().execute("""CREATE TABLE IF NOT EXISTS "user_tv_movie_data" ( - "id" INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, - "user" TEXT, - "imdb_id" TEXT, - "parent_imdb_id" TEXT, - "time" INTEGER, - "length" INTEGER, - "finished" INTEGER DEFAULT 0 - )""") - get_db().execute("CREATE INDEX IF NOT EXISTS path_index ON comics(path);") - get_db().execute("CREATE INDEX IF NOT EXISTS comic_id_index ON comic_thumbnails(comic_id);") - get_db().commit() - - get_imdb().execute("CREATE INDEX IF NOT EXISTS original_title_index ON title_basics(originalTitle)") - get_imdb().execute("CREATE INDEX IF NOT EXISTS primary_title_index ON title_basics(primaryTitle)") - get_imdb().execute("CREATE INDEX IF NOT EXISTS parent_tconst_index ON title_episode(parentTconst)") - get_imdb().commit() - - def update_user_tv_movie_data(imdb_id, parent_id, time, length, finished=False): session = Session() email = current_user.email @@ -390,12 +344,13 @@ def update_user_tv_movie_data(imdb_id, parent_id, time, length, finished=False): if user_data: user_data.time = time user_data.finished = finished + user_data.time_stamp = datetime.datetime.now() if not user_data.length > 0 and length > 0: user_data.length = length session.commit() return user_data else: - data = UserTvMovieData((email, imdb_id, parent_id, time, length, finished)) + data = UserTvMovieData((email, imdb_id, parent_id, time, length, finished, datetime.datetime.now())) session.add(data) session.commit() return data @@ -455,19 +410,29 @@ def add_comics(meta, thumbnails): for i in range(len(comic_data)): if comic_data[i] == "": comic_data[i] = None + dup_comics = [] for comic_data, images in zip(data, thumbnails): - comic = Comic(comic_data) - session.add(comic) - session.commit() - comic_id = session.query(Comic.id).order_by(Comic.id.desc()).first()[0] - for index in range(len(images)): - thumbnail = ComicThumbnail([comic_id, index, images[index][0], images[index][1]]) - session.add(thumbnail) - session.commit() - current_app.logger.info("{} comic{} added".format(len(meta), "s" if len(meta)>1 else "")) - print("#"*18) - print("# {} comic{} added #".format(len(meta), "s" if len(meta)>1 else "")) - print("#"*18) + try: + comic = Comic(comic_data) + session.add(comic) + session.commit() + comic_id = session.query(Comic.id).order_by(Comic.id.desc()).first()[0] + for index in range(len(images)): + thumbnail = ComicThumbnail([comic_id, index, images[index][0], images[index][1]]) + session.add(thumbnail) + session.commit() + except IntegrityError as e: + session.rollback() + dup_comics.append(comic_data) + for dup in dup_comics: + try: + dup_comic = DupComic(dup) + session.add(dup_comic) + session.commit() + except IntegrityError as e: + current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e)) + session.rollback() + current_app.logger.info("{} comic{} added".format(len(meta), "s" if len(meta)>1 or len(meta)<1 else "")) def db_get_all_comics(): @@ -478,14 +443,14 @@ def db_get_all_comics(): def db_get_series_by_publisher(publisher): session = Session() - result = session.query(Comic).filter(Comic.publisher == publisher).group_by(Comic.publisher, Comic.series, Comic.seriesYear).order_by(Comic.series, Comic.seriesYear).all() + result = session.query(Comic).filter(Comic.publisher == publisher).group_by(Comic.publisher, Comic.series, Comic.seriesyear).order_by(Comic.series, Comic.seriesyear).all() series = result return series def db_get_comics_in_series(series, publisher, series_year): session = Session() - result = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesYear == series_year)\ + result = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesyear == series_year)\ .order_by(Comic.issue).all() comics = result return comics @@ -506,13 +471,13 @@ def db_get_comic_by_id(comic_id): def db_get_thumbnail_by_id_page(comic_id, pageNumber): session = Session() - thumbnail = session.query(ComicThumbnail).filter(ComicThumbnail.comic_id == comic_id, ComicThumbnail.pageNumber == pageNumber).one_or_none() + thumbnail = session.query(ComicThumbnail).filter(ComicThumbnail.comic_id == comic_id, ComicThumbnail.pagenumber == pageNumber).one_or_none() return thumbnail def db_get_comic(publisher, series, series_year, issue): session = Session() - comic = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesYear == series_year, Comic.issue == issue).one_or_none() + comic = session.query(Comic).filter(Comic.publisher == publisher, Comic.series == series, Comic.seriesyear == series_year, Comic.issue == issue).one_or_none() return comic @@ -520,7 +485,8 @@ def comic_path_in_db(path): try: session = Session() result = session.query(Comic).filter(Comic.path == path).one_or_none() - if result: + result2 = session.query(DupComic).filter(DupComic.path == path).one_or_none() + if result or result2: return True except Exception as e: current_app.logger.info(path) @@ -564,26 +530,6 @@ def tv_episode_path_in_db(path): return False -def verify_paths(): - rows = get_db().execute("SELECT path FROM comics").fetchall() - get_db().commit() - for row in rows: - if not os.path.exists(row["path"]): - get_db().execute("DELETE FROM comic_thumbnails WHERE id IN (SELECT id FROM comics WHERE path=?)", [row["path"]]) - get_db().execute("DELETE FROM comics WHERE path=?", [row["path"]]) - get_db().commit() - - -def verify_path(path): - if not os.path.exists(path): - row = get_db().execute("SELECT path FROM comics WHERE path=?", [path]).fetchone() - get_db().commit() - if row: - get_db().execute("DELETE FROM comic_thumbnails WHERE id IN (SELECT id FROM comics WHERE path=?)", [path]) - get_db().execute("DELETE FROM comics WHERE path=?", [path]) - get_db().commit() - - def imdb_get_movie(title, year): row = get_imdb().execute("SELECT tconst, runtimeMinutes FROM title_basics WHERE (originalTitle LIKE ? OR primaryTitle LIKE ?) AND (titleType LIKE '%movie' OR titleType='video') AND startYear=?", (title, title, year)).fetchone() return row @@ -660,16 +606,49 @@ def db_get_user_tv_show_episodes_data(parent_imdb_id) -> list: session = Session() email = current_user.email result = session.query(UserTvMovieData).filter(UserTvMovieData.user == email, - UserTvMovieData.parent_imdb_id == parent_imdb_id).all() + UserTvMovieData.parent_imdb == parent_imdb_id).all() return result +def db_get_current_tv_show_episode_and_data(parent_imdb_id, episodes): + session = Session() + email = current_user.email + result = session.query(UserTvMovieData).filter(UserTvMovieData.user == email, + UserTvMovieData.parent_imdb == parent_imdb_id).all() + if not result: + return episodes[0], None + most_recent_data = result[0] + most_recent_episode = episodes[0] + for episode_data in result: + if episode_data.time_stamp and most_recent_data.time_stamp: + if episode_data.time_stamp > most_recent_data.time_stamp: + most_recent_data = episode_data + for episode in episodes: + if episode.imdb_id == most_recent_data.imdb_id: + if most_recent_data.finished: + if episode == episodes[-1]: + most_recent_episode = episodes[0] + most_recent_data = None + break + most_recent_episode = episodes[episodes.index(episode)+1] + for episode_data in result: + if most_recent_episode.imdb_id == episode_data.imdb_id: + most_recent_data = episode_data + break + else: + most_recent_data = None + break + most_recent_episode = episode + break + return most_recent_episode, most_recent_data + + def db_search_table_columns_by_query(query, table, columns, group=[], order=[]): session = Session() results = {} final_query = "%" + query.replace(" ", "%") + "%" for column in columns: - results[column.name] = session.query(table).filter(column.like(final_query)).group_by(*group).order_by(*order).all() + results[column.name] = [i[0] for i in session.query(table, over(func.rank(), partition_by=group, order_by=order)).filter(cast(column, sqlalchemy.String).ilike(final_query)).all()] return results @@ -680,7 +659,7 @@ def db_search_comics(query): results = db_search_table_columns_by_query(query, Comic, [Comic.publisher, Comic.title, Comic.series, Comic.year]) series_results = db_search_table_columns_by_query(query, Comic, [Comic.publisher, Comic.title, Comic.series, Comic.year], - group=[Comic.series, Comic.seriesYear], order=[Comic.issue]) + group=[Comic.series, Comic.seriesyear], order=[Comic.issue]) for row in results["publisher"]: if row["publisher"] not in publishers: publishers.append(row.publisher) @@ -745,14 +724,14 @@ def fix_thumbnails(): new_height = 256 new_width = 256 print("Start fix thumbnail size") - rows = get_db().execute("SELECT * FROM comic_thumbnails") + rows = get_db().execute("SELECT * FROM rpiwebapp.comic_thumbnails") print("got list of all thumbnails\n") for row in rows: image = Image(file=BytesIO(row["image"])) if image.width > new_width or image.height > new_height: print("id:", row["id"], "pageNumber:", row["pageNumber"]) - get_db().execute("UPDATE comic_thumbnails SET image=? WHERE comic_id=? AND pageNumber=?", + get_db().execute("UPDATE rpiwebapp.comic_thumbnails SET image=? WHERE comic_id=? AND pageNumber=?", [resize_image(image, new_width, new_height).make_blob(), row["id"], row["pageNumber"]]) get_db().commit() print("Finished fix thumbnail size") diff --git a/scripts/fix_thumbnail_size.py b/scripts/fix_thumbnail_size.py index f39d3b7..8947674 100644 --- a/scripts/fix_thumbnail_size.py +++ b/scripts/fix_thumbnail_size.py @@ -3,7 +3,7 @@ from wand.image import Image import sqlite3, os -RPI_DATABASE = "/usb/storage/rpiWebApp/database.db" +RPI_DATABASE = "/var/lib/rpiWebApp/database.db" MC_DATABASE = "***REMOVED***" diff --git a/scripts/func.py b/scripts/func.py index 18ac66d..51a164b 100644 --- a/scripts/func.py +++ b/scripts/func.py @@ -1,11 +1,10 @@ from flask import current_app -from flask import g from comicapi import comicarchive from blinker import Namespace from io import BytesIO from wand.image import Image -import os, sys, re +import os, re import inspect from scripts import database @@ -133,7 +132,7 @@ def open_comic(path): def get_movies(): current_app.logger.info("start load movies") - pattern = r"(.+) \((....)\)(\(extended\))?( Director's Cut)?(\.mkv)" + pattern = r"(?P.+) \((?P<year>\d+)\)(?P<extended>\(extended\))?(?P<directors_cut> Director's Cut)?(?P<extension>\.mkv)" movies = [] total_movies = 0 movies_in_db = 0 @@ -145,21 +144,21 @@ def get_movies(): path = os.path.join(root, f) if not database.movie_path_in_db(path): try: - match = re.fullmatch(pattern, f) + match = re.match(pattern, f) if not match: current_app.logger.info(f+" did not match regex.") continue current_app.logger.info("movie path: "+path) - title = match.group(1) + title = match.group("title") current_app.logger.info("movie title: "+title) - year = int(match.group(2)) + year = int(match.group("year")) extended = False directors_cut = False - if match.group(3): + if match.group("extended"): extended = True - imdb_data = database.imdb_get_movie(title.replace(match.group(3), ""), year) - elif match.group(4): - imdb_data = database.imdb_get_movie(title.replace(match.group(4), ""), year) + imdb_data = database.imdb_get_movie(title.replace(match.group("extended"), ""), year) + elif match.group("directors_cut"): + imdb_data = database.imdb_get_movie(title.replace(match.group("directors_cut"), ""), year) directors_cut = True else: imdb_data = database.imdb_get_movie(title, year) @@ -195,14 +194,14 @@ def get_movies(): def get_tv_shows(): - dir_pattern = r"(.+) \((....)\)" + dir_pattern = r"(?P<title>.+) \((?P<year>\d+)\)" for dir in sorted(os.listdir(TV_SHOWS_DIRECTORY)): - dir_match = re.fullmatch(dir_pattern, dir) + dir_match = re.match(dir_pattern, dir) if dir_match: path = TV_SHOWS_DIRECTORY+dir+"/" if not database.tv_show_path_in_db(path): - series_name = dir_match.group(1) - series_year = int(dir_match.group(2)) + series_name = dir_match.group("title") + series_year = int(dir_match.group("year")) imdb_data = database.imdb_get_tv_show(series_name, series_year) if not imdb_data: current_app.logger.info("could not get imdb data for:"+series_name+" "+str(series_year)) @@ -213,7 +212,7 @@ def get_tv_shows(): if not tmdb_data: current_app.logger.info("could not get tmdb data for:" + series_name + " " + str(series_year)) # print("could not get tmdb data for:", series_name, series_year) - with open("/usb/storage/rpiWebApp/log.txt", "a") as f: + with open("/var/lib/rpiWebApp/log.txt", "a") as f: f.write("could not get tmdb data for: " + imdb_id + " " + series_name + " " + str(series_year)+"\n") continue tmdb_id = tmdb_data[0] @@ -226,18 +225,18 @@ def get_tv_shows(): def get_tv_episodes(): try: - video_pattern = r"S(\d+)E(\d+) - (.+)(.mp4|.mkv)" + video_pattern = r"S(?P<season>\d+)E(?P<episode>\d+) - (?P<title>.+)(?P<extension>.mp4|.mkv)" rows = database.get_all_tv_shows() for tv_show in rows: episodes = [] for video in sorted(os.listdir(tv_show.path)): - video_match = re.fullmatch(video_pattern, video) + video_match = re.match(video_pattern, video) if video_match: path = tv_show.path + video if not database.tv_episode_path_in_db(path): - season = int(video_match.group(1)) - episode = int(video_match.group(2)) - episode_name = video_match.group(3) + season = int(video_match.group("season")) + episode = int(video_match.group("episode")) + episode_name = video_match.group("title") episode_imdb_data = database.imdb_get_tv_episode(tv_show.imdb_id, season, episode) if not episode_imdb_data: current_app.logger.info("could not get imdb data for: "+tv_show.title+" "+str(tv_show.year)+" "+str(season)+" "+str(episode)) @@ -247,7 +246,7 @@ def get_tv_episodes(): episode_tmdb_data = database.tmdb_get_tv_episode_by_imdb_id(episode_imdb_id) if not episode_tmdb_data: current_app.logger.info("could not get tmdb data for: "+tv_show.title+" "+str(tv_show.year)+" "+str(season)+" "+str(episode)) - with open("/usb/storage/rpiWebApp/log.txt", "w") as f: + with open("/var/lib/rpiWebApp/log.txt", "w") as f: f.write("could not get tmdb data for: " + episode_imdb_id + " " + tv_show.title + " " + str( tv_show.year) + " " + str(season) + " " + str(episode) + "\n") continue diff --git a/scripts/imdb_import.py b/scripts/imdb_import.py index 0e721c2..de463b3 100644 --- a/scripts/imdb_import.py +++ b/scripts/imdb_import.py @@ -1,7 +1,7 @@ import sqlite3, subprocess, os -RPI_IMDB_DATABASE = "/usb/storage/rpiWebApp/" -RPI_TSV_DIRECTORY = "/usb/storage/imdb-rename/" +RPI_IMDB_DATABASE = "/var/lib/rpiWebApp/" +RPI_TSV_DIRECTORY = "/var/lib/imdb-rename/" RPI_CSV_DIRECTORY = "/home/matt/" MC_IMDB_DATABASE = "***REMOVED***" diff --git a/scripts/tmdb.py b/scripts/tmdb.py index 5a251e9..e65f7ca 100644 --- a/scripts/tmdb.py +++ b/scripts/tmdb.py @@ -4,6 +4,7 @@ import inspect API_KEY = "***REMOVED***" TMDB_FIND_URL = "https://api.themoviedb.org/3/find/" +TMDB_GET_TV_URL = "https://api.themoviedb.org/3/tv/" TMDB_IMG_URL = "https://image.tmdb.org/t/p/original" @@ -67,18 +68,24 @@ def get_tv_episode_data(imdb_id): "external_source": "imdb_id" } r = requests.get(TMDB_FIND_URL+imdb_id, params=data) - info = dict(r.json()) - if "status_code" in info.keys(): - current_app.logger.info("error getting tmdb tv episode data, status code: " + str(info["status_code"])+" "+str(info["status_message"])) + episode_info = dict(r.json()) + if "status_code" in episode_info.keys(): + current_app.logger.info("error getting tmdb tv episode data, status code: " + str(episode_info["status_code"])+" "+str(episode_info["status_message"])) return None - if info["tv_episode_results"] == []: + if episode_info["tv_episode_results"] == []: current_app.logger.info("no tmdb results for: " + str(imdb_id)) return None - current_app.logger.info("tmdb tv_episode title: " + str(info["tv_episode_results"][0]["name"])) - tv_episode_id = info["tv_episode_results"][0]["id"] - name = info["tv_episode_results"][0]["name"] - overview = info["tv_episode_results"][0]["overview"] - still_path = info["tv_episode_results"][0]["still_path"] + data = { + "api_key": API_KEY, + "language": "en-US" + } + r = requests.get(TMDB_GET_TV_URL+str(episode_info["tv_episode_results"][0]["show_id"]), params=data) + show_name = dict(r.json())["name"] + current_app.logger.info("tmdb tv_episode title: " + show_name + ": " + str(episode_info["tv_episode_results"][0]["name"])) + tv_episode_id = episode_info["tv_episode_results"][0]["id"] + name = episode_info["tv_episode_results"][0]["name"] + overview = episode_info["tv_episode_results"][0]["overview"] + still_path = episode_info["tv_episode_results"][0]["still_path"] return tv_episode_id, overview, still_path except Exception as e: diff --git a/test.sql b/test.sql deleted file mode 100644 index d35c70e..0000000 --- a/test.sql +++ /dev/null @@ -1,20 +0,0 @@ -.open "/var/lib/rpiWebApp/database.db" - -.print "Start database query" -.print "" - -.print "Number of publishers:" -SELECT count(DISTINCT publisher) FROM comics; -.print "" -.print "Number of series:" -SELECT count(DISTINCT series) FROM comics; -.print "" -.print "Number of comics:" -SELECT count(id) FROM comics; -.print "" - -.print "Number of movies" -SELECT count(*) FROM movies; - -.print "" -.print "End database query" diff --git a/tv/templates/tv/episodeViewer.html b/tv/templates/tv/episodeViewer.html index 3e2ebf4..7588db4 100644 --- a/tv/templates/tv/episodeViewer.html +++ b/tv/templates/tv/episodeViewer.html @@ -76,8 +76,9 @@ videojs("player").ready(function(){ let myPlayer = videojs.getPlayer("player"); let saved_time = {{ user_data.time }}; + let finished = {{ user_data.finished|string|lower }}; //converts python bool into javascript bool let length = 0; - if (saved_time > 5) { + if (saved_time > 5 && !finished) { myPlayer.currentTime(saved_time); } @@ -86,7 +87,6 @@ } let time = myPlayer.currentTime(); - let timestamp = document.getElementById("timestamp"); myPlayer.on("timeupdate", function () { if (myPlayer.currentTime()-time >= 10) { length = myPlayer.duration(); diff --git a/tv/tv.py b/tv/tv.py index b969ff6..9a6f97e 100644 --- a/tv/tv.py +++ b/tv/tv.py @@ -3,6 +3,7 @@ from flask_login import login_required from scripts import database import inspect +import datetime TV = Blueprint("tv", __name__, template_folder="templates") @@ -55,16 +56,24 @@ def episode_viewer(imdb_id): return make_response("", 201) else: episodes = database.get_tv_show_episodes_by_imdb_id(imdb_id) - season_num = request.args.get("season", type=int, default=episodes[0].season) - episode_num = request.args.get("episode", type=int, default=episodes[0].episode) + season_num = request.args.get("season", type=int, default=None) + episode_num = request.args.get("episode", type=int, default=None) user_tv_show_data = database.db_get_user_tv_show_episodes_data(imdb_id) - current_episode = episodes[0] - for episode in episodes: - if episode.season == season_num and episode.episode == episode_num: - current_episode = episode - user_data = database.db_get_user_tv_movie_data(current_episode.imdb_id) - if not user_data: - user_data = database.update_user_tv_movie_data(current_episode.imdb_id, imdb_id, 0, 0) + if not season_num and not episode_num: + (current_episode, user_data) = database.db_get_current_tv_show_episode_and_data(imdb_id, episodes) + season_num = current_episode.season + episode_num = current_episode.episode + if not user_data: + user_data = database.update_user_tv_movie_data(current_episode.imdb_id, imdb_id, 0, 0) + else: + current_episode = episodes[0] + user_data = database.UserTvMovieData(("", "", "", 0, 0, False, datetime.datetime.now())) + for episode in episodes: + if episode.season == season_num and episode.episode == episode_num: + current_episode = episode + user_data = database.db_get_user_tv_movie_data(current_episode.imdb_id) + if not user_data: + user_data = database.update_user_tv_movie_data(current_episode.imdb_id, imdb_id, 0, 0) return render_template("tv/episodeViewer.html", title="Tv", episodes=episodes, season_num=season_num, episode_num=episode_num, episode=current_episode, user_data=user_data, user_tv_show_data=user_tv_show_data) except Exception as e: current_app.logger.info(inspect.stack()[0][3] + " " + str(type(e)) + " " + str(e))