diff --git a/app.py b/app.py index 2242f0a..9d566c0 100644 --- a/app.py +++ b/app.py @@ -1,15 +1,17 @@ # app.py -from flask import Flask, request, jsonify, send_file +from flask import Flask, request, jsonify, send_file, render_template import random -from datetime import date, datetime # used for daily verse +from datetime import date, datetime, timedelta # used for daily verse import sqlite3 -import string # for literals +import string # for literals shortcut import json # for dictionary import hunspell # for spell checking (not really used) import sys import logging +from tools import sanity_check -timestamp_runtime = datetime.now().strftime("%d%b%y-%H:%M") +sequential_purge_delta: timedelta = timedelta(days=90) +timestamp_runtime: datetime = datetime.now().strftime("%d%b%y-%H:%M") logging.basicConfig( filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log", encoding="utf-8", @@ -18,27 +20,38 @@ logging.basicConfig( logging.info(f"Started: {timestamp_runtime}") -app = Flask(__name__) +try: + sanity_check.check_sanity() + sanity: bool = True +except PermissionError as e: + logging.critical(f"PERMISSIONS FAILURE: {e}") + sanity: bool = False + +app = Flask(__name__, template_folder='html') """ Current work prefixed with * TODO: Replace send_file with something that loads headers and nav templates Search phrase alias dictionary -Response Bible book reading -Pretty navigation bar Search highlighting -Styles Bible Reading Plan /seq Going backwards (works but is buggy, disabled for now) /seq Wrap back to Gen 1:1 when you reach the end of Rev /seq Querying what your UID's next verse is going to be -/seq Purging old UIDs after a certain period of non-use +https://docs.python.org/3/library/sqlite3.html#sqlite3-placeholders """ -kjv_cur = sqlite3.connect("data/kjv.db").cursor() +if __name__ == "__main__": + # single user mode, running without uwsgi + usermode: str = "Single" + kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor() +else: + # you're running with uwsgi + usermode: str = "Multi" + kjv_cur = sqlite3.connect("data/kjv.db").cursor() -cached = { +cached: dict = { "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()], "book_names": [ i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall() @@ -61,17 +74,17 @@ cached = { ] cached["chapter_list"].sort() - -def build_html_reponse(): - html = "" - return html - - -def lookup_one_verse(verse_id): - logging.debug(f"lookup_one_verse({verse_id})") - return kjv_cur.execute(f"SELECT * FROM fts_kjv WHERE id = {verse_id};").fetchone() - - +def int32_to_id(n): + if n==0: return "0" + chars="0123456789ACEFHJKLMNPRTUVWXY" + length=len(chars) + result="" + remain=n + while remain>0: + pos = remain % length + remain = remain // length + result = chars[pos] + result + return result def lookup_bookname(book_id): logging.debug(f"lookup_bookname({book_id})") result = kjv_cur.execute( @@ -81,8 +94,6 @@ def lookup_bookname(book_id): result = result[0] logging.debug(f"lookup_bookname.result = '{result}'") return result - - def lookup_book_id(bookname): logging.debug(f"lookup_book_id({bookname})") result = kjv_cur.execute( @@ -92,13 +103,9 @@ def lookup_book_id(bookname): result = result[0] logging.debug(f"lookup_book_id.result = '{result}'") return result - - def lookup_fts(kwds): logging.debug(f"lookup_fts.kwds = '{kwds}'") return kjv_cur.execute(f"SELECT * FROM fts_kjv('{kwds}')").fetchall() - - def lookup_by_verse_id(start_verse, end_verse=None): if not end_verse: results = kjv_cur.execute( @@ -109,8 +116,6 @@ def lookup_by_verse_id(start_verse, end_verse=None): f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};" ).fetchall() return results - - def get_verse_id(book_id, chapter, verse): # verse_id is 00000000 (00 000 000) (book + chapter + verse) chapterid = chapter.zfill(3) @@ -118,8 +123,6 @@ def get_verse_id(book_id, chapter, verse): if verseid == "000": verseid = "001" # Avoid verse_ids list IndexErrors on :- searches return str(book_id) + chapterid + verseid - - def get_next_prev_chapter(verseid): current_chapter = verseid[:-3] current_chapter_index = cached["chapter_list"].index(int(current_chapter)) @@ -147,8 +150,6 @@ def get_next_prev_chapter(verseid): "next_chapter_link": next_chapter_link, } return result - - def ref_input_cleaning( ref, valid_chars=(string.ascii_letters + string.digits + ":-,"), ref_search=True ): @@ -185,8 +186,6 @@ def ref_input_cleaning( return prefix + "".join(cleaned_ref).capitalize() else: return "".join(cleaned_ref) - - def search_input_cleaning( query, valid_chars=string.ascii_letters + string.digits + '+^"* ' ): @@ -195,24 +194,22 @@ def search_input_cleaning( if character in valid_chars: cleaned_query.append(character) return "".join(cleaned_query) - - def ref_parse_book(ref): for book in reversed(cached["book_names"]): if book.startswith(ref): return book return False - - @app.get("/find") def parse_query(): query = request.args.get("kw", None) + if query: + query = query.strip() logging.info(f"Query: {query}") if not query: return send_file("html/find.html") if query.endswith(","): query = query[:-1] - result = show() + result = show(kw=query) if not hasattr(result, "__len__"): logging.debug(f"parse_query.show = '{result}'") return result @@ -242,8 +239,6 @@ def parse_query(): #result += search() logging.debug(f"parse_query.search = '{result}'") return result - - @app.get("/define") def define(word=None): ref = request.args.get("kw", None) @@ -274,15 +269,11 @@ def define(word=None): else: response = jsonify({ref: dictionary[ref]}) return response - - def return_dict_plain(ref, defs): response = "" for e in defs: response += e + "

" return ref.capitalize() + "

" + response + "\n" - - def return_dict_rich(ref, defs, simple): if simple: css = "" @@ -297,12 +288,12 @@ def return_dict_rich(ref, defs, simple): ) + css ) - - @app.get("/show") def show(kw=None, get_start_verse_id=False): ref = request.args.get("kw", None) - get_chapter = request.args.get("all", False) + get_chapter: bool = request.args.get("all", False) + if ref.endswith(","): + ref = ref[:-1] if kw: ref = kw if ref is None: @@ -320,6 +311,10 @@ def show(kw=None, get_start_verse_id=False): for ref in refs: logging.debug(f"show.reference({ref})") maybe_chapter = ref.split(":")[0][3:] + #FIXME: Not including : in a book reference should take you to the first book + # not just crash like a retard + if not ":" in maybe_chapter: + get_chapter = True ref_chapter = "" for maybedigit in maybe_chapter: if maybedigit.isdigit(): @@ -363,8 +358,6 @@ def show(kw=None, get_start_verse_id=False): else: response = jsonify(response_list) return response - - def response_dict(scripture): # used for JSON responses verse = { @@ -374,16 +367,12 @@ def response_dict(scripture): "text": scripture[4], } return verse - - def response_text(scripture): bookname = lookup_bookname(scripture[1]) chapter = str(scripture[2]) verse = str(scripture[3]) text = str(scripture[4]) return bookname + " " + chapter + ":" + verse + " " + text + "\n" - - def response_rich(scripture): # (passage, text) is the html formatting arguments bookname = lookup_bookname(scripture[1]) @@ -394,7 +383,6 @@ def response_rich(scripture): return cached["html_response"].format( chapter=passage.split(":")[0], passage=passage, text=text ) - def scripture_response(scriptures): response = [] arg_view = request.args.get("view", "json") @@ -428,33 +416,25 @@ def scripture_response(scriptures): for scripture in scriptures: response.append(response_dict(scripture)) return response - - @app.errorhandler(404) def page_not_found(e): message = { "error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness." } return jsonify(message) - - @app.get("/") def idx(): + if not sanity: + return("There is an issue with permissions. Check log, or please notify tyler@dinsmoor.us") logging.info("Index Served") return send_file("html/index.html") - - @app.get("/salvation") def salvation(): return send_file("html/heaven.html") - - -# backwards-compatibility -@app.get("/t/random", defaults={"view": "plain"}) +@app.get("/t/random", defaults={"view": "plain"}) # backwards-compatibility @app.get("/random") def random_verse(): rand_id = random.SystemRandom().choice(cached["verse_ids"]) - #scripture = lookup_one_verse(rand_id) scripture = lookup_by_verse_id(rand_id) logging.info(f"Random Served: {scripture[0]}") response_list = scripture_response(scripture) @@ -467,13 +447,10 @@ def random_verse(): response = jsonify(response_list) logging.debug(f"Search Response: {response}") return response - - -# backwards-compatibility -# plain takes 38ms -TODO can we cache this? -@app.get("/t/votd", defaults={"view": "plain"}) +@app.get("/t/votd", defaults={"view": "plain"}) # backwards-compatibility @app.get("/votd") def verse_of_the_day(): + # plain takes 38ms -TODO can we cache this? logging.info("VOTD Served") today = int(str(date.today()).replace("-", "")) # this is deterministic because of today's date being used as the seed @@ -489,7 +466,6 @@ def verse_of_the_day(): response = jsonify(response_list) logging.debug(f"Search Response: {response}") return response - @app.get("/search") def search(): keywords = request.args.get("kw", None) @@ -517,8 +493,6 @@ def search(): response = jsonify(response_list) logging.debug(f"Search Response: {response}") return response - - @app.get("/reader") def bible_reader(): return send_file("html/reader.html") @@ -529,8 +503,6 @@ def bible_reader(): # header should have a navigation bar and "next/previous chapter" # Footer should have next/previous as well. # Maybe preliminary theming support - - @app.get("/plan") def reading_plan(): return send_file("html/reading_plan.html") @@ -538,8 +510,41 @@ def reading_plan(): # should default to the entire Bible in 1 year. # should start today() and end today()+1year # Should return a calender with date and verse range. +@app.get("/status") +def status(): + import resource + return f"""Sanity: {sanity}
+UserMode: {usermode}
+Memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss} + """ +@app.get("/seqreport") +def sequential_report(uid=None): + if usermode == "Single": + seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False) + else: + seq_conn = sqlite3.connect("data/seq.db") + seq_cur = seq_conn.cursor() + uids = seq_cur.execute("select * from reading_tracker").fetchall() + return render_template("seq_report.html", uids=uids) +def sequential_purge(): + #TODO: Test this :^) + if usermode == "Single": + seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False) + else: + seq_conn = sqlite3.connect("data/seq.db") + seq_cur = seq_conn.cursor() + uids = seq_cur.execute("select * from reading_tracker").fetchmany(10) + evaluate_datetime = datetime.now() + for uid in uids: + last_used = datetime.fromisoformat(uid[2]) + since_use: timedelta = evaluate_datetime - last_used + if since_use > sequential_purge_delta: + logging.info(f"Purging UID: {uid[0]}") + seq_cur.execute(f"DELETE FROM reading_tracker WHERE uid = '{uid[0]}'") + seq_conn.commit() + seq_cur.close() @app.get("/seq") def sequential_read(start=0, num=1, uid=None, getuid=False): # /seq?uid=1 @@ -552,35 +557,35 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): return "Invalid num argument" if not uid and not getuid: return send_file("html/seq.html") - seq_conn = sqlite3.connect("data/seq.db") + if usermode == "Single": + seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False) + else: + seq_conn = sqlite3.connect("data/seq.db") seq_cur = seq_conn.cursor() + seq_cur.execute("CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)") last_used = str(datetime.now()) num -= 1 # 0 will mean "next" if num < 0: return jsonify({"error": "You may only increment forward."}) - if getuid: - import uuid - if start: - start = cached["verse_ids"].index( int(show(kw=start, get_start_verse_id=True)) ) - uid = str(uuid.uuid4()) + uid = int32_to_id(random.randrange(614_656,17_210_367)) # range returns a 5 digit id try: seq_cur.execute( f"INSERT INTO reading_tracker VALUES('{uid}', {start}, '{last_used}')" ) - except sqlite3.OperationalError: - logging.error(f"UID Database failed to insert {uid} into reading_tracker.") + except sqlite3.OperationalError as e: + logging.error(f"UID Database failed to insert {uid} into reading_tracker. {e}") return jsonify( { - "error": "Something went wrong with the UID database. Check permissions?" + "error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us" } ) seq_conn.commit() - logging.info(f"Created new SEQ UID: {uid}") + logging.info(f"New SEQ UID: {uid}") return f"Your new UID is: {uid}" uid = ref_input_cleaning(uid, ref_search=False) try: @@ -588,9 +593,10 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): f"SELECT verse FROM reading_tracker WHERE uid = '{uid}'" ).fetchone()[0] logging.debug(f"Queried UID {uid} on verse {nextverse}") + sequential_purge() except TypeError: - logging.debug(f"Invalid UID {uid}") - return jsonify({"error": "UID Invalid or purged"}) + logging.info(f"Invalid UID {uid}") + return jsonify({"error": "UID Invalid or purged (90 days of inactivity)"}) if len(cached["verse_ids"]) < (nextverse + num): # Correct our range to go to ceiling to prevent IndexError diff = len(cached["verse_ids"]) - (nextverse + num) @@ -602,8 +608,10 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): logging.info(f"Served {num+1} verses to UID {uid}") seq_cur.execute( - f"UPDATE reading_tracker SET verse = {nextverse+num+1}, last_used = '{last_used}' WHERE uid = '{uid}'" + f"REPLACE INTO reading_tracker (uid, verse, last_used) VALUES('{uid}', {nextverse+num+1}, '{last_used}')" ) + # Doing REPLACE INTO instead of UPDATE will make the updated row at the end of the table, making the purging + # only look at a small set of the oldest unused entries. seq_conn.commit() arg_view = request.args.get("view", None) if arg_view == "plain" or arg_view == "rich": @@ -613,7 +621,8 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): return response - if __name__ == "__main__": - print("You must run this program within a uwsgi environment.") - # app.run(host="127.0.0.1", port=1612) + print("You must run this program within a uwsgi environment if you are running this for multiple users.") + print("Otherwise, you may end up with database corruption if multiple writes happen.") + print("Operating in single user mode.") + app.run(host="127.0.0.1", port=1611) diff --git a/tools/sanity_check.py b/tools/sanity_check.py new file mode 100644 index 0000000..88676d7 --- /dev/null +++ b/tools/sanity_check.py @@ -0,0 +1,13 @@ +# Check if all the files needed by kjv-api (to write to) are also owned by the user running it. +# if this is not the same, writing to data/seq.db (and other undesirable behavior) is inevitable +# and will save a lot of headache in the future. + +#Additionally, proper CHMOD value is 664 for the database +from os import stat, getuid + +def check_sanity(): + dependancies = ['data/seq.db'] + current_uid = getuid() + for dependancy in dependancies: + if stat(dependancy).st_uid != current_uid: + raise PermissionError(dependancy) \ No newline at end of file