Sanity checks, single user mode, easier seq uids, fix comma EOL bug, status page, seq reporting, seq uid purging,

app.py

@@ -1,15 +1,17 @@
 # app.py
-from flask import Flask, request, jsonify, send_file
+from flask import Flask, request, jsonify, send_file, render_template
 import random
-from datetime import date, datetime  # used for daily verse
+from datetime import date, datetime, timedelta  # used for daily verse
 import sqlite3
-import string  # for literals
+import string  # for literals shortcut
 import json  # for dictionary
 import hunspell  # for spell checking (not really used)
 import sys
 import logging
+from tools import sanity_check
 
-timestamp_runtime = datetime.now().strftime("%d%b%y-%H:%M")
+sequential_purge_delta: timedelta = timedelta(days=90)
+timestamp_runtime: datetime = datetime.now().strftime("%d%b%y-%H:%M")
 logging.basicConfig(
     filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log",
     encoding="utf-8",
@@ -18,27 +20,38 @@ logging.basicConfig(
 
 logging.info(f"Started: {timestamp_runtime}")
 
-app = Flask(__name__)
+try:
+    sanity_check.check_sanity()
+    sanity: bool = True
+except PermissionError as e:
+    logging.critical(f"PERMISSIONS FAILURE: {e}")
+    sanity: bool = False
+
+app = Flask(__name__, template_folder='html')
 
 """
 Current work prefixed with *
 TODO:
 Replace send_file with something that loads headers and nav templates
 Search phrase alias dictionary
-Response Bible book reading
-Pretty navigation bar
 Search highlighting
-Styles
 Bible Reading Plan
 /seq Going backwards (works but is buggy, disabled for now)
 /seq Wrap back to Gen 1:1 when you reach the end of Rev
 /seq Querying what your UID's next verse is going to be
-/seq Purging old UIDs after a certain period of non-use
+https://docs.python.org/3/library/sqlite3.html#sqlite3-placeholders
 """
 
-kjv_cur = sqlite3.connect("data/kjv.db").cursor()
+if __name__ == "__main__":
+    # single user mode, running without uwsgi
+    usermode: str = "Single"
+    kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor()
+else:
+    # you're running with uwsgi
+    usermode: str = "Multi"
+    kjv_cur = sqlite3.connect("data/kjv.db").cursor()
 
-cached = {
+cached: dict = {
     "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()],
     "book_names": [
         i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall()
@@ -61,17 +74,17 @@ cached = {
 ]
 cached["chapter_list"].sort()
 
-
+def int32_to_id(n):
-def build_html_reponse():
+  if n==0: return "0"
-    html = ""
+  chars="0123456789ACEFHJKLMNPRTUVWXY"
-    return html
+  length=len(chars)
-
+  result=""
-
+  remain=n
-def lookup_one_verse(verse_id):
+  while remain>0:
-    logging.debug(f"lookup_one_verse({verse_id})")
+    pos = remain % length
-    return kjv_cur.execute(f"SELECT * FROM fts_kjv WHERE id = {verse_id};").fetchone()
+    remain = remain // length
-
+    result = chars[pos] + result
-
+  return result
 def lookup_bookname(book_id):
     logging.debug(f"lookup_bookname({book_id})")
     result = kjv_cur.execute(
@@ -81,8 +94,6 @@ def lookup_bookname(book_id):
         result = result[0]
     logging.debug(f"lookup_bookname.result = '{result}'")
     return result
-
-
 def lookup_book_id(bookname):
     logging.debug(f"lookup_book_id({bookname})")
     result = kjv_cur.execute(
@@ -92,13 +103,9 @@ def lookup_book_id(bookname):
         result = result[0]
     logging.debug(f"lookup_book_id.result = '{result}'")
     return result
-
-
 def lookup_fts(kwds):
     logging.debug(f"lookup_fts.kwds = '{kwds}'")
     return kjv_cur.execute(f"SELECT * FROM fts_kjv('{kwds}')").fetchall()
-
-
 def lookup_by_verse_id(start_verse, end_verse=None):
     if not end_verse:
         results = kjv_cur.execute(
@@ -109,8 +116,6 @@ def lookup_by_verse_id(start_verse, end_verse=None):
             f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};"
         ).fetchall()
     return results
-
-
 def get_verse_id(book_id, chapter, verse):
     # verse_id is 00000000 (00 000 000) (book + chapter + verse)
     chapterid = chapter.zfill(3)
@@ -118,8 +123,6 @@ def get_verse_id(book_id, chapter, verse):
     if verseid == "000":
         verseid = "001"  # Avoid verse_ids list IndexErrors on :- searches
     return str(book_id) + chapterid + verseid
-
-
 def get_next_prev_chapter(verseid):
     current_chapter = verseid[:-3]
     current_chapter_index = cached["chapter_list"].index(int(current_chapter))
@@ -147,8 +150,6 @@ def get_next_prev_chapter(verseid):
         "next_chapter_link": next_chapter_link,
     }
     return result
-
-
 def ref_input_cleaning(
     ref, valid_chars=(string.ascii_letters + string.digits + ":-,"), ref_search=True
 ):
@@ -185,8 +186,6 @@ def ref_input_cleaning(
         return prefix + "".join(cleaned_ref).capitalize()
     else:
         return "".join(cleaned_ref)
-
-
 def search_input_cleaning(
     query, valid_chars=string.ascii_letters + string.digits + '+^"* '
 ):
@@ -195,24 +194,22 @@ def search_input_cleaning(
         if character in valid_chars:
             cleaned_query.append(character)
     return "".join(cleaned_query)
-
-
 def ref_parse_book(ref):
     for book in reversed(cached["book_names"]):
         if book.startswith(ref):
             return book
     return False
-
-
 @app.get("/find")
 def parse_query():
     query = request.args.get("kw", None)
+    if query:
+        query = query.strip()
     logging.info(f"Query: {query}")
     if not query:
         return send_file("html/find.html")
     if query.endswith(","):
         query = query[:-1]
-    result = show()
+    result = show(kw=query)
     if not hasattr(result, "__len__"):
         logging.debug(f"parse_query.show = '{result}'")
         return result
@@ -242,8 +239,6 @@ def parse_query():
     #result += search()
     logging.debug(f"parse_query.search = '{result}'")
     return result
-
-
 @app.get("/define")
 def define(word=None):
     ref = request.args.get("kw", None)
@@ -274,15 +269,11 @@ def define(word=None):
     else:
         response = jsonify({ref: dictionary[ref]})
     return response
-
-
 def return_dict_plain(ref, defs):
     response = ""
     for e in defs:
         response += e + "<br><br>"
     return ref.capitalize() + "<br><br>" + response + "\n"
-
-
 def return_dict_rich(ref, defs, simple):
     if simple:
         css = ""
@@ -297,12 +288,12 @@ def return_dict_rich(ref, defs, simple):
         )
         + css
     )
-
-
 @app.get("/show")
 def show(kw=None, get_start_verse_id=False):
     ref = request.args.get("kw", None)
-    get_chapter = request.args.get("all", False)
+    get_chapter: bool = request.args.get("all", False)
+    if ref.endswith(","):
+        ref = ref[:-1]
     if kw:
         ref = kw
     if ref is None:
@@ -320,6 +311,10 @@ def show(kw=None, get_start_verse_id=False):
     for ref in refs:
         logging.debug(f"show.reference({ref})")
         maybe_chapter = ref.split(":")[0][3:]
+        #FIXME: Not including : in a book reference should take you to the first book
+        # not just crash like a retard
+        if not ":" in maybe_chapter:
+            get_chapter = True
         ref_chapter = ""
         for maybedigit in maybe_chapter:
             if maybedigit.isdigit():
@@ -363,8 +358,6 @@ def show(kw=None, get_start_verse_id=False):
     else:
         response = jsonify(response_list)
     return response
-
-
 def response_dict(scripture):
     # used for JSON responses
     verse = {
@@ -374,16 +367,12 @@ def response_dict(scripture):
         "text": scripture[4],
     }
     return verse
-
-
 def response_text(scripture):
     bookname = lookup_bookname(scripture[1])
     chapter = str(scripture[2])
     verse = str(scripture[3])
     text = str(scripture[4])
     return bookname + " " + chapter + ":" + verse + " " + text + "\n"
-
-
 def response_rich(scripture):
     # (passage, text) is the html formatting arguments
     bookname = lookup_bookname(scripture[1])
@@ -394,7 +383,6 @@ def response_rich(scripture):
     return cached["html_response"].format(
         chapter=passage.split(":")[0], passage=passage, text=text
     )
-
 def scripture_response(scriptures):
     response = []
     arg_view = request.args.get("view", "json")
@@ -428,33 +416,25 @@ def scripture_response(scriptures):
         for scripture in scriptures:
             response.append(response_dict(scripture))
     return response
-
-
 @app.errorhandler(404)
 def page_not_found(e):
     message = {
         "error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."
     }
     return jsonify(message)
-
-
 @app.get("/")
 def idx():
+    if not sanity:
+        return("There is an issue with permissions. Check log, or please notify tyler@dinsmoor.us")
     logging.info("Index Served")
     return send_file("html/index.html")
-
-
 @app.get("/salvation")
 def salvation():
     return send_file("html/heaven.html")
-
+@app.get("/t/random", defaults={"view": "plain"}) # backwards-compatibility
-
-# backwards-compatibility
-@app.get("/t/random", defaults={"view": "plain"})
 @app.get("/random")
 def random_verse():
     rand_id = random.SystemRandom().choice(cached["verse_ids"])
-    #scripture = lookup_one_verse(rand_id)
     scripture = lookup_by_verse_id(rand_id)
     logging.info(f"Random Served: {scripture[0]}")
     response_list = scripture_response(scripture)
@@ -467,13 +447,10 @@ def random_verse():
         response = jsonify(response_list)
     logging.debug(f"Search Response: {response}")
     return response
-
+@app.get("/t/votd", defaults={"view": "plain"}) # backwards-compatibility
-
-# backwards-compatibility
-# plain takes 38ms -TODO can we cache this?
-@app.get("/t/votd", defaults={"view": "plain"})
 @app.get("/votd")
 def verse_of_the_day():
+    # plain takes 38ms -TODO can we cache this?
     logging.info("VOTD Served")
     today = int(str(date.today()).replace("-", ""))
     # this is deterministic because of today's date being used as the seed
@@ -489,7 +466,6 @@ def verse_of_the_day():
         response = jsonify(response_list)
     logging.debug(f"Search Response: {response}")
     return response
-
 @app.get("/search")
 def search():
     keywords = request.args.get("kw", None)
@@ -517,8 +493,6 @@ def search():
         response = jsonify(response_list)
     logging.debug(f"Search Response: {response}")
     return response
-
-
 @app.get("/reader")
 def bible_reader():
     return send_file("html/reader.html")
@@ -529,8 +503,6 @@ def bible_reader():
     # header should have a navigation bar and "next/previous chapter"
     # Footer should have next/previous as well.
     # Maybe preliminary theming support
-
-
 @app.get("/plan")
 def reading_plan():
     return send_file("html/reading_plan.html")
@@ -538,8 +510,41 @@ def reading_plan():
     # should default to the entire Bible in 1 year.
     # should start today() and end today()+1year
     # Should return a calender with date and verse range.
+@app.get("/status")
+def status():
+    import resource
+    return f"""Sanity: {sanity} <br>
+UserMode: {usermode} <br>
+Memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}
+    """
+@app.get("/seqreport")
+def sequential_report(uid=None):
+    if usermode == "Single":
+        seq_conn = sqlite3.connect("data/seq.db",  check_same_thread=False)
+    else:
+        seq_conn = sqlite3.connect("data/seq.db")
+    seq_cur = seq_conn.cursor()
+    uids = seq_cur.execute("select * from reading_tracker").fetchall()
+    return render_template("seq_report.html", uids=uids)
+def sequential_purge():
+    #TODO: Test this :^)
+    if usermode == "Single":
+        seq_conn = sqlite3.connect("data/seq.db",  check_same_thread=False)
+    else:
+        seq_conn = sqlite3.connect("data/seq.db")
+    seq_cur = seq_conn.cursor()
+    uids = seq_cur.execute("select * from reading_tracker").fetchmany(10)
 
+    evaluate_datetime = datetime.now()
+    for uid in uids:
+        last_used = datetime.fromisoformat(uid[2])
+        since_use: timedelta = evaluate_datetime - last_used
+        if since_use > sequential_purge_delta:
+            logging.info(f"Purging UID: {uid[0]}")
+            seq_cur.execute(f"DELETE FROM reading_tracker WHERE uid = '{uid[0]}'")
 
+    seq_conn.commit()
+    seq_cur.close()
 @app.get("/seq")
 def sequential_read(start=0, num=1, uid=None, getuid=False):
     # /seq?uid=1
@@ -552,35 +557,35 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
         return "Invalid num argument"
     if not uid and not getuid:
         return send_file("html/seq.html")
-    seq_conn = sqlite3.connect("data/seq.db")
+    if usermode == "Single":
+        seq_conn = sqlite3.connect("data/seq.db",  check_same_thread=False)
+    else:
+        seq_conn = sqlite3.connect("data/seq.db")
     seq_cur = seq_conn.cursor()
+    seq_cur.execute("CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)")
     last_used = str(datetime.now())
     num -= 1  # 0 will mean "next"
     if num < 0:
         return jsonify({"error": "You may only increment forward."})
-
     if getuid:
-        import uuid
-
         if start:
-
             start = cached["verse_ids"].index(
                 int(show(kw=start, get_start_verse_id=True))
             )
-        uid = str(uuid.uuid4())
+        uid = int32_to_id(random.randrange(614_656,17_210_367)) # range returns a 5 digit id
         try:
             seq_cur.execute(
                 f"INSERT INTO reading_tracker VALUES('{uid}', {start}, '{last_used}')"
             )
-        except sqlite3.OperationalError:
+        except sqlite3.OperationalError as e:
-            logging.error(f"UID Database failed to insert {uid} into reading_tracker.")
+            logging.error(f"UID Database failed to insert {uid} into reading_tracker. {e}")
             return jsonify(
                 {
-                    "error": "Something went wrong with the UID database. Check permissions?"
+                    "error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"
                 }
             )
         seq_conn.commit()
-        logging.info(f"Created new SEQ UID: {uid}")
+        logging.info(f"New SEQ UID: {uid}")
         return f"Your new UID is: {uid}"
     uid = ref_input_cleaning(uid, ref_search=False)
     try:
@@ -588,9 +593,10 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
             f"SELECT verse FROM reading_tracker WHERE uid = '{uid}'"
         ).fetchone()[0]
         logging.debug(f"Queried UID {uid} on verse {nextverse}")
+        sequential_purge()
     except TypeError:
-        logging.debug(f"Invalid UID {uid}")
+        logging.info(f"Invalid UID {uid}")
-        return jsonify({"error": "UID Invalid or purged"})
+        return jsonify({"error": "UID Invalid or purged (90 days of inactivity)"})
     if len(cached["verse_ids"]) < (nextverse + num):
         # Correct our range to go to ceiling to prevent IndexError
         diff = len(cached["verse_ids"]) - (nextverse + num)
@@ -602,8 +608,10 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
 
     logging.info(f"Served {num+1} verses to UID {uid}")
     seq_cur.execute(
-        f"UPDATE reading_tracker SET verse = {nextverse+num+1}, last_used = '{last_used}' WHERE uid = '{uid}'"
+        f"REPLACE INTO reading_tracker (uid, verse, last_used) VALUES('{uid}', {nextverse+num+1}, '{last_used}')"
     )
+    # Doing REPLACE INTO instead of UPDATE will make the updated row at the end of the table, making the purging
+    # only look at a small set of the oldest unused entries. 
     seq_conn.commit()
     arg_view = request.args.get("view", None)
     if arg_view == "plain" or arg_view == "rich":
@@ -613,7 +621,8 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
 
     return response
 
-
 if __name__ == "__main__":
-    print("You must run this program within a uwsgi environment.")
+    print("You must run this program within a uwsgi environment if you are running this for multiple users.")
-    # app.run(host="127.0.0.1", port=1612)
+    print("Otherwise, you may end up with database corruption if multiple writes happen.")
+    print("Operating in single user mode.")
+    app.run(host="127.0.0.1", port=1611)

tools/sanity_check.py

@@ -0,0 +1,13 @@
+# Check if all the files needed by kjv-api (to write to) are also owned by the user running it.
+# if this is not the same, writing to data/seq.db (and other undesirable behavior) is inevitable
+# and will save a lot of headache in the future.
+
+#Additionally, proper CHMOD value is 664 for the database
+from os import stat, getuid
+
+def check_sanity():
+    dependancies = ['data/seq.db']
+    current_uid = getuid()
+    for dependancy in dependancies:
+        if stat(dependancy).st_uid != current_uid:
+            raise PermissionError(dependancy)