Sanity checks, single user mode, easier seq uids, fix comma EOL bug, status page, seq reporting, seq uid purging,
This commit is contained in:
@@ -1,15 +1,17 @@
|
||||
# app.py
|
||||
from flask import Flask, request, jsonify, send_file
|
||||
from flask import Flask, request, jsonify, send_file, render_template
|
||||
import random
|
||||
from datetime import date, datetime # used for daily verse
|
||||
from datetime import date, datetime, timedelta # used for daily verse
|
||||
import sqlite3
|
||||
import string # for literals
|
||||
import string # for literals shortcut
|
||||
import json # for dictionary
|
||||
import hunspell # for spell checking (not really used)
|
||||
import sys
|
||||
import logging
|
||||
from tools import sanity_check
|
||||
|
||||
timestamp_runtime = datetime.now().strftime("%d%b%y-%H:%M")
|
||||
sequential_purge_delta: timedelta = timedelta(days=90)
|
||||
timestamp_runtime: datetime = datetime.now().strftime("%d%b%y-%H:%M")
|
||||
logging.basicConfig(
|
||||
filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log",
|
||||
encoding="utf-8",
|
||||
@@ -18,27 +20,38 @@ logging.basicConfig(
|
||||
|
||||
logging.info(f"Started: {timestamp_runtime}")
|
||||
|
||||
app = Flask(__name__)
|
||||
try:
|
||||
sanity_check.check_sanity()
|
||||
sanity: bool = True
|
||||
except PermissionError as e:
|
||||
logging.critical(f"PERMISSIONS FAILURE: {e}")
|
||||
sanity: bool = False
|
||||
|
||||
app = Flask(__name__, template_folder='html')
|
||||
|
||||
"""
|
||||
Current work prefixed with *
|
||||
TODO:
|
||||
Replace send_file with something that loads headers and nav templates
|
||||
Search phrase alias dictionary
|
||||
Response Bible book reading
|
||||
Pretty navigation bar
|
||||
Search highlighting
|
||||
Styles
|
||||
Bible Reading Plan
|
||||
/seq Going backwards (works but is buggy, disabled for now)
|
||||
/seq Wrap back to Gen 1:1 when you reach the end of Rev
|
||||
/seq Querying what your UID's next verse is going to be
|
||||
/seq Purging old UIDs after a certain period of non-use
|
||||
https://docs.python.org/3/library/sqlite3.html#sqlite3-placeholders
|
||||
"""
|
||||
|
||||
kjv_cur = sqlite3.connect("data/kjv.db").cursor()
|
||||
if __name__ == "__main__":
|
||||
# single user mode, running without uwsgi
|
||||
usermode: str = "Single"
|
||||
kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor()
|
||||
else:
|
||||
# you're running with uwsgi
|
||||
usermode: str = "Multi"
|
||||
kjv_cur = sqlite3.connect("data/kjv.db").cursor()
|
||||
|
||||
cached = {
|
||||
cached: dict = {
|
||||
"verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()],
|
||||
"book_names": [
|
||||
i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall()
|
||||
@@ -61,17 +74,17 @@ cached = {
|
||||
]
|
||||
cached["chapter_list"].sort()
|
||||
|
||||
|
||||
def build_html_reponse():
|
||||
html = ""
|
||||
return html
|
||||
|
||||
|
||||
def lookup_one_verse(verse_id):
|
||||
logging.debug(f"lookup_one_verse({verse_id})")
|
||||
return kjv_cur.execute(f"SELECT * FROM fts_kjv WHERE id = {verse_id};").fetchone()
|
||||
|
||||
|
||||
def int32_to_id(n):
|
||||
if n==0: return "0"
|
||||
chars="0123456789ACEFHJKLMNPRTUVWXY"
|
||||
length=len(chars)
|
||||
result=""
|
||||
remain=n
|
||||
while remain>0:
|
||||
pos = remain % length
|
||||
remain = remain // length
|
||||
result = chars[pos] + result
|
||||
return result
|
||||
def lookup_bookname(book_id):
|
||||
logging.debug(f"lookup_bookname({book_id})")
|
||||
result = kjv_cur.execute(
|
||||
@@ -81,8 +94,6 @@ def lookup_bookname(book_id):
|
||||
result = result[0]
|
||||
logging.debug(f"lookup_bookname.result = '{result}'")
|
||||
return result
|
||||
|
||||
|
||||
def lookup_book_id(bookname):
|
||||
logging.debug(f"lookup_book_id({bookname})")
|
||||
result = kjv_cur.execute(
|
||||
@@ -92,13 +103,9 @@ def lookup_book_id(bookname):
|
||||
result = result[0]
|
||||
logging.debug(f"lookup_book_id.result = '{result}'")
|
||||
return result
|
||||
|
||||
|
||||
def lookup_fts(kwds):
|
||||
logging.debug(f"lookup_fts.kwds = '{kwds}'")
|
||||
return kjv_cur.execute(f"SELECT * FROM fts_kjv('{kwds}')").fetchall()
|
||||
|
||||
|
||||
def lookup_by_verse_id(start_verse, end_verse=None):
|
||||
if not end_verse:
|
||||
results = kjv_cur.execute(
|
||||
@@ -109,8 +116,6 @@ def lookup_by_verse_id(start_verse, end_verse=None):
|
||||
f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};"
|
||||
).fetchall()
|
||||
return results
|
||||
|
||||
|
||||
def get_verse_id(book_id, chapter, verse):
|
||||
# verse_id is 00000000 (00 000 000) (book + chapter + verse)
|
||||
chapterid = chapter.zfill(3)
|
||||
@@ -118,8 +123,6 @@ def get_verse_id(book_id, chapter, verse):
|
||||
if verseid == "000":
|
||||
verseid = "001" # Avoid verse_ids list IndexErrors on :- searches
|
||||
return str(book_id) + chapterid + verseid
|
||||
|
||||
|
||||
def get_next_prev_chapter(verseid):
|
||||
current_chapter = verseid[:-3]
|
||||
current_chapter_index = cached["chapter_list"].index(int(current_chapter))
|
||||
@@ -147,8 +150,6 @@ def get_next_prev_chapter(verseid):
|
||||
"next_chapter_link": next_chapter_link,
|
||||
}
|
||||
return result
|
||||
|
||||
|
||||
def ref_input_cleaning(
|
||||
ref, valid_chars=(string.ascii_letters + string.digits + ":-,"), ref_search=True
|
||||
):
|
||||
@@ -185,8 +186,6 @@ def ref_input_cleaning(
|
||||
return prefix + "".join(cleaned_ref).capitalize()
|
||||
else:
|
||||
return "".join(cleaned_ref)
|
||||
|
||||
|
||||
def search_input_cleaning(
|
||||
query, valid_chars=string.ascii_letters + string.digits + '+^"* '
|
||||
):
|
||||
@@ -195,24 +194,22 @@ def search_input_cleaning(
|
||||
if character in valid_chars:
|
||||
cleaned_query.append(character)
|
||||
return "".join(cleaned_query)
|
||||
|
||||
|
||||
def ref_parse_book(ref):
|
||||
for book in reversed(cached["book_names"]):
|
||||
if book.startswith(ref):
|
||||
return book
|
||||
return False
|
||||
|
||||
|
||||
@app.get("/find")
|
||||
def parse_query():
|
||||
query = request.args.get("kw", None)
|
||||
if query:
|
||||
query = query.strip()
|
||||
logging.info(f"Query: {query}")
|
||||
if not query:
|
||||
return send_file("html/find.html")
|
||||
if query.endswith(","):
|
||||
query = query[:-1]
|
||||
result = show()
|
||||
result = show(kw=query)
|
||||
if not hasattr(result, "__len__"):
|
||||
logging.debug(f"parse_query.show = '{result}'")
|
||||
return result
|
||||
@@ -242,8 +239,6 @@ def parse_query():
|
||||
#result += search()
|
||||
logging.debug(f"parse_query.search = '{result}'")
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/define")
|
||||
def define(word=None):
|
||||
ref = request.args.get("kw", None)
|
||||
@@ -274,15 +269,11 @@ def define(word=None):
|
||||
else:
|
||||
response = jsonify({ref: dictionary[ref]})
|
||||
return response
|
||||
|
||||
|
||||
def return_dict_plain(ref, defs):
|
||||
response = ""
|
||||
for e in defs:
|
||||
response += e + "<br><br>"
|
||||
return ref.capitalize() + "<br><br>" + response + "\n"
|
||||
|
||||
|
||||
def return_dict_rich(ref, defs, simple):
|
||||
if simple:
|
||||
css = ""
|
||||
@@ -297,12 +288,12 @@ def return_dict_rich(ref, defs, simple):
|
||||
)
|
||||
+ css
|
||||
)
|
||||
|
||||
|
||||
@app.get("/show")
|
||||
def show(kw=None, get_start_verse_id=False):
|
||||
ref = request.args.get("kw", None)
|
||||
get_chapter = request.args.get("all", False)
|
||||
get_chapter: bool = request.args.get("all", False)
|
||||
if ref.endswith(","):
|
||||
ref = ref[:-1]
|
||||
if kw:
|
||||
ref = kw
|
||||
if ref is None:
|
||||
@@ -320,6 +311,10 @@ def show(kw=None, get_start_verse_id=False):
|
||||
for ref in refs:
|
||||
logging.debug(f"show.reference({ref})")
|
||||
maybe_chapter = ref.split(":")[0][3:]
|
||||
#FIXME: Not including : in a book reference should take you to the first book
|
||||
# not just crash like a retard
|
||||
if not ":" in maybe_chapter:
|
||||
get_chapter = True
|
||||
ref_chapter = ""
|
||||
for maybedigit in maybe_chapter:
|
||||
if maybedigit.isdigit():
|
||||
@@ -363,8 +358,6 @@ def show(kw=None, get_start_verse_id=False):
|
||||
else:
|
||||
response = jsonify(response_list)
|
||||
return response
|
||||
|
||||
|
||||
def response_dict(scripture):
|
||||
# used for JSON responses
|
||||
verse = {
|
||||
@@ -374,16 +367,12 @@ def response_dict(scripture):
|
||||
"text": scripture[4],
|
||||
}
|
||||
return verse
|
||||
|
||||
|
||||
def response_text(scripture):
|
||||
bookname = lookup_bookname(scripture[1])
|
||||
chapter = str(scripture[2])
|
||||
verse = str(scripture[3])
|
||||
text = str(scripture[4])
|
||||
return bookname + " " + chapter + ":" + verse + " " + text + "\n"
|
||||
|
||||
|
||||
def response_rich(scripture):
|
||||
# (passage, text) is the html formatting arguments
|
||||
bookname = lookup_bookname(scripture[1])
|
||||
@@ -394,7 +383,6 @@ def response_rich(scripture):
|
||||
return cached["html_response"].format(
|
||||
chapter=passage.split(":")[0], passage=passage, text=text
|
||||
)
|
||||
|
||||
def scripture_response(scriptures):
|
||||
response = []
|
||||
arg_view = request.args.get("view", "json")
|
||||
@@ -428,33 +416,25 @@ def scripture_response(scriptures):
|
||||
for scripture in scriptures:
|
||||
response.append(response_dict(scripture))
|
||||
return response
|
||||
|
||||
|
||||
@app.errorhandler(404)
|
||||
def page_not_found(e):
|
||||
message = {
|
||||
"error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."
|
||||
}
|
||||
return jsonify(message)
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def idx():
|
||||
if not sanity:
|
||||
return("There is an issue with permissions. Check log, or please notify tyler@dinsmoor.us")
|
||||
logging.info("Index Served")
|
||||
return send_file("html/index.html")
|
||||
|
||||
|
||||
@app.get("/salvation")
|
||||
def salvation():
|
||||
return send_file("html/heaven.html")
|
||||
|
||||
|
||||
# backwards-compatibility
|
||||
@app.get("/t/random", defaults={"view": "plain"})
|
||||
@app.get("/t/random", defaults={"view": "plain"}) # backwards-compatibility
|
||||
@app.get("/random")
|
||||
def random_verse():
|
||||
rand_id = random.SystemRandom().choice(cached["verse_ids"])
|
||||
#scripture = lookup_one_verse(rand_id)
|
||||
scripture = lookup_by_verse_id(rand_id)
|
||||
logging.info(f"Random Served: {scripture[0]}")
|
||||
response_list = scripture_response(scripture)
|
||||
@@ -467,13 +447,10 @@ def random_verse():
|
||||
response = jsonify(response_list)
|
||||
logging.debug(f"Search Response: {response}")
|
||||
return response
|
||||
|
||||
|
||||
# backwards-compatibility
|
||||
# plain takes 38ms -TODO can we cache this?
|
||||
@app.get("/t/votd", defaults={"view": "plain"})
|
||||
@app.get("/t/votd", defaults={"view": "plain"}) # backwards-compatibility
|
||||
@app.get("/votd")
|
||||
def verse_of_the_day():
|
||||
# plain takes 38ms -TODO can we cache this?
|
||||
logging.info("VOTD Served")
|
||||
today = int(str(date.today()).replace("-", ""))
|
||||
# this is deterministic because of today's date being used as the seed
|
||||
@@ -489,7 +466,6 @@ def verse_of_the_day():
|
||||
response = jsonify(response_list)
|
||||
logging.debug(f"Search Response: {response}")
|
||||
return response
|
||||
|
||||
@app.get("/search")
|
||||
def search():
|
||||
keywords = request.args.get("kw", None)
|
||||
@@ -517,8 +493,6 @@ def search():
|
||||
response = jsonify(response_list)
|
||||
logging.debug(f"Search Response: {response}")
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/reader")
|
||||
def bible_reader():
|
||||
return send_file("html/reader.html")
|
||||
@@ -529,8 +503,6 @@ def bible_reader():
|
||||
# header should have a navigation bar and "next/previous chapter"
|
||||
# Footer should have next/previous as well.
|
||||
# Maybe preliminary theming support
|
||||
|
||||
|
||||
@app.get("/plan")
|
||||
def reading_plan():
|
||||
return send_file("html/reading_plan.html")
|
||||
@@ -538,8 +510,41 @@ def reading_plan():
|
||||
# should default to the entire Bible in 1 year.
|
||||
# should start today() and end today()+1year
|
||||
# Should return a calender with date and verse range.
|
||||
@app.get("/status")
|
||||
def status():
|
||||
import resource
|
||||
return f"""Sanity: {sanity} <br>
|
||||
UserMode: {usermode} <br>
|
||||
Memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}
|
||||
"""
|
||||
@app.get("/seqreport")
|
||||
def sequential_report(uid=None):
|
||||
if usermode == "Single":
|
||||
seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False)
|
||||
else:
|
||||
seq_conn = sqlite3.connect("data/seq.db")
|
||||
seq_cur = seq_conn.cursor()
|
||||
uids = seq_cur.execute("select * from reading_tracker").fetchall()
|
||||
return render_template("seq_report.html", uids=uids)
|
||||
def sequential_purge():
|
||||
#TODO: Test this :^)
|
||||
if usermode == "Single":
|
||||
seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False)
|
||||
else:
|
||||
seq_conn = sqlite3.connect("data/seq.db")
|
||||
seq_cur = seq_conn.cursor()
|
||||
uids = seq_cur.execute("select * from reading_tracker").fetchmany(10)
|
||||
|
||||
evaluate_datetime = datetime.now()
|
||||
for uid in uids:
|
||||
last_used = datetime.fromisoformat(uid[2])
|
||||
since_use: timedelta = evaluate_datetime - last_used
|
||||
if since_use > sequential_purge_delta:
|
||||
logging.info(f"Purging UID: {uid[0]}")
|
||||
seq_cur.execute(f"DELETE FROM reading_tracker WHERE uid = '{uid[0]}'")
|
||||
|
||||
seq_conn.commit()
|
||||
seq_cur.close()
|
||||
@app.get("/seq")
|
||||
def sequential_read(start=0, num=1, uid=None, getuid=False):
|
||||
# /seq?uid=1
|
||||
@@ -552,35 +557,35 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
|
||||
return "Invalid num argument"
|
||||
if not uid and not getuid:
|
||||
return send_file("html/seq.html")
|
||||
seq_conn = sqlite3.connect("data/seq.db")
|
||||
if usermode == "Single":
|
||||
seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False)
|
||||
else:
|
||||
seq_conn = sqlite3.connect("data/seq.db")
|
||||
seq_cur = seq_conn.cursor()
|
||||
seq_cur.execute("CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)")
|
||||
last_used = str(datetime.now())
|
||||
num -= 1 # 0 will mean "next"
|
||||
if num < 0:
|
||||
return jsonify({"error": "You may only increment forward."})
|
||||
|
||||
if getuid:
|
||||
import uuid
|
||||
|
||||
if start:
|
||||
|
||||
start = cached["verse_ids"].index(
|
||||
int(show(kw=start, get_start_verse_id=True))
|
||||
)
|
||||
uid = str(uuid.uuid4())
|
||||
uid = int32_to_id(random.randrange(614_656,17_210_367)) # range returns a 5 digit id
|
||||
try:
|
||||
seq_cur.execute(
|
||||
f"INSERT INTO reading_tracker VALUES('{uid}', {start}, '{last_used}')"
|
||||
)
|
||||
except sqlite3.OperationalError:
|
||||
logging.error(f"UID Database failed to insert {uid} into reading_tracker.")
|
||||
except sqlite3.OperationalError as e:
|
||||
logging.error(f"UID Database failed to insert {uid} into reading_tracker. {e}")
|
||||
return jsonify(
|
||||
{
|
||||
"error": "Something went wrong with the UID database. Check permissions?"
|
||||
"error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"
|
||||
}
|
||||
)
|
||||
seq_conn.commit()
|
||||
logging.info(f"Created new SEQ UID: {uid}")
|
||||
logging.info(f"New SEQ UID: {uid}")
|
||||
return f"Your new UID is: {uid}"
|
||||
uid = ref_input_cleaning(uid, ref_search=False)
|
||||
try:
|
||||
@@ -588,9 +593,10 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
|
||||
f"SELECT verse FROM reading_tracker WHERE uid = '{uid}'"
|
||||
).fetchone()[0]
|
||||
logging.debug(f"Queried UID {uid} on verse {nextverse}")
|
||||
sequential_purge()
|
||||
except TypeError:
|
||||
logging.debug(f"Invalid UID {uid}")
|
||||
return jsonify({"error": "UID Invalid or purged"})
|
||||
logging.info(f"Invalid UID {uid}")
|
||||
return jsonify({"error": "UID Invalid or purged (90 days of inactivity)"})
|
||||
if len(cached["verse_ids"]) < (nextverse + num):
|
||||
# Correct our range to go to ceiling to prevent IndexError
|
||||
diff = len(cached["verse_ids"]) - (nextverse + num)
|
||||
@@ -602,8 +608,10 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
|
||||
|
||||
logging.info(f"Served {num+1} verses to UID {uid}")
|
||||
seq_cur.execute(
|
||||
f"UPDATE reading_tracker SET verse = {nextverse+num+1}, last_used = '{last_used}' WHERE uid = '{uid}'"
|
||||
f"REPLACE INTO reading_tracker (uid, verse, last_used) VALUES('{uid}', {nextverse+num+1}, '{last_used}')"
|
||||
)
|
||||
# Doing REPLACE INTO instead of UPDATE will make the updated row at the end of the table, making the purging
|
||||
# only look at a small set of the oldest unused entries.
|
||||
seq_conn.commit()
|
||||
arg_view = request.args.get("view", None)
|
||||
if arg_view == "plain" or arg_view == "rich":
|
||||
@@ -613,7 +621,8 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
|
||||
|
||||
return response
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("You must run this program within a uwsgi environment.")
|
||||
# app.run(host="127.0.0.1", port=1612)
|
||||
print("You must run this program within a uwsgi environment if you are running this for multiple users.")
|
||||
print("Otherwise, you may end up with database corruption if multiple writes happen.")
|
||||
print("Operating in single user mode.")
|
||||
app.run(host="127.0.0.1", port=1611)
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
# Check if all the files needed by kjv-api (to write to) are also owned by the user running it.
|
||||
# if this is not the same, writing to data/seq.db (and other undesirable behavior) is inevitable
|
||||
# and will save a lot of headache in the future.
|
||||
|
||||
#Additionally, proper CHMOD value is 664 for the database
|
||||
from os import stat, getuid
|
||||
|
||||
def check_sanity():
|
||||
dependancies = ['data/seq.db']
|
||||
current_uid = getuid()
|
||||
for dependancy in dependancies:
|
||||
if stat(dependancy).st_uid != current_uid:
|
||||
raise PermissionError(dependancy)
|
||||
Reference in New Issue
Block a user