# app.py from flask import Flask, request, jsonify, send_file import random from datetime import date, datetime # used for daily verse import sqlite3 import string # for literals import json # for dictionary import hunspell # for spell checking import sys import logging timestamp_runtime = datetime.now().strftime("%d%b%y-%H:%M") logging.basicConfig( filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log", encoding="utf-8", level=logging.INFO, ) logging.info(f"Started: {timestamp_runtime}") app = Flask(__name__) """ Current work prefixed with * TODO: Replace send_file with something that loads headers and nav templates Search phrase alias dictionary Response Bible book reading *Forward and Backward Navigation ** add to rich responses Search highlighting Styles Bible Reading Plan Bible Reader /seq Going backwards (works but is buggy, disabled for now) /seq Wrap back to Gen 1:1 when you reach the end of Rev /seq Querying what your UID's next verse is going to be /seq Purging old UIDs after a certain period of non-use """ kjv_cur = sqlite3.connect("data/kjv.db").cursor() cached = { "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()], "book_names": [ i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall() ], "chapter_list": [], "json_dictionary": json.load(open("data/1828_Webster_KJV.json", "r")), "html_response": open("html/response.html", "r").read(), "html_navbar": open("html/navbar.html", "r").read(), "css_navbar": open("styles/header.css", "r").read(), "js_navbar": open("js/header.js", "r").read(), "css_response": open("styles/human_readable.css", "r").read(), "css_collapsible": open("styles/collapsible.css", "r").read(), "js_collapsible": open("js/collapsible.js", "r").read(), } # Fill the chapter list with unique chapters for seeking back and forward by index. #TODO Fix string-based..? sorting problem [ cached["chapter_list"].append(int(str(chap)[:-3])) for chap in cached.get("verse_ids") if int(str(chap)[:-3]) not in cached["chapter_list"] ] cached["chapter_list"].sort() def build_html_reponse(): html = "" return html def lookup_one_verse(verse_id): logging.debug(f"lookup_one_verse({verse_id})") return kjv_cur.execute(f"SELECT * FROM fts_kjv WHERE id = {verse_id};").fetchone() def lookup_bookname(book_id): logging.debug(f"lookup_bookname({book_id})") result = kjv_cur.execute( f"SELECT n FROM key_english WHERE b = {book_id};" ).fetchone() if result: result = result[0] logging.debug(f"lookup_bookname.result = '{result}'") return result def lookup_book_id(bookname): logging.debug(f"lookup_book_id({bookname})") result = kjv_cur.execute( f"SELECT b FROM key_english WHERE n = '{bookname}';" ).fetchone() if result: result = result[0] logging.debug(f"lookup_book_id.result = '{result}'") return result def lookup_fts(kwds): logging.debug(f"lookup_fts.kwds = '{kwds}'") return kjv_cur.execute(f"SELECT * FROM fts_kjv('{kwds}')").fetchall() def lookup_by_verse_id(start_verse, end_verse=None): if not end_verse: results = kjv_cur.execute( f"SELECT * FROM fts_kjv WHERE id = {start_verse};" ).fetchall() else: results = kjv_cur.execute( f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};" ).fetchall() return results def get_verse_id(book_id, chapter, verse): # verse_id is 00000000 (00 000 000) (book + chapter + verse) chapterid = chapter.zfill(3) verseid = verse.zfill(3) if verseid == "000": verseid = "001" # Avoid verse_ids list IndexErrors on :- searches return str(book_id) + chapterid + verseid def get_next_prev_chapter(verseid): current_chapter = verseid[:-3] current_chapter_index = cached["chapter_list"].index(int(current_chapter)) #next_chapter_id = str(cached["chapter_list"][current_chapter_index + 1]) next_chapter_id = str(cached["chapter_list"][(current_chapter_index + 1) % len(cached["chapter_list"])]) #prev_chapter_id = str(cached["chapter_list"][current_chapter_index - 1]) prev_chapter_id = str(cached["chapter_list"][(current_chapter_index - 1) % len(cached["chapter_list"])]) next_chapter_name = " ".join( (lookup_bookname(next_chapter_id[:-3]), str(next_chapter_id[-3:]).lstrip("0")) ) prev_chapter_name = " ".join( (lookup_bookname(prev_chapter_id[:-3]), str(prev_chapter_id[-3:]).lstrip("0")) ) next_chapter_link = f"/show?kw={next_chapter_name}&view=rich" prev_chapter_link = f"/show?kw={prev_chapter_name}&view=rich" result = { "prev_chapter_name": prev_chapter_name, "prev_chapter_link": prev_chapter_link, "next_chapter_name": next_chapter_name, "next_chapter_link": next_chapter_link, } return result def ref_input_cleaning( ref, valid_chars=(string.ascii_letters + string.digits + ":-,"), ref_search=True ): logging.debug(f"ref_input_cleaning({ref})") ref = ref.replace(" ", "") if ref.count(":") != 1 and ref_search: logging.debug("ref_input_cleaning.if ref.count(:): TRUE") logging.debug(" -> Assuming user wanted entire chapter") ref += ":-" if ref.endswith(":"): ref += "-" if ref.count("-") > 1 and ref_search: logging.debug("ref_input_cleaning.if ref.count(-): TRUE") return False if not ref[-1].isdigit() and ref[-1] != "-" and ref_search: logging.debug("ref_input_cleaning.if ref[-1] TRUE") return False ref = list(ref) # preserve book number if we can guess the first # character is a number and 3nd is a letter if ref[0].isdigit() and ref[1].isalpha() and ref_search: prefix = ref.pop(0) + " " else: prefix = "" # clean the rest of the reference cleaned_ref = [] for character in ref: if character in valid_chars: cleaned_ref.append(character) # Was there a problem? if len(cleaned_ref) == 0: return False if ref_search: return prefix + "".join(cleaned_ref).capitalize() else: return "".join(cleaned_ref) def search_input_cleaning( query, valid_chars=string.ascii_letters + string.digits + '+^"* ' ): cleaned_query = [] for character in query: if character in valid_chars: cleaned_query.append(character) return "".join(cleaned_query) def ref_parse_book(ref): for book in reversed(cached["book_names"]): if book.startswith(ref): return book return False @app.get("/find") def parse_query(): query = request.args.get("kw", None) logging.info(f"Query: {query}") if not query: return send_file("html/find.html") if query.endswith(","): query = query[:-1] result = show() if not hasattr(result, "__len__"): logging.debug(f"parse_query.show = '{result}'") return result if len(result) > 17: return result result = "" result += "" result += "

1828 Webster Definitions

" split_query = query.replace(",", " ").split(" ") if len(split_query) > 20: return "Your Query is too long. 20 Words or less." for word in split_query: definition = define(word) if not definition: continue result += ( "" ) result += "
" + definition + "
" logging.debug(f"parse_query.define = '{word}'") result += "" result += "
" result += search() logging.debug(f"parse_query.search = '{result}'") return result @app.get("/define") def define(word=None): ref = request.args.get("kw", None) if word: ref = word if ref: ref = ref.strip() if not ref: return send_file("html/dictionary.html") dictionary = cached["json_dictionary"] ref = ref.lower() if not ref in dictionary.keys(): if word: return False logging.warning(f"WEBSTER MISSING: {ref}") return f"{ref} not found." # search suggestions kind of suck # h = hunspell.HunSpell("data/kjv.dic", "data/kjv.aff") # suggestions = h.suggest(ref) # return "{} not found, try one of the following: {}".format( # ref, ", ".join(suggestions) # ) arg_view = request.args.get("view", None) if arg_view == "plain": response = return_dict_plain(ref, dictionary[ref]) elif arg_view == "rich": response = return_dict_rich(ref, dictionary[ref], simple=word) else: response = jsonify({ref: dictionary[ref]}) return response def return_dict_plain(ref, defs): response = "" for e in defs: response += e + "

" return ref.capitalize() + "

" + response + "\n" def return_dict_rich(ref, defs, simple): if simple: css = "" else: css = cached["css_response"] response = "" for e in defs: response += e + "

" return ( cached["html_response"].format( chapter=ref.split(":")[0], passage=ref.capitalize(), text=response ) + css ) @app.get("/show") def show(kw=None, get_start_verse_id=False): ref = request.args.get("kw", None) get_chapter = request.args.get("all", False) if kw: ref = kw if ref is None: return send_file("html/show.html") logging.debug(f"show({ref})") refs = [] for r in ref.split(","): r_ = ref_input_cleaning(r.strip()) if r_: refs.append(r_) if not refs: return "Invalid Query" verse_results = [] # check if the search is for multiple refs for ref in refs: logging.debug(f"show.reference({ref})") maybe_chapter = ref.split(":")[0][3:] ref_chapter = "" for maybedigit in maybe_chapter: if maybedigit.isdigit(): ref_chapter += maybedigit ref_verse = ref.split(":")[1] if ref.__contains__("-") or get_chapter: if ref.endswith("-") or get_chapter: start_verse = ref_verse.split("-")[0] end_verse = "500" # just get all the verses if it ends with a dash else: start_verse, end_verse = ( ref_verse.split("-")[0], ref_verse.split("-")[1], ) else: start_verse = ref_verse end_verse = None end_verse_id = None book_str = ref_input_cleaning(ref, string.ascii_letters) ref_bookname = ref_parse_book(book_str) ref_book_id = lookup_book_id(ref_bookname) if not ref_book_id: return "Invalid Query" start_verse_id = get_verse_id(ref_book_id, ref_chapter, start_verse) if get_start_verse_id: return start_verse_id if end_verse: end_verse_id = get_verse_id(ref_book_id, ref_chapter, end_verse) logging.debug(f"show.start_verse_id({start_verse_id})") logging.debug(f"show.end_verse_id({end_verse_id})") results = lookup_by_verse_id(start_verse_id, end_verse_id) logging.debug(f"show.results({results})") for result in results: verse_results.append(result) response_list = response_multi(verse_results) arg_view = request.args.get("view", None) if arg_view == "plain": response = "
".join(response_list) elif arg_view == "rich": response = "
".join(response_list) else: response = jsonify(response_list) return response def response_dict(scripture): # used for JSON responses verse = { "bookname": lookup_bookname(scripture[1]), "chapter": scripture[2], "verse": scripture[3], "text": scripture[4], } return verse def response_text(scripture): bookname = lookup_bookname(scripture[1]) chapter = str(scripture[2]) verse = str(scripture[3]) text = str(scripture[4]) return bookname + " " + chapter + ":" + verse + " " + text + "\n" def response_rich(scripture): # (passage, text) is the html formatting arguments bookname = lookup_bookname(scripture[1]) chapter = str(scripture[2]) verse = str(scripture[3]) text = str(scripture[4]) passage = bookname + " " + chapter + ":" + verse return cached["html_response"].format( chapter=passage.split(":")[0], passage=passage, text=text ) def response_single(scripture): logging.debug(f"response_single({scripture})") arg_view = request.args.get("view", "json") logging.debug(f"->requested_format = {arg_view}") if arg_view == "plain": return response_text(scripture) elif arg_view == "rich": l_r_nav = get_next_prev_chapter( str(scripture[0]) ) # get the first scripture, the id thereof return ( cached["html_navbar"].format( query=request.args.get("kw", ""), next_chapter_name=l_r_nav["next_chapter_name"], next_chapter_link=l_r_nav["next_chapter_link"], prev_chapter_name=l_r_nav["prev_chapter_name"], prev_chapter_link=l_r_nav["prev_chapter_link"], ) + cached["js_navbar"] + cached["css_navbar"] + cached["css_response"] + response_rich(scripture) ) else: response = response_dict(scripture) return jsonify(response) def response_multi(scriptures): response = [] arg_view = request.args.get("view", "json") logging.debug(f"search ->requested_format = {arg_view}") if arg_view == "plain": for scripture in scriptures: response.append(response_text(scripture)) elif arg_view == "rich": l_r_nav = get_next_prev_chapter( str(scriptures[0][0]) ) # get the first scripture, the id thereof response.append(cached["css_response"]) response.append( cached["html_navbar"].format( query=request.args.get("kw", ""), next_chapter_name=l_r_nav["next_chapter_name"], next_chapter_link=l_r_nav["next_chapter_link"], prev_chapter_name=l_r_nav["prev_chapter_name"], prev_chapter_link=l_r_nav["prev_chapter_link"], ) ) response.append(cached["css_navbar"]) response.append(cached["js_navbar"]) for scripture in scriptures: response.append(response_rich(scripture)) else: for scripture in scriptures: response.append(response_dict(scripture)) return response @app.errorhandler(404) def page_not_found(e): message = { "error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness." } return jsonify(message) @app.get("/") def idx(): logging.info("Index Served") return send_file("html/index.html") @app.get("/salvation") def salvation(): return send_file("html/heaven.html") # backwards-compatibility @app.get("/t/random", defaults={"view": "plain"}) @app.get("/random") def random_verse(): rand_id = random.SystemRandom().choice(cached["verse_ids"]) scripture = lookup_one_verse(rand_id) logging.info(f"Random Served: {scripture[0]}") return response_single(scripture) # backwards-compatibility # plain takes 38ms -TODO can we cache this? @app.get("/t/votd", defaults={"view": "plain"}) @app.get("/votd") def verse_of_the_day(): logging.info("VOTD Served") today = int(str(date.today()).replace("-", "")) # this is deterministic because of today's date being used as the seed random.seed(today) scripture = lookup_one_verse(random.choice(cached["verse_ids"])) return response_single(scripture) @app.get("/search") def search(): keywords = request.args.get("kw", None) logging.debug(f"search({keywords})") if keywords: cleaned_keywords = search_input_cleaning(keywords) if not cleaned_keywords: return send_file("html/search.html") else: return send_file("html/search.html") results = lookup_fts(cleaned_keywords) if not results: h = hunspell.HunSpell("data/kjv.dic", "data/kjv.aff") suggestions = h.suggest(cleaned_keywords) return "{} not found, try one of the following: {}".format( cleaned_keywords, ", ".join(suggestions) ) response_list = response_multi(results) arg_view = request.args.get("view", None) if arg_view == "plain": response = "
".join(response_list) elif arg_view == "rich": response = "
".join(response_list) else: response = jsonify(response_list) logging.debug(f"Search Response: {response}") return response @app.get("/reader") def bible_reader(): return send_file("html/reader.html") # We should return an index of all books with # drop down menus of each chapter, and a search # bar at the top for quick seeking, which can use # show(). Each page should return a header and footer. # header should have a navigation bar and "next/previous chapter" # Footer should have next/previous as well. # Maybe preliminary theming support @app.get("/plan") def reading_plan(): return send_file("html/reading_plan.html") # needs 2 sets of data, books and time frame. # should default to the entire Bible in 1 year. # should start today() and end today()+1year # Should return a calender with date and verse range. @app.get("/seq") def sequential_read(start=0, num=1, uid=None, getuid=False): # /seq?uid=1 uid = request.args.get("uid", uid) getuid = request.args.get("getuid", getuid) start = request.args.get("start", start) try: num = int(request.args.get("num", num)) except ValueError: return "Invalid num argument" if not uid and not getuid: return send_file("html/seq.html") seq_conn = sqlite3.connect("data/seq.db") seq_cur = seq_conn.cursor() last_used = str(datetime.now()) num -= 1 # 0 will mean "next" if num < 0: return jsonify({"error": "You may only increment forward."}) if getuid: import uuid if start: start = cached["verse_ids"].index( int(show(kw=start, get_start_verse_id=True)) ) uid = str(uuid.uuid4()) try: seq_cur.execute( f"INSERT INTO reading_tracker VALUES('{uid}', {start}, '{last_used}')" ) except sqlite3.OperationalError: logging.error(f"UID Database failed to insert {uid} into reading_tracker.") return jsonify( { "error": "Something went wrong with the UID database. Check permissions?" } ) seq_conn.commit() logging.info(f"Created new SEQ UID: {uid}") return f"Your new UID is: {uid}" uid = ref_input_cleaning(uid, ref_search=False) try: nextverse = seq_cur.execute( f"SELECT verse FROM reading_tracker WHERE uid = '{uid}'" ).fetchone()[0] logging.debug(f"Queried UID {uid} on verse {nextverse}") except TypeError: logging.debug(f"Invalid UID {uid}") return jsonify({"error": "UID Invalid or purged"}) if len(cached["verse_ids"]) < (nextverse + num): # Correct our range to go to ceiling to prevent IndexError diff = len(cached["verse_ids"]) - (nextverse + num) num = diff - num start_verse = cached["verse_ids"][nextverse] end_verse = cached["verse_ids"][nextverse + num] result_list = lookup_by_verse_id(start_verse, end_verse) response_list = response_multi(result_list) logging.info(f"Served {num+1} verses to UID {uid}") seq_cur.execute( f"UPDATE reading_tracker SET verse = {nextverse+num+1}, last_used = '{last_used}' WHERE uid = '{uid}'" ) seq_conn.commit() arg_view = request.args.get("view", None) if arg_view == "plain" or arg_view == "rich": response = "
".join(response_list) else: response = jsonify(response_list) return response if __name__ == "__main__": print("You must run this program within a uwsgi environment.") # app.run(host="127.0.0.1", port=1612)