Switched to sqlite virtual table searching, a built-in reference parser, and changed endpoints to be more sane.
This commit is contained in:
@@ -2,18 +2,11 @@
|
||||
from flask import Flask, request, jsonify, send_file
|
||||
import random
|
||||
from datetime import date # used for daily verse
|
||||
import subprocess # use of "bible" system application for parsing and returning Bible in KJV0
|
||||
import shlex # input security
|
||||
import sqlite3
|
||||
import string
|
||||
DEBUG = 2
|
||||
DEBUG_OUT = 0
|
||||
import string # for literals
|
||||
DEBUG = 0
|
||||
|
||||
"""
|
||||
NOTES
|
||||
- working on getting multi-search terms working right
|
||||
Lookup_bookname does not like it lol.
|
||||
|
||||
To-do:
|
||||
- Add data and function for Webster 1828 dictionary lookup
|
||||
--https://github.com/DataWar/1828-dictionary
|
||||
@@ -34,9 +27,9 @@ def dbg2(message):
|
||||
dbg("Debug Level 1 Enabled")
|
||||
dbg2("Debug Level 2 Enabled")
|
||||
|
||||
kjv_db = sqlite3.connect("kjv.db")
|
||||
kjv_cur = kjv_db.cursor()
|
||||
verse_ids = [i[0] for i in kjv_cur.execute("SELECT id FROM t_kjv;").fetchall()]
|
||||
kjv_cur = sqlite3.connect("kjv.db").cursor()
|
||||
verse_ids = [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()]
|
||||
book_names = [i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall()]
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
@@ -48,26 +41,182 @@ idx_text = 4
|
||||
|
||||
def lookup_one_verse(verse_id):
|
||||
dbg("lookup_one_verse({})".format(verse_id))
|
||||
return(kjv_cur.execute("SELECT * FROM t_kjv WHERE id = {};".format(verse_id)).fetchone())
|
||||
return(kjv_cur.execute("SELECT * FROM fts_kjv WHERE id = {};".format(verse_id)).fetchone())
|
||||
|
||||
def lookup_bookname(book_id):
|
||||
dbg2("lookup_bookname({})".format(book_id))
|
||||
result = kjv_cur.execute("SELECT n FROM key_english WHERE b = {}".format(book_id)).fetchone()[0]
|
||||
result = kjv_cur.execute("SELECT n FROM key_english WHERE b = {};".format(book_id)).fetchone()
|
||||
if result:
|
||||
result = result[0]
|
||||
dbg("lookup_bookname: {}".format(result))
|
||||
return result
|
||||
|
||||
def lookup_book_id(bookname):
|
||||
dbg2("lookup_book_id({})".format(bookname))
|
||||
result = kjv_cur.execute("SELECT b FROM key_english WHERE n = '{}';".format(bookname)).fetchone()
|
||||
if result:
|
||||
result = result[0]
|
||||
return result
|
||||
|
||||
def lookup_fts(kwds):
|
||||
return kjv_cur.execute("SELECT * FROM fts_kjv('{}')".format(kwds)).fetchall()
|
||||
|
||||
def lookup_contains(kw):
|
||||
results = kjv_cur.execute("SELECT * FROM t_kjv WHERE t LIKE '%{}%';".format(kw)).fetchall()
|
||||
results = kjv_cur.execute("SELECT * FROM fts_kjv WHERE t LIKE '%{}%';".format(kw)).fetchall()
|
||||
dbg("lookup_contains({}) {} results".format(kw,len(results)))
|
||||
return results
|
||||
|
||||
def lookup_by_verse_id(start_verse, end_verse = None):
|
||||
if not end_verse:
|
||||
results = kjv_cur.execute("SELECT * FROM fts_kjv WHERE id = {};".format(start_verse)).fetchall()
|
||||
else:
|
||||
results = kjv_cur.execute("SELECT * FROM fts_kjv WHERE id BETWEEN {} AND {};".format(start_verse, end_verse)).fetchall()
|
||||
return results
|
||||
|
||||
|
||||
def get_verse_id(book_id, chapter, verse):
|
||||
# verse_id is 00000000 (00 000 000) (book + chapter + verse)
|
||||
chapterid = chapter.zfill(3)
|
||||
verseid = verse.zfill(3)
|
||||
return str(book_id)+chapterid+verseid
|
||||
|
||||
def check_lone_word(text, kw):
|
||||
kw_length = len(kw)
|
||||
kw_location = text.find(kw)
|
||||
try:
|
||||
if (text[kw_location-1]).isalpha():
|
||||
return False
|
||||
except IndexError:
|
||||
# If it's an indexerror, it's likely that
|
||||
# the keyword is at the end or beginning
|
||||
# of the text.
|
||||
pass
|
||||
try:
|
||||
if (text[kw_location+kw_length]).isalpha():
|
||||
return False
|
||||
except IndexError:
|
||||
pass
|
||||
return True
|
||||
|
||||
def lookup_contains_exact(kw):
|
||||
results = kjv_cur.execute("SELECT * FROM t_kjv WHERE t LIKE '%{}%';".format(kw)).fetchall()
|
||||
preresult = lookup_contains(kw)
|
||||
results = []
|
||||
for result in preresult:
|
||||
if check_lone_word(result[idx_text], kw):
|
||||
results.append(result)
|
||||
dbg("lookup_contains({}) {} results".format(kw,len(results)))
|
||||
return results
|
||||
|
||||
def ref_input_cleaning(ref, valid_chars=(string.ascii_letters + string.digits + ":-,")):
|
||||
# removed invalid characters, whitespace
|
||||
ref = list(ref.replace(" ",""))
|
||||
|
||||
# basic validation checks
|
||||
# do we have a valid verse designator?
|
||||
if ref.count(':') != 1:
|
||||
return False
|
||||
# are we only checking one range?
|
||||
if ref.count("-") > 1:
|
||||
return False
|
||||
# does our argument end in a number or an endrange?
|
||||
if not ref[-1].isdigit() and ref[-1] != '-':
|
||||
return False
|
||||
|
||||
|
||||
# preserve book number if we can guess the first
|
||||
# character is a number and 3nd is a letter
|
||||
if ref[0].isdigit() and ref[1].isalpha():
|
||||
prefix = ref.pop(0) + " "
|
||||
else:
|
||||
prefix = ''
|
||||
|
||||
# clean the rest of the reference
|
||||
cleaned_ref = []
|
||||
for character in ref:
|
||||
if character in valid_chars:
|
||||
cleaned_ref.append(character)
|
||||
# Was there a problem?
|
||||
if len(cleaned_ref) == 0:
|
||||
return False
|
||||
return prefix + ''.join(cleaned_ref).capitalize()
|
||||
|
||||
def search_input_cleaning(query, valid_chars=string.ascii_letters + string.digits + "+^\" "):
|
||||
cleaned_query = []
|
||||
for character in query:
|
||||
if character in valid_chars:
|
||||
cleaned_query.append(character)
|
||||
return ''.join(cleaned_query)
|
||||
|
||||
|
||||
|
||||
def ref_parse_book(ref):
|
||||
for book in reversed(book_names):
|
||||
if book.startswith(ref):
|
||||
return book
|
||||
return False
|
||||
|
||||
@app.get("/show")
|
||||
def show():
|
||||
ref = request.args.get("kw", "John 3:16")
|
||||
dbg("show({})".format(ref))
|
||||
ref = ref_input_cleaning(ref)
|
||||
if not ref:
|
||||
return("Invalid Query")
|
||||
verse_results = []
|
||||
#check if the search is for multiple refs
|
||||
refs = ref.split(',')
|
||||
for ref in refs:
|
||||
dbg("show.reference({})".format(ref))
|
||||
|
||||
maybe_chapter = ref.split(':')[0][3:]
|
||||
ref_chapter = ''
|
||||
for maybedigit in maybe_chapter:
|
||||
if maybedigit.isdigit():
|
||||
ref_chapter += maybedigit
|
||||
|
||||
ref_verse = ref.split(':')[1]
|
||||
if ref.__contains__('-'):
|
||||
if ref.endswith('-'):
|
||||
start_verse = ref_verse.split("-")[0]
|
||||
end_verse = '500' # just get all the verses if it ends with a dash
|
||||
else:
|
||||
start_verse, end_verse = ref_verse.split("-")[0], ref_verse.split("-")[1]
|
||||
else:
|
||||
start_verse = ref_verse
|
||||
end_verse = None
|
||||
end_verse_id = None
|
||||
|
||||
book_str = ref_input_cleaning(ref, string.ascii_letters)
|
||||
ref_bookname = ref_parse_book(book_str)
|
||||
ref_book_id = lookup_book_id(ref_bookname)
|
||||
if not ref_book_id:
|
||||
return("Book reference invalid. Ex: 1John3:16")
|
||||
|
||||
start_verse_id = get_verse_id(ref_book_id, ref_chapter, start_verse)
|
||||
if end_verse:
|
||||
end_verse_id = get_verse_id(ref_book_id, ref_chapter, end_verse)
|
||||
|
||||
dbg2("show.start_verse_id({})".format(start_verse_id))
|
||||
dbg2("show.end_verse_id({})".format(end_verse_id))
|
||||
results = lookup_by_verse_id(start_verse_id, end_verse_id)
|
||||
dbg2("show.results({})".format(results))
|
||||
|
||||
for result in results:
|
||||
verse_results.append(result)
|
||||
|
||||
response_list = response_multi(verse_results)
|
||||
|
||||
arg_view = request.args.get("view", None)
|
||||
if arg_view == "plain" or arg_view == "rich":
|
||||
response = "<br>".join(response_list)
|
||||
else:
|
||||
response = jsonify(response_list)
|
||||
|
||||
return response
|
||||
|
||||
|
||||
def response_dict(scripture):
|
||||
# used for JSON responses
|
||||
verse = {
|
||||
"bookname": lookup_bookname(scripture[idx_book]),
|
||||
"chapter": scripture[idx_chapter],
|
||||
@@ -86,7 +235,7 @@ def response_text(scripture):
|
||||
def response_rich(scripture):
|
||||
css = open('human_readable.css', 'r')
|
||||
response_html = open('response.html', 'r')
|
||||
# passage, text
|
||||
# (passage, text) is the html formatting arguments
|
||||
|
||||
bookname = lookup_bookname(scripture[idx_book])
|
||||
chapter = str(scripture[idx_chapter])
|
||||
@@ -99,7 +248,7 @@ def response_rich(scripture):
|
||||
|
||||
def response_single(scripture):
|
||||
dbg2("response_single({})".format(scripture))
|
||||
arg_view = request.args.get("view", None)
|
||||
arg_view = request.args.get("view", 'json')
|
||||
dbg("->requested_format = {}".format(arg_view))
|
||||
if arg_view == "plain":
|
||||
return(response_text(scripture))
|
||||
@@ -112,20 +261,17 @@ def response_single(scripture):
|
||||
def response_multi(scriptures):
|
||||
dbg2("response_multi({})".format(scriptures))
|
||||
response = []
|
||||
arg_view = request.args.get("view", None)
|
||||
arg_view = request.args.get("view", 'json')
|
||||
dbg("search ->requested_format = {}".format(arg_view))
|
||||
|
||||
if arg_view == "plain":
|
||||
for scripture in scriptures:
|
||||
dbg2("for scripture in scriptures content: {}".format(scripture))
|
||||
response.append(response_text(scripture))
|
||||
elif arg_view == "rich":
|
||||
for scripture in scriptures:
|
||||
dbg2("for scripture in scriptures content: {}".format(scripture))
|
||||
response.append(response_rich(scripture))
|
||||
else:
|
||||
for scripture in scriptures:
|
||||
dbg2("for scripture in scriptures content: {}".format(scripture))
|
||||
response.append(response_dict(scripture))
|
||||
|
||||
return(response)
|
||||
@@ -146,6 +292,10 @@ def page_not_found(e):
|
||||
def idx():
|
||||
return send_file("index.html")
|
||||
|
||||
@app.get("/salvation")
|
||||
def salvation():
|
||||
return send_file("heaven.html")
|
||||
|
||||
#backwards-compatibility
|
||||
@app.get('/t/random', defaults={'view':'plain'})
|
||||
@app.get('/random')
|
||||
@@ -164,51 +314,18 @@ def verse_of_the_day():
|
||||
scripture = lookup_one_verse(random.choice(verse_ids))
|
||||
return(response_single(scripture))
|
||||
|
||||
# splits by delimiter then checks for any special characters
|
||||
# if special characters are found, the query is rejected
|
||||
def clean_keywords(kwds):
|
||||
kwds = kwds.split(',')
|
||||
for kwd in kwds:
|
||||
if not kwd.isalpha():
|
||||
return False
|
||||
dbg("clean_keywords = ({})".format(kwds))
|
||||
return kwds
|
||||
|
||||
|
||||
def search_and(cleaned_keywords):
|
||||
results = []
|
||||
for keyword in cleaned_keywords:
|
||||
dbg("search->for keyword in database({})".format(keyword))
|
||||
#This is groups of results per word. Must check between groups.
|
||||
results.append(lookup_contains(keyword))
|
||||
return (results)
|
||||
|
||||
def search_any(cleaned_keywords):
|
||||
results = []
|
||||
for keyword in cleaned_keywords:
|
||||
dbg("search->for keyword in database({})".format(keyword))
|
||||
for result in lookup_contains(keyword):
|
||||
results.append(result)
|
||||
dbg("==> search_any: {} results".format(len(results)))
|
||||
return(results)
|
||||
|
||||
|
||||
@app.get("/search")
|
||||
def search():
|
||||
keywords = request.args.get("kw", None)
|
||||
dbg("search({})".format(keywords))
|
||||
if keywords:
|
||||
cleaned_keywords = clean_keywords(keywords)
|
||||
if not clean_keywords:
|
||||
cleaned_keywords = search_input_cleaning(keywords)
|
||||
if not cleaned_keywords:
|
||||
return ("Invalid Query: Special Characters")
|
||||
else:
|
||||
return ("No Query")
|
||||
|
||||
if request.args.get("operator",None) == "and":
|
||||
results = search_and(cleaned_keywords)
|
||||
else:
|
||||
results = search_any(cleaned_keywords)
|
||||
|
||||
results = lookup_fts(cleaned_keywords)
|
||||
response_list = response_multi(results)
|
||||
|
||||
arg_view = request.args.get("view", None)
|
||||
@@ -219,128 +336,3 @@ def search():
|
||||
|
||||
dbg2("Search Response: {}".format(response))
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/qwerty")
|
||||
def test():
|
||||
style = css.read()
|
||||
return (style + "Hello! Testing!")
|
||||
|
||||
# plain
|
||||
@app.get("/<passage>")
|
||||
def passage_json(passage):
|
||||
try:
|
||||
command = "bible -f {}".format(shlex.quote(passage))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
return jsonify(result)
|
||||
except Exception as e:
|
||||
return jsonify(str(e))
|
||||
|
||||
# pretty
|
||||
@app.get("/p/<passage>")
|
||||
def passage_text(passage):
|
||||
try:
|
||||
command = "bible {}".format(shlex.quote(passage))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
return result
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
|
||||
#search
|
||||
@app.get("/s/<keyword>")
|
||||
def search_json(keyword):
|
||||
try:
|
||||
command = "bible -f ??{}".format(shlex.quote(keyword))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
if result.contains("??word"):
|
||||
raise Exception("Not found.")
|
||||
result = result.split()
|
||||
result = result[7:]
|
||||
return jsonify(result)
|
||||
except Exception as e:
|
||||
return jsonify(str(e))
|
||||
|
||||
#search
|
||||
@app.get("/sp/<keyword>")
|
||||
def search_text(keyword):
|
||||
try:
|
||||
command = "bible -f ??{}".format(shlex.quote(keyword))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
if result.contains("??word"):
|
||||
raise Exception("Not found.")
|
||||
result = result.split()
|
||||
result = result[7:]
|
||||
result = '<br>'.join(map(str, result))
|
||||
return result
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
#search text json
|
||||
@app.get("/sl/<keyword>")
|
||||
def search_json_return(keyword):
|
||||
try:
|
||||
command = "bible -f ??{}".format(shlex.quote(keyword))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
if result.contains("??word"):
|
||||
raise Exception("Not found.")
|
||||
result = result.split()
|
||||
result = result[7:]
|
||||
complete_result = []
|
||||
for verse in result:
|
||||
complete_result.append(passage_text(verse))
|
||||
return jsonify(complete_result)
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
#search pretty
|
||||
@app.get("/spl/<keyword>")
|
||||
def search_text_return(keyword):
|
||||
try:
|
||||
command = "bible -f ??{}".format(shlex.quote(keyword))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
if result.contains("??word"):
|
||||
raise Exception("Not found.")
|
||||
result = result.split()
|
||||
result = result[7:]
|
||||
complete_result = []
|
||||
for verse in result:
|
||||
complete_result.append(passage_text(verse))
|
||||
result = '<br>'.join(map(str, complete_result))
|
||||
return result
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
#search pretty multiple
|
||||
@app.get("/splm/<keywords>")
|
||||
def search_text_return_multiple(keywords):
|
||||
if len(keywords.split('&')) > 1:
|
||||
pass
|
||||
try:
|
||||
command = "bible -f ??{}".format(shlex.quote(keyword))
|
||||
result = str(subprocess.check_output(command, shell=True), "utf-8")
|
||||
if len(result) == 0:
|
||||
raise Exception("You have entered an incorrect syntax or your query returned no results.")
|
||||
if result.contains("??word"):
|
||||
raise Exception("Not found.")
|
||||
result = result.split()
|
||||
result = result[7:]
|
||||
complete_result = []
|
||||
for verse in result:
|
||||
complete_result.append(passage_text(verse))
|
||||
result = '<br>'.join(map(str, complete_result))
|
||||
return result
|
||||
except Exception as e:
|
||||
return str(e)
|
||||
|
||||
Reference in New Issue
Block a user