diff --git a/.env b/.env index fbbfaa8..9e976e5 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ -KJV_SERMON_DB="/home/tyler/kjv-api/sermon-analyzer/data/sermons.db" +KJV_SERMON_DB="./data/sermons.db" KJV_SECRET_API_KEY="testing" diff --git a/app.py b/app.py index 99c2b8a..040ce29 100644 --- a/app.py +++ b/app.py @@ -35,6 +35,7 @@ logging.basicConfig( ) logging.info(f"Started: {timestamp_runtime}") +logging.info(f"Using Sermon Database: {SERMON_DB}") try: sanity_check.check_sanity() @@ -687,6 +688,9 @@ def idx(shortcode=""): logging.info(f"/{shortcode}") return send_file("html/index.html") +@app.get("/fe") +def frontend(): + return send_file('kjv_fe.html') @app.get("/salvation") # a web page about going to heaven def salvation(): diff --git a/data/sermons.db b/data/sermons.db new file mode 100644 index 0000000..00b8644 Binary files /dev/null and b/data/sermons.db differ diff --git a/kjv_api.py b/kjv_api.py new file mode 100644 index 0000000..1c78dcd --- /dev/null +++ b/kjv_api.py @@ -0,0 +1,931 @@ +# kjv_api.py +from flask import ( + Flask, + request, + jsonify, +) +import random +from datetime import date # used for daily verse +import sqlite3 +import string # for literals shortcut +import json # for dictionary +import os +import socket +import json +from tools import sanity_check +from dotenv import load_dotenv +# this is python-dotenv package in pip3, 'dotenv' with pip doesn't work! +load_dotenv() + + +kjv_api = Flask(__name__, template_folder="html") +design_goals = """ + GOALS GOALS GOALS GOALS GOALS + +This rewrite should end up with a very flat and (ideally) stateless API... +EXCEPT for integration with the sermon database, which is a separate project. +AND the semantic search, which is also a separate project. + +Later, those projects may be integrated into this one (mostly via database merge), +but for now, they are separate. + +All endpoints that return verses should return a consistent JSON format, an array containing verse objects with the following fields: +- bookname: the name of the book + -- A string +- book_id: the internal book_id (1-66) + -- An integer +- chapter: the chapter number + -- An integer +- verse: the verse number + -- An integer +- text: the verse text + -- A string +- verse_id: the internal verse_id + -- A string +- full_ref: the full reference in the format "book chapter:verse" + -- A string +- chapter_length: the number of verses in the chapter + -- An integer +- book_length: the number of chapters in the book + -- An integer +- definitions: the definitions of the words in the verse (if available) + -- An array of words by the key of the word, the value containing the definitions of each word + +AND if the user has a valid token and a network connection, the following fields: +- similar_verses: an array of semantically similar verses (if available) + -- An array of verse objects +- sermons: an array of sermons that reference the verse (if available) + -- An array of sermon objects + (this should be filtered by the user's key, so they can only see sermons they have access to) + +An example response: +[ + { + "bookname": "Genesis", + "book_id": 1, + "chapter": 1, + "verse": 1, + "text": "In the beginning God created the heaven and the earth.", + "verse_id": "01001001", + "full_ref": "Genesis 1:1", + "chapter_length": 31, + "book_length": 50, + "definitions": { + "beginning": "The first part of time; the commencement; the first part of a series or sequence; the first part of a course.", + "God": "The Supreme Being; Jehovah; the eternal and infinite spirit, the creator, and the sovereign of the universe.", + "created": "Formed; made; produced; caused to exist; brought into being.", + "heaven": "The region of the air; the higher regions or the atmosphere; the place where the celestial orbs revolve; the firmament; the sky." + }, + "similar_verses": [ + { + "bookname": "Genesis", + "book_id": 1, + "chapter": 1, + "verse": 2, + "text": "And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.", + "verse_id": "01001002", + "full_ref": "Genesis 1:2", + "chapter_length": 31, + "book_length": 50 + }, + { + "bookname": "Genesis", + "book_id": 1, + "chapter": 1, + "verse": 3, + "text": "And God said, Let there be light: and there was light.", + "verse_id": "01001003", + "full_ref": "Genesis 1:3", + "chapter_length": 31, + "book_length": 50 + } + ], + "sermons": [ + { + "title": "The Creation of the World", + "source_url": "https://example.com/sermon/1", + "url": "https://example.com/sermon/1", + "preacher": "John Doe" + }, + { + "title": "The Beginning of All Things", + "source_url": "https://example.com/sermon/2", + "url": "https://example.com/sermon/2", + "preacher": "John Smith" + } + ] + } +] + + + +All endpoints that return sermons should return a consistent JSON format, an array containing sermon objects with the following fields: +- title: the title of the sermon + -- A string +- source_url: the URL of the source sermon + -- A string +- url: the URL of the cached sermon in the API + -- A string +- preacher: the name of the preacher + -- A string +- church_name: the name of the church + -- A string +- denomination: the denomination of the church + -- A string +- filename/sermon_id: the filename of the sermon (the filename variable name should be changed to sermon_id) + -- A string +- verses: an array of verse objects that the sermon references + -- An array of verse objects +- similar_sermons: an array of semantically similar sermons (if available) + -- An array of sermon objects +- text: the full text of the sermon + -- An array of objects containing: + -- timestamp: the timestamp of when the text was spoken in the recording + -- text: the text of the sermon at that timestamp + -- matches: an array of verse objects that the text references at that timestamp +- license: the license of the sermon + -- A string +- language: the language of the sermon + -- A string +- date: the date the sermon was given + -- A string +- duration: the duration of the sermon + -- A string +- views: the number of views the sermon has received on the API + -- An integer +- likes: the number of likes the sermon has received on the API + -- An integer +- notes: the notes of the sermon provided by the preacher or church + -- A string +- comments: the comments on the sermon provided by users + -- An array of objects containing: + -- user: the username of the commenter + -- comment: the comment text + -- timestamp: the timestamp of the comment + + +An example sermon response: +[ + { + "title": "The Power of Love", + "source_url": "https://example.com/sermon/3", + "url": "https://example.com/sermon/3", + "preacher": "John Smith", + "church_name": "Community Church", + "denomination": "Non-Denominational", + "sermon_id": "3", + "verses": [ + { + "bookname": "John", + "book_id": 43, + "chapter": 3, + "verse": 16, + "text": "For God so loved the world, that he gave his only begotten Son, that whosoever believeth in him should not perish, but have everlasting life.", + "verse_id": "43003016", + "full_ref": "John 3:16", + "chapter_length": 50, + "book_length": 21 + }, + { + "bookname": "1 Corinthians", + "book_id": 46, + "chapter": 13, + "verse": 4, + "text": "Charity suffereth long, and is kind; charity envieth not; charity vaunteth not itself, is not puffed up.", + "verse_id": "46013004", + "full_ref": "1 Corinthians 13:4", + "chapter_length": 16, + "book_length": 21 + } + ], + "similar_sermons": [ + { + "title": "The Love of God", + "source_url": "https://example.com/sermon/4", + "url": "https://example.com/sermon/4", + "preacher": "Jane Doe" + }, + { + "title": "Love Your Neighbor", + "source_url": "https://example.com/sermon/5", + "url": "https://example.com/sermon/5", + "preacher": "Mark Johnson" + } + ], + "text": [ + { + "timestamp": "00:00:00", + "text": "Welcome to today's sermon on the power of love.", + "matches": [ + { + "bookname": "John", + "book_id": 43, + "chapter": 3, + "verse": 16, + "text": "For God so loved the world, that he gave his only begotten Son, that whosoever believeth in him should not perish, but have everlasting life.", + "verse_id": "43003016", + "full_ref": "John 3:16", + "chapter_length": 50, + "book_length": 21 + }, + { + "bookname": "1 Corinthians", + "book_id": 46, + "chapter": 13, + "verse": 4, + "text": "Charity suffereth long, and is kind; charity envieth not; charity vaunteth not itself, is not puffed up.", + "verse_id": "46013004", + "full_ref": "1 Corinthians 13:4", + "chapter_length": 16, + "book_length": 21 + } + ] + } + ], + "license": "CC BY-NC-ND 4.0", + "language": "English", + "date": "2022-01-01", + "duration": "00:30:00", + "views": 100, + "likes": 50, + "notes": "This sermon explores the power of love in our lives and how it can transform us.", + "comments": [ + { + "user": "user1", + "comment": "Great sermon!", + "timestamp": "2022-01-01 12:00:00" + }, + { + "user": "user2", + "comment": "Thank you for sharing this message.", + "timestamp": "2022-01-01 12:30:00" + } + ] + } +] + + + +Sermon Endpoints: +- Add endpoint for listing all sermons + -- Endpoint name: /sermons + -- HTTP Method: GET +- Add endpoint for listing all preachers + -- Endpoint name: /preachers + -- HTTP Method: GET +- Add endpoint for listing all churches + -- Endpoint name: /churches + -- HTTP Method: GET +- Add endpoint for listing all denominations + -- Endpoint name: /denominations + -- HTTP Method: GET +- Add endpoint for searching sermons by scripture reference + -- Endpoint name: /sermon/search/ + -- HTTP Method: GET +- Add endpoint for searching sermons by preacher + -- Endpoint name: /sermons/preacher/ + -- HTTP Method: GET +- Add endpoint for searching sermons by church + -- Endpoint name: /sermons/church/ + -- HTTP Method: GET +- Add endpoint for searching sermons by denomination + -- Endpoint name: /sermons/denomination/ + -- HTTP Method: GET +- Add endpoint for searching sermons by filename + -- Endpoint name: /sermon/ + -- HTTP Method: GET +- Add endpoint for searching sermons by URL + -- Endpoint name: /sermon/url/ + -- HTTP Method: GET +- Add endpoint for searching sermons by keyword + -- Endpoint name: /sermon/search/ + -- HTTP Method: GET +- Add endpoint for searching sermon content by semantic search + -- Endpoint name: /sermon/search/semantic + -- HTTP Method: POST +- Add endpoint for searching sermons by random sermon + -- Endpoint name: /sermon/random + -- HTTP Method: GET +- Add endpoint for searching sermons by sermon of the day + -- Endpoint name: /sermon/votd + -- HTTP Method: GET +- Add endpoint for searching sermons by similar sermons + -- Endpoint name: /sermon/similar/ + -- HTTP Method: GET +- Add endpoint for removing sermons + -- Endpoint name: /sermon/remove/ + -- HTTP Method: DELETE +- Add endpoint for adding sermons + -- Endpoint name: /sermon/add + -- HTTP Method: POST +- Add endpoint for updating sermons + -- Endpoint name: /sermon/update/ + -- HTTP Method: PUT + + +Semantic Search Integration: +- Add endpoint for semantic search + -- Endpoint name: /search/semantic + -- HTTP Method: POST + -- Request Body: {"query":"myquerystring"} + -- Result: JSON object with search results as verse objects + + + +General API Improvements (see old app.py for more details): +(These should be implemented on the client and locally served but we still need to provide a standard) +(and can provide a fallback that just serves the json) +These can probaby call from the reference implementation we will make freely available. +- Add endpoint for serving the API documentation + -- Endpoint name: /docs + -- HTTP Method: GET + -- Result: HTML page with API documentation +- Add endpoint for searching by verse_id + -- Endpoint name: /verse_id/ + -- HTTP Method: GET +- Add endpoint for searching by verse reference +- Add endpoint for searching by keyword +- Add endpoint for searching by random verse +- Add endpoint for searching by verse of the day +- Add endpoint for searching by definition +- Add endpoint for searching by semantically +- Add endpoint for searching by similar verses + + +Key Control Implementation: +(All actions require a valid token, which is stored in the database and can be revoked at any time) +(Admins can generate new keys, revoke keys, and view usage metrics by giving a new user a unique single use token) +(Keys contain a prefix that identifies the key group subscriber, and a suffix that is a random string of characters per user of the key group) +(this is psuedo-secret and can be sniffed, but will rotate at a regualr intervals) +(This should allow for subscriber teirs, and rate limiting per key) +- Add endpoint for key generation + -- Endpoint name: /key + -- HTTP Method: POST + -- Reuqest Body: {"action":"generate", "token":"mytokenstring"} + -- Result: JSON object with new key + -- Result Example: {"key":"mykeystring"} +- Add endpoint for key revocation + -- Endpoint name: /key + -- HTTP Method: DELETE + -- Request Body: {"action":"revoke", "token":"mytokenstring"} + -- Result: JSON object with success message + -- Result Example: {"success":"Key revoked"} +- Add endpoint for key validation + -- Endpoint name: /key + -- HTTP Method: POST + -- Request Body: {"action":"validate", "token":"userkeystring"} + -- Result: JSON object with validation status + -- Result Example: {"valid":true} +- Add endpoint for listing active keys and their usage metrics + -- Endpoint name: /keys + -- HTTP Method: GET + -- Request Body: {"action":"list", "token":"mytokenstring"} + -- Result: JSON object with list of keys and their usage metrics (defined later) + -- Result Example: {"keys":[{"key":"mykeystring","usage":100},{"key":"myotherkeystring","usage":50}]} + + +Database Improvements: +- Merge all databases into one +- current databases are: + -- KJV Bible database + -- Sermon database + -- Semantic search database +- Evaluate using postgresql instead of sqlite + + +Sermon Analysis and Feature Improvements: +- Implement sentiment analysis on sermon text (is it positive, negative, encouraging, neutral?) +- Implement keyword extraction from sermon text (will be needed as database grows) +- Implement topic modeling on sermon text (what is the sermon about?) +- Implement named entity/event recognition on sermon text + - people, places, organizations, dates, etc. + - current events, historical events, etc. +- Implement sermon text summarization (MAYBE NOT! Could be accidentally heretical!) +- Implement sermon text translation (MAYBE NOT! Could be accidentally heretical!) +- Implement sermon text language detection +- Store all analysis results in the sermon database +- Store a copy of the sermon source file on local storage for user playback +- Store a copy of the sermon audio file on local storage for user playback (low bandwidth/offline mode) + + +Abuse Prevention: +- Implement abuse detection for high volume requests +- Implement rate limiting +- Implement key-based access control for compute-intensive operations +- Implement notification system for abuse detection + + +Logging and Metrics: +- Implement logging for all errors +- Implement logging for all queries +- we want to know what users are doing, what they're searching for, and what they're not finding +- we want to know what errors are happening, and when they're happening +- we want to know how many users are using the service, and how often + + +General Code Improvements: +- Implement a more verbose and readable code style +- Implement a more modular code structure +- Define an OpenAPI schema for the API (Critical) + +""" + +SERMON_DB = os.getenv("KJV_SERMON_DB") +# currently it would require a differnt instance of the API to run with a different sermon database +# but these can be rolled into one database later + +SECRET_API_KEY = os.getenv("KJV_SECRET_API_KEY") # this should end up being in the database +# we want to be able to revoke keys, and have multiple keys for different users +# this should be moved to the database, and the database should be able to handle multiple keys +# and revoke them as needed. This is currently just for testing purposes. + +# makes data fetched from db easier to process +# might ditch this if we move to postgres +def dict_factory(cursor, row): + fields = [column[0] for column in cursor.description] + return {key: value for key, value in zip(fields, row)} + +# prevent editor complaints about these being undefined +kjv_cur = None +sermon_cur = None +# where we are in our sqlite db + +if __name__ == "__main__": + # single user mode, running without uwsgi, for testing + usermode: str = "Single" + kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor() + # really as long as you aren't trying to write to the db, you can get away with this + if SERMON_DB: + sermon_con = sqlite3.connect(SERMON_DB, check_same_thread=False) + sermon_con.row_factory = dict_factory + sermon_cur = sermon_con.cursor() +else: + # you're running with uwsgi, as you should in a production environment + usermode: str = "Multi" + kjv_cur = sqlite3.connect("data/kjv.db").cursor() + if SERMON_DB: + sermon_con = sqlite3.connect(SERMON_DB) + sermon_con.row_factory = dict_factory + sermon_cur = sermon_con.cursor() + +if not kjv_cur: + exit("CRITICAL: Error connecting to KJV Database") +if not sermon_cur: + print("ERROR: Error connecting to Sermon Database") + +# this is a performance optimization, we don't want to have to query the database for this info every time +# we can just cache it in memory +cached: dict = { + "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()], + "book_names": [ + i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall() + ], + "chapter_list": [], + "books_chapter_lengths": {}, + "json_dictionary": json.load(open("data/1828_Webster_KJV.json", "r")), +} + +# Fill the chapter list with unique chapters for seeking back and forward by index. +# it is annoying to do it this way but makes it easier to seek back and forward by index +# to be honest I forgot why I did it this way, but it works. +[ + cached["chapter_list"].append(int(str(chap)[:-3])) + for chap in cached.get("verse_ids") + if int(str(chap)[:-3]) not in cached["chapter_list"] +] +cached["chapter_list"].sort() + +for book in range(1, 67): + cached["books_chapter_lengths"][book] = kjv_cur.execute( + "SELECT c FROM fts_kjv where b = ? ORDER BY id DESC LIMIT 1;", (book,) + ).fetchone()[0] + +# this was used for sharing verse groups with others, but will become a problem +# with multiple independent instances of the API by local users. +# It may be possible to share verse groups between users in the future by +# having it be a free service, but for now it's disabled and not in the API spec. +def generate_short_id(): + remain = random.randrange(614_656, 17_210_367) # resolves to length of 5 + chars = "0123456789ACEFHJKLMNPRTUVWXY" + length = len(chars) + result = "" + while remain > 0: + pos = remain % length + remain = remain // length + result = chars[pos] + result + return result + +# helper functions... +def lookup_bookname(book_id): + result = kjv_cur.execute( + "SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone() + if result: + result = result[0] + return result + + +def lookup_book_id(bookname) -> int: + result = kjv_cur.execute( + "SELECT b FROM key_english WHERE n = ?;",(bookname,)).fetchone() + if result: + result = result[0] + return result + +# fts is full text search, it's a virtual table that allows us to search the text of the bible quickly +# without having to scan the entire database, it's a performance optimization, but it's a feature +# of sqlite that we may have to find a workaround for if we move to postgres +def lookup_fts(kwds): + return kjv_cur.execute("SELECT * FROM fts_kjv(?)", (kwds,)).fetchall() + + +def lookup_by_verse_id(start_verse, end_verse=None) -> list: + # This is an internal function - we don't use placeholders because SQL injection should be impossible here + if not end_verse: + results = kjv_cur.execute( + f"SELECT * FROM fts_kjv WHERE id = {start_verse};" + ).fetchall() + else: + results = kjv_cur.execute( + f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};" + ).fetchall() + return results + +def get_sermons_by_verse_id(verse_id) -> list: + if not sermon_cur: + return + return sermon_cur.execute(f"select * from verse_refs where verse_id={verse_id}").fetchall() + +# this gives us a database key for a verse by book, chapter, and verse, because of the way the database is structured +def get_verse_id(book_id, chapter: str, verse: str) -> str: + # verse_id is 00000000 (00 000 000) (book + chapter + verse) + chapterid = chapter.zfill(3) + verseid = verse.zfill(3) + if verseid == "000": + verseid = "001" # Avoid verse_ids list IndexErrors on :- searches + return str(book_id) + chapterid + verseid + +# this gives us a human readable reference from a verse_id +def get_ref_from_verse_id(verse_id): + # verse_id is 00000000 (00 000 000) (book + chapter + verse) + verse_id = str(verse_id).zfill(8) + book_id = verse_id[:2] + chapter = verse_id[2:5] + verse = verse_id[5:8] + bookname = kjv_cur.execute("SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone()[0] + return bookname + " " + chapter + ":" + verse + +# this is for parsing a reference from a user query, it is complex but catches a lot of edge cases +# and makes it easier to search the database, as well as improving security by cleaning the input +def ref_input_cleaning(ref): + valid_chars = (string.ascii_letters + string.digits + ":-,") + ref = ref.replace(" ", "") + if ref.count(":") != 1: + if len(ref) == 0: + return "Invalid Query" + if ref[-1] in string.digits: + ref += ":-" + else: + ref += "1:-" + if ref.endswith(":"): + ref += "-" + if ref.count("-") > 1: + return + if not ref[-1].isdigit() and ref[-1] != "-": + return + ref = list(ref) + # preserve book number if we can guess the first + # character is a number and 3nd is a letter + if ref[0].isdigit() and ref[1].isalpha(): + prefix = ref.pop(0) + " " + else: + prefix = "" + # clean the rest of the reference + cleaned_ref = [] + for character in ref: + if character in valid_chars: + cleaned_ref.append(character) + # Was there a problem? + if len(cleaned_ref) == 0: + return + else: + return "".join(cleaned_ref) + +# similar to ref_input_cleaning, but for search queries, it is more permissive because +# sqlite3 has search operators that can be used to search the fts table +def search_input_cleaning(query): + valid_chars = string.ascii_letters + string.digits + "+^* " + cleaned_query = [] + for character in query: + if character in valid_chars: + cleaned_query.append(character) + return "".join(cleaned_query) + +# this finds books if someone puts in a partial book name +def ref_parse_book(maybe_book): + for book in reversed(cached["book_names"]): + if book.startswith(maybe_book): + return book + return False + +# this is a helper function for the semantic search, it sends a query to the semantic search server +# and returns the results. This is a separate project that will be integrated into this one later. +def request_semantic_search(query): + HOST = 'localhost' + PORT = 1613 + data = {"query": query} + + data_string = json.dumps(data).encode() + + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.connect((HOST, PORT)) + s.sendall(data_string) + data = b'' + while True: + part = s.recv(1024) + if not part: + break + data += part + return json.loads(data.decode()) + +# this explains the semantic search endpoint for developers +@kjv_api.get("/natural") +def explain_natural(): + res = 'Submit a POST request to this endpoint with the following params in the body: ' + res += '{"token": "mytokenstring", "query" : "myquerystring"}' + return jsonify({'error':res}) + +# this actually does the semantic search, it is a POST request because it is more secure +# and we require a token to access it, this is a feature of the API that will be expanded +# especially when we integrate the semantic search project into this one. +@kjv_api.post("/natural") # needs valid token and query in request body +def do_semantic_search(): + request_data = request.get_json() + if request_data is None: + # Handle error: No JSON data sent + return jsonify({'error': 'Invalid request format'}), 400 + + token_string = request_data.get('token') + query_string = request_data.get('query') + if token_string is None: + # Handle error: Missing token in request + return jsonify({'error': 'Missing token in request body'}), 401 + + if token_string != SECRET_API_KEY: + return jsonify({'error': "Invalid Authorization"}) + + result = request_semantic_search(query=query_string) + return jsonify(result) + +# this returns an internal verse_id by english scripture reference (1John5:7) +@kjv_api.get("/ref/") +def get_single_reference_verse_id(ref:str): + ref = ref.split(",")[0] + ref = ref.split("-")[0] + # only want the first single ref, not a range, and not a group. + ref = ref_input_cleaning(ref) + maybe_chapter: str = ref.split(":")[0][3:] + ref_chapter: str = "" + for maybedigit in maybe_chapter: + if maybedigit.isdigit(): + ref_chapter += maybedigit + ref_verse = ref.split(":")[1] + book_str: str = ref_input_cleaning(ref, string.ascii_letters) + ref_bookname: str = ref_parse_book(book_str) + ref_book_id: int = lookup_book_id(ref_bookname) + if not ref_book_id: + return + return get_verse_id(ref_book_id, ref_chapter, ref_verse) + +# this finds sermons that contain an english scripture reference (1John5:7) +@kjv_api.get("/sermon/search/") +def find_sermon_by_ref(ref:str=None): + verse_id = get_single_reference_verse_id(ref=ref) + if not sermon_cur: + return jsonify({'error':"Sermon database not found"}) + sermons = get_sermons_by_verse_id(verse_id=verse_id) + for sermon in sermons: + sermon.update({'kjv':find_by_verseid(sermon.get("verse_id"))[4]}) + return jsonify(sermons) + +# there are a lot of endpoints here, almost all of which are just database queries. Once the individial +# endpoint names are decided, these can change of course. + +@kjv_api.get("/sermons") # lists all the sermons in our database +def list_sermons(): + return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name,filename from verse_refs").fetchall()) + +@kjv_api.get("/sermon//verses") # show all matched info on a sermon by its filename (hash) +def list_sermon_references(filename): + return jsonify(sermon_cur.execute("select distinct verse_id,text from verse_refs where filename=?", (filename,)).fetchall()) + +@kjv_api.get("/preachers") # return a list of preachers stored in database, their denomination and church name +def list_preachers(): + return jsonify(sermon_cur.execute("select distinct denomination,preacher,church_name from verse_refs").fetchall()) + +@kjv_api.get("/churches") # return a list of churches with sermons in the database +def list_churches(): + return jsonify(sermon_cur.execute("select distinct denomination,church_name from verse_refs").fetchall()) + +@kjv_api.get("/denominations") # return a list of denominations included in the database +def list_denominations(): + return jsonify(sermon_cur.execute("select distinct denomination from verse_refs").fetchall()) + +@kjv_api.get("/sermons/preacher/") # show all sermons by a paticular preacher +def find_sermons_by_preacher(preacher): + return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where preacher=?", (preacher,)).fetchall()) + +@kjv_api.get("/sermons/church/") # show all sermons by a paticular church +def find_sermons_by_church(church_name): + return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where church_name=?", (church_name,)).fetchall()) + +@kjv_api.get("/sermons/denomination/") # show all sermons by a paticular denomination +def find_sermons_by_denomination(denomination): + return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where denomination=?", (denomination,)).fetchall()) + +@kjv_api.get("/sermon/") # show general information about a sermon by filename (hash) +def find_sermons_by_filename(filename): + return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where filename=?", (filename,)).fetchall()) + +@kjv_api.get("/sermon/url/") # show general information about a sermon by url +def find_sermons_by_url(url): + return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where url=?", (url,)).fetchall()) + +# this is in case someone searches for a verseid maliciously >:^) +def clean_verseid(verse_id): + verse_id = str(verse_id) + clean_verseid = '' + for char in verse_id: + if char in string.digits: + clean_verseid += char + return clean_verseid + + +# i do not remember why this is done this way, I was probably tired +def find_by_verseid(verse_id): + return lookup_by_verse_id(verse_id)[0] + +# I think this just returns a singular verse object +# as the API spec is changing, this will need to return an array of 1 verse object. +@kjv_api.get("/verse_id/") # return the database entry of an internal verse_id +def web_verseid(verse_id): + clean_verse_id = clean_verseid(verse_id=verse_id) + return jsonify(lookup_by_verse_id(clean_verse_id)[0]) + +# This used to be used for a direct search by word in the v1 API, but needs to change for v2 +def define(word=''): + # might have issues with case-matching + word = word.strip() + if not word: + return {'error':'No definition word provided'} + if not word in cached.get("json_dictionary").keys(): + return {'error':'Definition not found'} + return {word: cached.get("json_dictionary")[word]} + +# this checks for a valid request, and needs to check for a valid +# key in the database, it is a security feature +def is_valid_request(): + request_data = request.get_json() + if request_data is None: + return {'error': 'Invalid request format'} + token_string = request_data.get('token') + query_string = request_data.get('query') + mode_string = request_data.get('mode') + if token_string is None: + return {'error': 'Missing token in request body'} + if token_string != SECRET_API_KEY: + #TODO Replace SECRET_API_KEY with a list of valid keys in database + return {'error': "Invalid token"} + if query_string is None: + return {'error': 'Missing query parameter in request body'} + if mode_string is None: + return {'error': 'Missing mode parameter in request body'} + return {'success':"Authorized"} + + +# this is used for retrieving scripture by reference, it is a core feature of the API that +# is supposed to be implemented in the reference implementation, but is here for testing +# and development purposes. It is a complex function that handles a lot of edge cases. +# This will handle multiple references, and return an array of verse objects, as well +# as verse ranges and entire chapters. +# For the reference implementation, called functions should be simplified to +# make it easier to implement in other languages. +def get_kjv_verses_by_ref(ref=None, get_chapter=False): + if not ref: + return {'error':'Must provide reference query'} + if ref.endswith(","): + ref = ref[:-1] + refs: list = [] + for r in ref.split(","): + r_ = ref_input_cleaning(r.strip()) + if r_: + refs.append(r_) + if not refs: + return {'error':'Invalid Query'} + verse_results: list = [] + # check if the search is for multiple refs + for ref in refs: + if len(ref) < 2: + continue + maybe_chapter: str = ref.split(":")[0][3:] + ref_chapter: str = "" + for maybedigit in maybe_chapter: + if maybedigit.isdigit(): + ref_chapter += maybedigit + ref_verse = ref.split(":")[1] + if ref.__contains__("-") or get_chapter: + if ref.endswith("-") or get_chapter: + start_verse = ref_verse.split("-")[0] + end_verse = "500" # just get all the verses if it ends with a dash + else: + start_verse, end_verse = ( + ref_verse.split("-")[0], + ref_verse.split("-")[1], + ) + else: + start_verse = ref_verse + end_verse = None + end_verse_id = None + book_str: str = ref_input_cleaning(ref, string.ascii_letters) + ref_bookname: str = ref_parse_book(book_str) + ref_book_id: int = lookup_book_id(ref_bookname) + if not ref_book_id: + return {'error':'Invalid Query'} + maximum_chapters: int = cached.get("books_chapter_lengths").get(ref_book_id) + if int(ref_chapter) > maximum_chapters: + ref_chapter = str(maximum_chapters) + start_verse_id = get_verse_id(ref_book_id, ref_chapter, start_verse) + if end_verse: + end_verse_id = get_verse_id(ref_book_id, ref_chapter, end_verse) + results = lookup_by_verse_id(start_verse_id, end_verse_id) + for result in results: + verse_results.append(result) + response_list = scripture_response(verse_results) + return response_list + +# just packages the verse object into a dictionary for the API response +def response_dict(scripture): + verse = { + "bookname": lookup_bookname(scripture[1]), + "chapter": scripture[2], + "verse": scripture[3], + "text": scripture[4], + } + return verse + +# this packages a group of verse objects into an array of dictionaries for the API response +# this is mostly used for the get_kjv_verses_by_ref function and other reference implementation functions +def scripture_response(scriptures): + response = [] + for scripture in scriptures: + response.append(response_dict(scripture)) + return response + + +# might be good to implement detection of malicious endpoint 'testing' +# we don't currently have any secret endpoints, but may in the future +@kjv_api.errorhandler(404) +def page_not_found(e): + return jsonify({"error": "Invalid api endpoint"}) + +@kjv_api.get("/") # index +def idx(): + return jsonify({'error':'Nothing at this top level'}) + +# believe it or not, this is the most important endpoint in the API +# additionally, normal random.choice isn't random enough! +# This will be retained as a single verse object at this endpoint for backwards compatibility +@kjv_api.get("/random") # truely random Bible verse +def random_verse(): + rand_id = random.SystemRandom().choice(cached["verse_ids"]) + scripture = lookup_by_verse_id(rand_id) + response_list = scripture_response(scripture) + return jsonify(response_list) + +# same story as the random verse, but this is for the verse of the day and has a seed +@kjv_api.get("/votd") # today's Bible verse +def verse_of_the_day(): + today = int(str(date.today()).replace("-", "")) + random.seed(today) + scripture = lookup_by_verse_id(random.choice(cached["verse_ids"])) + return jsonify(scripture_response(scripture)) + + +def kjv_keyword_search(keywords): + if keywords: + cleaned_keywords = search_input_cleaning(keywords) + if not cleaned_keywords: + return {'error':'Cleaning of search parameters returned nothing'} + else: + return {'error':'No search parameters supplied'} + results = lookup_fts(cleaned_keywords) + response_list = scripture_response(results) + return response_list + +# if we run this in single user mode, we can test the API by running it locally +# but we will not have the security features of uwsgi, and we will not have the +# performance features of uwsgi. This is just for testing and development purposes. +if __name__ == "__main__": + print("Operating in single user debug mode.") + # port 1612 is our test port, in production we will use 1611 because that's + # the year the KJV was published! + kjv_api.run(host="0.0.0.0", port=1612, debug=True, threaded=True) diff --git a/kjv_fe.html b/kjv_fe.html new file mode 100644 index 0000000..1bfd570 --- /dev/null +++ b/kjv_fe.html @@ -0,0 +1,115 @@ + + + + + + Search + + + +

Bible Semantic Search

+

This is NOT a keyword search. This means searching single words will not have the results you expect. For that, use the normal keyword search at api.1611.social

+

Use a sentence like "The Lord will kill his enemies without mercy" - you are supposed to search an idea, not individual words.

+

Recent searches:

+
    + + +
      + + + + \ No newline at end of file