tyler/kjv_api
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

mostly a hosted interfaces rewrite and better documentation

Browse Source
This commit is contained in:
Tyler
2024-05-13 14:40:18 -07:00
parent 0ab32bf4e2
commit ab15eeaebe
5 changed files with 1051 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1 -1
View File
@@ -1,2 +1,2 @@
KJV_SERMON_DB="/home/tyler/kjv-api/sermon-analyzer/data/sermons.db" KJV_SERMON_DB="./data/sermons.db"
KJV_SECRET_API_KEY="testing" KJV_SECRET_API_KEY="testing"
app.py
+4
View File
@@ -35,6 +35,7 @@ logging.basicConfig(
) )
logging.info(f"Started: {timestamp_runtime}") logging.info(f"Started: {timestamp_runtime}")
logging.info(f"Using Sermon Database: {SERMON_DB}")
try: try:
sanity_check.check_sanity() sanity_check.check_sanity()
@@ -687,6 +688,9 @@ def idx(shortcode=""):
logging.info(f"/{shortcode}") logging.info(f"/{shortcode}")
return send_file("html/index.html") return send_file("html/index.html")
@app.get("/fe")
def frontend():
return send_file('kjv_fe.html')
@app.get("/salvation") # a web page about going to heaven @app.get("/salvation") # a web page about going to heaven
def salvation(): def salvation():
data/sermons.db
BIN
View File
Binary file not shown.
kjv_api.py
+931
View File
@@ -0,0 +1,931 @@
# kjv_api.py
from flask import (
Flask,
request,
jsonify,
)
import random
from datetime import date # used for daily verse
import sqlite3
import string # for literals shortcut
import json # for dictionary
import os
import socket
import json
from tools import sanity_check
from dotenv import load_dotenv
# this is python-dotenv package in pip3, 'dotenv' with pip doesn't work!
load_dotenv()
kjv_api = Flask(__name__, template_folder="html")
design_goals = """
GOALS GOALS GOALS GOALS GOALS
This rewrite should end up with a very flat and (ideally) stateless API...
EXCEPT for integration with the sermon database, which is a separate project.
AND the semantic search, which is also a separate project.
Later, those projects may be integrated into this one (mostly via database merge),
but for now, they are separate.
All endpoints that return verses should return a consistent JSON format, an array containing verse objects with the following fields:
- bookname: the name of the book
-- A string
- book_id: the internal book_id (1-66)
-- An integer
- chapter: the chapter number
-- An integer
- verse: the verse number
-- An integer
- text: the verse text
-- A string
- verse_id: the internal verse_id
-- A string
- full_ref: the full reference in the format "book chapter:verse"
-- A string
- chapter_length: the number of verses in the chapter
-- An integer
- book_length: the number of chapters in the book
-- An integer
- definitions: the definitions of the words in the verse (if available)
-- An array of words by the key of the word, the value containing the definitions of each word
AND if the user has a valid token and a network connection, the following fields:
- similar_verses: an array of semantically similar verses (if available)
-- An array of verse objects
- sermons: an array of sermons that reference the verse (if available)
-- An array of sermon objects
(this should be filtered by the user's key, so they can only see sermons they have access to)
An example response:
[
{
"bookname": "Genesis",
"book_id": 1,
"chapter": 1,
"verse": 1,
"text": "In the beginning God created the heaven and the earth.",
"verse_id": "01001001",
"full_ref": "Genesis 1:1",
"chapter_length": 31,
"book_length": 50,
"definitions": {
"beginning": "The first part of time; the commencement; the first part of a series or sequence; the first part of a course.",
"God": "The Supreme Being; Jehovah; the eternal and infinite spirit, the creator, and the sovereign of the universe.",
"created": "Formed; made; produced; caused to exist; brought into being.",
"heaven": "The region of the air; the higher regions or the atmosphere; the place where the celestial orbs revolve; the firmament; the sky."
},
"similar_verses": [
{
"bookname": "Genesis",
"book_id": 1,
"chapter": 1,
"verse": 2,
"text": "And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.",
"verse_id": "01001002",
"full_ref": "Genesis 1:2",
"chapter_length": 31,
"book_length": 50
},
{
"bookname": "Genesis",
"book_id": 1,
"chapter": 1,
"verse": 3,
"text": "And God said, Let there be light: and there was light.",
"verse_id": "01001003",
"full_ref": "Genesis 1:3",
"chapter_length": 31,
"book_length": 50
}
],
"sermons": [
{
"title": "The Creation of the World",
"source_url": "https://example.com/sermon/1",
"url": "https://example.com/sermon/1",
"preacher": "John Doe"
},
{
"title": "The Beginning of All Things",
"source_url": "https://example.com/sermon/2",
"url": "https://example.com/sermon/2",
"preacher": "John Smith"
}
]
}
]
All endpoints that return sermons should return a consistent JSON format, an array containing sermon objects with the following fields:
- title: the title of the sermon
-- A string
- source_url: the URL of the source sermon
-- A string
- url: the URL of the cached sermon in the API
-- A string
- preacher: the name of the preacher
-- A string
- church_name: the name of the church
-- A string
- denomination: the denomination of the church
-- A string
- filename/sermon_id: the filename of the sermon (the filename variable name should be changed to sermon_id)
-- A string
- verses: an array of verse objects that the sermon references
-- An array of verse objects
- similar_sermons: an array of semantically similar sermons (if available)
-- An array of sermon objects
- text: the full text of the sermon
-- An array of objects containing:
-- timestamp: the timestamp of when the text was spoken in the recording
-- text: the text of the sermon at that timestamp
-- matches: an array of verse objects that the text references at that timestamp
- license: the license of the sermon
-- A string
- language: the language of the sermon
-- A string
- date: the date the sermon was given
-- A string
- duration: the duration of the sermon
-- A string
- views: the number of views the sermon has received on the API
-- An integer
- likes: the number of likes the sermon has received on the API
-- An integer
- notes: the notes of the sermon provided by the preacher or church
-- A string
- comments: the comments on the sermon provided by users
-- An array of objects containing:
-- user: the username of the commenter
-- comment: the comment text
-- timestamp: the timestamp of the comment
An example sermon response:
[
{
"title": "The Power of Love",
"source_url": "https://example.com/sermon/3",
"url": "https://example.com/sermon/3",
"preacher": "John Smith",
"church_name": "Community Church",
"denomination": "Non-Denominational",
"sermon_id": "3",
"verses": [
{
"bookname": "John",
"book_id": 43,
"chapter": 3,
"verse": 16,
"text": "For God so loved the world, that he gave his only begotten Son, that whosoever believeth in him should not perish, but have everlasting life.",
"verse_id": "43003016",
"full_ref": "John 3:16",
"chapter_length": 50,
"book_length": 21
},
{
"bookname": "1 Corinthians",
"book_id": 46,
"chapter": 13,
"verse": 4,
"text": "Charity suffereth long, and is kind; charity envieth not; charity vaunteth not itself, is not puffed up.",
"verse_id": "46013004",
"full_ref": "1 Corinthians 13:4",
"chapter_length": 16,
"book_length": 21
}
],
"similar_sermons": [
{
"title": "The Love of God",
"source_url": "https://example.com/sermon/4",
"url": "https://example.com/sermon/4",
"preacher": "Jane Doe"
},
{
"title": "Love Your Neighbor",
"source_url": "https://example.com/sermon/5",
"url": "https://example.com/sermon/5",
"preacher": "Mark Johnson"
}
],
"text": [
{
"timestamp": "00:00:00",
"text": "Welcome to today's sermon on the power of love.",
"matches": [
{
"bookname": "John",
"book_id": 43,
"chapter": 3,
"verse": 16,
"text": "For God so loved the world, that he gave his only begotten Son, that whosoever believeth in him should not perish, but have everlasting life.",
"verse_id": "43003016",
"full_ref": "John 3:16",
"chapter_length": 50,
"book_length": 21
},
{
"bookname": "1 Corinthians",
"book_id": 46,
"chapter": 13,
"verse": 4,
"text": "Charity suffereth long, and is kind; charity envieth not; charity vaunteth not itself, is not puffed up.",
"verse_id": "46013004",
"full_ref": "1 Corinthians 13:4",
"chapter_length": 16,
"book_length": 21
}
]
}
],
"license": "CC BY-NC-ND 4.0",
"language": "English",
"date": "2022-01-01",
"duration": "00:30:00",
"views": 100,
"likes": 50,
"notes": "This sermon explores the power of love in our lives and how it can transform us.",
"comments": [
{
"user": "user1",
"comment": "Great sermon!",
"timestamp": "2022-01-01 12:00:00"
},
{
"user": "user2",
"comment": "Thank you for sharing this message.",
"timestamp": "2022-01-01 12:30:00"
}
]
}
]
Sermon Endpoints:
- Add endpoint for listing all sermons
-- Endpoint name: /sermons
-- HTTP Method: GET
- Add endpoint for listing all preachers
-- Endpoint name: /preachers
-- HTTP Method: GET
- Add endpoint for listing all churches
-- Endpoint name: /churches
-- HTTP Method: GET
- Add endpoint for listing all denominations
-- Endpoint name: /denominations
-- HTTP Method: GET
- Add endpoint for searching sermons by scripture reference
-- Endpoint name: /sermon/search/<ref>
-- HTTP Method: GET
- Add endpoint for searching sermons by preacher
-- Endpoint name: /sermons/preacher/<preacher>
-- HTTP Method: GET
- Add endpoint for searching sermons by church
-- Endpoint name: /sermons/church/<church_name>
-- HTTP Method: GET
- Add endpoint for searching sermons by denomination
-- Endpoint name: /sermons/denomination/<denomination>
-- HTTP Method: GET
- Add endpoint for searching sermons by filename
-- Endpoint name: /sermon/<filename>
-- HTTP Method: GET
- Add endpoint for searching sermons by URL
-- Endpoint name: /sermon/url/<url>
-- HTTP Method: GET
- Add endpoint for searching sermons by keyword
-- Endpoint name: /sermon/search/<keyword>
-- HTTP Method: GET
- Add endpoint for searching sermon content by semantic search
-- Endpoint name: /sermon/search/semantic
-- HTTP Method: POST
- Add endpoint for searching sermons by random sermon
-- Endpoint name: /sermon/random
-- HTTP Method: GET
- Add endpoint for searching sermons by sermon of the day
-- Endpoint name: /sermon/votd
-- HTTP Method: GET
- Add endpoint for searching sermons by similar sermons
-- Endpoint name: /sermon/similar/<filename>
-- HTTP Method: GET
- Add endpoint for removing sermons
-- Endpoint name: /sermon/remove/<filename>
-- HTTP Method: DELETE
- Add endpoint for adding sermons
-- Endpoint name: /sermon/add
-- HTTP Method: POST
- Add endpoint for updating sermons
-- Endpoint name: /sermon/update/<filename>
-- HTTP Method: PUT
Semantic Search Integration:
- Add endpoint for semantic search
-- Endpoint name: /search/semantic
-- HTTP Method: POST
-- Request Body: {"query":"myquerystring"}
-- Result: JSON object with search results as verse objects
General API Improvements (see old app.py for more details):
(These should be implemented on the client and locally served but we still need to provide a standard)
(and can provide a fallback that just serves the json)
These can probaby call from the reference implementation we will make freely available.
- Add endpoint for serving the API documentation
-- Endpoint name: /docs
-- HTTP Method: GET
-- Result: HTML page with API documentation
- Add endpoint for searching by verse_id
-- Endpoint name: /verse_id/<verse_id>
-- HTTP Method: GET
- Add endpoint for searching by verse reference
- Add endpoint for searching by keyword
- Add endpoint for searching by random verse
- Add endpoint for searching by verse of the day
- Add endpoint for searching by definition
- Add endpoint for searching by semantically
- Add endpoint for searching by similar verses
Key Control Implementation:
(All actions require a valid token, which is stored in the database and can be revoked at any time)
(Admins can generate new keys, revoke keys, and view usage metrics by giving a new user a unique single use token)
(Keys contain a prefix that identifies the key group subscriber, and a suffix that is a random string of characters per user of the key group)
(this is psuedo-secret and can be sniffed, but will rotate at a regualr intervals)
(This should allow for subscriber teirs, and rate limiting per key)
- Add endpoint for key generation
-- Endpoint name: /key
-- HTTP Method: POST
-- Reuqest Body: {"action":"generate", "token":"mytokenstring"}
-- Result: JSON object with new key
-- Result Example: {"key":"mykeystring"}
- Add endpoint for key revocation
-- Endpoint name: /key
-- HTTP Method: DELETE
-- Request Body: {"action":"revoke", "token":"mytokenstring"}
-- Result: JSON object with success message
-- Result Example: {"success":"Key revoked"}
- Add endpoint for key validation
-- Endpoint name: /key
-- HTTP Method: POST
-- Request Body: {"action":"validate", "token":"userkeystring"}
-- Result: JSON object with validation status
-- Result Example: {"valid":true}
- Add endpoint for listing active keys and their usage metrics
-- Endpoint name: /keys
-- HTTP Method: GET
-- Request Body: {"action":"list", "token":"mytokenstring"}
-- Result: JSON object with list of keys and their usage metrics (defined later)
-- Result Example: {"keys":[{"key":"mykeystring","usage":100},{"key":"myotherkeystring","usage":50}]}
Database Improvements:
- Merge all databases into one
- current databases are:
-- KJV Bible database
-- Sermon database
-- Semantic search database
- Evaluate using postgresql instead of sqlite
Sermon Analysis and Feature Improvements:
- Implement sentiment analysis on sermon text (is it positive, negative, encouraging, neutral?)
- Implement keyword extraction from sermon text (will be needed as database grows)
- Implement topic modeling on sermon text (what is the sermon about?)
- Implement named entity/event recognition on sermon text
- people, places, organizations, dates, etc.
- current events, historical events, etc.
- Implement sermon text summarization (MAYBE NOT! Could be accidentally heretical!)
- Implement sermon text translation (MAYBE NOT! Could be accidentally heretical!)
- Implement sermon text language detection
- Store all analysis results in the sermon database
- Store a copy of the sermon source file on local storage for user playback
- Store a copy of the sermon audio file on local storage for user playback (low bandwidth/offline mode)
Abuse Prevention:
- Implement abuse detection for high volume requests
- Implement rate limiting
- Implement key-based access control for compute-intensive operations
- Implement notification system for abuse detection
Logging and Metrics:
- Implement logging for all errors
- Implement logging for all queries
- we want to know what users are doing, what they're searching for, and what they're not finding
- we want to know what errors are happening, and when they're happening
- we want to know how many users are using the service, and how often
General Code Improvements:
- Implement a more verbose and readable code style
- Implement a more modular code structure
- Define an OpenAPI schema for the API (Critical)
"""
SERMON_DB = os.getenv("KJV_SERMON_DB")
# currently it would require a differnt instance of the API to run with a different sermon database
# but these can be rolled into one database later
SECRET_API_KEY = os.getenv("KJV_SECRET_API_KEY") # this should end up being in the database
# we want to be able to revoke keys, and have multiple keys for different users
# this should be moved to the database, and the database should be able to handle multiple keys
# and revoke them as needed. This is currently just for testing purposes.
# makes data fetched from db easier to process
# might ditch this if we move to postgres
def dict_factory(cursor, row):
fields = [column[0] for column in cursor.description]
return {key: value for key, value in zip(fields, row)}
# prevent editor complaints about these being undefined
kjv_cur = None
sermon_cur = None
# where we are in our sqlite db
if __name__ == "__main__":
# single user mode, running without uwsgi, for testing
usermode: str = "Single"
kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor()
# really as long as you aren't trying to write to the db, you can get away with this
if SERMON_DB:
sermon_con = sqlite3.connect(SERMON_DB, check_same_thread=False)
sermon_con.row_factory = dict_factory
sermon_cur = sermon_con.cursor()
else:
# you're running with uwsgi, as you should in a production environment
usermode: str = "Multi"
kjv_cur = sqlite3.connect("data/kjv.db").cursor()
if SERMON_DB:
sermon_con = sqlite3.connect(SERMON_DB)
sermon_con.row_factory = dict_factory
sermon_cur = sermon_con.cursor()
if not kjv_cur:
exit("CRITICAL: Error connecting to KJV Database")
if not sermon_cur:
print("ERROR: Error connecting to Sermon Database")
# this is a performance optimization, we don't want to have to query the database for this info every time
# we can just cache it in memory
cached: dict = {
"verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()],
"book_names": [
i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall()
],
"chapter_list": [],
"books_chapter_lengths": {},
"json_dictionary": json.load(open("data/1828_Webster_KJV.json", "r")),
}
# Fill the chapter list with unique chapters for seeking back and forward by index.
# it is annoying to do it this way but makes it easier to seek back and forward by index
# to be honest I forgot why I did it this way, but it works.
[
cached["chapter_list"].append(int(str(chap)[:-3]))
for chap in cached.get("verse_ids")
if int(str(chap)[:-3]) not in cached["chapter_list"]
]
cached["chapter_list"].sort()
for book in range(1, 67):
cached["books_chapter_lengths"][book] = kjv_cur.execute(
"SELECT c FROM fts_kjv where b = ? ORDER BY id DESC LIMIT 1;", (book,)
).fetchone()[0]
# this was used for sharing verse groups with others, but will become a problem
# with multiple independent instances of the API by local users.
# It may be possible to share verse groups between users in the future by
# having it be a free service, but for now it's disabled and not in the API spec.
def generate_short_id():
remain = random.randrange(614_656, 17_210_367) # resolves to length of 5
chars = "0123456789ACEFHJKLMNPRTUVWXY"
length = len(chars)
result = ""
while remain > 0:
pos = remain % length
remain = remain // length
result = chars[pos] + result
return result
# helper functions...
def lookup_bookname(book_id):
result = kjv_cur.execute(
"SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone()
if result:
result = result[0]
return result
def lookup_book_id(bookname) -> int:
result = kjv_cur.execute(
"SELECT b FROM key_english WHERE n = ?;",(bookname,)).fetchone()
if result:
result = result[0]
return result
# fts is full text search, it's a virtual table that allows us to search the text of the bible quickly
# without having to scan the entire database, it's a performance optimization, but it's a feature
# of sqlite that we may have to find a workaround for if we move to postgres
def lookup_fts(kwds):
return kjv_cur.execute("SELECT * FROM fts_kjv(?)", (kwds,)).fetchall()
def lookup_by_verse_id(start_verse, end_verse=None) -> list:
# This is an internal function - we don't use placeholders because SQL injection should be impossible here
if not end_verse:
results = kjv_cur.execute(
f"SELECT * FROM fts_kjv WHERE id = {start_verse};"
).fetchall()
else:
results = kjv_cur.execute(
f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};"
).fetchall()
return results
def get_sermons_by_verse_id(verse_id) -> list:
if not sermon_cur:
return
return sermon_cur.execute(f"select * from verse_refs where verse_id={verse_id}").fetchall()
# this gives us a database key for a verse by book, chapter, and verse, because of the way the database is structured
def get_verse_id(book_id, chapter: str, verse: str) -> str:
# verse_id is 00000000 (00 000 000) (book + chapter + verse)
chapterid = chapter.zfill(3)
verseid = verse.zfill(3)
if verseid == "000":
verseid = "001" # Avoid verse_ids list IndexErrors on :- searches
return str(book_id) + chapterid + verseid
# this gives us a human readable reference from a verse_id
def get_ref_from_verse_id(verse_id):
# verse_id is 00000000 (00 000 000) (book + chapter + verse)
verse_id = str(verse_id).zfill(8)
book_id = verse_id[:2]
chapter = verse_id[2:5]
verse = verse_id[5:8]
bookname = kjv_cur.execute("SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone()[0]
return bookname + " " + chapter + ":" + verse
# this is for parsing a reference from a user query, it is complex but catches a lot of edge cases
# and makes it easier to search the database, as well as improving security by cleaning the input
def ref_input_cleaning(ref):
valid_chars = (string.ascii_letters + string.digits + ":-,")
ref = ref.replace(" ", "")
if ref.count(":") != 1:
if len(ref) == 0:
return "Invalid Query"
if ref[-1] in string.digits:
ref += ":-"
else:
ref += "1:-"
if ref.endswith(":"):
ref += "-"
if ref.count("-") > 1:
return
if not ref[-1].isdigit() and ref[-1] != "-":
return
ref = list(ref)
# preserve book number if we can guess the first
# character is a number and 3nd is a letter
if ref[0].isdigit() and ref[1].isalpha():
prefix = ref.pop(0) + " "
else:
prefix = ""
# clean the rest of the reference
cleaned_ref = []
for character in ref:
if character in valid_chars:
cleaned_ref.append(character)
# Was there a problem?
if len(cleaned_ref) == 0:
return
else:
return "".join(cleaned_ref)
# similar to ref_input_cleaning, but for search queries, it is more permissive because
# sqlite3 has search operators that can be used to search the fts table
def search_input_cleaning(query):
valid_chars = string.ascii_letters + string.digits + "+^* "
cleaned_query = []
for character in query:
if character in valid_chars:
cleaned_query.append(character)
return "".join(cleaned_query)
# this finds books if someone puts in a partial book name
def ref_parse_book(maybe_book):
for book in reversed(cached["book_names"]):
if book.startswith(maybe_book):
return book
return False
# this is a helper function for the semantic search, it sends a query to the semantic search server
# and returns the results. This is a separate project that will be integrated into this one later.
def request_semantic_search(query):
HOST = 'localhost'
PORT = 1613
data = {"query": query}
data_string = json.dumps(data).encode()
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((HOST, PORT))
s.sendall(data_string)
data = b''
while True:
part = s.recv(1024)
if not part:
break
data += part
return json.loads(data.decode())
# this explains the semantic search endpoint for developers
@kjv_api.get("/natural")
def explain_natural():
res = 'Submit a POST request to this endpoint with the following params in the body: '
res += '{"token": "mytokenstring", "query" : "myquerystring"}'
return jsonify({'error':res})
# this actually does the semantic search, it is a POST request because it is more secure
# and we require a token to access it, this is a feature of the API that will be expanded
# especially when we integrate the semantic search project into this one.
@kjv_api.post("/natural") # needs valid token and query in request body
def do_semantic_search():
request_data = request.get_json()
if request_data is None:
# Handle error: No JSON data sent
return jsonify({'error': 'Invalid request format'}), 400
token_string = request_data.get('token')
query_string = request_data.get('query')
if token_string is None:
# Handle error: Missing token in request
return jsonify({'error': 'Missing token in request body'}), 401
if token_string != SECRET_API_KEY:
return jsonify({'error': "Invalid Authorization"})
result = request_semantic_search(query=query_string)
return jsonify(result)
# this returns an internal verse_id by english scripture reference (1John5:7)
@kjv_api.get("/ref/<ref>")
def get_single_reference_verse_id(ref:str):
ref = ref.split(",")[0]
ref = ref.split("-")[0]
# only want the first single ref, not a range, and not a group.
ref = ref_input_cleaning(ref)
maybe_chapter: str = ref.split(":")[0][3:]
ref_chapter: str = ""
for maybedigit in maybe_chapter:
if maybedigit.isdigit():
ref_chapter += maybedigit
ref_verse = ref.split(":")[1]
book_str: str = ref_input_cleaning(ref, string.ascii_letters)
ref_bookname: str = ref_parse_book(book_str)
ref_book_id: int = lookup_book_id(ref_bookname)
if not ref_book_id:
return
return get_verse_id(ref_book_id, ref_chapter, ref_verse)
# this finds sermons that contain an english scripture reference (1John5:7)
@kjv_api.get("/sermon/search/<ref>")
def find_sermon_by_ref(ref:str=None):
verse_id = get_single_reference_verse_id(ref=ref)
if not sermon_cur:
return jsonify({'error':"Sermon database not found"})
sermons = get_sermons_by_verse_id(verse_id=verse_id)
for sermon in sermons:
sermon.update({'kjv':find_by_verseid(sermon.get("verse_id"))[4]})
return jsonify(sermons)
# there are a lot of endpoints here, almost all of which are just database queries. Once the individial
# endpoint names are decided, these can change of course.
@kjv_api.get("/sermons") # lists all the sermons in our database
def list_sermons():
return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name,filename from verse_refs").fetchall())
@kjv_api.get("/sermon/<filename>/verses") # show all matched info on a sermon by its filename (hash)
def list_sermon_references(filename):
return jsonify(sermon_cur.execute("select distinct verse_id,text from verse_refs where filename=?", (filename,)).fetchall())
@kjv_api.get("/preachers") # return a list of preachers stored in database, their denomination and church name
def list_preachers():
return jsonify(sermon_cur.execute("select distinct denomination,preacher,church_name from verse_refs").fetchall())
@kjv_api.get("/churches") # return a list of churches with sermons in the database
def list_churches():
return jsonify(sermon_cur.execute("select distinct denomination,church_name from verse_refs").fetchall())
@kjv_api.get("/denominations") # return a list of denominations included in the database
def list_denominations():
return jsonify(sermon_cur.execute("select distinct denomination from verse_refs").fetchall())
@kjv_api.get("/sermons/preacher/<preacher>") # show all sermons by a paticular preacher
def find_sermons_by_preacher(preacher):
return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where preacher=?", (preacher,)).fetchall())
@kjv_api.get("/sermons/church/<church_name>") # show all sermons by a paticular church
def find_sermons_by_church(church_name):
return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where church_name=?", (church_name,)).fetchall())
@kjv_api.get("/sermons/denomination/<denomination>") # show all sermons by a paticular denomination
def find_sermons_by_denomination(denomination):
return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where denomination=?", (denomination,)).fetchall())
@kjv_api.get("/sermon/<filename>") # show general information about a sermon by filename (hash)
def find_sermons_by_filename(filename):
return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where filename=?", (filename,)).fetchall())
@kjv_api.get("/sermon/url/<url>") # show general information about a sermon by url
def find_sermons_by_url(url):
return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where url=?", (url,)).fetchall())
# this is in case someone searches for a verseid maliciously >:^)
def clean_verseid(verse_id):
verse_id = str(verse_id)
clean_verseid = ''
for char in verse_id:
if char in string.digits:
clean_verseid += char
return clean_verseid
# i do not remember why this is done this way, I was probably tired
def find_by_verseid(verse_id):
return lookup_by_verse_id(verse_id)[0]
# I think this just returns a singular verse object
# as the API spec is changing, this will need to return an array of 1 verse object.
@kjv_api.get("/verse_id/<verse_id>") # return the database entry of an internal verse_id
def web_verseid(verse_id):
clean_verse_id = clean_verseid(verse_id=verse_id)
return jsonify(lookup_by_verse_id(clean_verse_id)[0])
# This used to be used for a direct search by word in the v1 API, but needs to change for v2
def define(word=''):
# might have issues with case-matching
word = word.strip()
if not word:
return {'error':'No definition word provided'}
if not word in cached.get("json_dictionary").keys():
return {'error':'Definition not found'}
return {word: cached.get("json_dictionary")[word]}
# this checks for a valid request, and needs to check for a valid
# key in the database, it is a security feature
def is_valid_request():
request_data = request.get_json()
if request_data is None:
return {'error': 'Invalid request format'}
token_string = request_data.get('token')
query_string = request_data.get('query')
mode_string = request_data.get('mode')
if token_string is None:
return {'error': 'Missing token in request body'}
if token_string != SECRET_API_KEY:
#TODO Replace SECRET_API_KEY with a list of valid keys in database
return {'error': "Invalid token"}
if query_string is None:
return {'error': 'Missing query parameter in request body'}
if mode_string is None:
return {'error': 'Missing mode parameter in request body'}
return {'success':"Authorized"}
# this is used for retrieving scripture by reference, it is a core feature of the API that
# is supposed to be implemented in the reference implementation, but is here for testing
# and development purposes. It is a complex function that handles a lot of edge cases.
# This will handle multiple references, and return an array of verse objects, as well
# as verse ranges and entire chapters.
# For the reference implementation, called functions should be simplified to
# make it easier to implement in other languages.
def get_kjv_verses_by_ref(ref=None, get_chapter=False):
if not ref:
return {'error':'Must provide reference query'}
if ref.endswith(","):
ref = ref[:-1]
refs: list = []
for r in ref.split(","):
r_ = ref_input_cleaning(r.strip())
if r_:
refs.append(r_)
if not refs:
return {'error':'Invalid Query'}
verse_results: list = []
# check if the search is for multiple refs
for ref in refs:
if len(ref) < 2:
continue
maybe_chapter: str = ref.split(":")[0][3:]
ref_chapter: str = ""
for maybedigit in maybe_chapter:
if maybedigit.isdigit():
ref_chapter += maybedigit
ref_verse = ref.split(":")[1]
if ref.__contains__("-") or get_chapter:
if ref.endswith("-") or get_chapter:
start_verse = ref_verse.split("-")[0]
end_verse = "500" # just get all the verses if it ends with a dash
else:
start_verse, end_verse = (
ref_verse.split("-")[0],
ref_verse.split("-")[1],
)
else:
start_verse = ref_verse
end_verse = None
end_verse_id = None
book_str: str = ref_input_cleaning(ref, string.ascii_letters)
ref_bookname: str = ref_parse_book(book_str)
ref_book_id: int = lookup_book_id(ref_bookname)
if not ref_book_id:
return {'error':'Invalid Query'}
maximum_chapters: int = cached.get("books_chapter_lengths").get(ref_book_id)
if int(ref_chapter) > maximum_chapters:
ref_chapter = str(maximum_chapters)
start_verse_id = get_verse_id(ref_book_id, ref_chapter, start_verse)
if end_verse:
end_verse_id = get_verse_id(ref_book_id, ref_chapter, end_verse)
results = lookup_by_verse_id(start_verse_id, end_verse_id)
for result in results:
verse_results.append(result)
response_list = scripture_response(verse_results)
return response_list
# just packages the verse object into a dictionary for the API response
def response_dict(scripture):
verse = {
"bookname": lookup_bookname(scripture[1]),
"chapter": scripture[2],
"verse": scripture[3],
"text": scripture[4],
}
return verse
# this packages a group of verse objects into an array of dictionaries for the API response
# this is mostly used for the get_kjv_verses_by_ref function and other reference implementation functions
def scripture_response(scriptures):
response = []
for scripture in scriptures:
response.append(response_dict(scripture))
return response
# might be good to implement detection of malicious endpoint 'testing'
# we don't currently have any secret endpoints, but may in the future
@kjv_api.errorhandler(404)
def page_not_found(e):
return jsonify({"error": "Invalid api endpoint"})
@kjv_api.get("/") # index
def idx():
return jsonify({'error':'Nothing at this top level'})
# believe it or not, this is the most important endpoint in the API
# additionally, normal random.choice isn't random enough!
# This will be retained as a single verse object at this endpoint for backwards compatibility
@kjv_api.get("/random") # truely random Bible verse
def random_verse():
rand_id = random.SystemRandom().choice(cached["verse_ids"])
scripture = lookup_by_verse_id(rand_id)
response_list = scripture_response(scripture)
return jsonify(response_list)
# same story as the random verse, but this is for the verse of the day and has a seed
@kjv_api.get("/votd") # today's Bible verse
def verse_of_the_day():
today = int(str(date.today()).replace("-", ""))
random.seed(today)
scripture = lookup_by_verse_id(random.choice(cached["verse_ids"]))
return jsonify(scripture_response(scripture))
def kjv_keyword_search(keywords):
if keywords:
cleaned_keywords = search_input_cleaning(keywords)
if not cleaned_keywords:
return {'error':'Cleaning of search parameters returned nothing'}
else:
return {'error':'No search parameters supplied'}
results = lookup_fts(cleaned_keywords)
response_list = scripture_response(results)
return response_list
# if we run this in single user mode, we can test the API by running it locally
# but we will not have the security features of uwsgi, and we will not have the
# performance features of uwsgi. This is just for testing and development purposes.
if __name__ == "__main__":
print("Operating in single user debug mode.")
# port 1612 is our test port, in production we will use 1611 because that's
# the year the KJV was published!
kjv_api.run(host="0.0.0.0", port=1612, debug=True, threaded=True)
kjv_fe.html
+115
View File
@@ -0,0 +1,115 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Search</title>
<style>
#search-box {
padding: 10px;
border: 1px solid #ccc;
border-radius: 4px;
width: 100%;
margin-bottom: 10px;
}
#results {
list-style: none;
padding: 0;
}
#results li {
margin-bottom: 10px;
border-bottom: 1px solid #eee;
padding: 5px;
}
</style>
</head>
<body>
<h1>Bible Semantic Search</h1>
<p>This is NOT a keyword search. This means searching single words will not have the results you expect. For that, use the normal keyword search at <a href='/'>api.1611.social</a></p>
<p>Use a sentence like "The Lord will kill his enemies without mercy" - you are supposed to search an idea, not individual words.</p>
<h3>Recent searches:</h3>
<ul id="last-searched"></ul>
<input type="text" id="search-box" placeholder="The lord god is a man of war, just in all his dealings">
<button id="search-button">Search</button>
<ul id="results"></ul>
<script>
const searchButton = document.getElementById('search-button');
const searchBox = document.getElementById('search-box');
const resultsList = document.getElementById('results');
const lastSearched = document.getElementById('last-searched');
window.addEventListener('DOMContentLoaded', async () => {
const lastSearchedURL = `https://api.1611.social/natural`;
const data = {
token: 'testing',
query: '__last_searched__'
};
const response = await fetch(lastSearchedURL, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(data)
});
if (response.ok) {
const results = await response.json();
lastSearched.innerHTML = ''; // Clear previous results
results.forEach(result => {
const listItem = document.createElement('li');
listItem.innerHTML = `
${result.query}<br>
`;
lastSearched.appendChild(listItem);
});
} else {
console.error('Error fetching data:', response.statusText);
}
});
searchButton.addEventListener('click', async () => {
const query = searchBox.value;
if (query.length >= 100){
alert("Query too long");
return;}
const url = `https://api.1611.social/natural`; // Replace with your actual URL
const data = {
token: 'testing',
query: query
};
const response = await fetch(url, {
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify(data)
});
if (response.ok) {
const results = await response.json();
resultsList.innerHTML = ''; // Clear previous results
results.forEach(result => {
const listItem = document.createElement('li');
const uriRef = encodeURIComponent(result.ref);
listItem.innerHTML = `
<strong>Verse ID:</strong> ${result.verse_id}<br>
<strong>Text:</strong> ${result.text}<br>
<strong>Ref:</strong> <a href=/find?kw=${uriRef}>${result.ref}</a><br>
<strong>Score:</strong> ${result.score}
`;
resultsList.appendChild(listItem);
});
} else {
console.error('Error fetching data:', response.statusText);
alert('An error occurred while searching. Please try again later.');
}
});
</script>
</body>
</html>