diff --git a/INSTALL.txt b/INSTALL.txt index 0788982..06d1346 100644 --- a/INSTALL.txt +++ b/INSTALL.txt @@ -9,11 +9,12 @@ python3-hunspell:0.5.5-2build6 uwsgi-core:2.0.20-4 uwsgi-plugin-python3:2.0.20-4 apache2:2.4.52-1ubuntu4.2 +python3-requests:2.25.1+dfsg-2 Installation ====== -This software expects to be installed at /opt/kjv-api/ +This software expects to be installed at /opt/kjv-api/ (for uwsgi use on a reverse proxy) This installation method is intended to play nice among other Virtual Hosts 0. Install dependancies listed at the head of this document. @@ -43,4 +44,4 @@ This installation method is intended to play nice among other Virtual Hosts - It is up to you to use https or not, but if you know nginx or haproxy you should be able to set up local proxying to the WSGI server that runs the flask. - There are WSGI modules for apache avaliable, which may be better for larger scaled - deployments, but http proxying to the local uwsgi served flask works fine. + deployments, but http proxying to the local uwsgi served flask works fine for smaller uses. diff --git a/README.txt b/README.txt index 128573a..94edee0 100644 --- a/README.txt +++ b/README.txt @@ -41,32 +41,37 @@ CONTENTS │   ├── kjv.aff - KJV Spellchecking Description file (hunspell) │   ├── kjv.db - KJV sqlite3 database │   └── kjv.dic - KJV Spellchecking dictionary file (hunspell) +│   └── seq.db - Sqlite3 Database with user's ids for sequenced scripture +│   └── shortcodes.db - Sqlite3 Database with saved shortcodes for scripture groups. ├── favicon.ico ├── html │   ├── dictionary.html - dictionary search landing page /define │   ├── find.html - unified search bar landing page /find -│   ├── header.html - navigation bar added programatically to rich results +│   ├── navbar.html - navigation bar added programatically to rich results │   ├── heaven.html - Bible way to heaven, static page /salvation +│   ├── on_reading_bible.html - Introduction on good reading practices │   ├── index.html - / landing page with unified search bar │   ├── reader.html - not yet implemented Bible browser end point │   ├── reading_plan.html - not yet implemented Bible reading plan generator │   ├── response.html - Template for rich verse responses │   ├── search.html - Bible keyword search landing page /search +│   ├── search_bar.html - Unused │   ├── show.html - Bible verse search landing page /show +│   ├── seq.html - Bible sequential verse landing page /seq │   └── votd.html - Matty's test VOTD rich landing page ├── install │   ├── apache.site.conf.example - example installation configuration │   └── kjv-api.service - example systemd service file ├── INSTALL.txt - installation instructions +├── logs - log folder (populated at runtime - there is NO log rotation) ├── js │   └── collapsible.js - js for the definition drop-down menu used in unified search -├── kjv-api.sh - runs flask app on port 1611 -├── kjv-api-test.sh - runs flask app on port 1612 -├── media -│   └── kjv_background.jpg - Matty's test rich media +├── kjv-api.sh - runs uwsgi flask app on port 1611 +├── kjv-api-test.sh - runs uwsgi flask app on port 1612 ├── README.txt - This document ├── styles │   ├── collapsible.css +│   ├── header.css │   ├── fonts │   │   ├── Arizonia-Regular.ttf │   │   └── Parisienne-Regular.ttf @@ -75,6 +80,9 @@ CONTENTS │   └── votd.css └── tools └── av1611_dictionary_scraper.py - builds a dictionary json for the 1828 dictionary + └── sanity_check.py - Integrated during runtime and also as endpoint testing when run alone. + └── search_keywords.txt - endpoint testing queries + └── sanity_results.txt - results of running sanity_check.py as a standalone AUTHOR ====== diff --git a/app.py b/app.py index fc81a5e..b6479ba 100644 --- a/app.py +++ b/app.py @@ -23,7 +23,7 @@ timestamp_runtime: datetime = datetime.now().strftime("%d%b%y-%H:%M") logging.basicConfig( filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log", encoding="utf-8", - level=logging.DEBUG, + level=logging.INFO, ) logging.info(f"Started: {timestamp_runtime}") @@ -88,7 +88,7 @@ cached: dict = { if int(str(chap)[:-3]) not in cached["chapter_list"] ] cached["chapter_list"].sort() -for book in range(1, 66): +for book in range(1, 67): cached["books_chapter_lengths"][book] = kjv_cur.execute( "SELECT c FROM fts_kjv where b = ? ORDER BY id DESC LIMIT 1;", (book,) ).fetchone()[0] @@ -108,8 +108,7 @@ def generate_short_id(): def lookup_bookname(book_id): result = kjv_cur.execute( - f"SELECT n FROM key_english WHERE b = {book_id};" - ).fetchone() + "SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone() if result: result = result[0] return result @@ -117,8 +116,7 @@ def lookup_bookname(book_id): def lookup_book_id(bookname) -> int: result = kjv_cur.execute( - f"SELECT b FROM key_english WHERE n = '{bookname}';" - ).fetchone() + "SELECT b FROM key_english WHERE n = ?;",(bookname,)).fetchone() if result: result = result[0] return result @@ -126,10 +124,11 @@ def lookup_book_id(bookname) -> int: def lookup_fts(kwds): logging.debug(f"lookup_fts.kwds = '{kwds}'") - return kjv_cur.execute(f"SELECT * FROM fts_kjv('{kwds}')").fetchall() + return kjv_cur.execute("SELECT * FROM fts_kjv(?)", (kwds,)).fetchall() -def lookup_by_verse_id(start_verse, end_verse=None): +def lookup_by_verse_id(start_verse, end_verse=None) -> list: + # This is an internal function - we don't use placeholders because SQL injection should be impossible here if not end_verse: results = kjv_cur.execute( f"SELECT * FROM fts_kjv WHERE id = {start_verse};" @@ -191,7 +190,8 @@ def ref_input_cleaning( logging.debug(f"ref_input_cleaning({ref})") ref = ref.replace(" ", "") if ref.count(":") != 1 and ref_search: - logging.debug("ref_input_cleaning: Assuming user wanted entire chapter") + if len(ref) == 0: + return "Invalid Query" if ref[-1] in string.digits: ref += ":-" else: @@ -227,8 +227,9 @@ def ref_input_cleaning( def search_input_cleaning( - query, valid_chars=string.ascii_letters + string.digits + "+^* " + query, valid_chars=string.ascii_letters + string.digits + "+^* ", extra_chars='' ): + valid_chars += extra_chars # should protect against SQL injection, maybe. Should have another person review. cleaned_query = [] for character in query: @@ -294,7 +295,7 @@ def parse_query(query_override: str = None): result += "" result += "
" # result += search() - logging.debug(f"parse_query.search = '{result}'") + #logging.debug(f"parse_query.search = '{result}'") return result @@ -357,13 +358,13 @@ def return_dict_rich(ref, defs, simple): def show(kw=None, get_start_verse_id=False, human_readable=False): ref: str = request.args.get("kw", None) get_chapter: bool = request.args.get("all", False) - if ref.endswith(","): - ref = ref[:-1] if kw: ref = kw - if ref is None: + if not ref: return send_file("html/show.html") logging.debug(f"show({ref})") + if ref.endswith(","): + ref = ref[:-1] refs: list = [] for r in ref.split(","): r_ = ref_input_cleaning(r.strip()) @@ -374,6 +375,8 @@ def show(kw=None, get_start_verse_id=False, human_readable=False): verse_results: list = [] # check if the search is for multiple refs for ref in refs: + if len(ref) < 2: + continue logging.debug(f"show.reference({ref})") maybe_chapter: str = ref.split(":")[0][3:] ref_chapter: str = "" @@ -410,7 +413,7 @@ def show(kw=None, get_start_verse_id=False, human_readable=False): logging.debug(f"show.start_verse_id({start_verse_id})") logging.debug(f"show.end_verse_id({end_verse_id})") results = lookup_by_verse_id(start_verse_id, end_verse_id) - logging.debug(f"show.results(len({results}))") + logging.debug(f"show.results len({len(results)})") for result in results: verse_results.append(result) response_list = scripture_response(verse_results, human_readable) @@ -507,10 +510,11 @@ def page_not_found(e): @app.get("/") def idx(shortcode=""): if not sanity: - return "Sanity check failed. Check log, or please notify tyler@dinsmoor.us" + return "Sanity check failed. Check database ownership or please notify tyler@dinsmoor.us" if shortcode: + logging.info(f"/{shortcode}") return resolve_shortcode(shortcode) - logging.info("Index Served") + logging.info(f"/{shortcode}") return send_file("html/index.html") @@ -533,7 +537,7 @@ def random_verse(): response = "
".join(response_list) else: response = jsonify(response_list) - logging.debug(f"Search Response: {response}") + #logging.debug(f"Search Response: {response}") return response @@ -554,7 +558,7 @@ def verse_of_the_day(): response = "
".join(response_list) else: response = jsonify(response_list) - logging.debug(f"Search Response: {response}") + #logging.debug(f"Search Response: {response}") return response @@ -595,7 +599,7 @@ def search(): response = "
".join(response_list) else: response = jsonify(response_list) - logging.debug(f"Search Response: {response}") + #logging.debug(f"Search Response: {response}") return response @@ -640,6 +644,18 @@ def sequential_report(uid=None): uids = seq_cur.execute("select * from reading_tracker").fetchall() return render_template("seq_report.html", uids=uids) +@app.get("/screport") +def shortcode_report(shortcode=None): + if usermode == "Single": + short_conn = sqlite3.connect("data/shortcodes.db", check_same_thread=False) + else: + short_conn = sqlite3.connect("data/shortcodes.db") + short_cur = short_conn.cursor() + shortcodes = short_cur.execute( + "SELECT * FROM shortcodes" + ).fetchall() + return render_template("shortcode_report.html", shortcodes=shortcodes) + def sequential_purge(): if usermode == "Single": @@ -655,7 +671,7 @@ def sequential_purge(): since_use: timedelta = evaluate_datetime - last_used if since_use > sequential_purge_delta: logging.info(f"Purging UID: {uid[0]}") - seq_cur.execute(f"DELETE FROM reading_tracker WHERE uid = '{uid[0]}'") + seq_cur.execute("DELETE FROM reading_tracker WHERE uid = ?", (uid[0],)) seq_conn.commit() seq_cur.close() @@ -693,7 +709,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): uid = generate_short_id() # range returns a 5 digit id try: seq_cur.execute( - f"INSERT INTO reading_tracker VALUES('{uid}', {start}, '{last_used}')" + "INSERT INTO reading_tracker VALUES(?, ?, ?)", (uid, start, last_used) ) except sqlite3.OperationalError as e: logging.error( @@ -710,7 +726,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): uid = ref_input_cleaning(uid, ref_search=False) try: nextverse = seq_cur.execute( - f"SELECT verse FROM reading_tracker WHERE uid = '{uid}'" + "SELECT verse FROM reading_tracker WHERE uid = ?", (uid,) ).fetchone()[0] logging.debug(f"Queried UID {uid} on verse {nextverse}") sequential_purge() @@ -728,7 +744,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): logging.info(f"Served {num+1} verses to UID {uid}") seq_cur.execute( - f"REPLACE INTO reading_tracker (uid, verse, last_used) VALUES('{uid}', {nextverse+num+1}, '{last_used}')" + "REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", (uid, (nextverse+num+1), last_used) ) # Doing REPLACE INTO instead of UPDATE will make the updated row at the end of the table, making the purging # only look at a small set of the oldest unused entries. @@ -744,7 +760,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False): @app.get("/mksc/") def generate_shortcode(query: str = ""): - query = search_input_cleaning(query) + query = search_input_cleaning(query, extra_chars=":-,") if not query: return "Empty query." if usermode == "Single": @@ -759,7 +775,7 @@ def generate_shortcode(query: str = ""): creation_date = str(datetime.now()) shortcode = generate_short_id() short_cur.execute( - f"REPLACE INTO shortcodes(shortcode,query,creation_date) VALUES('{shortcode}', '{query}', '{creation_date}')" + "REPLACE INTO shortcodes(shortcode,query,creation_date) VALUES(?, ?, ?)", (shortcode, query, creation_date) ) short_conn.commit() short_cur.close() @@ -779,7 +795,7 @@ def resolve_shortcode(shortcode: str = ""): short_conn = sqlite3.connect("data/shortcodes.db") short_cur = short_conn.cursor() query = short_cur.execute( - f"SELECT query FROM shortcodes WHERE shortcode = '{shortcode}'" + "SELECT query FROM shortcodes WHERE shortcode = ?", (shortcode,) ).fetchone()[0] short_cur.close() logging.info(f"Resolved Shortcode {shortcode} for query '{query}'") diff --git a/data/seq.db b/data/seq.db new file mode 100644 index 0000000..aaa187e Binary files /dev/null and b/data/seq.db differ diff --git a/data/shortcodes.db b/data/shortcodes.db new file mode 100644 index 0000000..8092c1d Binary files /dev/null and b/data/shortcodes.db differ diff --git a/html/navbar.html b/html/navbar.html index 8945159..42d8482 100644 --- a/html/navbar.html +++ b/html/navbar.html @@ -13,10 +13,23 @@ + + +
← {prev_chapter_name} | {current_chapter_name} | {next_chapter_name} → + \ No newline at end of file diff --git a/html/shortcode_report.html b/html/shortcode_report.html new file mode 100644 index 0000000..2862337 --- /dev/null +++ b/html/shortcode_report.html @@ -0,0 +1,14 @@ + + + + + + + {% for code in shortcodes %} + + + + + + {% endfor %} +
shortcode |query |creation_date
{{ code[0] }} |{{ code[1] }} |{{ code[2] }}
\ No newline at end of file diff --git a/media/kjv_background.jpg b/media/kjv_background.jpg deleted file mode 100644 index 63958c8..0000000 Binary files a/media/kjv_background.jpg and /dev/null differ