INSTALL.txt

@@ -9,11 +9,12 @@ python3-hunspell:0.5.5-2build6
 uwsgi-core:2.0.20-4
 uwsgi-plugin-python3:2.0.20-4
 apache2:2.4.52-1ubuntu4.2
+python3-requests:2.25.1+dfsg-2
 
 
 Installation
 ======
-This software expects to be installed at /opt/kjv-api/
+This software expects to be installed at /opt/kjv-api/ (for uwsgi use on a reverse proxy)
 This installation method is intended to play nice among other Virtual Hosts
 
 0. Install dependancies listed at the head of this document.
@@ -43,4 +44,4 @@ This installation method is intended to play nice among other Virtual Hosts
  - It is up to you to use https or not, but if you know nginx or haproxy you should be able
    to set up local proxying to the WSGI server that runs the flask.
  - There are WSGI modules for apache avaliable, which may be better for larger scaled
-   deployments, but http proxying to the local uwsgi served flask works fine.
+   deployments, but http proxying to the local uwsgi served flask works fine for smaller uses.

README.txt

@@ -41,32 +41,37 @@ CONTENTS
 │   ├── kjv.aff - KJV Spellchecking Description file (hunspell)
 │   ├── kjv.db - KJV sqlite3 database
 │   └── kjv.dic - KJV Spellchecking dictionary file (hunspell)
+│   └── seq.db - Sqlite3 Database with user's ids for sequenced scripture
+│   └── shortcodes.db - Sqlite3 Database with saved shortcodes for scripture groups.
 ├── favicon.ico
 ├── html
 │   ├── dictionary.html - dictionary search landing page /define
 │   ├── find.html - unified search bar landing page /find
-│   ├── header.html - navigation bar added programatically to rich results
+│   ├── navbar.html - navigation bar added programatically to rich results
 │   ├── heaven.html - Bible way to heaven, static page /salvation
+│   ├── on_reading_bible.html - Introduction on good reading practices
 │   ├── index.html - / landing page with unified search bar
 │   ├── reader.html - not yet implemented Bible browser end point
 │   ├── reading_plan.html - not yet implemented Bible reading plan generator
 │   ├── response.html - Template for rich verse responses
 │   ├── search.html - Bible keyword search landing page /search
+│   ├── search_bar.html - Unused
 │   ├── show.html - Bible verse search landing page /show
+│   ├── seq.html - Bible sequential verse landing page /seq
 │   └── votd.html - Matty's test VOTD rich landing page
 ├── install
 │   ├── apache.site.conf.example - example installation configuration
 │   └── kjv-api.service - example systemd service file
 ├── INSTALL.txt - installation instructions
+├── logs - log folder (populated at runtime - there is NO log rotation)
 ├── js
 │   └── collapsible.js - js for the definition drop-down menu used in unified search
-├── kjv-api.sh - runs flask app on port 1611
+├── kjv-api.sh - runs uwsgi flask app on port 1611
-├── kjv-api-test.sh - runs flask app on port 1612
+├── kjv-api-test.sh - runs uwsgi flask app on port 1612
-├── media
-│   └── kjv_background.jpg - Matty's test rich media
 ├── README.txt - This document
 ├── styles
 │   ├── collapsible.css
+│   ├── header.css
 │   ├── fonts
 │   │   ├── Arizonia-Regular.ttf
 │   │   └── Parisienne-Regular.ttf
@@ -75,6 +80,9 @@ CONTENTS
 │   └── votd.css
 └── tools
     └── av1611_dictionary_scraper.py - builds a dictionary json for the 1828 dictionary
+    └── sanity_check.py - Integrated during runtime and also as endpoint testing when run alone.
+    └── search_keywords.txt - endpoint testing queries
+    └── sanity_results.txt - results of running sanity_check.py as a standalone
 
 AUTHOR
 ======

app.py

@@ -23,7 +23,7 @@ timestamp_runtime: datetime = datetime.now().strftime("%d%b%y-%H:%M")
 logging.basicConfig(
     filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log",
     encoding="utf-8",
-    level=logging.DEBUG,
+    level=logging.INFO,
 )
 
 logging.info(f"Started: {timestamp_runtime}")
@@ -88,7 +88,7 @@ cached: dict = {
     if int(str(chap)[:-3]) not in cached["chapter_list"]
 ]
 cached["chapter_list"].sort()
-for book in range(1, 66):
+for book in range(1, 67):
     cached["books_chapter_lengths"][book] = kjv_cur.execute(
         "SELECT c FROM fts_kjv where b = ? ORDER BY id DESC LIMIT 1;", (book,)
     ).fetchone()[0]
@@ -108,8 +108,7 @@ def generate_short_id():
 
 def lookup_bookname(book_id):
     result = kjv_cur.execute(
-        f"SELECT n FROM key_english WHERE b = {book_id};"
+        "SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone()
-    ).fetchone()
     if result:
         result = result[0]
     return result
@@ -117,8 +116,7 @@ def lookup_bookname(book_id):
 
 def lookup_book_id(bookname) -> int:
     result = kjv_cur.execute(
-        f"SELECT b FROM key_english WHERE n = '{bookname}';"
+        "SELECT b FROM key_english WHERE n = ?;",(bookname,)).fetchone()
-    ).fetchone()
     if result:
         result = result[0]
     return result
@@ -126,10 +124,11 @@ def lookup_book_id(bookname) -> int:
 
 def lookup_fts(kwds):
     logging.debug(f"lookup_fts.kwds = '{kwds}'")
-    return kjv_cur.execute(f"SELECT * FROM fts_kjv('{kwds}')").fetchall()
+    return kjv_cur.execute("SELECT * FROM fts_kjv(?)", (kwds,)).fetchall()
 
 
-def lookup_by_verse_id(start_verse, end_verse=None):
+def lookup_by_verse_id(start_verse, end_verse=None) -> list:
+    # This is an internal function - we don't use placeholders because SQL injection should be impossible here
     if not end_verse:
         results = kjv_cur.execute(
             f"SELECT * FROM fts_kjv WHERE id = {start_verse};"
@@ -191,7 +190,8 @@ def ref_input_cleaning(
     logging.debug(f"ref_input_cleaning({ref})")
     ref = ref.replace(" ", "")
     if ref.count(":") != 1 and ref_search:
-        logging.debug("ref_input_cleaning: Assuming user wanted entire chapter")
+        if len(ref) == 0:
+            return "Invalid Query"
         if ref[-1] in string.digits:
             ref += ":-"
         else:
@@ -227,8 +227,9 @@ def ref_input_cleaning(
 
 
 def search_input_cleaning(
-    query, valid_chars=string.ascii_letters + string.digits + "+^* "
+    query, valid_chars=string.ascii_letters + string.digits + "+^* ", extra_chars=''
 ):
+    valid_chars += extra_chars
     # should protect against SQL injection, maybe. Should have another person review.
     cleaned_query = []
     for character in query:
@@ -294,7 +295,7 @@ def parse_query(query_override: str = None):
     result += "<script>" + cached["js_collapsible"] + "</script>"
     result += "<hr>"
     # result += search()
-    logging.debug(f"parse_query.search = '{result}'")
+    #logging.debug(f"parse_query.search = '{result}'")
     return result
 
 
@@ -357,13 +358,13 @@ def return_dict_rich(ref, defs, simple):
 def show(kw=None, get_start_verse_id=False, human_readable=False):
     ref: str = request.args.get("kw", None)
     get_chapter: bool = request.args.get("all", False)
-    if ref.endswith(","):
-        ref = ref[:-1]
     if kw:
         ref = kw
-    if ref is None:
+    if not ref:
         return send_file("html/show.html")
     logging.debug(f"show({ref})")
+    if ref.endswith(","):
+        ref = ref[:-1]
     refs: list = []
     for r in ref.split(","):
         r_ = ref_input_cleaning(r.strip())
@@ -374,6 +375,8 @@ def show(kw=None, get_start_verse_id=False, human_readable=False):
     verse_results: list = []
     # check if the search is for multiple refs
     for ref in refs:
+        if len(ref) < 2:
+            continue
         logging.debug(f"show.reference({ref})")
         maybe_chapter: str = ref.split(":")[0][3:]
         ref_chapter: str = ""
@@ -410,7 +413,7 @@ def show(kw=None, get_start_verse_id=False, human_readable=False):
         logging.debug(f"show.start_verse_id({start_verse_id})")
         logging.debug(f"show.end_verse_id({end_verse_id})")
         results = lookup_by_verse_id(start_verse_id, end_verse_id)
-        logging.debug(f"show.results(len({results}))")
+        logging.debug(f"show.results len({len(results)})")
         for result in results:
             verse_results.append(result)
     response_list = scripture_response(verse_results, human_readable)
@@ -507,10 +510,11 @@ def page_not_found(e):
 @app.get("/<shortcode>")
 def idx(shortcode=""):
     if not sanity:
-        return "Sanity check failed. Check log, or please notify tyler@dinsmoor.us"
+        return "Sanity check failed. Check database ownership or please notify tyler@dinsmoor.us"
     if shortcode:
+        logging.info(f"/{shortcode}")
         return resolve_shortcode(shortcode)
-    logging.info("Index Served")
+    logging.info(f"/{shortcode}")
     return send_file("html/index.html")
 
 
@@ -533,7 +537,7 @@ def random_verse():
         response = "<br>".join(response_list)
     else:
         response = jsonify(response_list)
-    logging.debug(f"Search Response: {response}")
+    #logging.debug(f"Search Response: {response}")
     return response
 
 
@@ -554,7 +558,7 @@ def verse_of_the_day():
         response = "<br>".join(response_list)
     else:
         response = jsonify(response_list)
-    logging.debug(f"Search Response: {response}")
+    #logging.debug(f"Search Response: {response}")
     return response
 
 
@@ -595,7 +599,7 @@ def search():
         response = "<br>".join(response_list)
     else:
         response = jsonify(response_list)
-    logging.debug(f"Search Response: {response}")
+    #logging.debug(f"Search Response: {response}")
     return response
 
 
@@ -640,6 +644,18 @@ def sequential_report(uid=None):
     uids = seq_cur.execute("select * from reading_tracker").fetchall()
     return render_template("seq_report.html", uids=uids)
 
+@app.get("/screport")
+def shortcode_report(shortcode=None):
+    if usermode == "Single":
+        short_conn = sqlite3.connect("data/shortcodes.db", check_same_thread=False)
+    else:
+        short_conn = sqlite3.connect("data/shortcodes.db")
+    short_cur = short_conn.cursor()
+    shortcodes = short_cur.execute(
+        "SELECT * FROM shortcodes"
+    ).fetchall()
+    return render_template("shortcode_report.html", shortcodes=shortcodes)
+
 
 def sequential_purge():
     if usermode == "Single":
@@ -655,7 +671,7 @@ def sequential_purge():
         since_use: timedelta = evaluate_datetime - last_used
         if since_use > sequential_purge_delta:
             logging.info(f"Purging UID: {uid[0]}")
-            seq_cur.execute(f"DELETE FROM reading_tracker WHERE uid = '{uid[0]}'")
+            seq_cur.execute("DELETE FROM reading_tracker WHERE uid = ?", (uid[0],))
 
     seq_conn.commit()
     seq_cur.close()
@@ -693,7 +709,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
         uid = generate_short_id()  # range returns a 5 digit id
         try:
             seq_cur.execute(
-                f"INSERT INTO reading_tracker VALUES('{uid}', {start}, '{last_used}')"
+                "INSERT INTO reading_tracker VALUES(?, ?, ?)", (uid, start, last_used)
             )
         except sqlite3.OperationalError as e:
             logging.error(
@@ -710,7 +726,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
     uid = ref_input_cleaning(uid, ref_search=False)
     try:
         nextverse = seq_cur.execute(
-            f"SELECT verse FROM reading_tracker WHERE uid = '{uid}'"
+            "SELECT verse FROM reading_tracker WHERE uid = ?", (uid,)
         ).fetchone()[0]
         logging.debug(f"Queried UID {uid} on verse {nextverse}")
         sequential_purge()
@@ -728,7 +744,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
 
     logging.info(f"Served {num+1} verses to UID {uid}")
     seq_cur.execute(
-        f"REPLACE INTO reading_tracker (uid, verse, last_used) VALUES('{uid}', {nextverse+num+1}, '{last_used}')"
+        "REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", (uid, (nextverse+num+1), last_used)
     )
     # Doing REPLACE INTO instead of UPDATE will make the updated row at the end of the table, making the purging
     # only look at a small set of the oldest unused entries.
@@ -744,7 +760,7 @@ def sequential_read(start=0, num=1, uid=None, getuid=False):
 
 @app.get("/mksc/<query>")
 def generate_shortcode(query: str = ""):
-    query = search_input_cleaning(query)
+    query = search_input_cleaning(query, extra_chars=":-,")
     if not query:
         return "Empty query."
     if usermode == "Single":
@@ -759,7 +775,7 @@ def generate_shortcode(query: str = ""):
     creation_date = str(datetime.now())
     shortcode = generate_short_id()
     short_cur.execute(
-        f"REPLACE INTO shortcodes(shortcode,query,creation_date) VALUES('{shortcode}', '{query}', '{creation_date}')"
+        "REPLACE INTO shortcodes(shortcode,query,creation_date) VALUES(?, ?, ?)", (shortcode, query, creation_date)
     )
     short_conn.commit()
     short_cur.close()
@@ -779,7 +795,7 @@ def resolve_shortcode(shortcode: str = ""):
         short_conn = sqlite3.connect("data/shortcodes.db")
     short_cur = short_conn.cursor()
     query = short_cur.execute(
-        f"SELECT query FROM shortcodes WHERE shortcode = '{shortcode}'"
+        "SELECT query FROM shortcodes WHERE shortcode = ?", (shortcode,)
     ).fetchone()[0]
     short_cur.close()
     logging.info(f"Resolved Shortcode {shortcode} for query '{query}'")

html/navbar.html

@@ -13,10 +13,23 @@
         <input type="submit" value="Search">
         <input type="hidden" name="view" value="rich">
     </form>
+    <button onclick="make_shortcode()">Get Shortcode: </button>
+    <a id="context_href" href=''></a>
+    <br>
     &leftarrow;
     <a href="{prev_chapter_link}">{prev_chapter_name}</a> |
     {current_chapter_name} |
     <a href="{next_chapter_link}">{next_chapter_name}</a>
     &rightarrow;
 </div>
+<script>
+    function make_shortcode(){{
+        var xmlHttp = new XMLHttpRequest();
+        var theUrl = "/mksc/" + document.getElementById("searchBar").value;
+        xmlHttp.open( "GET", theUrl, false ); // false for synchronous request
+        xmlHttp.send( null );
+        document.getElementById("context_href").innerHTML = xmlHttp.responseText;
+        document.getElementById('context_href').setAttribute("href", "/" + xmlHttp.responseText);
+    }}
+</script>
 </header>

html/shortcode_report.html

@@ -0,0 +1,14 @@
+<table id="shortcode_report">
+  <tr>
+    <td>shortcode |</td>
+    <td>query |</td>
+    <td>creation_date</td>
+  </tr>
+    {% for code in shortcodes %}
+    <tr>
+      <td>{{ code[0] }} |</td>
+      <td>{{ code[1] }} |</td>
+      <td>{{ code[2] }}</td>
+    </tr>
+  {% endfor %}
+  </table>