Search landing page, compiled dictionary and wrote scraper

app.py

@@ -321,9 +321,11 @@ def search():
     if keywords:
         cleaned_keywords = search_input_cleaning(keywords)
         if not cleaned_keywords:
-            return ("Invalid Query: Special Characters")
+            return send_file("html/search.html")
+            #return ("Invalid Query: Special Characters")
     else:
-        return ("No Query")
+        return send_file("html/search.html")
+        #return ("No Query")
 
     results = lookup_fts(cleaned_keywords)
     response_list = response_multi(results)

av1611_dictionary_scraper.py

@@ -0,0 +1,74 @@
+import urllib.request # library for connecting to a URL and getting it's content
+from bs4 import BeautifulSoup # library for parsing the HTML - requires python3-bs4 package
+import string # handy predefined shortcuts which saves us time rather than enumerating letters by hand
+import json # we are going to output the finished dictionary as JSON since it's easy to work with
+import time
+
+letterindex_urls = [] # we initialize an empty list that we will populate with entires below
+dict_base_url = "https://av1611.com/kjbp/kjv-dictionary/"
+finished_dictionary = {}
+
+# This function takes a list of HTML tags separated into a list, one tag per list element.
+# The result is a dictionary where the key is a string, the word, and the value is a list
+# with corresponding values as a list. Words with one definition will have a single
+# element list, whereas words with multiple definitions will have a multi-element list.
+def tags_to_dict(tags):
+    h2_keys = [] # temporary key storage so we remember our last key when we iterate
+                 # through multiple word definitions
+    result = {} # an empty dictionary where we will store our words and definitions
+    for element in tags:
+        soup = BeautifulSoup(element, "lxml") # it's easier to use new soup instance for this. Not fast.
+        if soup.h2: # is our element a header? (key)
+            result[soup.h2.text] = [] # initialize a new dictionary key with an empty list
+            h2_keys.append(soup.h2.text) # store our key temporarily until the next key
+        elif soup.find_all('p'): # get our list of definitions in paragraph tags
+            for r in soup.find_all('p'): # We need to add each paragraph tag (definition)
+                result[h2_keys[-1]].append(r.text) # Add it to the LAST key we used (-1)
+    return result
+
+
+for letter in string.ascii_uppercase: # for A, B, C..... assign "letter = A" ... "B"... etc
+    letterindex_urls.append("https://av1611.com/kjbp/kjv-dictionary/index-{}.html".format(letter))
+    # add https://.../index-A.html .... B.html .... C.html... to the list
+# now letterindex_urls has URLS for A-Z for us to use.
+
+word_urls = [] # we are going to store the results of our parsed indexes here
+
+
+# This for loop will first get all the html document name that we will need to grab
+for url in letterindex_urls: # for every URL in our list we just made....
+    print("Getting URL: {}".format(url))
+    response = urllib.request.urlopen(url) # grab the HTML as a response object
+    html_str = response.read().decode('UTF-8') # this is the actual text of the response as a string
+    soup = BeautifulSoup(html_str, "lxml")
+    #
+    # all word URLS are located in the <div role="main">
+    # we can search for this tag, and it returns a 1-element list containing the html in question
+    # It is not iterable (each line as one entry) and it returns as a "tag" datatype, so we must
+    # get the first element in the list (zero) and cast that to a normal string, which can be used
+    # to parse it again and only get the actual links, since doing it again will result in something
+    # that is both filtered for the references we want, and also iterable.
+    str_of_links = str(soup.find_all("div", {"role":"main"})[0])
+    soup2 = BeautifulSoup(str_of_links, "lxml")
+    for link in soup2.find_all('a'):
+        word_urls.append(dict_base_url + link.get('href'))
+
+for word_url in word_urls:
+    time.sleep(0.2) # try to slow down a little bit, rate limit so we don't cause excess load
+                    # on us or the av1611 web server.
+    print("Getting URL: {}".format(word_url))
+    response = urllib.request.urlopen(word_url) # grab the HTML as a response object
+    html_str = response.read().decode('UTF-8') # this is the actual text of the response as a string
+    soup = BeautifulSoup(html_str, "lxml")
+    #
+    # all definition and variation data is located in the <div role="main">
+    # The format is <h2>(word) <p>(word_def) <h2>(word_var_2) <p>(word_var_2_def) <p>(word_var_2_def_2)...
+    # for as many variations as there are. See "addict" for variations like addicted/addicting/etc.
+    str_of_tags = str(soup.find_all("div", {"role":"main"})[0])
+    list_of_tags = str_of_tags.split("\n")
+    word_definitions = tags_to_dict(list_of_tags)
+    print("Progress: {}".format(word_definitions.keys()))
+    finished_dictionary.update(word_definitions)
+
+with open("1828_Webster_KJV.json", 'w') as output:
+    json.dump(finished_dictionary, output)

html/search.html

@@ -0,0 +1,21 @@
+<h2>Bible Search</h2>
+<form action="/search">
+ <label for='kw'>Key Words</label><br>
+ <input type="text" name="kw"><br><br>
+
+ <label for='view'>View<label><br> 
+ <input type="radio" name="view" value="rich">Rich<br>
+ <input type="radio" name="view" value="plain">Plain<br>
+ <input type="radio" name="view" value="json">JSON<br>
+
+ <label for='ext'>Extended Search<label><br> 
+ <input type="checkbox" name="ext" value="on">Extended<br>
+
+ <input type="submit">
+<form>
+
+<hr>
+<p>You can use standard search operators such as <b>AND</b>, <b>OR</b>, and <b>NOT</b>.</p>
+<p>Such as: "reprobate <b>NOT</b> silver".</p>
+<p>Multiple word queries without operators will be assumed to be AND.<p>
+<p><b>Extended Search</b> uses a thesaurus table to return additional like-word results.</p>