import urllib.request # library for connecting to a URL and getting it's content
from bs4 import BeautifulSoup # library for parsing the HTML - requires python3-bs4 package
import string # handy predefined shortcuts which saves us time rather than enumerating letters by hand
import json # we are going to output the finished dictionary as JSON since it's easy to work with
import time

letterindex_urls = [] # we initialize an empty list that we will populate with entires below
dict_base_url = "https://av1611.com/kjbp/kjv-dictionary/"
finished_dictionary = {}

# This function takes a list of HTML tags separated into a list, one tag per list element.
# The result is a dictionary where the key is a string, the word, and the value is a list
# with corresponding values as a list. Words with one definition will have a single
# element list, whereas words with multiple definitions will have a multi-element list.
def tags_to_dict(tags):
    h2_keys = [] # temporary key storage so we remember our last key when we iterate
                 # through multiple word definitions
    result = {} # an empty dictionary where we will store our words and definitions
    for element in tags:
        soup = BeautifulSoup(element, "lxml") # it's easier to use new soup instance for this. Not fast.
        if soup.h2: # is our element a header? (key)
            result[soup.h2.text] = [] # initialize a new dictionary key with an empty list
            h2_keys.append(soup.h2.text) # store our key temporarily until the next key
        elif soup.find_all('p'): # get our list of definitions in paragraph tags
            for r in soup.find_all('p'): # We need to add each paragraph tag (definition)
                result[h2_keys[-1]].append(r.text) # Add it to the LAST key we used (-1)
    return result


for letter in string.ascii_uppercase: # for A, B, C..... assign "letter = A" ... "B"... etc
    letterindex_urls.append("https://av1611.com/kjbp/kjv-dictionary/index-{}.html".format(letter))
    # add https://.../index-A.html .... B.html .... C.html... to the list
# now letterindex_urls has URLS for A-Z for us to use.

word_urls = [] # we are going to store the results of our parsed indexes here


# This for loop will first get all the html document name that we will need to grab
for url in letterindex_urls: # for every URL in our list we just made....
    print("Getting URL: {}".format(url))
    response = urllib.request.urlopen(url) # grab the HTML as a response object
    html_str = response.read().decode('UTF-8') # this is the actual text of the response as a string
    soup = BeautifulSoup(html_str, "lxml")
    #
    # all word URLS are located in the <div role="main">
    # we can search for this tag, and it returns a 1-element list containing the html in question
    # It is not iterable (each line as one entry) and it returns as a "tag" datatype, so we must
    # get the first element in the list (zero) and cast that to a normal string, which can be used
    # to parse it again and only get the actual links, since doing it again will result in something
    # that is both filtered for the references we want, and also iterable.
    str_of_links = str(soup.find_all("div", {"role":"main"})[0])
    soup2 = BeautifulSoup(str_of_links, "lxml")
    for link in soup2.find_all('a'):
        word_urls.append(dict_base_url + link.get('href'))

for word_url in word_urls:
    time.sleep(0.2) # try to slow down a little bit, rate limit so we don't cause excess load
                    # on us or the av1611 web server.
    print("Getting URL: {}".format(word_url))
    response = urllib.request.urlopen(word_url) # grab the HTML as a response object
    html_str = response.read().decode('UTF-8') # this is the actual text of the response as a string
    soup = BeautifulSoup(html_str, "lxml")
    #
    # all definition and variation data is located in the <div role="main">
    # The format is <h2>(word) <p>(word_def) <h2>(word_var_2) <p>(word_var_2_def) <p>(word_var_2_def_2)...
    # for as many variations as there are. See "addict" for variations like addicted/addicting/etc.
    str_of_tags = str(soup.find_all("div", {"role":"main"})[0])
    list_of_tags = str_of_tags.split("\n")
    word_definitions = tags_to_dict(list_of_tags)
    print("Progress: {}".format(word_definitions.keys()))
    finished_dictionary.update(word_definitions)

with open("1828_Webster_KJV.json", 'w') as output:
    json.dump(finished_dictionary, output)