75 lines
4.4 KiB
Python
75 lines
4.4 KiB
Python
import urllib.request # library for connecting to a URL and getting it's content
|
|
from bs4 import BeautifulSoup # library for parsing the HTML - requires python3-bs4 package
|
|
import string # handy predefined shortcuts which saves us time rather than enumerating letters by hand
|
|
import json # we are going to output the finished dictionary as JSON since it's easy to work with
|
|
import time
|
|
|
|
letterindex_urls = [] # we initialize an empty list that we will populate with entires below
|
|
dict_base_url = "https://av1611.com/kjbp/kjv-dictionary/"
|
|
finished_dictionary = {}
|
|
|
|
# This function takes a list of HTML tags separated into a list, one tag per list element.
|
|
# The result is a dictionary where the key is a string, the word, and the value is a list
|
|
# with corresponding values as a list. Words with one definition will have a single
|
|
# element list, whereas words with multiple definitions will have a multi-element list.
|
|
def tags_to_dict(tags):
|
|
h2_keys = [] # temporary key storage so we remember our last key when we iterate
|
|
# through multiple word definitions
|
|
result = {} # an empty dictionary where we will store our words and definitions
|
|
for element in tags:
|
|
soup = BeautifulSoup(element, "lxml") # it's easier to use new soup instance for this. Not fast.
|
|
if soup.h2: # is our element a header? (key)
|
|
result[soup.h2.text] = [] # initialize a new dictionary key with an empty list
|
|
h2_keys.append(soup.h2.text) # store our key temporarily until the next key
|
|
elif soup.find_all('p'): # get our list of definitions in paragraph tags
|
|
for r in soup.find_all('p'): # We need to add each paragraph tag (definition)
|
|
result[h2_keys[-1]].append(r.text) # Add it to the LAST key we used (-1)
|
|
return result
|
|
|
|
|
|
for letter in string.ascii_uppercase: # for A, B, C..... assign "letter = A" ... "B"... etc
|
|
letterindex_urls.append("https://av1611.com/kjbp/kjv-dictionary/index-{}.html".format(letter))
|
|
# add https://.../index-A.html .... B.html .... C.html... to the list
|
|
# now letterindex_urls has URLS for A-Z for us to use.
|
|
|
|
word_urls = [] # we are going to store the results of our parsed indexes here
|
|
|
|
|
|
# This for loop will first get all the html document name that we will need to grab
|
|
for url in letterindex_urls: # for every URL in our list we just made....
|
|
print("Getting URL: {}".format(url))
|
|
response = urllib.request.urlopen(url) # grab the HTML as a response object
|
|
html_str = response.read().decode('UTF-8') # this is the actual text of the response as a string
|
|
soup = BeautifulSoup(html_str, "lxml")
|
|
#
|
|
# all word URLS are located in the <div role="main">
|
|
# we can search for this tag, and it returns a 1-element list containing the html in question
|
|
# It is not iterable (each line as one entry) and it returns as a "tag" datatype, so we must
|
|
# get the first element in the list (zero) and cast that to a normal string, which can be used
|
|
# to parse it again and only get the actual links, since doing it again will result in something
|
|
# that is both filtered for the references we want, and also iterable.
|
|
str_of_links = str(soup.find_all("div", {"role":"main"})[0])
|
|
soup2 = BeautifulSoup(str_of_links, "lxml")
|
|
for link in soup2.find_all('a'):
|
|
word_urls.append(dict_base_url + link.get('href'))
|
|
|
|
for word_url in word_urls:
|
|
time.sleep(0.2) # try to slow down a little bit, rate limit so we don't cause excess load
|
|
# on us or the av1611 web server.
|
|
print("Getting URL: {}".format(word_url))
|
|
response = urllib.request.urlopen(word_url) # grab the HTML as a response object
|
|
html_str = response.read().decode('UTF-8') # this is the actual text of the response as a string
|
|
soup = BeautifulSoup(html_str, "lxml")
|
|
#
|
|
# all definition and variation data is located in the <div role="main">
|
|
# The format is <h2>(word) <p>(word_def) <h2>(word_var_2) <p>(word_var_2_def) <p>(word_var_2_def_2)...
|
|
# for as many variations as there are. See "addict" for variations like addicted/addicting/etc.
|
|
str_of_tags = str(soup.find_all("div", {"role":"main"})[0])
|
|
list_of_tags = str_of_tags.split("\n")
|
|
word_definitions = tags_to_dict(list_of_tags)
|
|
print("Progress: {}".format(word_definitions.keys()))
|
|
finished_dictionary.update(word_definitions)
|
|
|
|
with open("1828_Webster_KJV.json", 'w') as output:
|
|
json.dump(finished_dictionary, output)
|