From f523c79618711d7eade509976d60adbf414a910e Mon Sep 17 00:00:00 2001 From: Tyler Date: Mon, 26 Oct 2026 20:30:43 -0700 Subject: [PATCH] claude go reimplement --- .env.example | 3 - .gitignore | 19 +- INSTALL.txt | 55 - Makefile | 28 + README.txt | 249 ++- app.py | 1914 ----------------- {data => assets/data}/1828_Webster_KJV.json | 0 {data => assets/data}/book_descriptions.json | 0 {data => assets/data}/chapter_summaries.json | 0 {data => assets/data}/concept_mappings.json | 0 .../data}/grammatical_variants.json | 0 {data => assets/data}/kjv.aff | 0 {data => assets/data}/kjv.dic | 0 .../data}/theological_synonyms.json | 0 {html => assets/html}/chapter_nav.html | 0 {html => assets/html}/dictionary.html | 0 {html => assets/html}/find.html | 0 {html => assets/html}/heaven.html | 0 {html => assets/html}/index.html | 0 {html => assets/html}/navbar.html | 0 {html => assets/html}/on_reading_bible.html | 0 {html => assets/html}/plan.html | 0 {html => assets/html}/response.html | 0 {html => assets/html}/saved.html | 0 {html => assets/html}/search.html | 0 {html => assets/html}/search_bar.html | 0 {html => assets/html}/seq.html | 0 {html => assets/html}/show.html | 0 {html => assets/html}/votd.html | 0 {js => assets/js}/book_suggest.js | 0 {js => assets/js}/collapsible.js | 0 {js => assets/js}/shortcode.js | 0 {js => assets/js}/verse_menu.js | 0 kjv_fe.html => assets/kjv_fe.html | 0 favicon.ico => assets/static/favicon.ico | Bin {static => assets/static}/favicon.svg | 0 {styles => assets/styles}/collapsible.css | 0 .../styles}/fonts/Arizonia-Regular.ttf | Bin .../styles}/fonts/Parisienne-Regular.ttf | Bin {styles => assets/styles}/header.css | 0 {styles => assets/styles}/header_dark.css | 0 {styles => assets/styles}/header_light.css | 0 {styles => assets/styles}/human_readable.css | 0 {styles => assets/styles}/page.css | 0 {styles => assets/styles}/search_rich.css | 0 {styles => assets/styles}/votd.css | 0 cmd/kjv-api/main.go | 93 + data/t_kjv.db | 0 {data => db}/kjv.db | Bin dist/kjv-api.apache.conf | 16 + dist/kjv-api.nginx.conf | 21 + dist/kjv-api.service | 19 + data/notes => doc/notes.txt | 0 go.mod | 17 + go.sum | 50 + html/reader.html | 24 - html/seq_report.html | 26 - html/shortcode_report.html | 31 - install.sh | 119 - install/apache.site.conf.example | 23 - install/kjv-api.service | 18 - install/nginx.example | 24 - internal/kjv/app.go | 430 ++++ internal/kjv/assets.go | 113 + internal/kjv/difflib.go | 76 + internal/kjv/logdb.go | 198 ++ internal/kjv/mt.go | 86 + internal/kjv/pages.go | 512 +++++ internal/kjv/refs.go | 357 +++ internal/kjv/sermons.go | 163 ++ internal/kjv/server.go | 313 +++ internal/kjv/spell.go | 253 +++ internal/kjv/state.go | 521 +++++ internal/kjv/store.go | 227 ++ kjv-api-test.sh | 10 - kjv-api.sh | 3 - kjv_api.py | 1009 --------- logs/.gitkeep | 0 logs/.include | 0 scripts/kjv-api-test.sh | 11 + scripts/kjv-api.sh | 6 + tools/fuzz_refs.py | 110 - uninstall.sh | 21 - 83 files changed, 3714 insertions(+), 3454 deletions(-) delete mode 100644 .env.example delete mode 100644 INSTALL.txt create mode 100644 Makefile delete mode 100644 app.py rename {data => assets/data}/1828_Webster_KJV.json (100%) rename {data => assets/data}/book_descriptions.json (100%) rename {data => assets/data}/chapter_summaries.json (100%) rename {data => assets/data}/concept_mappings.json (100%) rename {data => assets/data}/grammatical_variants.json (100%) rename {data => assets/data}/kjv.aff (100%) rename {data => assets/data}/kjv.dic (100%) rename {data => assets/data}/theological_synonyms.json (100%) rename {html => assets/html}/chapter_nav.html (100%) rename {html => assets/html}/dictionary.html (100%) rename {html => assets/html}/find.html (100%) rename {html => assets/html}/heaven.html (100%) rename {html => assets/html}/index.html (100%) rename {html => assets/html}/navbar.html (100%) rename {html => assets/html}/on_reading_bible.html (100%) rename {html => assets/html}/plan.html (100%) rename {html => assets/html}/response.html (100%) rename {html => assets/html}/saved.html (100%) rename {html => assets/html}/search.html (100%) rename {html => assets/html}/search_bar.html (100%) rename {html => assets/html}/seq.html (100%) rename {html => assets/html}/show.html (100%) rename {html => assets/html}/votd.html (100%) rename {js => assets/js}/book_suggest.js (100%) rename {js => assets/js}/collapsible.js (100%) rename {js => assets/js}/shortcode.js (100%) rename {js => assets/js}/verse_menu.js (100%) rename kjv_fe.html => assets/kjv_fe.html (100%) rename favicon.ico => assets/static/favicon.ico (100%) rename {static => assets/static}/favicon.svg (100%) rename {styles => assets/styles}/collapsible.css (100%) rename {styles => assets/styles}/fonts/Arizonia-Regular.ttf (100%) rename {styles => assets/styles}/fonts/Parisienne-Regular.ttf (100%) rename {styles => assets/styles}/header.css (100%) rename {styles => assets/styles}/header_dark.css (100%) rename {styles => assets/styles}/header_light.css (100%) rename {styles => assets/styles}/human_readable.css (100%) rename {styles => assets/styles}/page.css (100%) rename {styles => assets/styles}/search_rich.css (100%) rename {styles => assets/styles}/votd.css (100%) create mode 100644 cmd/kjv-api/main.go delete mode 100644 data/t_kjv.db rename {data => db}/kjv.db (100%) create mode 100644 dist/kjv-api.apache.conf create mode 100644 dist/kjv-api.nginx.conf create mode 100644 dist/kjv-api.service rename data/notes => doc/notes.txt (100%) create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 html/reader.html delete mode 100644 html/seq_report.html delete mode 100644 html/shortcode_report.html delete mode 100755 install.sh delete mode 100644 install/apache.site.conf.example delete mode 100644 install/kjv-api.service delete mode 100644 install/nginx.example create mode 100644 internal/kjv/app.go create mode 100644 internal/kjv/assets.go create mode 100644 internal/kjv/difflib.go create mode 100644 internal/kjv/logdb.go create mode 100644 internal/kjv/mt.go create mode 100644 internal/kjv/pages.go create mode 100644 internal/kjv/refs.go create mode 100644 internal/kjv/sermons.go create mode 100644 internal/kjv/server.go create mode 100644 internal/kjv/spell.go create mode 100644 internal/kjv/state.go create mode 100644 internal/kjv/store.go delete mode 100755 kjv-api-test.sh delete mode 100755 kjv-api.sh delete mode 100644 kjv_api.py delete mode 100644 logs/.gitkeep delete mode 100644 logs/.include create mode 100755 scripts/kjv-api-test.sh create mode 100755 scripts/kjv-api.sh delete mode 100644 tools/fuzz_refs.py delete mode 100755 uninstall.sh diff --git a/.env.example b/.env.example deleted file mode 100644 index 8fc4956..0000000 --- a/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -KJV_SERMON_DB="./data/sermons.db" -# this does not do anything unless sermon_analyzer.py is running in tcp socket mode as a separate process -KJV_SECRET_API_KEY="change-me" diff --git a/.gitignore b/.gitignore index 08b9184..04c166c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,15 @@ -logs/*.log -*__pycache__* -data/shortcodes.db -data/seq.db -data/sermons.db +# build output: dist/ holds what gets deployed, the unit and proxy configs are +# tracked, the two built files (and the database journal of a local run) are not +/bin/ +/dist/kjv-api +/dist/kjv.db +/dist/kjv.db-* + +# local settings, may hold KJV_SECRET_API_KEY .env +*.env + +# tools/ +__pycache__/ + +*.swp diff --git a/INSTALL.txt b/INSTALL.txt deleted file mode 100644 index a339d4e..0000000 --- a/INSTALL.txt +++ /dev/null @@ -1,55 +0,0 @@ - -Tested with Dependancy:Version -======================= -python3:3.10.6-1~22.04 -python3-flask:2.0.1-2ubuntu1 -sqlite3:3.37.2-2ubuntu0.1 -libhunspell-dev:1.7.0-4build1 -python3-hunspell:0.5.5-2build6 -uwsgi-core:2.0.20-4 -uwsgi-plugin-python3:2.0.20-4 -apache2:2.4.52-1ubuntu4.2 -python3-requests:2.25.1+dfsg-2 - -However, most up to date packages should be fine. - -Installation -====== -This software expects to be installed at /opt/kjv-api/ (for uwsgi use on a reverse proxy) -If you change this installation directoy, make sure kjv-api.sh has its path updated. -This installation method is intended to play nice among other Virtual Hosts - -0. Install dependancies listed at the head of this document. - - I used ubuntu repositories, but many can be installed via pip in a venv. it - would probably be a better way than with system repositories. uwsgi however is - a system package, I'm pretty sure. - - If you use pip/venv, you will need to modify the kjv-api.sh script to activate - the venv before executing it, I'm pretty sure. - - You may substitute the webserver for one of your choice, as long as it's capable - of reverse proxy. Do not run it with the flask development server or serve directly - as a uswgi thing. - -1. For security, create a new user with the name 'kjv' - - sudo useradd -r -s /bin/false kjv - -2. Change ownership of the /opt/kjv-api/ folder to the kjv user - - sudo chmod -R kjv:kjv /opt/kjv-api/ - -3. Test the software runs by running kjv-api.sh as a non-root user (preferably as kjv user) - and that you are able to query it locally. - - term1. sudo -Hu kjv ./kjv-api.sh - - term2. curl -XGET 127.0.0.1:1611/random - -4. If you intend to set up the software as a persistant service, copy kjv-api.service from - the install folder to /etc/systemd/system/kjv-api.service if you're using systemd. - - sudo cp /opt/kjv-api/install/kjv-api.service /etc/systemd/system/ - - sudo systemctl enable kjv-api.service - - sudo systemctl start kjv-api.service - -5. Configure your webserver as a reverse proxy to point to 127.0.0.1:1611 for where you - wish to have the api hosted. - - A sample configuration for apache is provided in the install folder. - - It is up to you to use https or not, but if you know nginx or haproxy you should be able - to set up local proxying to the WSGI server that runs the flask. - - There are WSGI modules for apache avaliable, which may be better for larger scaled - deployments, but http proxying to the local uwsgi served flask works fine for smaller uses. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1da6fd6 --- /dev/null +++ b/Makefile @@ -0,0 +1,28 @@ +# Builds the two-file distribution: dist/kjv-api (static binary) and dist/kjv.db +# (the Bible, every page/script/style/data file, plus runtime tables and logs). +# dist/ also holds the user service and the nginx/apache configs, which are kept in git. + +GO ?= go +SOURCES := $(shell find cmd internal -name '*.go') go.mod go.sum + +.PHONY: all build dist test clean +all: dist + +build: bin/kjv-api + +bin/kjv-api: $(SOURCES) + CGO_ENABLED=0 $(GO) build -trimpath -ldflags="-s -w" -o $@ ./cmd/kjv-api + +# kjv.db is a fresh copy of db/kjv.db with everything under assets/ loaded into it +dist: bin/kjv-api db/kjv.db $(shell find assets -type f) + mkdir -p dist + cp bin/kjv-api dist/kjv-api + cp db/kjv.db dist/kjv.db + bin/kjv-api import-assets -db dist/kjv.db assets + @ls -l dist + +test: + $(GO) vet ./... && $(GO) test ./... + +clean: + rm -rf bin dist/kjv-api dist/kjv.db diff --git a/README.txt b/README.txt index 94edee0..a520c99 100644 --- a/README.txt +++ b/README.txt @@ -4,6 +4,9 @@ King James Version web API is a machine and human callable service to query the King James Bible and provides useful features, such as random verses and a quote of the day. +It is a single dependancyless program (kjv-api) and its companion sqlite3 +database (kjv.db). Those two files are the entire distribution. + EXTERNAL DATA ============= @@ -11,79 +14,207 @@ The software uses a sqlite3 database as external data dependancy, saved locally and modified to remove false Bible translations (included) https://github.com/scrollmapper/bible_databases -For software dependancies see INSTALL.txt +kjv.db also holds everything else the program serves, so there is nothing else +to copy next to it: the pages, scripts and styles (the assets table), the 1828 +Webster dictionary, the spellchecking word list, users' reading plans and +shortcodes, and the logs. INSTALLATION ============ -See INSTALL.TXT +Nothing needs to be installed to run it: no runtime, no packages, no root. +There are two ways to run it, pick one. Both start from the dist folder: + kjv-api - the program + kjv.db - its database + kjv-api.service - systemd user service, for way 2 + kjv-api.nginx.conf - nginx reverse proxy site, for way 2 + kjv-api.apache.conf - the same for apache, if you use that instead +Get the two files in it by building them ($ make dist, see BUILDING) or from +whoever gave them to you. + +1. Run it locally, just to use it or try it +------------------------------------------- +$ dist/kjv-api +Then browse to http://127.0.0.1:1611 , Ctrl-C stops it. That's all. +It uses the kjv.db next to it, which keeps your reading plans, shortcodes and logs +(make dist replaces dist/kjv.db with a fresh one). On another port, for example +when the service (way 2) already has 1611 on this machine: +$ dist/kjv-api -addr 127.0.0.1:1612 + +2. Install it as a persistent service behind nginx +-------------------------------------------------- +It runs as a systemd user service under your own account on 127.0.0.1:1611, +and nginx forwards your domain to it. There is no root service and no special user. + + a. Put the program and database in ~/kjv-api, and start the service: + $ mkdir -p ~/kjv-api ~/.config/systemd/user + $ cp dist/kjv-api dist/kjv.db ~/kjv-api/ + $ cp dist/kjv-api.service ~/.config/systemd/user/ + $ systemctl --user daemon-reload + $ systemctl --user enable --now kjv-api + $ sudo loginctl enable-linger $USER # keeps it running when you log out, starts it at boot + $ curl 127.0.0.1:1611/random # a Bible verse in json means it works + + b. Point nginx at it (change example.com in the file to your domain first): + $ sudo cp dist/kjv-api.nginx.conf /etc/nginx/sites-available/kjv-api + $ sudo ln -s /etc/nginx/sites-available/kjv-api /etc/nginx/sites-enabled/ + $ sudo nginx -t && sudo systemctl reload nginx + $ sudo certbot --nginx -d example.com # https, optional but recommended + Already have a site on that domain? Copy just the location block from the file into it. + + Settings: optional, see the environment variables below. Put them in + ~/kjv-api/kjv-api.env (one NAME=value per line), chmod 600 it, and + systemctl --user restart kjv-api. + + Updating: replace only the program, never kjv.db (it holds your users' reading + plans and shortcodes, and your logs). From the source folder: + $ make dist + $ install -m 755 dist/kjv-api ~/kjv-api/kjv-api + $ ~/kjv-api/kjv-api import-assets -db ~/kjv-api/kjv.db assets # only if pages changed + $ systemctl --user restart kjv-api + + Logs: journalctl --user -u kjv-api -f + or: ~/kjv-api/kjv-api logs -db ~/kjv-api/kjv.db -n 100 + + Uninstall: systemctl --user disable --now kjv-api, then delete ~/kjv-api, + ~/.config/systemd/user/kjv-api.service and the nginx site. USAGE ===== -The program may be invoked by running kjv-api.sh in a non-root account, or via -systemd daemon process in a non-root account. -Alternately for testing, the program may be run on another port by running: -$ uwsgi_python3 --http-socket : --wsgi-file /opt/kjv-api/app.py --callable app -or by executing kjv-api-test.sh which will listen on 0.0.0.0:1612 +It only ever listens on localhost (127.0.0.1:1611 unless told otherwise), it is +meant to sit behind a reverse proxy. Asking it to listen on any other address is refused. +For development, scripts/kjv-api-test.sh builds it, copies the database, and +listens on 127.0.0.1:1612 Visiting the base directory will show the user-facing documentation via web browser or cURL. -When testing or doing a deployment, the data directory must be owned by whatever user is -running the program in order to prevent a sqlite3.OperationalError for writing to a "readonly" -database. +Changing a page, script, style or data file means editing it in the source tree +and loading it into the database again, then restarting the program: +$ ./kjv-api import-assets -db kjv.db assets + +The logs are in the database, there is no log rotation but the program deletes +lines older than 90 days when it starts: +$ ./kjv-api logs -db kjv.db -n 100 +$ sqlite3 kjv.db "select * from logs where level = 'ERROR'" +When run by systemd the same lines also go to the journal (journalctl --user -u kjv-api), +and KJV_LOG_STDERR=1 does that outside of systemd. + +When testing or doing a deployment, the directory holding kjv.db must be owned by +whatever user is running the program (sqlite3 writes its journal beside the +database), otherwise reading plans, shortcodes and logs will not be saved. + +Optional environment variables +- KJV_DB - path to kjv.db instead of the -db flag +- KJV_ADDR - localhost address to listen on instead of the -addr flag +- KJV_SECRET_API_KEY - the token POST /natural expects, and the bearer token the /seqreport and + /screport listings need (without it those two do not exist). +- KJV_LOG_STDERR - set to 1 to also print the log lines to stderr (automatic under systemd) +- KJV_SEMANTIC_ADDR - where the semantic search service listens, default localhost:1613 +- KJV_SERMON_DB - path to the separate sermon database. Without it the sermon + endpoints answer 503. + +BUILDING +======== +Only the machine that compiles needs anything: go 1.26 or newer, and make (built +and tested with go 1.27.3, get it from https://go.dev/dl/ or your package manager). +The sqlite3 driver (modernc.org/sqlite) is pure Go, no C compiler needed, and go +downloads it the first time it builds. +$ make build builds bin/kjv-api +$ make dist builds dist/kjv-api and dist/kjv.db, the two files to distribute +$ make test go vet, and any tests +Optional: sqlite3 (the command line tool) for looking at the logs and database. CONTENTS ======== . -├── app.py - main program -├── data -│   ├── 1828_Webster_KJV.json - Dictionary -│   ├── kjv.aff - KJV Spellchecking Description file (hunspell) -│   ├── kjv.db - KJV sqlite3 database -│   └── kjv.dic - KJV Spellchecking dictionary file (hunspell) -│   └── seq.db - Sqlite3 Database with user's ids for sequenced scripture -│   └── shortcodes.db - Sqlite3 Database with saved shortcodes for scripture groups. -├── favicon.ico -├── html -│   ├── dictionary.html - dictionary search landing page /define -│   ├── find.html - unified search bar landing page /find -│   ├── navbar.html - navigation bar added programatically to rich results -│   ├── heaven.html - Bible way to heaven, static page /salvation -│   ├── on_reading_bible.html - Introduction on good reading practices -│   ├── index.html - / landing page with unified search bar -│   ├── reader.html - not yet implemented Bible browser end point -│   ├── reading_plan.html - not yet implemented Bible reading plan generator -│   ├── response.html - Template for rich verse responses -│   ├── search.html - Bible keyword search landing page /search -│   ├── search_bar.html - Unused -│   ├── show.html - Bible verse search landing page /show -│   ├── seq.html - Bible sequential verse landing page /seq -│   └── votd.html - Matty's test VOTD rich landing page -├── install -│   ├── apache.site.conf.example - example installation configuration -│   └── kjv-api.service - example systemd service file -├── INSTALL.txt - installation instructions -├── logs - log folder (populated at runtime - there is NO log rotation) -├── js -│   └── collapsible.js - js for the definition drop-down menu used in unified search -├── kjv-api.sh - runs uwsgi flask app on port 1611 -├── kjv-api-test.sh - runs uwsgi flask app on port 1612 -├── README.txt - This document -├── styles -│   ├── collapsible.css -│   ├── header.css -│   ├── fonts -│   │   ├── Arizonia-Regular.ttf -│   │   └── Parisienne-Regular.ttf -│   ├── human_readable.css -│   ├── search_rich.css -│   └── votd.css -└── tools - └── av1611_dictionary_scraper.py - builds a dictionary json for the 1828 dictionary - └── sanity_check.py - Integrated during runtime and also as endpoint testing when run alone. - └── search_keywords.txt - endpoint testing queries - └── sanity_results.txt - results of running sanity_check.py as a standalone +├── cmd +│ └── kjv-api +│ └── main.go - command line: run the server, import-assets, logs +├── internal +│ └── kjv - the program, one package +│ ├── server.go - startup, localhost listener, routes +│ ├── app.go - loads the assets from the database, builds the rich page pieces and verse rendering +│ ├── pages.go - /show /find /search /define /random /votd +│ ├── refs.go - reference resolver ("1 jn 3.16", "II Kings 2:1-4", "song of songs 2") +│ ├── state.go - /seq /plan and shortcodes, saved in the database +│ ├── sermons.go - sermon database and semantic search endpoints, optional +│ ├── store.go - the kjv.db queries and the Bible held in memory +│ ├── assets.go - import-assets, putting the files in assets/ into the database +│ ├── logdb.go - logs saved in the database +│ ├── spell.go - spellchecking suggestions from the word list +│ ├── difflib.go - fuzzy book name matching +│ └── mt.go - the random number generator that picks the verse of the day +├── db +│ └── kjv.db - KJV sqlite3 database, the start of the companion database +├── assets - everything that gets loaded into kjv.db by import-assets +│ ├── data +│ │ ├── 1828_Webster_KJV.json - Dictionary +│ │ ├── kjv.aff - KJV Spellchecking Description file +│ │ ├── kjv.dic - KJV Spellchecking dictionary file +│ │ └── *.json - book descriptions, chapter summaries, search synonyms and variants +│ ├── html +│ │ ├── dictionary.html - dictionary search landing page /define +│ │ ├── find.html - unified search bar landing page /find +│ │ ├── navbar.html - navigation bar added programatically to rich results +│ │ ├── chapter_nav.html - previous/next chapter bar +│ │ ├── heaven.html - Bible way to heaven, static page /salvation +│ │ ├── on_reading_bible.html - Introduction on good reading practices +│ │ ├── index.html - / landing page with unified search bar, the program fills in the book lists +│ │ ├── plan.html - Bible reading plan setup /plan +│ │ ├── saved.html - favorites, definitions and cross references kept in the browser /saved +│ │ ├── response.html - Template for rich verse responses +│ │ ├── search.html - Bible keyword search landing page /search +│ │ ├── search_bar.html - Unused +│ │ ├── show.html - Bible verse search landing page /show +│ │ ├── seq.html - Bible sequential verse landing page /seq +│ │ └── votd.html - Matty's test VOTD rich landing page +│ ├── js +│ │ ├── collapsible.js - js for the definition drop-down menu used in unified search +│ │ └── book_suggest.js, shortcode.js, verse_menu.js +│ ├── static +│ │ └── favicon.svg, favicon.ico +│ ├── styles +│ │ ├── collapsible.css +│ │ ├── header.css +│ │ ├── fonts +│ │ │ ├── Arizonia-Regular.ttf +│ │ │ └── Parisienne-Regular.ttf +│ │ ├── human_readable.css +│ │ ├── search_rich.css +│ │ └── votd.css +│ └── kjv_fe.html - /fe frontend page +├── dist - what gets deployed, see INSTALLATION +│ ├── kjv-api, kjv.db - built by make dist (not kept in git) +│ ├── kjv-api.service - systemd user service, port 1611 +│ ├── kjv-api.nginx.conf - nginx reverse proxy site +│ └── kjv-api.apache.conf - apache reverse proxy site +├── scripts +│ ├── kjv-api.sh - builds if needed and runs the program on localhost port 1611 +│ └── kjv-api-test.sh - builds and runs the program on a copy of the database on port 1612 +├── tools +│ ├── av1611_dictionary_scraper.py - builds a dictionary json for the 1828 dictionary +│ ├── sanity_check.py - Endpoint testing, run alone against a running instance (needs python3-requests) +│ └── search_keywords.txt - endpoint testing queries +├── doc +│ ├── openapi.yaml +│ └── notes.txt +├── Makefile - make, make dist, make test +├── go.mod, go.sum +└── README.txt - This document AUTHOR ====== -Copyright Tyler R Dinsmoor & Matty Sneederson 2023 +Copyright Tyler R Dinsmoor & Matty Sneederson 2022-2026 +Reimplemented in Go by demon machine claude. + + +SECURITY +======== +- Listens on localhost only, and refuses any other address. +- The user service runs as your account, so it can read and write whatever you can. For more + separation, make an account just for it (sudo useradd -m kjv; sudo loginctl enable-linger kjv) + and do step 2a logged in as that account. +- Reading plan ids and shortcodes are random but they are the only secret; anyone holding a plan id can + move that plan. Making new ones is rate limited per visitor (by the X-Real-IP the proxy sends, + the nginx and apache files do). Plans unused for 90 days are deleted. +- Please report anything you find to the author. diff --git a/app.py b/app.py deleted file mode 100644 index a990201..0000000 --- a/app.py +++ /dev/null @@ -1,1914 +0,0 @@ -# app.py -from flask import ( - Flask, - request, - jsonify, - send_file, - render_template, - redirect, - Response, -) -import random -from datetime import date, datetime, timedelta # used for daily verse -import sqlite3 -import string # for literals shortcut -import json # for dictionary -import hunspell # for spell checking (not really used) -import os -import sys -import logging -import socket -import json -import hashlib -import re -import difflib -import math -import urllib.parse -import html as html_lib -from tools import sanity_check -try: - from dotenv import load_dotenv - load_dotenv() - SERMON_DB = os.getenv("KJV_SERMON_DB") - SECRET_API_KEY = os.getenv("KJV_SECRET_API_KEY") -except: - KJV_SERMON_DB="./data/sermons.db" - # this does not do anything unless sermon_analyzer.py is running in tcp socket mode as a separate process - KJV_SECRET_API_KEY="testing" - -sequential_purge_delta: timedelta = timedelta(days=90) -timestamp_runtime: datetime = datetime.now().strftime("%d%b%y-%H:%M") -logging.basicConfig( - filename=f"logs/kjv-api_{datetime.now().strftime('%d%b%y')}.log", - encoding="utf-8", - level=logging.INFO, -) - -logging.info(f"Started: {timestamp_runtime}") -logging.info(f"Using Sermon Database: {SERMON_DB}") - -try: - sanity_check.check_sanity() - sanity: bool = True -except PermissionError as e: - logging.critical(f"PERMISSIONS FAILURE: {e}") - sanity: bool = False -except Exception as e: - logging.critical(e) - sanity: bool = False -finally: - logging.info(f"Sanity: {sanity}") - -app = Flask(__name__, template_folder="html") - -""" -Current work prefixed with * -TODO: -Replace send_file with something that loads headers and nav templates -Search phrase alias dictionary -Search highlighting -Bible Reading Plan -/seq Going backwards (works but is buggy, disabled for now) -/seq Wrap back to Gen 1:1 when you reach the end of Rev -/seq Querying what your UID's next verse is going to be -https://docs.python.org/3/library/sqlite3.html#sqlite3-placeholders -URL Shortener for sharing groups of scripture. -""" - -def dict_factory(cursor, row): - fields = [column[0] for column in cursor.description] - return {key: value for key, value in zip(fields, row)} - -kjv_cur = None -sermon_cur = None -# --------------------------------------------------------------------------- -# Verse menu helpers (favorites, definitions and cross references are saved by -# the browser itself; the server only supplies the lookups) -# --------------------------------------------------------------------------- - -DICT_STOPWORDS = { - "the", "and", "that", "for", "him", "his", "her", "but", "not", "with", "are", - "was", "were", "unto", "thou", "thee", "thy", "ye", "you", "all", "from", "they", - "them", "this", "have", "had", "shall", "will", "which", "into", "upon", "then", -} - - -def format_verse_ref(verse_id) -> str: - """12345678 style id -> 'John 3:16'""" - verse_id = int(verse_id) - return f"{lookup_bookname(verse_id // 1_000_000)} {verse_id // 1000 % 1000}:{verse_id % 1000}" - - -@app.get("/xref/") # cross references for one verse, best ranked first -def cross_references(verse_id: int): - rows = kjv_cur.execute( - "SELECT sv, ev FROM cross_reference WHERE vid = ? ORDER BY r DESC LIMIT 15", - (verse_id,), - ).fetchall() - results = [] - for start_id, end_id in rows: - ref = format_verse_ref(start_id) - if end_id and end_id > start_id: - end_ref = format_verse_ref(end_id) - # "John 3:16-18" when the range stays in one chapter - same_chapter = start_id // 1000 == end_id // 1000 - ref += "-" + (str(end_id % 1000) if same_chapter else end_ref) - verses = lookup_by_verse_id(start_id, end_id) - else: - verses = lookup_by_verse_id(start_id) - results.append({ - "ref": ref, - "verse_id": start_id, - "text": " ".join(v[4] for v in verses), - }) - return jsonify(results) - - -@app.get("/defs") # 1828 Webster definitions for several words at once: /defs?words=grace,loved -def definitions(): - words = request.args.get("words", "").lower().replace(",", " ").split()[:40] - dictionary = cached["json_dictionary"] - found = {} - for word in words: - word = "".join(c for c in word if c.isalpha()) - if word and word not in DICT_STOPWORDS and word in dictionary: - found[word] = dictionary[word] - return jsonify(found) - - -@app.get("/saved") # favorites, definitions and cross references kept in this browser -def saved_page(): - return static_page("html/saved.html") - - -# --------------------------------------------------------------------------- -# Bible reading plan: a /seq style tracker that can go backwards. The plan is -# its UID; the page URL (/plan/?n=) is what gets bookmarked. -# --------------------------------------------------------------------------- - -PLAN_UID_CHARS = "0123456789ACEFHJKLMNPRTUVWXY" -PLAN_MAX_PER_DAY = 500 - - -def plan_connection(): - if usermode == "Single": - conn = sqlite3.connect("data/seq.db", check_same_thread=False) - else: - conn = sqlite3.connect("data/seq.db") - conn.execute( - "CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)" - ) - return conn - - -def plan_per_day(raw, default=25) -> int: - try: - return max(1, min(PLAN_MAX_PER_DAY, int(raw))) - except (TypeError, ValueError): - return default - - -def plan_start_index(ref: str): - """Index into verse_ids for a reference like 'Matthew 1:1'; None if it can't be parsed.""" - ref = (ref or "").strip() - if not ref: - return 0 - try: - verse_id = int(show(kw=ref, get_start_verse_id=True)) - return cached["verse_ids"].index(verse_id) - except Exception: - return None - - -def plan_error(message: str): - return static_html_doc( - "Reading Plan", - f"

Reading Plan

{html_lib.escape(message)}

Back to plan setup

", - ), 400 - - -@app.get("/plan") # setup form for a new reading plan -def reading_plan(): - return static_page("html/plan.html") - - -@app.post("/plan/new") -def plan_create(): - total = len(cached["verse_ids"]) - start = plan_start_index(request.form.get("start", "")) - if start is None: - return plan_error("We couldn't find that starting point. Try something like 'Matthew 1:1'.") - if request.form.get("pace") == "verses": - per_day = plan_per_day(request.form.get("verses")) - else: - try: - days = max(1, min(3650, int(request.form.get("days", 365)))) - except ValueError: - days = 365 - per_day = plan_per_day(math.ceil((total - start) / days)) - conn = plan_connection() - for _ in range(5): # a clash on a random 5 character id is very unlikely - uid = generate_short_id() - try: - conn.execute( - "INSERT INTO reading_tracker VALUES(?, ?, ?)", (uid, start, str(datetime.now())) - ) - break - except sqlite3.IntegrityError: - continue - else: - return plan_error("Could not create a plan, please try again.") - conn.commit() - conn.close() - logging.info(f"New reading plan {uid} start={start} per_day={per_day}") - return redirect(f"/plan/{uid}?n={per_day}", code=303) - - -def plan_uid_or_none(raw: str): - uid = search_input_cleaning(raw, valid_chars=PLAN_UID_CHARS) - return uid if uid and uid == raw else None - - -@app.post("/plan//move") # back / next / jump; POST so a page prefetch can never move your place -def plan_move(uid: str): - uid = plan_uid_or_none(uid) - per_day = plan_per_day(request.form.get("n")) - if not uid: - return plan_error("That reading plan id is not valid.") - conn = plan_connection() - row = conn.execute("SELECT verse FROM reading_tracker WHERE uid = ?", (uid,)).fetchone() - if row is None: - conn.close() - return plan_error("That reading plan was not found (plans unused for 90 days are removed).") - total = len(cached["verse_ids"]) - position = row[0] - action = request.form.get("action") - if action == "next": - position = min(total, position + per_day) - elif action == "back": - position = max(0, position - per_day) - elif action == "restart": - position = 0 - elif action == "goto": - target = plan_start_index(request.form.get("kw", "")) - if target is None: - conn.close() - return plan_error("We couldn't find that reference. Try something like 'John 3:1'.") - position = target - conn.execute( - "REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", - (uid, position, str(datetime.now())), - ) - conn.commit() - conn.close() - return redirect(f"/plan/{uid}?n={per_day}", code=303) - - -@app.get("/plan/") # today's reading for a plan. Viewing never moves your place. -def plan_view(uid: str): - uid = plan_uid_or_none(uid) - if not uid: - return plan_error("That reading plan id is not valid.") - per_day = plan_per_day(request.args.get("n")) - conn = plan_connection() - row = conn.execute("SELECT verse FROM reading_tracker WHERE uid = ?", (uid,)).fetchone() - if row is None: - conn.close() - return plan_error("That reading plan was not found (plans unused for 90 days are removed).") - position = row[0] - # keeps an active plan from being purged as unused - conn.execute( - "REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", - (uid, position, str(datetime.now())), - ) - conn.commit() - conn.close() - - verse_ids = cached["verse_ids"] - total = len(verse_ids) - finished = position >= total - last_index = min(total, position + per_day) - 1 - percent = round(100 * min(position, total) / total, 1) - esc = html_lib.escape - - controls = f"""
-""" - back_disabled = "disabled" if position <= 0 else "" - next_html = ( - "" if finished else - "" - ) - controls += f"""{next_html} - -
-
- - - -
-
-
""" - - if finished: - summary = ( - "
You've finished the plan!" - f"
You read all {total:,} verses. Use Previous reading to look back, or start over.
" - ) - verses_html = "" - controls += f"""
- -
""" - else: - first_ref = format_verse_ref(verse_ids[position]) - last_ref = format_verse_ref(verse_ids[last_index]) - count = last_index - position + 1 - summary = ( - "
Today's reading" - f"
{esc(first_ref)} – {esc(last_ref)}
" - f"
{count} verses · {percent}% of the Bible complete
" - f"
" - ) - verses = lookup_by_verse_id(verse_ids[position], verse_ids[last_index]) - verses_html = "".join(response_rich(v) for v in verses) - - bookmark_note = ( - "
Bookmark this page — it is your reading plan. " - "Come back each day, then press Done to move on.
" - ) - store_script = ( - "" - ) - return ( - render_navbar("", bar_html="") - + "
" - + summary - + verses_html - + f"
{controls}
" - + bookmark_note - + store_script - + "
" - ) - - - -if __name__ == "__main__": - # single user mode, running without uwsgi - usermode: str = "Single" - kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor() - if SERMON_DB: - sermon_con = sqlite3.connect(SERMON_DB, check_same_thread=False) - sermon_con.row_factory = dict_factory - sermon_cur = sermon_con.cursor() -else: - # you're running with uwsgi - usermode: str = "Multi" - kjv_cur = sqlite3.connect("data/kjv.db").cursor() - if SERMON_DB: - sermon_con = sqlite3.connect(SERMON_DB) - sermon_con.row_factory = dict_factory - sermon_cur = sermon_con.cursor() - -if not kjv_cur: - exit("CRITICAL: Error connecting to KJV Database") -if not sermon_cur: - print("WARN: Error connecting to Sermon Database") - -# Build grammatical variants lookup from word groups -variant_groups = json.load(open("data/grammatical_variants.json", "r")) -grammatical_variants_map = {} -for group in variant_groups: - for word in group: - # Map each word to all other words in its group - grammatical_variants_map[word] = [w for w in group if w != word] - -# Build theological synonyms lookup from word groups -theological_synonym_groups = json.load(open("data/theological_synonyms.json", "r")) -theological_synonyms_map = {} -for group in theological_synonym_groups: - for word in group: - # Map each word to other words in its group - theological_synonyms_map[word] = [w for w in group if w != word] - -# Load concept mappings for common theological terms not in KJV -concept_mappings = json.load(open("data/concept_mappings.json", "r")) -concept_mappings_dict = {mapping["query"].lower(): mapping for mapping in concept_mappings} - -# Static assets: each file's URL carries a hash of its own contents ("?v=..."). -# A URL whose hash matches is served as immutable for a year; the hash changes -# whenever the file does, so a deploy is picked up immediately and only the -# files that actually changed get downloaded again. Any other URL for the same -# file (no hash, or an old one) is served with "no-cache", so it is revalidated -# every time instead of lingering in a browser for a year. -ASSET_SOURCES = { - "/styles/page.css": ("styles/page.css", "text/css"), - "/js/collapsible.js": ("js/collapsible.js", "application/javascript"), - "/js/shortcode.js": ("js/shortcode.js", "application/javascript"), - "/js/verse_menu.js": ("js/verse_menu.js", "application/javascript"), - "/js/book_suggest.js": ("js/book_suggest.js", "application/javascript"), -} -ASSET_VERSIONS: dict = { - url: hashlib.md5(open(path, "rb").read()).hexdigest()[:8] - for url, (path, _mime) in ASSET_SOURCES.items() -} - - -def asset_url(url: str) -> str: - return f"{url}?v={ASSET_VERSIONS[url]}" - - -app.jinja_env.globals["asset_url"] = asset_url - - -def load_html(path: str) -> str: - """Read an html file, filling each '/js/x.js?v=__V__' with that file's own version.""" - return re.sub( - r"(/(?:js|styles)/[\w.\-]+)\?v=__V__", - lambda m: asset_url(m.group(1)), - open(path, "r").read(), - ) - - -cached: dict = { - "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()], - "book_names": [ - i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall() - ], - "chapter_list": [], - "books_chapter_lengths": {}, - "json_dictionary": json.load(open("data/1828_Webster_KJV.json", "r")), - "grammatical_variants": grammatical_variants_map, - "theological_synonyms": theological_synonyms_map, - "concept_mappings": concept_mappings_dict, - "html_response": open("html/response.html", "r").read(), - "html_navbar": load_html("html/navbar.html"), - "book_descriptions": json.load(open("data/book_descriptions.json", "r")), - "chapter_summaries": json.load(open("data/chapter_summaries.json", "r")), - "html_chapter_nav": open("html/chapter_nav.html", "r").read(), - "pages": {}, - "css_navbar": open("styles/header.css", "r").read(), - "css_response": open("styles/human_readable.css", "r").read(), - "css_collapsible": open("styles/collapsible.css", "r").read(), - "css_page": open("styles/page.css", "r").read(), - "js_collapsible": open("js/collapsible.js", "r").read(), - "js_shortcode": open("js/shortcode.js", "r").read(), - "js_verse_menu": open("js/verse_menu.js", "r").read(), - "favicon_svg": open("static/favicon.svg", "r").read(), - "book_info": kjv_cur.execute( - 'SELECT "order", title_short, category, otnt, chapters FROM book_info ORDER BY "order"' - ).fetchall(), -} - -# Book name suggestions: the list of names is baked into the script once -cached["js_book_suggest"] = open("js/book_suggest.js", "r").read().replace( - "__BOOKS__", json.dumps(cached["book_names"]) -) - -# Fill the chapter list with unique chapters for seeking back and forward by index. -[ - cached["chapter_list"].append(int(str(chap)[:-3])) - for chap in cached.get("verse_ids") - if int(str(chap)[:-3]) not in cached["chapter_list"] -] -cached["chapter_list"].sort() -for book in range(1, 67): - cached["books_chapter_lengths"][book] = kjv_cur.execute( - "SELECT c FROM fts_kjv where b = ? ORDER BY id DESC LIMIT 1;", (book,) - ).fetchone()[0] - -# Cache book_id to bookname mapping for fast lookups -cached["book_id_to_name"] = {} -for row in kjv_cur.execute("SELECT b, n FROM key_english").fetchall(): - cached["book_id_to_name"][row[0]] = row[1] - -# Organize books by testament and category for index page -from collections import defaultdict -testament_data = {"OT": defaultdict(list), "NT": defaultdict(list)} -for book in cached["book_info"]: - book_order, title, category, testament, num_chapters = book - testament_data[testament][category].append({ - "title": title, - "chapters": num_chapters, - "about": cached["book_descriptions"].get(title, ""), - "summaries": cached["chapter_summaries"].get(title, []), - }) -cached["book_info_for_index"] = dict(testament_data) - -# Render the index page once and cache it -with app.app_context(): - from flask import render_template_string - template_content = load_html("html/index.html") - cached["html_index"] = render_template_string(template_content, testament_data=cached["book_info_for_index"]) - -def _make_asset_view(url_path: str, mimetype: str): - def view(): - response = Response(cached["assets"][url_path], mimetype=mimetype) - response.add_etag() - if request.args.get("v") == ASSET_VERSIONS[url_path]: - response.headers["Cache-Control"] = "public, max-age=31536000, immutable" - else: - response.headers["Cache-Control"] = "no-cache" - return response.make_conditional(request) - view.__name__ = "asset_" + url_path.strip("/").replace("/", "_").replace(".", "_") - return view - - -cached["assets"] = { - "/styles/page.css": cached["css_page"], - "/js/collapsible.js": cached["js_collapsible"], - "/js/shortcode.js": cached["js_shortcode"], - "/js/verse_menu.js": cached["js_verse_menu"], - "/js/book_suggest.js": cached["js_book_suggest"], -} -for _url, (_path, _mime) in ASSET_SOURCES.items(): - app.add_url_rule(_url, view_func=_make_asset_view(_url, _mime), methods=["GET"]) - - -@app.after_request -def html_pages_are_never_stale(response): - """Pages reference versioned assets, so they must always be revalidated to see new versions.""" - if response.mimetype == "text/html" and "Cache-Control" not in response.headers: - response.headers["Cache-Control"] = "no-cache" - return response - -@app.get("/favicon.ico") -@app.get("/favicon.svg") -def serve_favicon(): - response = Response(cached.get("favicon_svg"), mimetype='image/svg+xml') - response.headers['Cache-Control'] = 'public, max-age=86400' - return response - -def static_page(path: str): - """Serve one of the simple html/ pages with its asset URLs versioned.""" - if path not in cached["pages"]: - cached["pages"][path] = load_html(path) - return Response(cached["pages"][path], mimetype="text/html") - - -def static_html_doc(title: str, body: str): - """Wrap a small html snippet in the standard page shell.""" - return ( - "" - "" - f"{title} - KJV API" - f"" - "
Home Quick Search
" - f"{body}" - ) - - -def chapter_find_link(name: str) -> str: - """Same link the verse headers use: the whole chapter, in the rich view.""" - return "/find?kw=" + urllib.parse.quote(name) + "&view=rich" - - -def render_navbar(query: str = "", chapter_nav: dict = None, bar_html: str = "") -> str: - """Opens the rich html document: , site header and the chapter bar. - - The caller adds the body content after it. bar_html replaces the chapter - bar for pages (like the reading plan) that bring their own. - """ - esc = lambda value: html_lib.escape(str(value), quote=True) - query = query or "" - title = esc(query.strip() or (chapter_nav or {}).get("current_chapter_name") or "Scripture") - chapter_html = bar_html - if chapter_nav: - values = {key: esc(value) for key, value in chapter_nav.items()} - values["current_chapter_link"] = esc(chapter_find_link(chapter_nav["current_chapter_name"])) - chapter_html = cached["html_chapter_nav"].format(**values) - return cached["html_navbar"].format( - title=title, css=cached["css_navbar"], query=esc(query), chapter_nav=chapter_html - ) - - -def generate_short_id(): - remain = random.randrange(614_656, 17_210_367) # resolves to length of 5 - chars = "0123456789ACEFHJKLMNPRTUVWXY" - length = len(chars) - result = "" - while remain > 0: - pos = remain % length - remain = remain // length - result = chars[pos] + result - return result - - -def lookup_bookname(book_id): - # Ensure book_id is an integer for cache lookup - if isinstance(book_id, str): - book_id = int(book_id) - return cached["book_id_to_name"].get(book_id) - - -def lookup_book_id(bookname) -> int: - result = kjv_cur.execute( - "SELECT b FROM key_english WHERE n = ?;",(bookname,)).fetchone() - if result: - result = result[0] - return result - - -def lookup_fts(kwds): - logging.debug(f"lookup_fts.kwds = '{kwds}'") - return kjv_cur.execute("SELECT * FROM fts_kjv(?)", (kwds,)).fetchall() - - -def lookup_by_verse_id(start_verse, end_verse=None) -> list: - # This is an internal function - we don't use placeholders because SQL injection should be impossible here - if not end_verse: - results = kjv_cur.execute( - f"SELECT * FROM fts_kjv WHERE id = {start_verse};" - ).fetchall() - else: - results = kjv_cur.execute( - f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};" - ).fetchall() - return results - -def get_sermons_by_verse_id(verse_id) -> list: - if not sermon_cur: - return - return sermon_cur.execute(f"select * from verse_refs where verse_id={verse_id}").fetchall() - -def get_verse_id(book_id, chapter: str, verse: str) -> str: - # verse_id is 00000000 (00 000 000) (book + chapter + verse) - chapterid = chapter.zfill(3) - verseid = verse.zfill(3) - if verseid == "000": - verseid = "001" # Avoid verse_ids list IndexErrors on :- searches - return str(book_id) + chapterid + verseid - -def get_ref_from_verse_id(verse_id): - # verse_id is 00000000 (00 000 000) (book + chapter + verse) - verse_id = str(verse_id).zfill(8) - book_id = verse_id[:2] - chapter = verse_id[2:5] - verse = verse_id[5:8] - bookname = kjv_cur.execute("SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone()[0] - return bookname + " " + chapter + ":" + verse - - -def get_next_prev_chapter(verseid) -> dict: - current_chapter = verseid[:-3] - current_chapter_index = cached["chapter_list"].index(int(current_chapter)) - - next_chapter_id = str( - cached["chapter_list"][ - (current_chapter_index + 1) % len(cached["chapter_list"]) - ] - ) - prev_chapter_id = str( - cached["chapter_list"][ - (current_chapter_index - 1) % len(cached["chapter_list"]) - ] - ) - current_chapter_name = " ".join( - (lookup_bookname(current_chapter[:-3]), str(current_chapter[-3:]).lstrip("0")) - ) - next_chapter_name = " ".join( - (lookup_bookname(next_chapter_id[:-3]), str(next_chapter_id[-3:]).lstrip("0")) - ) - prev_chapter_name = " ".join( - (lookup_bookname(prev_chapter_id[:-3]), str(prev_chapter_id[-3:]).lstrip("0")) - ) - next_chapter_link = f"/show?kw={next_chapter_name}&view=rich" - prev_chapter_link = f"/show?kw={prev_chapter_name}&view=rich" - result = { - "current_chapter_name": current_chapter_name, - "prev_chapter_name": prev_chapter_name, - "prev_chapter_link": prev_chapter_link, - "next_chapter_name": next_chapter_name, - "next_chapter_link": next_chapter_link, - } - return result - - -def ref_input_cleaning( - ref, valid_chars=(string.ascii_letters + string.digits + ":-,"), ref_search=True -): - logging.debug(f"ref_input_cleaning({ref})") - ref = ref.replace(" ", "") - if ref.count(":") != 1 and ref_search: - if len(ref) == 0: - return "Invalid Query" - if ref[-1] in string.digits: - ref += ":-" - else: - # TODO: This should return something like /browse and list the chapters - ref += "1:-" - if ref.endswith(":"): - ref += "-" - if ref.count("-") > 1 and ref_search: - logging.debug("ref_input_cleaning.if ref.count(-): TRUE") - return False - if not ref[-1].isdigit() and ref[-1] != "-" and ref_search: - logging.debug("ref_input_cleaning.if ref[-1] TRUE") - return False - ref = list(ref) - # preserve book number if we can guess the first - # character is a number and 3nd is a letter - if ref[0].isdigit() and ref[1].isalpha() and ref_search: - prefix = ref.pop(0) + " " - else: - prefix = "" - # clean the rest of the reference - cleaned_ref = [] - for character in ref: - if character in valid_chars: - cleaned_ref.append(character) - # Was there a problem? - if len(cleaned_ref) == 0: - return False - if ref_search: - return prefix + "".join(cleaned_ref).capitalize() - else: - return "".join(cleaned_ref) - - -def search_input_cleaning( - query, valid_chars=string.ascii_letters + string.digits + "+^* ", extra_chars='' -): - valid_chars += extra_chars - # should protect against SQL injection, maybe. Should have another person review. - cleaned_query = [] - for character in query: - if character in valid_chars: - cleaned_query.append(character) - return "".join(cleaned_query) - - -def ref_parse_book(ref): - for book in reversed(cached["book_names"]): - if book.startswith(ref): - return book - return False - -def request_semantic_search(query): - HOST = 'localhost' - PORT = 1613 - data = {"query": query} - - data_string = json.dumps(data).encode() - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((HOST, PORT)) - s.sendall(data_string) - data = b'' - while True: - part = s.recv(1024) - if not part: - break - data += part - return json.loads(data.decode()) - -@app.get("/natural") -def explain_natural(): - res = 'Submit a POST request to this endpoint with the following params in the body:\n' - res += '{"token": "mytokenstring", "query" : "myquerystring"}' - return res - -@app.post("/natural") # needs valid token and query in request body -def do_semantic_search(): - request_data = request.get_json() - if request_data is None: - # Handle error: No JSON data sent - return jsonify({'error': 'Invalid request format'}), 400 - - token_string = request_data.get('token') - query_string = request_data.get('query') - if token_string is None: - # Handle error: Missing token in request - return jsonify({'error': 'Missing token in request body'}), 401 - - if token_string != SECRET_API_KEY: - return jsonify({'error': "Invalid Authorization"}) - - result = request_semantic_search(query=query_string) - return jsonify(result) - -@app.get("/ref/") # find database verse_id by english scripture reference (1John5:7) -def get_single_reference_verse_id(ref:str): - ref = ref.split(",")[0] - ref = ref.split("-")[0] - # only want the first single ref, not a range, and not a group. - ref = ref_input_cleaning(ref) - maybe_chapter: str = ref.split(":")[0][3:] - ref_chapter: str = "" - for maybedigit in maybe_chapter: - if maybedigit.isdigit(): - ref_chapter += maybedigit - ref_verse = ref.split(":")[1] - book_str: str = ref_input_cleaning(ref, string.ascii_letters) - ref_bookname: str = ref_parse_book(book_str) - ref_book_id: int = lookup_book_id(ref_bookname) - if not ref_book_id: - return - return get_verse_id(ref_book_id, ref_chapter, ref_verse) - -@app.get("/sermon/search/") # find a sermon by english scripture reference (1John5:7) -def find_sermon_by_ref(ref:str=None): - verse_id = get_single_reference_verse_id(ref=ref) - if not sermon_cur: - return jsonify({'error':"Sermon database not found"}) - sermons = get_sermons_by_verse_id(verse_id=verse_id) - for sermon in sermons: - sermon.update({'kjv':find_by_verseid(sermon.get("verse_id"))[4]}) - return jsonify(sermons) - -@app.get("/sermons") # lists all the sermons in our database -def list_sermons(): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name,filename from verse_refs").fetchall()) - -@app.get("/sermon//verses") # show all matched info on a sermon by its filename (hash) -def list_sermon_references(filename): - return jsonify(sermon_cur.execute("select distinct verse_id,text from verse_refs where filename=?", (filename,)).fetchall()) - -@app.get("/preachers") # return a list of preachers stored in database, their denomination and church name -def list_preachers(): - return jsonify(sermon_cur.execute("select distinct denomination,preacher,church_name from verse_refs").fetchall()) - -@app.get("/churches") # return a list of churches with sermons in the database -def list_churches(): - return jsonify(sermon_cur.execute("select distinct denomination,church_name from verse_refs").fetchall()) - -@app.get("/denominations") # return a list of denominations included in the database -def list_denominations(): - return jsonify(sermon_cur.execute("select distinct denomination from verse_refs").fetchall()) - -@app.get("/sermons/preacher/") # show all sermons by a paticular preacher -def find_sermons_by_preacher(preacher): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where preacher=?", (preacher,)).fetchall()) - -@app.get("/sermons/church/") # show all sermons by a paticular church -def find_sermons_by_church(church_name): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where church_name=?", (church_name,)).fetchall()) - -@app.get("/sermons/denomination/") # show all sermons by a paticular denomination -def find_sermons_by_denomination(denomination): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where denomination=?", (denomination,)).fetchall()) - -@app.get("/sermon/") # show general information about a sermon by filename (hash) -def find_sermons_by_filename(filename): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where filename=?", (filename,)).fetchall()) - -@app.get("/sermon/url/") # show general information about a sermon by url -def find_sermons_by_url(url): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where url=?", (url,)).fetchall()) - -def clean_verseid(verse_id): - verse_id = str(verse_id) - clean_verseid = '' - for char in verse_id: - if char in string.digits: - clean_verseid += char - return clean_verseid - - -def find_by_verseid(verse_id): - return lookup_by_verse_id(verse_id)[0] -# wow lol I just wrote this and forgot how it works already lmao - - -@app.get("/verse_id/") # return the database entry of an internal verse_id -def web_verseid(verse_id): - clean_verse_id = clean_verseid(verse_id=verse_id) - return jsonify(lookup_by_verse_id(clean_verse_id)[0]) - -@app.get("/find") # user-facing general search parser. This will move to an independant frontend. -def parse_query(query_override: str = None): - query = request.args.get("kw", None) - # /find should always return human readable (rich) but some users may try - # to use it with other view arguments, we can try to support that. - if not request.args.get("view", False): - human_readable = True - else: - human_readable = False - if query_override: - query = query_override - if query: - query = query.strip() - if request.args.get("view", "rich") != "rich": - return jsonify({"Error": "/find endpoint only supports rich responses"}) - logging.info(f"Query: {query}") - if not query: - return static_page("html/find.html") - if query.endswith(","): - query = query[:-1] - - # A query that reads as scripture references ("Exodus", "1 jn 3.16") is shown - # as scripture; anything else is a keyword search. - resolved = resolve_references(query) - if resolved: - result = show(kw=query, human_readable=human_readable) - if isinstance(result, str) and result != "Invalid Query": - notices = "" - for original, name in resolved["notes"]: - notices += ( - f"
Showing {html_lib.escape(name)} " - f"(interpreted from '{html_lib.escape(original)}')
" - ) - if resolved["bare"]: - keyword = urllib.parse.quote_plus(search_input_cleaning(query)) - notices += ( - f"
Showing {html_lib.escape(resolved['label'])}, " - f"chapter 1. Looking for the word instead? " - f"Search the text for '{html_lib.escape(query)}'
" - ) - if notices: - result = result.replace("
", "
" + notices, 1) - return result - if not search_input_cleaning(query).strip(): - # nothing searchable left (punctuation, accents): show the search form - return static_page("html/find.html") - result = search() - # Remove the closing
from search() so we can add dictionary content inside .responses - if result.endswith("
"): - result = result[:-6] # Remove last - - # Check if we have any dictionary definitions before showing the section - split_query = query.replace(",", " ").split(" ") - if len(split_query) > 20: - return "Your Query is too long. 20 Words or less." - - definitions_html = "" - has_definitions = False - for word in split_query: - definition = define(word) - if not definition: - continue - has_definitions = True - definitions_html += ( - "" - ) - definitions_html += "
" + definition + "
" - logging.debug(f"parse_query.define = '{word}'") - - # Only add the dictionary section if we found at least one definition - if has_definitions: - result += "" - result += "

1828 Webster Definitions

Click on the word to get the definition.

" - result += definitions_html - result += f"" - - result += "" - return result - - -@app.get("/define") # user-facing Webster Dictionary, this will move to an independent frontend. -def define(word=None): - ref = request.args.get("kw", None) - if word: - ref = word - if ref: - ref = ref.strip() - if not ref: - return static_page("html/dictionary.html") - dictionary = cached["json_dictionary"] - ref = ref.lower() - if not ref in dictionary.keys(): - if word: - return False - logging.warning(f"WEBSTER MISSING: {ref}") - return f"{ref} not found." - # search suggestions kind of suck - # h = hunspell.HunSpell("data/kjv.dic", "data/kjv.aff") - # suggestions = h.suggest(ref) - # return "{} not found, try one of the following: {}".format( - # ref, ", ".join(suggestions) - # ) - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = return_dict_plain(ref, dictionary[ref]) - elif arg_view == "rich": - response = return_dict_rich(ref, dictionary[ref], simple=word) - else: - response = jsonify({ref: dictionary[ref]}) - return response - - -def return_dict_plain(ref, defs): - response = "" - for e in defs: - response += e + "

" - return ref.capitalize() + "

" + response + "\n" - - -def return_dict_rich(ref, defs, simple): - if simple: - css = "" - head = "" - else: - # standalone page: needs its own document head so phones scale it properly - css = "" - head = ( - "" - "" - "" - f"{html_lib.escape(ref.capitalize())} - KJV API" - + cached["css_navbar"] - + "
" - ) - response = "" - for e in defs: - response += e + "

" - return ( - head - + cached["html_response"].format( - chapter=ref.split(":")[0], passage=ref.capitalize(), text=response, verse_id="" - ) - + css - ) - - -# --------------------------------------------------------------------------- -# Scripture reference resolver. Understands the many ways people type a -# reference ("1 jn 3.16", "II Kings 2:1-4", "song of songs 2", "Exodus ") and -# turns it into ranges of verse ids (book*1000000 + chapter*1000 + verse). -# --------------------------------------------------------------------------- - -# Extra spellings people use that the abbreviations table does not know. -EXTRA_BOOK_ALIASES = { - "Song of Solomon": ["song of songs", "songs of solomon", "songs of songs", "song of sol", - "song of sol", "canticles", "canticle of canticles", "sos", "sng", "ss"], - "Psalms": ["psalm", "psalter", "pslm", "psa", "pss", "psm", "ps"], - "Revelation": ["revelations", "revelation of john", "revelation of st john", "the revelation", "apocalypse"], - "Ecclesiastes": ["ecclesiasts", "eccles", "ecc", "qoheleth"], - "Philemon": ["phlm", "phm", "philem"], - "Judges": ["jdg", "jdgs", "jg"], - "Mark": ["mrk", "mk", "mr"], - "John": ["jhn", "jn"], - "Acts": ["acts of the apostles", "act"], - "James": ["jas", "jm"], -} - -# Words that are books but also ordinary search words: a bare "numbers" or -# "mark" still shows the book, but /find offers a keyword search as well. -BARE_BOOK_EXTRA = { - "psalm", "songofsongs", "songsofsolomon", "songofsol", "canticles", "canticleofcanticles", - "revelations", "revelationofjohn", "therevelation", -} - -ROMAN_PREFIX = r"(?:iii|ii|i)(?=[\s.])" -ORDINAL_PREFIX = r"(?:1st|2nd|3rd|first|second|third)" -BOOK_PIECE_RE = re.compile( - r"^\s*(?P(?:(?:[1-3]|" + ROMAN_PREFIX + r"|" + ORDINAL_PREFIX + r")\.?\s*)?[a-z][a-z.\s]*?)" - r"\s*(?P\d[\d\s:.\-–—]*)?\s*$", - re.I, -) -CONTINUATION_RE = re.compile(r"^\s*\d[\d\s:.\-–—]*$") - - -def normalize_book_text(text: str) -> str: - """'I Cor.' / '1st Corinthians' / '1 corinthians' -> '1corinthians'""" - text = text.lower().strip() - text = re.sub(r"^(the\s+)?(book\s+of\s+)", "", text) - text = re.sub(r"^(1st|first)(?=[\s.a-z]|$)", "1", text) - text = re.sub(r"^(2nd|second)(?=[\s.a-z]|$)", "2", text) - text = re.sub(r"^(3rd|third)(?=[\s.a-z]|$)", "3", text) - text = re.sub(r"^iii(?=[\s.]|$)", "3", text) - text = re.sub(r"^ii(?=[\s.]|$)", "2", text) - text = re.sub(r"^i(?=[\s.])", "1", text) - return re.sub(r"[^a-z0-9]", "", text) - - -def build_book_aliases(): - """alias key -> book id, plus the canonical keys on their own""" - canonical = {} - aliases = {} - for book_id, name in cached["book_id_to_name"].items(): - canonical[normalize_book_text(name)] = book_id - aliases.update(canonical) - for abbreviation, book_id in kjv_cur.execute("SELECT a, b FROM key_abbreviations_english").fetchall(): - aliases.setdefault(normalize_book_text(abbreviation), book_id) - for name, extras in EXTRA_BOOK_ALIASES.items(): - book_id = lookup_book_id(name) - for extra in extras: - aliases[normalize_book_text(extra)] = book_id - return canonical, aliases - - -BOOK_CANONICAL, BOOK_ALIASES = build_book_aliases() -BOOK_ALIAS_KEYS = sorted(BOOK_ALIASES) -MAX_CHAPTER_ID_SPAN = 999 - - -def resolve_book(token: str, loose: bool = True): - """Returns (book_id, how) with how in exact|prefix|fuzzy, or None. - - loose=False only accepts full names and spellings that are unmistakably - a book (used for a bare word with no chapter number). - """ - key = normalize_book_text(token) - if not key: - return None - if not loose: - if key in BOOK_CANONICAL or key in BARE_BOOK_EXTRA: - return (BOOK_ALIASES[key], "exact") - # a clear misspelling of a long full name ("genisis") - if len(key) >= 6 and not key[0].isdigit(): - close = difflib.get_close_matches(key, list(BOOK_CANONICAL), n=1, cutoff=0.85) - if close: - return (BOOK_CANONICAL[close[0]], "fuzzy") - return None - if key in BOOK_ALIASES: - return (BOOK_ALIASES[key], "exact") - # unique (or best) prefix: "deut", "1cor", "matth" - prefixed = [k for k in BOOK_ALIAS_KEYS if k.startswith(key)] - if prefixed: - books = {BOOK_ALIASES[k] for k in prefixed} - canonical_hits = {BOOK_CANONICAL[k] for k in BOOK_CANONICAL if k.startswith(key)} - if len(canonical_hits) == 1: - return (next(iter(canonical_hits)), "prefix") - if len(books) == 1: - return (next(iter(books)), "prefix") - return (max(canonical_hits or books), "prefix") # ambiguous: the later book, as before - if len(key) >= 4: - close = difflib.get_close_matches(key, BOOK_ALIAS_KEYS, n=1, cutoff=0.78) - if close: - return (BOOK_ALIASES[close[0]], "fuzzy") - return None - - -def chapter_bounds(book_id: int, chapter: int) -> int: - """Clamp a chapter number into the book (0 -> 1, past the end -> last chapter).""" - return max(1, min(chapter, cached["books_chapter_lengths"][book_id])) - - -def parse_ref_rest(rest: str, book_id: int, previous=None): - """'3:16-18', '3 16', '3.16', '3-4', '3:16-4:2', '3', '3:-' -> (start_id, end_id|None, chapter, verse)""" - rest = rest.replace("–", "-").replace("—", "-") - rest = re.sub(r"\s*([:\-])\s*", r"\1", rest.strip()) - base = book_id * 1_000_000 - m = re.fullmatch(r"(\d+)", rest) - if m: - n = int(m.group(1)) - if previous and previous["verse"]: # "John 3:16, 18": a verse of the same chapter - chapter = previous["chapter"] - return (base + chapter * 1000 + n, None, chapter, n) - chapter = chapter_bounds(book_id, n) - return (base + chapter * 1000 + 1, base + chapter * 1000 + MAX_CHAPTER_ID_SPAN, chapter, 0) - m = re.fullmatch(r"(\d+)(?::-|-)", rest) # "3:-" or "3-": the whole chapter - if m: - chapter = chapter_bounds(book_id, int(m.group(1))) - return (base + chapter * 1000 + 1, base + chapter * 1000 + MAX_CHAPTER_ID_SPAN, chapter, 0) - m = re.fullmatch(r"(\d+)-(\d+)", rest) # chapters 3 to 4 - if m: - first, last = chapter_bounds(book_id, int(m.group(1))), chapter_bounds(book_id, int(m.group(2))) - if last < first: - first, last = last, first - return (base + first * 1000 + 1, base + last * 1000 + MAX_CHAPTER_ID_SPAN, first, 0) - m = re.fullmatch(r"(\d+)[:.\s](\d+)", rest) - if m: - chapter = chapter_bounds(book_id, int(m.group(1))) - verse = max(1, int(m.group(2))) - return (base + chapter * 1000 + verse, None, chapter, verse) - m = re.fullmatch(r"(\d+)[:.\s](\d+)-", rest) # "3:16-": from verse 16 to the end - if m: - chapter = chapter_bounds(book_id, int(m.group(1))) - return (base + chapter * 1000 + max(1, int(m.group(2))), base + chapter * 1000 + MAX_CHAPTER_ID_SPAN, chapter, int(m.group(2))) - m = re.fullmatch(r"(\d+)[:.\s](\d+)-(\d+)", rest) - if m: - chapter = chapter_bounds(book_id, int(m.group(1))) - first, last = sorted((max(1, int(m.group(2))), max(1, int(m.group(3))))) - return (base + chapter * 1000 + first, base + chapter * 1000 + last, chapter, first) - m = re.fullmatch(r"(\d+)[:.\s](\d+)-(\d+)[:.](\d+)", rest) # 3:16-4:2 across chapters - if m: - c1, c2 = chapter_bounds(book_id, int(m.group(1))), chapter_bounds(book_id, int(m.group(3))) - start = base + c1 * 1000 + max(1, int(m.group(2))) - end = base + c2 * 1000 + max(1, int(m.group(4))) - if end < start: - return None - return (start, end, c1, int(m.group(2))) - return None - - -def resolve_references(text: str, loose_bare: bool = False): - """Parse a whole query ('John 3:16, Romans 5:8') into verse id ranges. - - Returns None when it does not read as scripture references, so the - caller can treat it as a keyword search. Otherwise a dict with: - refs - list of (start_id, end_id or None) - notes - short explanations of guesses that were made - bare - the query was just a book name (shows its first chapter) - label - the name of the first book, for messages - """ - if not text or not text.strip(): - return None - pieces = [p for p in re.split(r"[,;\n]+", text) if p.strip()] - if not pieces or len(pieces) > 20: - return None - refs, notes = [], [] - bare = False - previous = None - label = None - for piece in pieces: - m = BOOK_PIECE_RE.match(piece) - if m: - token, rest = m.group("book"), m.group("rest") - if rest: - found = resolve_book(token) - else: - found = resolve_book(token, loose=loose_bare) - if not found: - return None - book_id, how = found - name = lookup_bookname(book_id) - if how == "fuzzy": - notes.append((token.strip(), name)) - if not rest: - if len(pieces) == 1: - bare = True - start, end, chapter, verse = (book_id * 1_000_000 + 1001, book_id * 1_000_000 + 1999, 1, 0) - else: - parsed = parse_ref_rest(rest, book_id, None) - if not parsed: - return None - start, end, chapter, verse = parsed - elif CONTINUATION_RE.match(piece) and previous: - book_id = previous["book"] - name = lookup_bookname(book_id) - parsed = parse_ref_rest(piece, book_id, previous) - if not parsed: - return None - start, end, chapter, verse = parsed - else: - return None - previous = {"book": book_id, "chapter": chapter, "verse": verse} - label = label or name - refs.append((start, end)) - return {"refs": refs, "notes": notes, "bare": bare, "label": label} - - -@app.get("/show") # user facing to show a group of scripture. This will be deprecated and moved to backend. -def show(kw=None, get_start_verse_id=False, human_readable=False): - ref: str = request.args.get("kw", None) - get_chapter: bool = request.args.get("all", False) - if kw: - ref = kw - if not ref: - return static_page("html/show.html") - logging.debug(f"show({ref})") - resolved = resolve_references(ref, loose_bare=True) - if not resolved: - return "Invalid Query" - if get_start_verse_id: - return str(resolved["refs"][0][0]) - verse_results: list = [] - for start_id, end_id in resolved["refs"]: - if get_chapter: # ?all= : from the given verse to the end of its chapter - end_id = start_id // 1000 * 1000 + MAX_CHAPTER_ID_SPAN - results = lookup_by_verse_id(start_id, end_id) - logging.debug(f"show.results len({len(results)})") - verse_results.extend(results) - response_list = scripture_response(verse_results, human_readable) - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = "
".join(response_list) - elif arg_view == "rich" or human_readable: - response = "".join(response_list) - else: - response = jsonify(response_list) - return response - - -def response_dict(scripture): - # used for JSON responses - verse = { - "bookname": lookup_bookname(scripture[1]), - "chapter": scripture[2], - "verse": scripture[3], - "text": scripture[4], - } - return verse - - -def response_text(scripture): - bookname = lookup_bookname(scripture[1]) - chapter = str(scripture[2]) - verse = str(scripture[3]) - text = str(scripture[4]) - return bookname + " " + chapter + ":" + verse + " " + text + "\n" - - -def response_rich(scripture): - # (passage, text) is the html formatting arguments - bookname = lookup_bookname(scripture[1]) - chapter = str(scripture[2]) - verse = str(scripture[3]) - text = str(scripture[4]) - passage = bookname + " " + chapter + ":" + verse - return cached["html_response"].format( - chapter=passage.split(":")[0], passage=passage, text=text, verse_id=scripture[0] - ) - - -def scripture_response(scriptures, human_readable=False): - response = [] - arg_view = request.args.get("view", "rich") - logging.debug(f"search ->requested_format = {arg_view}") - if arg_view == "plain": - for scripture in scriptures: - response.append(response_text(scripture)) - elif arg_view == "rich" or human_readable: - try: - scriptures[0][0] - except IndexError: - logging.error(f"Malformed query: {request.args.get('kw',None)}") - return(("Malformed Query - if you believe this to be in error (your query should resolve), email tyler@dinsmoor.us",)) - # Prev/next chapter links only make sense when reading a single chapter, - # not for a keyword search that spans books. - l_r_nav = None - if len({str(scripture[0])[:-3] for scripture in scriptures}) == 1: - l_r_nav = get_next_prev_chapter( - str(scriptures[0][0]) - ) # get the first scripture, the id thereof - # response.append(cached["css_response"]) - response.append( - render_navbar(request.args.get("kw", ""), l_r_nav) - ) - - response.append("
") - response.append(f"
{len(scriptures)} Results
") - for scripture in scriptures: - response.append(response_rich(scripture)) - response.append("
") - else: - for scripture in scriptures: - response.append(response_dict(scripture)) - return response - - -@app.errorhandler(404) -def page_not_found(e): - message = { - "error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness." - } - return jsonify(message) - - -@app.get("/") # index -@app.get("/") # resolves a generated shortcode -def idx(shortcode=""): - if not sanity: - return "Sanity check failed. Check database ownership or please notify tyler@dinsmoor.us" - if shortcode: - logging.info(f"/{shortcode}") - return resolve_shortcode(shortcode) - logging.info(f"/{shortcode}") - return cached["html_index"] - -@app.get("/fe") -def frontend(): - return static_page("kjv_fe.html") - -@app.get("/salvation") # a web page about going to heaven -def salvation(): - return static_page("html/heaven.html") - - -@app.get("/t/random", defaults={"view": "plain"}) # backwards-compatibility plaintext random Bible verse -@app.get("/random") # truely random Bible verse -def random_verse(): - rand_id = random.SystemRandom().choice(cached["verse_ids"]) - scripture = lookup_by_verse_id(rand_id) - logging.info(f"Random Served: {scripture[0]}") - response_list = scripture_response(scripture) - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = "
".join(response_list) - elif arg_view == "rich": - response = "".join(response_list) - else: - response = jsonify(response_list) - #logging.debug(f"Search Response: {response}") - return response - - -@app.get("/t/votd", defaults={"view": "plain"}) # backwards-compatibility plaintext "Today's" Bible verse -@app.get("/votd") # "today's" Bible verse -def verse_of_the_day(): - # plain takes 38ms -TODO can we cache this? - logging.info("VOTD Served") - today = int(str(date.today()).replace("-", "")) - # this is deterministic because of today's date being used as the seed - random.seed(today) - scripture = lookup_by_verse_id(random.choice(cached["verse_ids"])) - response_list = scripture_response(scripture) - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = "
".join(response_list) - elif arg_view == "rich": - response = "".join(response_list) - else: - response = jsonify(response_list) - #logging.debug(f"Search Response: {response}") - return response - - -def make_view(response_list=[]): - # TODO: Maybe use flask session to make this easier to deal with - # https://stackoverflow.com/a/53152394 - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = "
".join(response_list) - elif arg_view == "rich": - response = "
".join(response_list) - else: - response = jsonify(response_list) - -@app.get("/search") # user-facing keyword search. To be moved to backend, possibly replaced with semantic search. -def search(): - keywords = request.args.get("kw", None) - logging.debug(f"search({keywords})") - if keywords: - cleaned_keywords = search_input_cleaning(keywords) - if not cleaned_keywords: - return static_page("html/search.html") - else: - return static_page("html/search.html") - - # Check if this is a common theological concept not in KJV - concept_mapping = cached["concept_mappings"].get(cleaned_keywords.lower(), None) - concept_notice = None - if concept_mapping: - # Build hyperlinks for all search terms - original_query = cleaned_keywords - search_links = [] - for term in concept_mapping["search_terms"]: - search_link = f"/search?kw={term.replace(' ', '+')}&view=rich" - search_links.append(f"{term}") - - links_html = ", ".join(search_links) - concept_notice = f"
{original_query.capitalize()}: {concept_mapping['explanation']}
You may find what you are looking for with these search terms: {links_html}
" - - # Use the first search term for the actual search - cleaned_keywords = concept_mapping["search_terms"][0] - - def score_result_by_proximity(verse_text, query_words): - """ - Score a verse based on how close together the query words appear. - Higher score = words appear closer together. - """ - verse_lower = verse_text.lower() - - # Find all positions of each query word - word_positions = {} - for word in query_words: - word_lower = word.lower() - positions = [] - start = 0 - while True: - pos = verse_lower.find(word_lower, start) - if pos == -1: - break - positions.append(pos) - start = pos + 1 - word_positions[word] = positions - - # If any word is missing, score is 0 - if any(len(positions) == 0 for positions in word_positions.values()): - return 0 - - # Calculate minimum distance between all query words - # For each combination of positions, find the span (max_pos - min_pos) - # Smaller span = higher score - import itertools - - # Get all combinations of positions for all words - all_position_combos = list(itertools.product(*word_positions.values())) - - if not all_position_combos: - return 0 - - # Find the combination with minimum span - min_span = float('inf') - for combo in all_position_combos: - span = max(combo) - min(combo) - if span < min_span: - min_span = span - - # Convert span to score (smaller span = higher score) - # Use inverse: 1000 / (span + 1) so adjacent words score ~1000, far apart score low - score = 1000.0 / (min_span + 1) - - return score - - results = lookup_fts(cleaned_keywords) - - # Sort results by proximity score if we have multiple query words - if results and len(cleaned_keywords.split()) > 1: - query_words = cleaned_keywords.split() - scored_results = [] - for result in results: - verse_text = result[4] # The text is at index 4 - score = score_result_by_proximity(verse_text, query_words) - scored_results.append((score, result)) - - # Sort by score descending, then keep just the results - scored_results.sort(key=lambda x: x[0], reverse=True) - results = [result for score, result in scored_results] - - if not results: - # PRIORITY 1: Try grammatical variant expansion first - words = cleaned_keywords.split() - - def sort_variants_by_similarity(original, variants): - """Sort variants by morphological similarity: forwards (longer/more specific) then backwards (shorter/simpler)""" - def get_common_prefix_length(w1, w2): - """Find how many characters match from the start""" - prefix_len = 0 - for i in range(min(len(w1), len(w2))): - if w1[i].lower() == w2[i].lower(): - prefix_len += 1 - else: - break - return prefix_len - - scored_variants = [] - for variant in variants: - prefix_len = get_common_prefix_length(original, variant) - - # Reject variants that don't share the stem (beginning doesn't match) - if prefix_len < min(2, len(original) // 2): - continue - - # Score: prioritize common prefix length (stem preservation) heavily, - # then prefer longer variants (forward/more specific) over shorter (backward/simpler) - # Prefix is weighted 1000x to ensure stem-preservation is primary - score = (prefix_len * 1000) + len(variant) - scored_variants.append((score, variant)) - - # Sort by score descending: highest score = most similar with longest form first - return [v for _, v in sorted(scored_variants, key=lambda x: x[0], reverse=True)] - - for word in words: - variants = cached["grammatical_variants"].get(word.lower(), []) - if variants: - # Sort variants: forwards in likeness (longer/specific) then backwards (shorter/simple) - sorted_variants = sort_variants_by_similarity(word.lower(), variants) - - # Try each variant in order of similarity - for variant in sorted_variants: - variant_query = cleaned_keywords.replace(word, variant, 1) - results = lookup_fts(variant_query) - if results: - response_list = scripture_response(results) - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = f"Showing results for '{variant_query}' (grammatical variant of '{cleaned_keywords}')\n
" + "
".join(response_list) - elif arg_view == "rich": - variant_notice = f"
Showing results for '{variant_query}' (grammatical variant of '{cleaned_keywords}')
" - response_str = "".join(response_list) - response = response_str.replace("
", f"
{variant_notice}", 1) - else: - response = jsonify({"original": cleaned_keywords, "variant": variant_query, "results": response_list}) - return response - - # PRIORITY 2: If grammatical variants didn't help, try spell-check - h = hunspell.HunSpell("data/kjv.dic", "data/kjv.aff") - corrected_words = [] - had_corrections = False - - for word in words: - if h.spell(word): - # Word is spelled correctly - corrected_words.append(word) - else: - # Word is misspelled, get suggestions - word_suggestions = h.suggest(word) - if word_suggestions: - corrected_words.append(word_suggestions[0]) - had_corrections = True - else: - # No suggestions, keep original - corrected_words.append(word) - - if had_corrections: - corrected_keyword = " ".join(corrected_words) - results = lookup_fts(corrected_keyword) - if results: - response_list = scripture_response(results) - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = f"Showing results for '{corrected_keyword}' (corrected from '{cleaned_keywords}')\n
" + "
".join(response_list) - elif arg_view == "rich": - correction_notice = f"
Showing results for '{corrected_keyword}' (corrected from '{cleaned_keywords}')
" - response_str = "".join(response_list) - response = response_str.replace("
", f"
{correction_notice}", 1) - else: - response = jsonify({"corrected_from": cleaned_keywords, "corrected_to": corrected_keyword, "results": response_list}) - return response - - # PRIORITY 3: Try theological synonym expansion - for word in words: - synonyms = cached["theological_synonyms"].get(word.lower(), None) - if synonyms: - for synonym in synonyms: - synonym_query = cleaned_keywords.replace(word, synonym, 1) - results = lookup_fts(synonym_query) - if results: - response_list = scripture_response(results) - arg_view = request.args.get("view", None) - explanation = f"In the KJV, '{word}' and '{synonym}' often refer to similar theological concepts. Consider also searching for related verses." - if arg_view == "plain": - response = f"Showing results for '{synonym_query}' (theological equivalent of '{cleaned_keywords}')\n{explanation}\n
" + "
".join(response_list) - elif arg_view == "rich": - theological_notice = f"
Showing results for '{synonym_query}' (theological equivalent of '{cleaned_keywords}')
{explanation}
" - response_str = "".join(response_list) - response = response_str.replace("
", f"
{theological_notice}", 1) - else: - response = jsonify({"original": cleaned_keywords, "theological_equivalent": synonym_query, "explanation": explanation, "results": response_list}) - return response - - # PRIORITY 4: If we still have no results, show spelling suggestions - suggestions = h.suggest(cleaned_keywords) - arg_view = request.args.get("view", "rich") - - if arg_view == "rich": - # Build a nice no-results page with header - no_results_html = render_navbar(cleaned_keywords) - no_results_html += "
" - no_results_html += f"
No results found for '{cleaned_keywords}'.
" - - if suggestions: - no_results_html += "

Did you mean?

" - no_results_html += "

We couldn't find any verses matching your search. Here are some spelling suggestions:

" - for suggestion in suggestions[:5]: # Limit to 5 suggestions - search_link = f"/search?kw={suggestion}&view=rich" - no_results_html += f"
{suggestion}
" - no_results_html += "
" - else: - no_results_html += "

Search Tips

" - no_results_html += "

We couldn't find any verses matching your search. Try:

" - no_results_html += "
    " - no_results_html += "
  • Checking your spelling
    • " - no_results_html += "
  • Using different keywords
    • " - no_results_html += "
  • Searching for love, faith, or hope
    • " - no_results_html += "
" - - no_results_html += "
" - return no_results_html - else: - # Plain text fallback - return "{} not found, try one of the following: {}".format( - cleaned_keywords, ", ".join(suggestions) if suggestions else "No suggestions available" - ) - response_list = scripture_response(results) - - # Check if any words in the search have theological synonyms to suggest - theological_suggestions = [] - words = cleaned_keywords.split() - for word in words: - synonyms = cached["theological_synonyms"].get(word.lower(), None) - if synonyms: - for synonym in synonyms: - theological_suggestions.append(synonym) - - arg_view = request.args.get("view", "rich") - if arg_view == "plain": - response = "
".join(response_list) - if concept_notice: - response = concept_notice + "
" + response - if theological_suggestions: - response += "

Also consider searching: " - for suggestion in theological_suggestions: - response += f"{suggestion}
" - elif arg_view == "rich": - response = "".join(response_list) - # Insert concept notice at the top if it exists - if concept_notice: - # Find the first
after the opening
and insert after it - responses_start = response.find("
") - if responses_start != -1: - # Find the results-count div closing - insert_pos = response.find("
", responses_start + len("
")) - if insert_pos != -1: - insert_pos += len("
") - response = response[:insert_pos] + concept_notice + response[insert_pos:] - if theological_suggestions: - # Add suggestions before the LAST closing
which closes the responses container - suggestions_html = "

Related Theological Concepts

" - suggestions_html += "

In the KJV, these terms often refer to similar theological concepts. Consider also searching for related verses.

" - for suggestion in theological_suggestions: - search_link = f"/search?kw={suggestion}&view=rich" - suggestions_html += f"
" - suggestions_html += f"{suggestion.capitalize()}" - suggestions_html += "
" - suggestions_html += "
" - # Find the last
and insert before it - last_div_pos = response.rfind("
") - if last_div_pos != -1: - response = response[:last_div_pos] + suggestions_html + response[last_div_pos:] - else: - response = jsonify(response_list) - #logging.debug(f"Search Response: {response}") - return response - - -@app.get("/browse") # not implemented. TO be moved to frontend. -def bible_reader(): - return render_template("reader.html", books=cached.get("book_names")) - # We should return an index of all books with - # drop down menus of each chapter, and a search - # bar at the top for quick seeking, which can use - # show(). Each page should return a header and footer. - # header should have a navigation bar and "next/previous chapter" - # Footer should have next/previous as well. - # Maybe preliminary theming support - - -@app.get("/status") # basic instance info -def status(): - import resource - - return static_html_doc( - "Status", - f"""

Status

-

Sanity: {sanity}

-

UserMode: {usermode}

-

Memory: {resource.getrusage(resource.RUSAGE_SELF).ru_maxrss}

""", - ) - - -@app.get("/seqreport") # shows current tracked scripture sequences -def sequential_report(uid=None): - if usermode == "Single": - seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False) - else: - seq_conn = sqlite3.connect("data/seq.db") - seq_cur = seq_conn.cursor() - uids = seq_cur.execute("select * from reading_tracker").fetchall() - return render_template("seq_report.html", uids=uids) - -@app.get("/screport") # shows current remembered shortcodes -def shortcode_report(shortcode=None): - if usermode == "Single": - short_conn = sqlite3.connect("data/shortcodes.db", check_same_thread=False) - else: - short_conn = sqlite3.connect("data/shortcodes.db") - short_cur = short_conn.cursor() - shortcodes = short_cur.execute( - "SELECT * FROM shortcodes" - ).fetchall() - return render_template("shortcode_report.html", shortcodes=shortcodes) - - -def sequential_purge(): - if usermode == "Single": - seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False) - else: - seq_conn = sqlite3.connect("data/seq.db") - seq_cur = seq_conn.cursor() - uids = seq_cur.execute("select * from reading_tracker").fetchmany(10) - - evaluate_datetime = datetime.now() - for uid in uids: - last_used = datetime.fromisoformat(uid[2]) - since_use: timedelta = evaluate_datetime - last_used - if since_use > sequential_purge_delta: - logging.info(f"Purging UID: {uid[0]}") - seq_cur.execute("DELETE FROM reading_tracker WHERE uid = ?", (uid[0],)) - - seq_conn.commit() - seq_cur.close() - - -@app.get("/seq") # allows you to get groups of scripture sequentially -def sequential_read(start=0, num=1, uid=None, getuid=False): - # /seq?uid=1 - uid = request.args.get("uid", uid) - getuid = request.args.get("getuid", getuid) - start = request.args.get("start", start) - try: - num = int(request.args.get("num", num)) - except ValueError: - return "Invalid num argument" - if not uid and not getuid: - return static_page("html/seq.html") - if usermode == "Single": - seq_conn = sqlite3.connect("data/seq.db", check_same_thread=False) - else: - seq_conn = sqlite3.connect("data/seq.db") - seq_cur = seq_conn.cursor() - seq_cur.execute( - "CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)" - ) - last_used = str(datetime.now()) - num -= 1 # 0 will mean "next" - if num < 0: - return jsonify({"error": "You may only increment forward."}) - if getuid: - if start: - start = cached["verse_ids"].index( - int(show(kw=start, get_start_verse_id=True)) - ) - uid = generate_short_id() # range returns a 5 digit id - try: - seq_cur.execute( - "INSERT INTO reading_tracker VALUES(?, ?, ?)", (uid, start, last_used) - ) - except sqlite3.OperationalError as e: - logging.error( - f"UID Database failed to insert {uid} into reading_tracker. {e}" - ) - return jsonify( - { - "error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us" - } - ) - seq_conn.commit() - logging.info(f"New SEQ UID: {uid}") - return f"Your new UID is: {uid}" - uid = ref_input_cleaning(uid, ref_search=False) - try: - nextverse = seq_cur.execute( - "SELECT verse FROM reading_tracker WHERE uid = ?", (uid,) - ).fetchone()[0] - logging.debug(f"Queried UID {uid} on verse {nextverse}") - sequential_purge() - except TypeError: - logging.info(f"Invalid UID {uid}") - return jsonify({"error": "UID Invalid or purged (90 days of inactivity)"}) - if len(cached["verse_ids"]) < (nextverse + num): - # Correct our range to go to ceiling to prevent IndexError - diff = len(cached["verse_ids"]) - (nextverse + num) - num = diff - num - start_verse = cached["verse_ids"][nextverse] - end_verse = cached["verse_ids"][nextverse + num] - result_list = lookup_by_verse_id(start_verse, end_verse) - response_list = scripture_response(result_list) - - logging.info(f"Served {num+1} verses to UID {uid}") - seq_cur.execute( - "REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", (uid, (nextverse+num+1), last_used) - ) - # Doing REPLACE INTO instead of UPDATE will make the updated row at the end of the table, making the purging - # only look at a small set of the oldest unused entries. - seq_conn.commit() - arg_view = request.args.get("view", None) - if arg_view == "plain": - response = "
".join(response_list) - elif arg_view == "rich": - response = "".join(response_list) - else: - response = jsonify(response_list) - - return response - - -@app.get("/mksc/") # create and remember a new shortcode -def generate_shortcode(query: str = ""): - query = search_input_cleaning(query, extra_chars=":-,") - if not query: - return "Empty query." - if usermode == "Single": - short_conn = sqlite3.connect("data/shortcodes.db", check_same_thread=False) - else: - short_conn = sqlite3.connect("data/shortcodes.db") - short_cur = short_conn.cursor() - short_cur.execute( - "CREATE TABLE IF NOT EXISTS shortcodes (shortcode TEXT PRIMARY KEY, query TEXT UNIQUE, creation_date TEXT)" - ) - short_conn.commit() - creation_date = str(datetime.now()) - shortcode = generate_short_id() - short_cur.execute( - "REPLACE INTO shortcodes(shortcode,query,creation_date) VALUES(?, ?, ?)", (shortcode, query, creation_date) - ) - short_conn.commit() - short_cur.close() - logging.info(f"Generated Shortcode {shortcode} for query '{query}'") - return shortcode - - -def resolve_shortcode(shortcode: str = ""): - shortcode = search_input_cleaning( - shortcode, valid_chars="0123456789ACEFHJKLMNPRTUVWXY" - ) - if not shortcode: - return "Empty shortcode." - if usermode == "Single": - short_conn = sqlite3.connect("data/shortcodes.db", check_same_thread=False) - else: - short_conn = sqlite3.connect("data/shortcodes.db") - short_cur = short_conn.cursor() - query = short_cur.execute( - "SELECT query FROM shortcodes WHERE shortcode = ?", (shortcode,) - ).fetchone()[0] - short_cur.close() - logging.info(f"Resolved Shortcode {shortcode} for query '{query}'") - - return redirect(f"/find?kw={query}&view=rich") - - -if __name__ == "__main__": - print( - "You must run this program within a uwsgi environment if you are running this for multiple users." - ) - print( - "Otherwise, you may end up with database corruption if multiple writes happen." - ) - print("Operating in single user mode.") - app.run(host="0.0.0.0", port=1612, debug=True, threaded=True) diff --git a/data/1828_Webster_KJV.json b/assets/data/1828_Webster_KJV.json similarity index 100% rename from data/1828_Webster_KJV.json rename to assets/data/1828_Webster_KJV.json diff --git a/data/book_descriptions.json b/assets/data/book_descriptions.json similarity index 100% rename from data/book_descriptions.json rename to assets/data/book_descriptions.json diff --git a/data/chapter_summaries.json b/assets/data/chapter_summaries.json similarity index 100% rename from data/chapter_summaries.json rename to assets/data/chapter_summaries.json diff --git a/data/concept_mappings.json b/assets/data/concept_mappings.json similarity index 100% rename from data/concept_mappings.json rename to assets/data/concept_mappings.json diff --git a/data/grammatical_variants.json b/assets/data/grammatical_variants.json similarity index 100% rename from data/grammatical_variants.json rename to assets/data/grammatical_variants.json diff --git a/data/kjv.aff b/assets/data/kjv.aff similarity index 100% rename from data/kjv.aff rename to assets/data/kjv.aff diff --git a/data/kjv.dic b/assets/data/kjv.dic similarity index 100% rename from data/kjv.dic rename to assets/data/kjv.dic diff --git a/data/theological_synonyms.json b/assets/data/theological_synonyms.json similarity index 100% rename from data/theological_synonyms.json rename to assets/data/theological_synonyms.json diff --git a/html/chapter_nav.html b/assets/html/chapter_nav.html similarity index 100% rename from html/chapter_nav.html rename to assets/html/chapter_nav.html diff --git a/html/dictionary.html b/assets/html/dictionary.html similarity index 100% rename from html/dictionary.html rename to assets/html/dictionary.html diff --git a/html/find.html b/assets/html/find.html similarity index 100% rename from html/find.html rename to assets/html/find.html diff --git a/html/heaven.html b/assets/html/heaven.html similarity index 100% rename from html/heaven.html rename to assets/html/heaven.html diff --git a/html/index.html b/assets/html/index.html similarity index 100% rename from html/index.html rename to assets/html/index.html diff --git a/html/navbar.html b/assets/html/navbar.html similarity index 100% rename from html/navbar.html rename to assets/html/navbar.html diff --git a/html/on_reading_bible.html b/assets/html/on_reading_bible.html similarity index 100% rename from html/on_reading_bible.html rename to assets/html/on_reading_bible.html diff --git a/html/plan.html b/assets/html/plan.html similarity index 100% rename from html/plan.html rename to assets/html/plan.html diff --git a/html/response.html b/assets/html/response.html similarity index 100% rename from html/response.html rename to assets/html/response.html diff --git a/html/saved.html b/assets/html/saved.html similarity index 100% rename from html/saved.html rename to assets/html/saved.html diff --git a/html/search.html b/assets/html/search.html similarity index 100% rename from html/search.html rename to assets/html/search.html diff --git a/html/search_bar.html b/assets/html/search_bar.html similarity index 100% rename from html/search_bar.html rename to assets/html/search_bar.html diff --git a/html/seq.html b/assets/html/seq.html similarity index 100% rename from html/seq.html rename to assets/html/seq.html diff --git a/html/show.html b/assets/html/show.html similarity index 100% rename from html/show.html rename to assets/html/show.html diff --git a/html/votd.html b/assets/html/votd.html similarity index 100% rename from html/votd.html rename to assets/html/votd.html diff --git a/js/book_suggest.js b/assets/js/book_suggest.js similarity index 100% rename from js/book_suggest.js rename to assets/js/book_suggest.js diff --git a/js/collapsible.js b/assets/js/collapsible.js similarity index 100% rename from js/collapsible.js rename to assets/js/collapsible.js diff --git a/js/shortcode.js b/assets/js/shortcode.js similarity index 100% rename from js/shortcode.js rename to assets/js/shortcode.js diff --git a/js/verse_menu.js b/assets/js/verse_menu.js similarity index 100% rename from js/verse_menu.js rename to assets/js/verse_menu.js diff --git a/kjv_fe.html b/assets/kjv_fe.html similarity index 100% rename from kjv_fe.html rename to assets/kjv_fe.html diff --git a/favicon.ico b/assets/static/favicon.ico similarity index 100% rename from favicon.ico rename to assets/static/favicon.ico diff --git a/static/favicon.svg b/assets/static/favicon.svg similarity index 100% rename from static/favicon.svg rename to assets/static/favicon.svg diff --git a/styles/collapsible.css b/assets/styles/collapsible.css similarity index 100% rename from styles/collapsible.css rename to assets/styles/collapsible.css diff --git a/styles/fonts/Arizonia-Regular.ttf b/assets/styles/fonts/Arizonia-Regular.ttf similarity index 100% rename from styles/fonts/Arizonia-Regular.ttf rename to assets/styles/fonts/Arizonia-Regular.ttf diff --git a/styles/fonts/Parisienne-Regular.ttf b/assets/styles/fonts/Parisienne-Regular.ttf similarity index 100% rename from styles/fonts/Parisienne-Regular.ttf rename to assets/styles/fonts/Parisienne-Regular.ttf diff --git a/styles/header.css b/assets/styles/header.css similarity index 100% rename from styles/header.css rename to assets/styles/header.css diff --git a/styles/header_dark.css b/assets/styles/header_dark.css similarity index 100% rename from styles/header_dark.css rename to assets/styles/header_dark.css diff --git a/styles/header_light.css b/assets/styles/header_light.css similarity index 100% rename from styles/header_light.css rename to assets/styles/header_light.css diff --git a/styles/human_readable.css b/assets/styles/human_readable.css similarity index 100% rename from styles/human_readable.css rename to assets/styles/human_readable.css diff --git a/styles/page.css b/assets/styles/page.css similarity index 100% rename from styles/page.css rename to assets/styles/page.css diff --git a/styles/search_rich.css b/assets/styles/search_rich.css similarity index 100% rename from styles/search_rich.css rename to assets/styles/search_rich.css diff --git a/styles/votd.css b/assets/styles/votd.css similarity index 100% rename from styles/votd.css rename to assets/styles/votd.css diff --git a/cmd/kjv-api/main.go b/cmd/kjv-api/main.go new file mode 100644 index 0000000..ba7deae --- /dev/null +++ b/cmd/kjv-api/main.go @@ -0,0 +1,93 @@ +// kjv-api: the King James Bible web API as one program plus one SQLite +// database (kjv.db) that holds the text, every page and asset, user data and +// the logs. +package main + +import ( + "flag" + "fmt" + "os" + "path/filepath" + + "kjv-api/internal/kjv" +) + +const usage = `usage: + kjv-api [-db kjv.db] [-addr 127.0.0.1:1611] run the server + kjv-api import-assets [-db kjv.db] DIR load the pages, scripts, styles and data files from the assets folder into the database + kjv-api logs [-db kjv.db] [-n 100] print the newest log lines stored in the database +` + +func findDB(flagValue string) string { + if flagValue != "" { + return flagValue + } + if v := os.Getenv("KJV_DB"); v != "" { + return v + } + candidates := []string{"kjv.db"} + if exe, err := os.Executable(); err == nil { + candidates = append(candidates, filepath.Join(filepath.Dir(exe), "kjv.db")) + } + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + return c + } + } + return "kjv.db" +} + +func main() { + args := os.Args[1:] + if len(args) > 0 { + switch args[0] { + case "import-assets": + fs := flag.NewFlagSet("import-assets", flag.ExitOnError) + db := fs.String("db", "", "path to kjv.db") + fs.Parse(args[1:]) + if fs.NArg() != 1 { + fmt.Fprint(os.Stderr, usage) + os.Exit(2) + } + dbPath := findDB(*db) + if _, err := os.Stat(dbPath); err != nil { + fatal(err) + } + if err := kjv.ImportAssets(dbPath, fs.Arg(0)); err != nil { + fatal(err) + } + return + case "logs": + fs := flag.NewFlagSet("logs", flag.ExitOnError) + db := fs.String("db", "", "path to kjv.db") + n := fs.Int("n", 100, "number of lines") + fs.Parse(args[1:]) + if err := kjv.ShowLogs(findDB(*db), *n); err != nil { + fatal(err) + } + return + case "help", "-h", "--help": + fmt.Print(usage) + return + } + } + fs := flag.NewFlagSet("kjv-api", flag.ExitOnError) + dbFlag := fs.String("db", "", "path to kjv.db (default: $KJV_DB, ./kjv.db, next to the program)") + addr := fs.String("addr", envOr("KJV_ADDR", "127.0.0.1:1611"), "listen address (localhost only)") + fs.Parse(args) + if err := kjv.Run(findDB(*dbFlag), *addr); err != nil { + fatal(err) + } +} + +func fatal(err error) { + fmt.Fprintln(os.Stderr, "kjv-api:", err) + os.Exit(1) +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} diff --git a/data/t_kjv.db b/data/t_kjv.db deleted file mode 100644 index e69de29..0000000 diff --git a/data/kjv.db b/db/kjv.db similarity index 100% rename from data/kjv.db rename to db/kjv.db diff --git a/dist/kjv-api.apache.conf b/dist/kjv-api.apache.conf new file mode 100644 index 0000000..c82cab3 --- /dev/null +++ b/dist/kjv-api.apache.conf @@ -0,0 +1,16 @@ +# apache site for kjv-api (needs mod_proxy, mod_proxy_http and mod_headers: +# sudo a2enmod proxy proxy_http headers) +# Change domain.tld to your domain, copy to /etc/apache2/sites-available/ and a2ensite it. +# For https: sudo certbot --apache -d domain.tld + + + ServerName domain.tld + + ProxyPreserveHost On + ProxyRequests Off + ProxyVia Off + # the program limits how fast each visitor can make plans and shortcodes by this address + RequestHeader set X-Real-IP "%{REMOTE_ADDR}s" + ProxyPass / http://127.0.0.1:1611/ + ProxyPassReverse / http://127.0.0.1:1611/ + diff --git a/dist/kjv-api.nginx.conf b/dist/kjv-api.nginx.conf new file mode 100644 index 0000000..a8f5f09 --- /dev/null +++ b/dist/kjv-api.nginx.conf @@ -0,0 +1,21 @@ +# nginx site for kjv-api. Change example.com to your domain, then: +# sudo cp kjv-api.nginx.conf /etc/nginx/sites-available/kjv-api +# sudo ln -s /etc/nginx/sites-available/kjv-api /etc/nginx/sites-enabled/ +# sudo nginx -t && sudo systemctl reload nginx +# sudo certbot --nginx -d example.com (adds https to this file for you) +# +# Already have a site? Copy just the location block into it. + +server { + listen 80; + server_name example.com; + + location / { + proxy_pass http://127.0.0.1:1611; + proxy_set_header Host $host; + # the program limits how fast each visitor can make plans and shortcodes by this address + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } +} diff --git a/dist/kjv-api.service b/dist/kjv-api.service new file mode 100644 index 0000000..4cfdc53 --- /dev/null +++ b/dist/kjv-api.service @@ -0,0 +1,19 @@ +[Unit] +Description=KJV API endpoint at localhost port 1611 + +# A systemd user service, no root needed. It runs ~/kjv-api/kjv-api with the +# database next to it. Install with: +# mkdir -p ~/.config/systemd/user && cp kjv-api.service ~/.config/systemd/user/ +# systemctl --user daemon-reload && systemctl --user enable --now kjv-api +# sudo loginctl enable-linger $USER (keeps it running while logged out, and starts it at boot) + +[Service] +ExecStart=%h/kjv-api/kjv-api -db %h/kjv-api/kjv.db -addr 127.0.0.1:1611 +# Optional settings (KJV_SECRET_API_KEY, KJV_SERMON_DB, KJV_SEMANTIC_ADDR), one NAME=value per line +EnvironmentFile=-%h/kjv-api/kjv-api.env +Restart=on-failure +NoNewPrivileges=yes +UMask=0077 + +[Install] +WantedBy=default.target diff --git a/data/notes b/doc/notes.txt similarity index 100% rename from data/notes rename to doc/notes.txt diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..2a19ef5 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module kjv-api + +go 1.26.0 + +require modernc.org/sqlite v1.61.0 + +require ( + github.com/dustin/go-humanize v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.1.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + golang.org/x/sys v0.49.0 // indirect + modernc.org/libc v1.79.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..f39acd7 --- /dev/null +++ b/go.sum @@ -0,0 +1,50 @@ +github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs= +github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= +github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg= +golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs= +golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA= +golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs= +golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs= +modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys= +modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4= +modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o= +modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/html/reader.html b/html/reader.html deleted file mode 100644 index 8831a66..0000000 --- a/html/reader.html +++ /dev/null @@ -1,24 +0,0 @@ - - - - - -Books - KJV API - - - -
-Home Quick Search -
-

Books

-
- - {% for book in books %} - - - - {% endfor %} -
{{ book[0] }}
-
- - diff --git a/html/seq_report.html b/html/seq_report.html deleted file mode 100644 index f4d8054..0000000 --- a/html/seq_report.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - -Sequence Report - KJV API - - - -
-Home Quick Search -
-

Sequence Report

-
- - {% for uid in uids %} - - - - - - {% endfor %} -
{{ uid[0] }}{{ uid[1] }}{{ uid[2] }}
-
- - diff --git a/html/shortcode_report.html b/html/shortcode_report.html deleted file mode 100644 index 372996a..0000000 --- a/html/shortcode_report.html +++ /dev/null @@ -1,31 +0,0 @@ - - - - - -Shortcode Report - KJV API - - - -
-Home Quick Search -
-

Shortcode Report

-
- - - - - - - {% for code in shortcodes %} - - - - - - {% endfor %} -
shortcodequerycreation_date
{{ code[0] }}{{ code[1] }}{{ code[2] }}
-
- - diff --git a/install.sh b/install.sh deleted file mode 100755 index ff0f01d..0000000 --- a/install.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash - -# Check for root privileges -if [[ $EUID -ne 0 ]]; then - echo "This install script requires root privileges. Please run with sudo." - exit 1 -fi - -script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -dest=/opt/kjv-api - -echo "Attempting to install dependancies" -apt install python3-flask sqlite3 libhunspell-dev python3-hunspell uwsgi-core uwsgi-plugin-python3 python3-requests python3-dotenv curl rsync - -if id kjv >/dev/null 2>&1; then - echo "User kjv already exists, leaving it alone" -else - echo "Creating new user kjv with no login shell" - useradd -r -s /bin/false kjv -fi - -# Stop a running install so files are not swapped underneath it -if systemctl is-active --quiet kjv-api.service; then - echo "Stopping running kjv-api service for the update" - systemctl stop kjv-api.service -fi - -mkdir -p "$dest" - -# Never overwrite local state on a re-run: the .env, the runtime databases -# (seq/shortcodes/sermons, including any -wal/-shm files), logs, and dev cruft. -# Files are only added or updated, never deleted. -if [[ "$script_dir" == "$dest" ]]; then - echo "Running from $dest, nothing to copy" -else - echo "Copying server files to $dest (existing .env, databases and logs are kept)" - rsync -rlt \ - --exclude='.git/' --exclude='__pycache__/' --exclude='*.pyc' \ - --exclude='/.env' \ - --exclude='/data/seq.db*' --exclude='/data/shortcodes.db*' --exclude='/data/sermons.db*' \ - --exclude='/logs/*' \ - "${script_dir}"/ "$dest"/ -fi - -mkdir -p "$dest/logs" "$dest/data" - -if [[ ! -f "$dest/.env" ]]; then - echo "Creating $dest/.env from .env.example (edit it to set real values)" - cp "$dest/.env.example" "$dest/.env" -else - echo "Keeping existing $dest/.env" -fi - -# Admins manage the install through the kjv group (the service's own group). -admin_user="${KJV_ADMIN_USER:-$SUDO_USER}" -if [[ -n "$admin_user" && "$admin_user" != root ]] && id "$admin_user" >/dev/null 2>&1; then - echo "Adding $admin_user to group kjv (log out and back in to apply)" - usermod -aG kjv "$admin_user" -else - echo "No admin user detected; set KJV_ADMIN_USER=name to add one to the kjv group" -fi - -echo "Setting permissions (code group-writable by kjv; .env, logs and data limited)" -chown -R root:kjv "$dest" -find "$dest" -type d -exec chmod 2775 {} + -find "$dest" -type f -exec chmod 664 {} + -chmod 775 "$dest"/*.sh -chmod 640 "$dest/.env" -# SQLite needs a writable directory for journal files -chown kjv:kjv "$dest/logs" "$dest/data" -chmod 750 "$dest/logs" "$dest/data" -chown -R kjv:kjv "$dest/logs" -chmod -R u+rwX,g+rX,o-rwx "$dest/logs" -# Runtime-written databases belong to kjv; shipped read-only databases stay root-owned -for db in "$dest"/data/seq.db* "$dest"/data/shortcodes.db*; do - if [[ -f "$db" ]]; then - chown kjv:kjv "$db" - chmod 640 "$db" - fi -done - -echo "Installing systemd service task" -unit=/etc/systemd/system/kjv-api.service -if [[ -f "$unit" ]] && ! cmp -s "$dest/install/kjv-api.service" "$unit"; then - echo "Existing service file differs, saving it as $unit.bak" - cp "$unit" "$unit.bak" -fi -cp "$dest/install/kjv-api.service" "$unit" -systemctl daemon-reload - -echo "Enabling auto-start and starting kjv-api" -systemctl enable kjv-api.service -systemctl restart kjv-api.service - -echo "Waiting for the service to initialize" -sleep 3 - -echo "If you see a random Bible verse below in json format, your installation was successful:" -echo -echo -curl -XGET 127.0.0.1:1611/random -echo -echo - -YEL='\033[0;33m' -NC='\033[0m' # No Color - -echo -e "${YEL}You must now set your webserver as a reverse proxy for your domain and point it to 127.0.0.1:1611 ${NC}" -echo "See the install folder for example configuration files." -echo -echo "Notes:" -echo " - Thank you for running this software to bring the Bible to people." -echo " - You can change the pages (it's not fancy) in /opt/kjv-api/html/. There's no fancy libraries." -echo " - Logs are located in two places (application logs and uwsgi logs)." -echo " - /opt/kjv-api/logs/" -echo " - journalctl -e -u kjv-api.service" -echo " - If this install failed (you didn't get a Bible verse above) - try running /opt/kjv-api/kjv-api-test.sh" -echo " after stopping the kjv-api.service with systemctl, it should give you a traceback in your console." diff --git a/install/apache.site.conf.example b/install/apache.site.conf.example deleted file mode 100644 index 5cec1ed..0000000 --- a/install/apache.site.conf.example +++ /dev/null @@ -1,23 +0,0 @@ -# Example proxy configuration - - - - ServerName domain.tld - RewriteEngine On - RewriteCond %{HTTPS} off - RewriteRule (.*) https://%{SERVER_NAME}/$1 [R,L] - - - - ServerName domain.tld - - SSLEngine on - SSLCertificateFile /etc/letsencrypt/live/domain.tld/fullchain.pem - SSLCertificateKeyFile /etc/letsencrypt/live/domain.tld/privkey.pem - - ProxyPreserveHost On - ProxyRequests Off - ProxyVia Off - ProxyPass / http://127.0.0.1:1611/ - ProxyPassReverse / http://127.0.0.1:1611/ - diff --git a/install/kjv-api.service b/install/kjv-api.service deleted file mode 100644 index 1c9f3ec..0000000 --- a/install/kjv-api.service +++ /dev/null @@ -1,18 +0,0 @@ -[Unit] -Description=KJV API endpoint at localhost port 1611 - -[Service] -#ExecStart=uwsgi_python3 --http-socket :1611 --wsgi-file /opt/kjv-api/app.py --callable app --stats 127.0.0.1:9191 -ExecStart=/opt/kjv-api/kjv-api.sh -User=kjv -EnvironmentFile=/etc/environment -KillMode=mixed -# Hardening: the app may only write to its databases and logs, never its own code -NoNewPrivileges=yes -ProtectSystem=strict -ReadWritePaths=/opt/kjv-api/data /opt/kjv-api/logs -ProtectHome=yes -PrivateTmp=yes - -[Install] -WantedBy=multi-user.target diff --git a/install/nginx.example b/install/nginx.example deleted file mode 100644 index d039408..0000000 --- a/install/nginx.example +++ /dev/null @@ -1,24 +0,0 @@ -server { - listen 80; - server_name domain.tld; - - return 301 https://$host$request_uri; -} - -server { - listen 443 ssl; - server_name domain.tld; - - ssl_certificate /etc/letsencrypt/live/domain.tld/fullchain.pem; - ssl_certificate_key /etc/letsencrypt/live/domain.tld/privkey.pem;   - - - location / { - proxy_pass http://127.0.0.1:1611; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme;   - - } -} diff --git a/internal/kjv/app.go b/internal/kjv/app.go new file mode 100644 index 0000000..5384499 --- /dev/null +++ b/internal/kjv/app.go @@ -0,0 +1,430 @@ +package kjv + +import ( + "bytes" + "crypto/sha256" + "database/sql" + "encoding/json" + "fmt" + "html" + "net/http" + "net/url" + "os" + "regexp" + "sort" + "strings" + "time" +) + +// App is the whole web application. Everything it serves comes from the +// companion database (see assets.go), loaded once at startup. +type App struct { + store *Store + res *Resolver + speller *Speller + started time.Time + + dictionary map[string][]string // 1828 Webster + variants map[string][]string + synonyms map[string][]string + concepts map[string]concept + bookDescs map[string]string + chapSums map[string][]string + + served map[string]servedAsset // versioned static files: /styles/page.css ... + pages map[string]string // simple html pages, asset urls versioned + + htmlResponse, htmlNavbar, htmlChapterNav string + cssNavbar, cssCollapsible string + favicon []byte + indexHTML string + + sermonDB *sql.DB + secretKey string + semantic string // host:port of the optional semantic search service +} + +type concept struct { + Query string `json:"query"` + Explanation string `json:"explanation"` + SearchTerms []string `json:"search_terms"` +} + +type servedAsset struct { + body []byte + mime string + version string + etag string +} + +var servedPaths = map[string]string{ // url -> asset path + "/styles/page.css": "styles/page.css", + "/js/collapsible.js": "js/collapsible.js", + "/js/shortcode.js": "js/shortcode.js", + "/js/verse_menu.js": "js/verse_menu.js", + "/js/book_suggest.js": "js/book_suggest.js", +} + +func NewApp(store *Store) (*App, error) { + assets, err := store.LoadAssets() + if err != nil { + return nil, err + } + need := func(p string) ([]byte, error) { + b, ok := assets[p] + if !ok { + return nil, fmt.Errorf("asset %s is missing from the database (run: kjv-api import-assets )", p) + } + return b, nil + } + a := &App{store: store, res: NewResolver(store), started: time.Now(), pages: map[string]string{}, served: map[string]servedAsset{}} + + loadJSON := func(p string, into any) error { + b, err := need(p) + if err != nil { + return err + } + if err := json.Unmarshal(b, into); err != nil { + return fmt.Errorf("%s: %w", p, err) + } + return nil + } + if err := loadJSON("data/1828_Webster_KJV.json", &a.dictionary); err != nil { + return nil, err + } + if err := loadJSON("data/book_descriptions.json", &a.bookDescs); err != nil { + return nil, err + } + if err := loadJSON("data/chapter_summaries.json", &a.chapSums); err != nil { + return nil, err + } + var groups [][]string + if err := loadJSON("data/grammatical_variants.json", &groups); err != nil { + return nil, err + } + a.variants = groupMap(groups) + if err := loadJSON("data/theological_synonyms.json", &groups); err != nil { + return nil, err + } + a.synonyms = groupMap(groups) + var concepts []concept + if err := loadJSON("data/concept_mappings.json", &concepts); err != nil { + return nil, err + } + a.concepts = map[string]concept{} + for _, c := range concepts { + a.concepts[strings.ToLower(c.Query)] = c + } + dic, err := need("data/kjv.dic") + if err != nil { + return nil, err + } + aff, err := need("data/kjv.aff") + if err != nil { + return nil, err + } + a.speller = NewSpeller(dic, aff) + + // static assets, each URL carrying a hash of its own contents + for u, p := range servedPaths { + b, err := need(p) + if err != nil { + return nil, err + } + if p == "js/book_suggest.js" { + names, _ := json.Marshal(store.bookNames) + b = bytes.ReplaceAll(b, []byte("__BOOKS__"), names) + } + sum := sha256.Sum256(b) + mime := assetMIME[p[strings.LastIndexByte(p, '.'):]] + a.served[u] = servedAsset{b, mime, fmt.Sprintf("%x", sum[:4]), fmt.Sprintf(`"%x"`, sum[:8])} + } + text := func(p string) (string, error) { b, err := need(p); return string(b), err } + if a.htmlResponse, err = text("html/response.html"); err != nil { + return nil, err + } + if a.htmlChapterNav, err = text("html/chapter_nav.html"); err != nil { + return nil, err + } + if a.cssNavbar, err = text("styles/header.css"); err != nil { + return nil, err + } + if a.cssCollapsible, err = text("styles/collapsible.css"); err != nil { + return nil, err + } + nav, err := text("html/navbar.html") + if err != nil { + return nil, err + } + a.htmlNavbar = a.loadHTML(nav) + if a.favicon, err = need("static/favicon.svg"); err != nil { + return nil, err + } + for _, p := range []string{"html/find.html", "html/show.html", "html/search.html", "html/seq.html", "html/dictionary.html", + "html/saved.html", "html/plan.html", "html/heaven.html", "kjv_fe.html"} { + t, err := text(p) + if err != nil { + return nil, err + } + a.pages[p] = a.loadHTML(t) + } + index, err := text("html/index.html") + if err != nil { + return nil, err + } + a.indexHTML = a.renderIndex(a.loadHTML(index)) + return a, nil +} + +func groupMap(groups [][]string) map[string][]string { + m := map[string][]string{} + for _, g := range groups { + for _, w := range g { + var others []string + for _, o := range g { + if o != w { + others = append(others, o) + } + } + m[w] = others + } + } + return m +} + +var assetRefRE = regexp.MustCompile(`(/(?:js|styles)/[\w.\-]+)\?v=__V__`) + +// loadHTML fills each '/js/x.js?v=__V__' with that file's own version. +func (a *App) loadHTML(s string) string { + return assetRefRE.ReplaceAllStringFunc(s, func(m string) string { + u := assetRefRE.FindStringSubmatch(m)[1] + if as, ok := a.served[u]; ok { + return u + "?v=" + as.version + } + return m + }) +} + +func (a *App) assetURL(u string) string { return u + "?v=" + a.served[u].version } + +var indexLoopRE = regexp.MustCompile(`(?s)\{% for category, books in testament_data\.(OT|NT)\.items\(\) %\}.*?\n \{% endfor %\}`) + +// renderIndex fills the book lists of the home page. +func (a *App) renderIndex(tmpl string) string { + esc := html.EscapeString + return indexLoopRE.ReplaceAllStringFunc(tmpl, func(m string) string { + testament := indexLoopRE.FindStringSubmatch(m)[1] + var order []string + byCat := map[string][]BookInfo{} + for _, b := range a.store.bookInfo { + if b.OTNT != testament { + continue + } + if _, ok := byCat[b.Category]; !ok { + order = append(order, b.Category) + } + byCat[b.Category] = append(byCat[b.Category], b) + } + var sb strings.Builder + for _, cat := range order { + fmt.Fprintf(&sb, "\n

%s

\n
", esc(cat)) + for _, b := range byCat[cat] { + fmt.Fprintf(&sb, "\n
\n
", esc(b.Title), esc(b.Title)) + if about := a.bookDescs[b.Title]; about != "" { + fmt.Fprintf(&sb, "

%s

", esc(about)) + } + sums := a.chapSums[b.Title] + for c := 1; c <= b.Chapters; c++ { + fmt.Fprintf(&sb, "\n%d", esc(b.Title), c, c) + if len(sums) >= c { + fmt.Fprintf(&sb, "%s", esc(sums[c-1])) + } + sb.WriteString("") + } + sb.WriteString("\n
") + } + sb.WriteString("\n
") + } + return sb.String() + }) +} + +// --------------------------------------------------------------------------- +// responses + +type Resp struct { + Body string + Type string + Status int + Header map[string]string +} + +func htmlR(s string) Resp { return Resp{Body: s, Type: "text/html; charset=utf-8"} } + +func jsonR(v any) Resp { return jsonStatus(v, 0) } + +func jsonStatus(v any, status int) Resp { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + enc.Encode(v) + return Resp{Body: buf.String(), Type: "application/json", Status: status} +} + +func errJSON(status int, msg string) Resp { return jsonStatus(map[string]string{"error": msg}, status) } + +func (r Resp) write(w http.ResponseWriter) { + h := w.Header() + h.Set("Content-Type", r.Type) + if strings.HasPrefix(r.Type, "text/html") && r.Header["Cache-Control"] == "" { + h.Set("Cache-Control", "no-cache") + } + for k, v := range r.Header { + h.Set(k, v) + } + if r.Status != 0 { + w.WriteHeader(r.Status) + } + w.Write([]byte(r.Body)) +} + +func (a *App) staticPage(p string) Resp { return htmlR(a.pages[p]) } + +// staticHTMLDoc wraps a small html snippet in the standard page shell. +func (a *App) staticHTMLDoc(title, body string) string { + return "" + + "" + + "" + title + " - KJV API" + + "" + + "
Home Quick Search
" + + body + "" +} + +// --------------------------------------------------------------------------- +// verse rendering + +func (a *App) responseRich(v Verse) string { + passage := fmt.Sprintf("%s %d:%d", a.store.BookName(v.Book), v.Chapter, v.Verse) + return strings.NewReplacer( + "{chapter}", passage[:strings.LastIndexByte(passage, ':')], + "{passage}", passage, + "{text}", v.Text, + "{verse_id}", fmt.Sprint(v.ID), + ).Replace(a.htmlResponse) +} + +func chapterFindLink(name string) string { return "/find?kw=" + url.PathEscape(name) + "&view=rich" } + +type chapterNav struct{ current, prev, next string } + +func (a *App) chapterName(key int64) string { + return fmt.Sprintf("%s %d", a.store.BookName(int(key/1000)), key%1000) +} + +func (a *App) nextPrevChapter(verseID int64) chapterNav { + key := verseID / 1000 + n := len(a.store.chapterList) + i := sort.Search(n, func(i int) bool { return a.store.chapterList[i] >= key }) + return chapterNav{ + current: a.chapterName(key), + prev: a.chapterName(a.store.chapterList[(i-1+n)%n]), + next: a.chapterName(a.store.chapterList[(i+1)%n]), + } +} + +// renderNavbar opens the rich html document: , site header and the +// chapter bar. The caller adds the body content after it. +func (a *App) renderNavbar(query string, nav *chapterNav, barHTML string) string { + esc := html.EscapeString + title := strings.TrimSpace(query) + if title == "" && nav != nil { + title = nav.current + } + if title == "" { + title = "Scripture" + } + chapterHTML := barHTML + if nav != nil { + chapterHTML = strings.NewReplacer( + "{current_chapter_name}", esc(nav.current), + "{prev_chapter_name}", esc(nav.prev), + "{prev_chapter_link}", esc("/show?kw="+nav.prev+"&view=rich"), + "{next_chapter_name}", esc(nav.next), + "{next_chapter_link}", esc("/show?kw="+nav.next+"&view=rich"), + "{current_chapter_link}", esc(chapterFindLink(nav.current)), + ).Replace(a.htmlChapterNav) + } + return strings.NewReplacer( + "{title}", esc(title), "{css}", a.cssNavbar, "{query}", esc(query), "{chapter_nav}", chapterHTML, + ).Replace(a.htmlNavbar) +} + +type rendered struct { + text string // plain and rich + list []map[string]any + isJSON bool +} + +func (r rendered) resp() Resp { + if r.isJSON { + return jsonR(r.list) + } + return htmlR(r.text) +} + +// render lays out verses as view=plain, view=rich or (anything else) json. +func (a *App) render(verses []Verse, view, kw string) rendered { + switch view { + case "plain": + parts := make([]string, len(verses)) + for i, v := range verses { + parts[i] = fmt.Sprintf("%s %d:%d %s\n", a.store.BookName(v.Book), v.Chapter, v.Verse, v.Text) + } + return rendered{text: strings.Join(parts, "
")} + case "rich": + if len(verses) == 0 { + lg.Error("Malformed query: %s", kw) + return rendered{text: "Malformed Query - if you believe this to be in error (your query should resolve), email tyler@dinsmoor.us"} + } + // Prev/next chapter links only make sense when reading a single chapter, + // not for a keyword search that spans books. + var nav *chapterNav + first := verses[0].ID / 1000 + single := true + for _, v := range verses { + if v.ID/1000 != first { + single = false + break + } + } + if single { + n := a.nextPrevChapter(verses[0].ID) + nav = &n + } + var sb strings.Builder + sb.WriteString(a.renderNavbar(kw, nav, "")) + sb.WriteString("
") + fmt.Fprintf(&sb, "
%d Results
", len(verses)) + for _, v := range verses { + sb.WriteString(a.responseRich(v)) + } + sb.WriteString("
") + return rendered{text: sb.String()} + } + list := make([]map[string]any, len(verses)) + for i, v := range verses { + list[i] = map[string]any{"bookname": a.store.BookName(v.Book), "chapter": v.Chapter, "verse": v.Verse, "text": v.Text} + } + return rendered{list: list, isJSON: true} +} + +func envOr(key, def string) string { + if v := os.Getenv(key); v != "" { + return v + } + return def +} + +func itoa64(n int64) string { return fmt.Sprint(n) } +func urlQuery(s string) string { return url.QueryEscape(s) } +func htmlEsc(s string) string { return html.EscapeString(s) } diff --git a/internal/kjv/assets.go b/internal/kjv/assets.go new file mode 100644 index 0000000..02212b7 --- /dev/null +++ b/internal/kjv/assets.go @@ -0,0 +1,113 @@ +package kjv + +import ( + "database/sql" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "strings" +) + +// Everything the program serves or reads besides the Bible itself (pages, +// scripts, styles, the 1828 dictionary, word lists) lives in the assets table +// of the companion database, so the program and the database are the only two +// files that need to be distributed. + +var assetMIME = map[string]string{ + ".html": "text/html; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".js": "application/javascript; charset=utf-8", + ".json": "application/json", + ".svg": "image/svg+xml", + ".dic": "text/plain; charset=utf-8", + ".aff": "text/plain; charset=utf-8", +} + +func isAssetFile(rel string) bool { + dir := path.Dir(rel) + switch { + case rel == "kjv_fe.html": + return true + case dir == "html", dir == "js", dir == "static": + return true + case dir == "styles": + return strings.HasSuffix(rel, ".css") + case dir == "data": + ext := path.Ext(rel) + return ext == ".json" || ext == ".dic" || ext == ".aff" + } + return false +} + +// ImportAssets replaces the assets table with the files of a source checkout. +func ImportAssets(dbPath, srcDir string) error { + db, err := sql.Open("sqlite", "file:"+dbPath+"?_pragma=busy_timeout(5000)") + if err != nil { + return err + } + defer db.Close() + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + for _, stmt := range []string{ + "CREATE TABLE IF NOT EXISTS assets (path TEXT PRIMARY KEY, mime TEXT NOT NULL, body BLOB NOT NULL)", + "DELETE FROM assets", + } { + if _, err := tx.Exec(stmt); err != nil { + return err + } + } + n := 0 + err = filepath.WalkDir(srcDir, func(p string, d fs.DirEntry, err error) error { + if err != nil || !d.Type().IsRegular() { // no directories, and never follow a symlink out of the tree + return err + } + rel, _ := filepath.Rel(srcDir, p) + rel = filepath.ToSlash(rel) + if !isAssetFile(rel) { + return nil + } + body, err := os.ReadFile(p) + if err != nil { + return err + } + mime := assetMIME[path.Ext(rel)] + if mime == "" { + mime = "application/octet-stream" + } + n++ + _, err = tx.Exec("INSERT INTO assets (path, mime, body) VALUES (?, ?, ?)", rel, mime, body) + return err + }) + if err != nil { + return err + } + if n == 0 { + return fmt.Errorf("no assets found under %s (expected html/, js/, styles/, data/*.json ...)", srcDir) + } + fmt.Printf("imported %d assets into %s\n", n, dbPath) + return tx.Commit() +} + +// LoadAssets reads every asset into memory. +func (s *Store) LoadAssets() (map[string][]byte, error) { + rows, err := s.db.Query("SELECT path, body FROM assets") + if err != nil { + return nil, fmt.Errorf("no assets table (run: kjv-api import-assets ): %w", err) + } + defer rows.Close() + out := map[string][]byte{} + for rows.Next() { + var p string + var b []byte + if err := rows.Scan(&p, &b); err != nil { + return nil, err + } + out[p] = b + } + return out, rows.Err() +} diff --git a/internal/kjv/difflib.go b/internal/kjv/difflib.go new file mode 100644 index 0000000..f7780e3 --- /dev/null +++ b/internal/kjv/difflib.go @@ -0,0 +1,76 @@ +package kjv + +// Ratcliff/Obershelp string similarity (the ratio of matching characters) and +// a closest match lookup, which the fuzzy book name matching relies on. + +type matchBlock struct{ a, b, size int } + +func longestMatch(a, b []rune, alo, ahi, blo, bhi int) matchBlock { + besti, bestj, bestsize := alo, blo, 0 + prev := make([]int, len(b)+1) + cur := make([]int, len(b)+1) + for i := alo; i < ahi; i++ { + for j := blo; j < bhi; j++ { + if a[i] != b[j] { + cur[j+1] = 0 + continue + } + k := prev[j] + 1 + cur[j+1] = k + if k > bestsize { + besti, bestj, bestsize = i-k+1, j-k+1, k + } + } + prev, cur = cur, prev + for j := range cur { + cur[j] = 0 + } + } + return matchBlock{besti, bestj, bestsize} +} + +func matchedChars(a, b []rune) int { + type span struct{ alo, ahi, blo, bhi int } + queue := []span{{0, len(a), 0, len(b)}} + total := 0 + for len(queue) > 0 { + q := queue[len(queue)-1] + queue = queue[:len(queue)-1] + m := longestMatch(a, b, q.alo, q.ahi, q.blo, q.bhi) + if m.size == 0 { + continue + } + total += m.size + if q.alo < m.a && q.blo < m.b { + queue = append(queue, span{q.alo, m.a, q.blo, m.b}) + } + if m.a+m.size < q.ahi && m.b+m.size < q.bhi { + queue = append(queue, span{m.a + m.size, q.ahi, m.b + m.size, q.bhi}) + } + } + return total +} + +func similarity(a, b string) float64 { + ra, rb := []rune(a), []rune(b) + if len(ra)+len(rb) == 0 { + return 1 + } + return 2.0 * float64(matchedChars(ra, rb)) / float64(len(ra)+len(rb)) +} + +// closestMatch is the best scoring possibility with a ratio of at least cutoff. +// Ties on score go to the lexicographically larger possibility. +func closestMatch(word string, possibilities []string, cutoff float64) (string, bool) { + best, bestScore, found := "", 0.0, false + for _, p := range possibilities { + score := similarity(p, word) // difflib compares possibility (a) against word (b) + if score < cutoff { + continue + } + if !found || score > bestScore || (score == bestScore && p > best) { + best, bestScore, found = p, score, true + } + } + return best, found +} diff --git a/internal/kjv/logdb.go b/internal/kjv/logdb.go new file mode 100644 index 0000000..675ee46 --- /dev/null +++ b/internal/kjv/logdb.go @@ -0,0 +1,198 @@ +package kjv + +import ( + "database/sql" + "fmt" + "os" + "strings" + "sync" + "syscall" + "time" + "unicode/utf8" +) + +// DBLog writes log lines into the logs table of the companion database. Lines +// are queued and inserted in batches so requests never wait on a write. Until +// a database is attached (and if it turns out to be read-only) lines go to stderr. +type DBLog struct { + mu sync.Mutex + db *sql.DB + ch chan logLine + done chan struct{} + started bool + retain int +} + +type logLine struct { + ts, level, msg string +} + +var lg = &DBLog{ch: make(chan logLine, 4096), done: make(chan struct{})} + +const ( + logTimeFmt = "2006-01-02 15:04:05.000" + maxLogLine = 400 // request URLs and queries come from strangers +) + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + for n > 0 && !utf8.RuneStart(s[n]) { // don't cut a character in half + n-- + } + return s[:n] + "..." +} + +func (l *DBLog) Attach(db *sql.DB, retainDays int) { + if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS logs (id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT NOT NULL, level TEXT NOT NULL, msg TEXT NOT NULL)`); err != nil { + fmt.Fprintln(os.Stderr, "WARN: cannot write logs to the database, logging to stderr:", err) + return + } + l.retain = retainDays + l.prune(db) + l.mu.Lock() + l.db, l.started = db, true + l.mu.Unlock() + go l.run() +} + +func (l *DBLog) prune(db *sql.DB) { + db.Exec("DELETE FROM logs WHERE ts < ?", time.Now().AddDate(0, 0, -l.retain).Format(logTimeFmt)) +} + +func (l *DBLog) run() { + tick := time.NewTicker(time.Second) + defer tick.Stop() + daily := time.NewTicker(6 * time.Hour) + defer daily.Stop() + var batch []logLine + flush := func() { + if len(batch) == 0 { + return + } + tx, err := l.db.Begin() + if err == nil { + for _, ln := range batch { + if _, err = tx.Exec("INSERT INTO logs (ts, level, msg) VALUES (?, ?, ?)", ln.ts, ln.level, ln.msg); err != nil { + break + } + } + if err == nil { + err = tx.Commit() + } else { + tx.Rollback() + } + } + if err != nil { + for _, ln := range batch { + fmt.Fprintf(os.Stderr, "%s %s %s\n", ln.ts, ln.level, ln.msg) + } + } + batch = batch[:0] + } + for { + select { + case ln := <-l.ch: + batch = append(batch, ln) + if len(batch) >= 200 { + flush() + } + case <-tick.C: + flush() + case <-daily.C: + l.prune(l.db) + case <-l.done: + for { + select { + case ln := <-l.ch: + batch = append(batch, ln) + default: + flush() + l.done <- struct{}{} + return + } + } + } + } +} + +// Close flushes queued lines. +func (l *DBLog) Close() { + l.mu.Lock() + started := l.started + l.mu.Unlock() + if !started { + return + } + l.done <- struct{}{} + <-l.done +} + +// Under systemd (stderr is the journal) or with KJV_LOG_STDERR=1 every line is +// also written to stderr, where the journal collects it. The "" prefix is +// the syslog priority journald understands; it adds its own timestamps. +var teeStderr = stderrIsJournal() || os.Getenv("KJV_LOG_STDERR") == "1" +var journalPriority = map[string]int{"ERROR": 3, "WARN": 4, "INFO": 6} + +// stderrIsJournal: JOURNAL_STREAM is inherited by everything a systemd +// started desktop session runs, terminals included, so it only counts when it +// names the device:inode stderr really is (as systemd.exec(5) advises). +func stderrIsJournal() bool { + want := os.Getenv("JOURNAL_STREAM") + if want == "" { + return false + } + fi, err := os.Stderr.Stat() + if err != nil { + return false + } + st, ok := fi.Sys().(*syscall.Stat_t) + return ok && fmt.Sprintf("%d:%d", st.Dev, st.Ino) == want +} + +func (l *DBLog) log(level, format string, args ...any) { + ln := logLine{time.Now().Format(logTimeFmt), level, truncate(fmt.Sprintf(format, args...), maxLogLine)} + if teeStderr { + fmt.Fprintf(os.Stderr, "<%d>%s\n", journalPriority[level], strings.ReplaceAll(ln.msg, "\n", " ")) + } + l.mu.Lock() + started := l.started + l.mu.Unlock() + if started { + select { + case l.ch <- ln: + return + default: // queue full: don't block a request + } + } + if !teeStderr { // not in the database and not already on stderr + fmt.Fprintf(os.Stderr, "%s %s %s\n", ln.ts, ln.level, ln.msg) + } +} + +func (l *DBLog) Info(format string, args ...any) { l.log("INFO", format, args...) } +func (l *DBLog) Warn(format string, args ...any) { l.log("WARN", format, args...) } +func (l *DBLog) Error(format string, args ...any) { l.log("ERROR", format, args...) } + +// ShowLogs prints the newest n log lines (oldest first). +func ShowLogs(dbPath string, n int) error { + db, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro&_pragma=busy_timeout(5000)") + if err != nil { + return err + } + defer db.Close() + rows, err := db.Query("SELECT ts, level, msg FROM (SELECT * FROM logs ORDER BY id DESC LIMIT ?) ORDER BY id", n) + if err != nil { + return err + } + defer rows.Close() + for rows.Next() { + var ts, level, msg string + if err := rows.Scan(&ts, &level, &msg); err != nil { + return err + } + fmt.Printf("%s %-5s %s\n", ts, level, msg) + } + return rows.Err() +} diff --git a/internal/kjv/mt.go b/internal/kjv/mt.go new file mode 100644 index 0000000..e4fceb4 --- /dev/null +++ b/internal/kjv/mt.go @@ -0,0 +1,86 @@ +package kjv + +// MT19937 seeded with init_by_array from one integer, and a choice made by +// rejection sampling over its bits, so the verse of the day for any given date +// never changes. + +type mtRandom struct { + mt [624]uint32 + idx int +} + +func newMTRandom(seed uint32) *mtRandom { + r := &mtRandom{} + r.initGenrand(19650218) + key := []uint32{seed} + i, j := 1, 0 + k := max(624, len(key)) + for ; k > 0; k-- { + r.mt[i] = (r.mt[i] ^ ((r.mt[i-1] ^ (r.mt[i-1] >> 30)) * 1664525)) + key[j] + uint32(j) + i++ + j++ + if i >= 624 { + r.mt[0] = r.mt[623] + i = 1 + } + if j >= len(key) { + j = 0 + } + } + for k = 623; k > 0; k-- { + r.mt[i] = (r.mt[i] ^ ((r.mt[i-1] ^ (r.mt[i-1] >> 30)) * 1566083941)) - uint32(i) + i++ + if i >= 624 { + r.mt[0] = r.mt[623] + i = 1 + } + } + r.mt[0] = 0x80000000 + r.idx = 624 + return r +} + +func (r *mtRandom) initGenrand(s uint32) { + r.mt[0] = s + for i := 1; i < 624; i++ { + r.mt[i] = 1812433253*(r.mt[i-1]^(r.mt[i-1]>>30)) + uint32(i) + } + r.idx = 624 +} + +func (r *mtRandom) next() uint32 { + if r.idx >= 624 { + for k := 0; k < 624; k++ { + y := (r.mt[k] & 0x80000000) | (r.mt[(k+1)%624] & 0x7fffffff) + v := r.mt[(k+397)%624] ^ (y >> 1) + if y&1 != 0 { + v ^= 0x9908b0df + } + r.mt[k] = v + } + r.idx = 0 + } + y := r.mt[r.idx] + r.idx++ + y ^= y >> 11 + y ^= (y << 7) & 0x9d2c5680 + y ^= (y << 15) & 0xefc60000 + y ^= y >> 18 + return y +} + +// choiceIndex is random.choice's _randbelow(n) for n < 2**32. +func (r *mtRandom) choiceIndex(n int) int { + if n <= 0 { + return 0 + } + k := 0 + for v := n; v > 0; v >>= 1 { + k++ + } + for { + if x := int(r.next() >> (32 - k)); x < n { + return x + } + } +} diff --git a/internal/kjv/pages.go b/internal/kjv/pages.go new file mode 100644 index 0000000..c53552c --- /dev/null +++ b/internal/kjv/pages.go @@ -0,0 +1,512 @@ +package kjv + +import ( + "fmt" + "html" + "math/rand/v2" + "net/url" + "sort" + "strconv" + "strings" + "time" + "unicode" +) + +const ( + maxSearchWords = 20 + maxSearchLen = 200 +) + +const searchChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+^* " + +// cleanInput keeps only the allowed characters. +func cleanInput(s, valid string) string { + return strings.Map(func(r rune) rune { + if strings.ContainsRune(valid, r) { + return r + } + return -1 + }, s) +} + +func searchClean(s string) string { return cleanInput(s, searchChars) } + +// --------------------------------------------------------------------------- +// /show + +func (a *App) show(kw, view string, all bool) (Resp, bool) { + resolved := a.res.Resolve(kw, true) + if resolved == nil { + return htmlR("Invalid Query"), false + } + var verses []Verse + for _, r := range resolved.Refs { + end := r.End + if all { // ?all= : from the given verse to the end of its chapter + end = r.Start/1000*1000 + maxChapterIDSpan + } + verses = append(verses, a.store.Range(r.Start, end)...) + } + return a.render(verses, view, kw).resp(), true +} + +func viewOf(q url.Values, def string) string { + if q.Has("view") { + return q.Get("view") + } + return def +} + +func (a *App) handleShow(q url.Values) Resp { + kw := q.Get("kw") + if kw == "" { + return a.staticPage("html/show.html") + } + resp, _ := a.show(kw, viewOf(q, "json"), q.Get("all") != "") + return resp +} + +// --------------------------------------------------------------------------- +// /random and /votd + +func (a *App) singleVerse(id int64, q url.Values, defView string) Resp { + return a.render(a.store.Range(id, 0), viewOf(q, defView), "").resp() +} + +func (a *App) handleRandom(q url.Values, defView string) Resp { + id := a.store.verseIDs[rand.IntN(len(a.store.verseIDs))] + lg.Info("Random served: %d", id) + return a.singleVerse(id, q, defView) +} + +// votdID is deterministic: the date seeds the generator, so everyone gets the same verse that day. +func (a *App) votdID(t time.Time) int64 { + seed, _ := strconv.Atoi(t.Format("20060102")) + r := newMTRandom(uint32(seed)) + return a.store.verseIDs[r.choiceIndex(len(a.store.verseIDs))] +} + +func (a *App) handleVotd(q url.Values, defView string) Resp { + lg.Info("VOTD served") + return a.singleVerse(a.votdID(time.Now()), q, defView) +} + +// --------------------------------------------------------------------------- +// dictionary + +func (a *App) defineRich(ref string, defs []string, simple bool) string { + head := "" + if !simple { + head = "" + + "" + + "" + + "" + html.EscapeString(capitalize(ref)) + " - KJV API" + + a.cssNavbar + "
" + } + text := "" + for _, d := range defs { + text += d + "

" + } + out := head + strings.NewReplacer( + "{chapter}", strings.SplitN(ref, ":", 2)[0], "{passage}", capitalize(ref), "{text}", text, "{verse_id}", "", + ).Replace(a.htmlResponse) + if !simple { + out += "
" + } + return out +} + +func (a *App) handleDefine(q url.Values) Resp { + ref := strings.ToLower(strings.TrimSpace(q.Get("kw"))) + if ref == "" { + return a.staticPage("html/dictionary.html") + } + defs, ok := a.dictionary[ref] + if !ok { + lg.Warn("WEBSTER MISSING: %s", ref) + return htmlR(html.EscapeString(ref) + " not found.") + } + switch q.Get("view") { + case "plain": + text := "" + for _, d := range defs { + text += d + "

" + } + return htmlR(capitalize(ref) + "

" + text + "\n") + case "rich": + return htmlR(a.defineRich(ref, defs, false)) + } + return jsonR(map[string][]string{ref: defs}) +} + +var dictStopwords = map[string]bool{} + +func init() { + for _, w := range strings.Fields(`the and that for him his her but not with are was were unto thou thee thy ye you all from + they them this have had shall will which into upon then`) { + dictStopwords[w] = true + } +} + +// /defs?words=grace,loved : several definitions at once +func (a *App) handleDefs(q url.Values) Resp { + words := strings.Fields(strings.ReplaceAll(strings.ToLower(q.Get("words")), ",", " ")) + if len(words) > 40 { + words = words[:40] + } + found := map[string][]string{} + for _, w := range words { + w = strings.Map(func(r rune) rune { + if unicode.IsLetter(r) { + return r + } + return -1 + }, w) + if d, ok := a.dictionary[w]; ok && w != "" && !dictStopwords[w] { + found[w] = d + } + } + return jsonR(found) +} + +// --------------------------------------------------------------------------- +// /search + +// proximityScore: higher when the query words appear closer together. +func proximityScore(text string, words []string) float64 { + lower := strings.ToLower(text) + type hit struct{ pos, list int } + var hits []hit + seen := map[string]int{} + for _, w := range words { + if _, dup := seen[w]; dup { + continue + } + li := len(seen) + seen[w] = li + w = strings.ToLower(w) + n := 0 + for at := 0; ; { + i := strings.Index(lower[at:], w) + if i < 0 { + break + } + hits = append(hits, hit{at + i, li}) + n++ + at += i + 1 + } + if n == 0 { + return 0 + } + } + lists := len(seen) + sort.SliceStable(hits, func(i, j int) bool { return hits[i].pos < hits[j].pos }) + // smallest window holding at least one position of every word + counts := make([]int, lists) + have, left, best := 0, 0, -1 + for right := range hits { + if counts[hits[right].list]++; counts[hits[right].list] == 1 { + have++ + } + for have == lists { + if span := hits[right].pos - hits[left].pos; best < 0 || span < best { + best = span + } + if counts[hits[left].list]--; counts[hits[left].list] == 0 { + have-- + } + left++ + } + } + if best < 0 { + return 0 + } + return 1000.0 / float64(best+1) +} + +func sortVariants(original string, variants []string) []string { + type sv struct { + score int + v string + } + var scored []sv + orig := strings.ToLower(original) + for _, v := range variants { + lv := strings.ToLower(v) + prefix := 0 + for prefix < len(orig) && prefix < len(lv) && orig[prefix] == lv[prefix] { + prefix++ + } + if prefix < min(2, len(orig)/2) { // doesn't share the stem + continue + } + scored = append(scored, sv{prefix*1000 + len(v), v}) + } + sort.SliceStable(scored, func(i, j int) bool { return scored[i].score > scored[j].score }) + out := make([]string, len(scored)) + for i, s := range scored { + out[i] = s.v + } + return out +} + +func (a *App) fts(q string) []Verse { + v, err := a.store.FTS(q) + if err != nil { // FTS5 syntax the user typed by accident: just no results + lg.Info("fts %q: %v", q, err) + } + return v +} + +func insertAfterResponsesOpen(page, notice string) string { + return strings.Replace(page, "
", "
"+notice, 1) +} + +// fallbackResp shows the results of a rewritten query with an explanation. +func (a *App) fallbackResp(results []Verse, view, kw, plainNote, richNotice string, jsonObj map[string]any) Resp { + r := a.render(results, view, kw) + switch view { + case "plain": + return htmlR(plainNote + "\n
" + r.text) + case "rich": + return htmlR(insertAfterResponsesOpen(r.text, richNotice)) + } + jsonObj["results"] = r.list + return jsonR(jsonObj) +} + +func (a *App) handleSearch(kw, view string) Resp { + cleaned := searchClean(kw) + if words := strings.Fields(cleaned); len(words) > maxSearchWords || len(cleaned) > maxSearchLen { + return htmlR("Your query is too long. 20 words or less.") + } + if cleaned == "" { + return a.staticPage("html/search.html") + } + esc := html.EscapeString + + // a common theological concept that is not in the KJV? + conceptNotice := "" + if c, ok := a.concepts[strings.ToLower(cleaned)]; ok { + var links []string + for _, t := range c.SearchTerms { + links = append(links, fmt.Sprintf("%s", strings.ReplaceAll(t, " ", "+"), t)) + } + conceptNotice = fmt.Sprintf("
%s: %s
You may find what you are looking for with these search terms: %s
", + capitalize(cleaned), c.Explanation, strings.Join(links, ", ")) + cleaned = c.SearchTerms[0] + } + + results := a.fts(cleaned) + words := strings.Fields(cleaned) + if len(results) > 0 && len(words) > 1 { + scores := make(map[int64]float64, len(results)) + for _, v := range results { + scores[v.ID] = proximityScore(v.Text, words) + } + sort.SliceStable(results, func(i, j int) bool { return scores[results[i].ID] > scores[results[j].ID] }) + } + + if len(results) == 0 { + // 1. grammatical variants + for _, word := range words { + for _, variant := range sortVariants(word, a.variants[strings.ToLower(word)]) { + vq := strings.Replace(cleaned, word, variant, 1) + if r := a.fts(vq); len(r) > 0 { + return a.fallbackResp(r, view, kw, + fmt.Sprintf("Showing results for '%s' (grammatical variant of '%s')", vq, cleaned), + fmt.Sprintf("
Showing results for '%s' (grammatical variant of '%s')
", vq, cleaned), + map[string]any{"original": cleaned, "variant": vq}) + } + } + } + // 2. spell check + corrected := make([]string, len(words)) + changed := false + for i, w := range words { + corrected[i] = w + if !a.speller.Spell(w) { + if sug := a.speller.Suggest(w); len(sug) > 0 { + corrected[i], changed = sug[0], true + } + } + } + if changed { + ck := strings.Join(corrected, " ") + if r := a.fts(ck); len(r) > 0 { + return a.fallbackResp(r, view, kw, + fmt.Sprintf("Showing results for '%s' (corrected from '%s')", ck, cleaned), + fmt.Sprintf("
Showing results for '%s' (corrected from '%s')
", esc(ck), cleaned), + map[string]any{"corrected_from": cleaned, "corrected_to": ck}) + } + } + // 3. theological equivalents + for _, word := range words { + for _, syn := range a.synonyms[strings.ToLower(word)] { + sq := strings.Replace(cleaned, word, syn, 1) + if r := a.fts(sq); len(r) > 0 { + explanation := fmt.Sprintf("In the KJV, '%s' and '%s' often refer to similar theological concepts. Consider also searching for related verses.", word, syn) + return a.fallbackResp(r, view, kw, + fmt.Sprintf("Showing results for '%s' (theological equivalent of '%s')\n%s", sq, cleaned, explanation), + fmt.Sprintf("
Showing results for '%s' (theological equivalent of '%s')
%s
", sq, cleaned, explanation), + map[string]any{"original": cleaned, "theological_equivalent": sq, "explanation": explanation}) + } + } + } + // 4. nothing: offer spelling suggestions + var suggestions []string + for i, w := range words { + if a.speller.Spell(w) { + continue + } + for _, s := range a.speller.Suggest(w) { + if len(suggestions) >= 5 { + break + } + alt := append([]string(nil), words...) + alt[i] = s + suggestions = append(suggestions, strings.Join(alt, " ")) + } + } + if view != "rich" { + list := "No suggestions available" + if len(suggestions) > 0 { + list = strings.Join(suggestions, ", ") + } + return htmlR(fmt.Sprintf("%s not found, try one of the following: %s", cleaned, list)) + } + var sb strings.Builder + sb.WriteString(a.renderNavbar(cleaned, nil, "")) + sb.WriteString("
") + fmt.Fprintf(&sb, "
No results found for '%s'.
", cleaned) + if len(suggestions) > 0 { + sb.WriteString("

Did you mean?

") + sb.WriteString("

We couldn't find any verses matching your search. Here are some spelling suggestions:

") + for _, s := range suggestions { + fmt.Fprintf(&sb, "
%s
", url.QueryEscape(s), esc(s)) + } + sb.WriteString("
") + } else { + sb.WriteString("

Search Tips

") + sb.WriteString("

We couldn't find any verses matching your search. Try:

") + sb.WriteString("
    ") + sb.WriteString("
  • Checking your spelling
  • Using different keywords
    • ") + sb.WriteString("
  • Searching for love, faith, or hope
    • ") + sb.WriteString("
") + } + sb.WriteString("
") + return htmlR(sb.String()) + } + + r := a.render(results, view, kw) + var related []string + for _, w := range words { + related = append(related, a.synonyms[strings.ToLower(w)]...) + } + switch view { + case "plain": + out := r.text + if conceptNotice != "" { + out = conceptNotice + "
" + out + } + if len(related) > 0 { + out += "

Also consider searching: " + for _, s := range related { + out += s + "
" + } + } + return htmlR(out) + case "rich": + out := r.text + if conceptNotice != "" { + if start := strings.Index(out, "
"); start != -1 { + if end := strings.Index(out[start+len("
"):], "
"); end != -1 { + at := start + len("
") + end + len("
") + out = out[:at] + conceptNotice + out[at:] + } + } + } + if len(related) > 0 { + var sb strings.Builder + sb.WriteString("

Related Theological Concepts

") + sb.WriteString("

In the KJV, these terms often refer to similar theological concepts. Consider also searching for related verses.

") + for _, s := range related { + fmt.Fprintf(&sb, "
%s
", url.QueryEscape(s), capitalize(s)) + } + sb.WriteString("
") + if at := strings.LastIndex(out, "
"); at != -1 { + out = out[:at] + sb.String() + out[at:] + } + } + return htmlR(out) + } + return r.resp() +} + +// --------------------------------------------------------------------------- +// /find : scripture if it reads as scripture, otherwise a keyword search with definitions + +func (a *App) handleFind(q url.Values) Resp { + query := strings.TrimSpace(q.Get("kw")) + if viewOf(q, "rich") != "rich" { + return jsonR(map[string]string{"Error": "/find endpoint only supports rich responses"}) + } + lg.Info("Query: %s", query) + if query == "" { + return a.staticPage("html/find.html") + } + query = strings.TrimSuffix(query, ",") + + // A query that reads as scripture references ("Exodus", "1 jn 3.16") is shown + // as scripture; anything else is a keyword search. + if resolved := a.res.Resolve(query, false); resolved != nil { + if resp, ok := a.show(query, "rich", false); ok { + notices := "" + for _, n := range resolved.Notes { + notices += fmt.Sprintf("
Showing %s (interpreted from '%s')
", + html.EscapeString(n.Name), html.EscapeString(n.Original)) + } + if resolved.Bare { + keyword := url.QueryEscape(searchClean(query)) + notices += fmt.Sprintf("
Showing %s, chapter 1. Looking for the word instead? "+ + "Search the text for '%s'
", + html.EscapeString(resolved.Label), keyword, html.EscapeString(query)) + } + if notices != "" { + resp.Body = insertAfterResponsesOpen(resp.Body, notices) + } + return resp + } + } + if strings.TrimSpace(searchClean(query)) == "" { + return a.staticPage("html/find.html") // nothing searchable left (punctuation, accents) + } + var tokens []string + for _, w := range strings.Split(strings.ReplaceAll(query, ",", " "), " ") { + if w != "" { + tokens = append(tokens, w) + } + } + if len(tokens) > 20 { + return htmlR("Your Query is too long. 20 Words or less.") + } + result := a.handleSearch(q.Get("kw"), "rich") + body := strings.TrimSuffix(result.Body, "
") // reopen .responses to add the dictionary inside it + + var defs strings.Builder + for _, word := range tokens { + entry, ok := a.dictionary[strings.ToLower(word)] + if !ok { + continue + } + defs.WriteString("") + defs.WriteString("
" + a.defineRich(strings.ToLower(word), entry, true) + "
") + } + if defs.Len() > 0 { + body += "" + body += "

1828 Webster Definitions

Click on the word to get the definition.

" + body += defs.String() + body += "" + } + return htmlR(body + "
") +} diff --git a/internal/kjv/refs.go b/internal/kjv/refs.go new file mode 100644 index 0000000..069d56a --- /dev/null +++ b/internal/kjv/refs.go @@ -0,0 +1,357 @@ +package kjv + +// Scripture reference resolver. Understands the many ways people type a +// reference ("1 jn 3.16", "II Kings 2:1-4", "song of songs 2", "Exodus ") and +// turns it into ranges of verse ids (book*1000000 + chapter*1000 + verse). + +import ( + "regexp" + "sort" + "strconv" + "strings" +) + +// Extra spellings people use that the abbreviations table does not know. +var extraBookAliases = map[string][]string{ + "Song of Solomon": {"song of songs", "songs of solomon", "songs of songs", "song of sol", + "song of sol", "canticles", "canticle of canticles", "sos", "sng", "ss"}, + "Psalms": {"psalm", "psalter", "pslm", "psa", "pss", "psm", "ps"}, + "Revelation": {"revelations", "revelation of john", "revelation of st john", "the revelation", "apocalypse"}, + "Ecclesiastes": {"ecclesiasts", "eccles", "ecc", "qoheleth"}, + "Philemon": {"phlm", "phm", "philem"}, + "Judges": {"jdg", "jdgs", "jg"}, + "Mark": {"mrk", "mk", "mr"}, + "John": {"jhn", "jn"}, + "Acts": {"acts of the apostles", "act"}, + "James": {"jas", "jm"}, +} + +// Words that are books but also ordinary search words: a bare "numbers" or +// "mark" still shows the book, but /find offers a keyword search as well. +var bareBookExtra = map[string]bool{ + "psalm": true, "songofsongs": true, "songsofsolomon": true, "songofsol": true, "canticles": true, + "canticleofcanticles": true, "revelations": true, "revelationofjohn": true, "therevelation": true, +} + +const maxChapterIDSpan = 999 + +// A reference is never this long; longer text goes to the keyword search. Bounds the fuzzy matching work. +const maxRefQueryLen = 300 + +// Go's regexp has no lookaheads, so they are written as a consumed [\s.] here, which matches the same text. +var ( + bookPieceRE = regexp.MustCompile(`(?i)^\s*(?P(?:(?:[1-3]|(?:iii|ii|i)[\s.]|1st|2nd|3rd|first|second|third)\.?\s*)?[a-z][a-z.\s]*?)` + + `\s*(?P\d[\d\s:.\-–—]*)?\s*$`) + continuationRE = regexp.MustCompile(`^\s*\d[\d\s:.\-–—]*$`) + + reBookOf = regexp.MustCompile(`^(?:the\s+)?(?:book\s+of\s+)`) + reFirst = regexp.MustCompile(`^(?:1st|first)([\s.a-z]|$)`) + reSecond = regexp.MustCompile(`^(?:2nd|second)([\s.a-z]|$)`) + reThird = regexp.MustCompile(`^(?:3rd|third)([\s.a-z]|$)`) + reRoman3 = regexp.MustCompile(`^iii([\s.]|$)`) + reRoman2 = regexp.MustCompile(`^ii([\s.]|$)`) + reRoman1 = regexp.MustCompile(`^i([\s.])`) + reNotAlnum = regexp.MustCompile(`[^a-z0-9]`) + reSepSpace = regexp.MustCompile(`\s*([:\-])\s*`) + + reChapter = regexp.MustCompile(`^(\d+)$`) + reChapterAll = regexp.MustCompile(`^(\d+)(?::-|-)$`) + reChapterRange = regexp.MustCompile(`^(\d+)-(\d+)$`) + reVerse = regexp.MustCompile(`^(\d+)[:.\s](\d+)$`) + reVerseOn = regexp.MustCompile(`^(\d+)[:.\s](\d+)-$`) + reVerseRange = regexp.MustCompile(`^(\d+)[:.\s](\d+)-(\d+)$`) + reVerseAcrossCh = regexp.MustCompile(`^(\d+)[:.\s](\d+)-(\d+)[:.](\d+)$`) +) + +// normalizeBookText: 'I Cor.' / '1st Corinthians' / '1 corinthians' -> '1corinthians' +func normalizeBookText(text string) string { + text = strings.ToLower(strings.TrimSpace(text)) + text = reBookOf.ReplaceAllString(text, "") + text = reFirst.ReplaceAllString(text, "1${1}") + text = reSecond.ReplaceAllString(text, "2${1}") + text = reThird.ReplaceAllString(text, "3${1}") + text = reRoman3.ReplaceAllString(text, "3${1}") + text = reRoman2.ReplaceAllString(text, "2${1}") + text = reRoman1.ReplaceAllString(text, "1${1}") + return reNotAlnum.ReplaceAllString(text, "") +} + +type Resolver struct { + store *Store + canonical map[string]int + aliases map[string]int + aliasKeys []string + canonKeys []string +} + +func NewResolver(s *Store) *Resolver { + r := &Resolver{store: s, canonical: map[string]int{}, aliases: map[string]int{}} + for id, name := range s.bookName { + r.canonical[normalizeBookText(name)] = id + } + for k, v := range r.canonical { + r.aliases[k] = v + } + for _, a := range s.abbrevs { + if k := normalizeBookText(a.Name); r.aliases[k] == 0 { + r.aliases[k] = a.Book + } + } + for name, extras := range extraBookAliases { + id := s.bookByName[name] + for _, extra := range extras { + r.aliases[normalizeBookText(extra)] = id + } + } + for k := range r.aliases { + r.aliasKeys = append(r.aliasKeys, k) + } + sort.Strings(r.aliasKeys) + for k := range r.canonical { + r.canonKeys = append(r.canonKeys, k) + } + sort.Strings(r.canonKeys) + return r +} + +type bookMatch struct { + ID int + How string // exact | prefix | fuzzy +} + +// resolveBook finds a book from a typed name. loose=false only accepts full +// names and spellings that are unmistakably a book (used for a bare word with +// no chapter number). +func (r *Resolver) resolveBook(token string, loose bool) (bookMatch, bool) { + key := normalizeBookText(token) + if key == "" { + return bookMatch{}, false + } + if !loose { + if _, ok := r.canonical[key]; ok || bareBookExtra[key] { + return bookMatch{r.aliases[key], "exact"}, true + } + // a clear misspelling of a long full name ("genisis") + if len(key) >= 6 && len(key) <= 40 && !(key[0] >= '0' && key[0] <= '9') { + if close, ok := closestMatch(key, r.canonKeys, 0.85); ok { + return bookMatch{r.canonical[close], "fuzzy"}, true + } + } + return bookMatch{}, false + } + if id, ok := r.aliases[key]; ok { + return bookMatch{id, "exact"}, true + } + // unique (or best) prefix: "deut", "1cor", "matth" + books := map[int]bool{} + for _, k := range r.aliasKeys { + if strings.HasPrefix(k, key) { + books[r.aliases[k]] = true + } + } + if len(books) > 0 { + canonHits := map[int]bool{} + for _, k := range r.canonKeys { + if strings.HasPrefix(k, key) { + canonHits[r.canonical[k]] = true + } + } + if len(canonHits) == 1 { + return bookMatch{anyKey(canonHits), "prefix"}, true + } + if len(books) == 1 { + return bookMatch{anyKey(books), "prefix"}, true + } + pool := books + if len(canonHits) > 0 { + pool = canonHits + } + best := 0 + for id := range pool { + if id > best { + best = id // ambiguous: the later book, as before + } + } + return bookMatch{best, "prefix"}, true + } + if len(key) >= 4 && len(key) <= 40 { + if close, ok := closestMatch(key, r.aliasKeys, 0.78); ok { + return bookMatch{r.aliases[close], "fuzzy"}, true + } + } + return bookMatch{}, false +} + +func anyKey(m map[int]bool) int { + for k := range m { + return k + } + return 0 +} + +// clampChapter: 0 -> 1, past the end -> the last chapter. +func (r *Resolver) clampChapter(book, chapter int) int { + last := r.store.chapterLen[book] + if chapter > last { + chapter = last + } + if chapter < 1 { + chapter = 1 + } + return chapter +} + +// atoi never fails: absurdly long numbers saturate. +func atoi(s string) int { + n, err := strconv.Atoi(s) + if err != nil { + return 1_000_000 + } + if n > 1_000_000 { + return 1_000_000 + } + return n +} + +type refPos struct{ book, chapter, verse int } + +// parseRefRest: '3:16-18', '3 16', '3.16', '3-4', '3:16-4:2', '3', '3:-' -> (start, end or 0, position) +func (r *Resolver) parseRefRest(rest string, book int, previous *refPos) (start, end int64, pos refPos, ok bool) { + rest = strings.NewReplacer("–", "-", "—", "-").Replace(rest) + rest = reSepSpace.ReplaceAllString(strings.TrimSpace(rest), "$1") + base := int64(book) * 1_000_000 + at := func(chapter, verse int) int64 { return base + int64(chapter)*1000 + int64(verse) } + pos.book = book + if m := reChapter.FindStringSubmatch(rest); m != nil { + n := atoi(m[1]) + if previous != nil && previous.verse != 0 { // "John 3:16, 18": a verse of the same chapter + pos.chapter, pos.verse = previous.chapter, n + return at(previous.chapter, n), 0, pos, true + } + ch := r.clampChapter(book, n) + pos.chapter = ch + return at(ch, 1), at(ch, maxChapterIDSpan), pos, true + } + if m := reChapterAll.FindStringSubmatch(rest); m != nil { // "3:-" or "3-": the whole chapter + ch := r.clampChapter(book, atoi(m[1])) + pos.chapter = ch + return at(ch, 1), at(ch, maxChapterIDSpan), pos, true + } + if m := reChapterRange.FindStringSubmatch(rest); m != nil { // chapters 3 to 4 + first, last := r.clampChapter(book, atoi(m[1])), r.clampChapter(book, atoi(m[2])) + if last < first { + first, last = last, first + } + pos.chapter = first + return at(first, 1), at(last, maxChapterIDSpan), pos, true + } + if m := reVerse.FindStringSubmatch(rest); m != nil { + ch := r.clampChapter(book, atoi(m[1])) + verse := max(1, atoi(m[2])) + pos.chapter, pos.verse = ch, verse + return at(ch, verse), 0, pos, true + } + if m := reVerseOn.FindStringSubmatch(rest); m != nil { // "3:16-": from verse 16 to the end + ch := r.clampChapter(book, atoi(m[1])) + pos.chapter, pos.verse = ch, atoi(m[2]) + return at(ch, max(1, atoi(m[2]))), at(ch, maxChapterIDSpan), pos, true + } + if m := reVerseRange.FindStringSubmatch(rest); m != nil { + ch := r.clampChapter(book, atoi(m[1])) + first, last := max(1, atoi(m[2])), max(1, atoi(m[3])) + if last < first { + first, last = last, first + } + pos.chapter, pos.verse = ch, first + return at(ch, first), at(ch, last), pos, true + } + if m := reVerseAcrossCh.FindStringSubmatch(rest); m != nil { // 3:16-4:2 across chapters + c1, c2 := r.clampChapter(book, atoi(m[1])), r.clampChapter(book, atoi(m[3])) + s, e := at(c1, max(1, atoi(m[2]))), at(c2, max(1, atoi(m[4]))) + if e < s { + return 0, 0, pos, false + } + pos.chapter, pos.verse = c1, atoi(m[2]) + return s, e, pos, true + } + return 0, 0, pos, false +} + +type RefRange struct{ Start, End int64 } // End == 0: a single verse + +type Note struct{ Original, Name string } + +// Resolved is a whole parsed query. +type Resolved struct { + Refs []RefRange + Notes []Note // short explanations of guesses that were made + Bare bool // the query was just a book name (shows its first chapter) + Label string // the name of the first book, for messages +} + +// Resolve parses a whole query ('John 3:16, Romans 5:8') into verse id ranges. +// It returns nil when the text does not read as scripture references, so the +// caller can treat it as a keyword search. +func (r *Resolver) Resolve(text string, looseBare bool) *Resolved { + if strings.TrimSpace(text) == "" || len(text) > maxRefQueryLen { + return nil + } + var pieces []string + for _, p := range regexp.MustCompile(`[,;\n]+`).Split(text, -1) { + if strings.TrimSpace(p) != "" { + pieces = append(pieces, p) + } + } + if len(pieces) == 0 || len(pieces) > 20 { + return nil + } + out := &Resolved{} + var previous *refPos + for _, piece := range pieces { + var start, end int64 + var pos refPos + var name string + if m := bookPieceRE.FindStringSubmatchIndex(piece); m != nil { + bi, ri := bookPieceRE.SubexpIndex("book"), bookPieceRE.SubexpIndex("rest") + token := piece[m[2*bi]:m[2*bi+1]] + hasRest := m[2*ri] >= 0 + found, ok := r.resolveBook(token, hasRest || looseBare) + if !ok { + return nil + } + name = r.store.BookName(found.ID) + if found.How == "fuzzy" { + out.Notes = append(out.Notes, Note{strings.TrimSpace(token), name}) + } + if !hasRest { + if len(pieces) == 1 { + out.Bare = true + } + base := int64(found.ID) * 1_000_000 + start, end, pos = base+1001, base+1999, refPos{found.ID, 1, 0} + } else { + var ok bool + start, end, pos, ok = r.parseRefRest(piece[m[2*ri]:m[2*ri+1]], found.ID, nil) + if !ok { + return nil + } + } + } else if continuationRE.MatchString(piece) && previous != nil { + name = r.store.BookName(previous.book) + var ok bool + start, end, pos, ok = r.parseRefRest(piece, previous.book, previous) + if !ok { + return nil + } + } else { + return nil + } + p := pos + previous = &p + if out.Label == "" { + out.Label = name + } + out.Refs = append(out.Refs, RefRange{start, end}) + } + return out +} diff --git a/internal/kjv/sermons.go b/internal/kjv/sermons.go new file mode 100644 index 0000000..c2c8bc3 --- /dev/null +++ b/internal/kjv/sermons.go @@ -0,0 +1,163 @@ +package kjv + +import ( + "crypto/subtle" + "database/sql" + "encoding/json" + "io" + "net" + "net/http" + "strings" + "time" +) + +// Optional endpoints backed by the separate sermon database (KJV_SERMON_DB) +// and the separate semantic search service (KJV_SEMANTIC_ADDR, default +// localhost:1613). Neither is part of the distribution; without them these +// endpoints answer with an error. + +func rowsToMaps(rows *sql.Rows) ([]map[string]any, error) { + defer rows.Close() + cols, err := rows.Columns() + if err != nil { + return nil, err + } + out := []map[string]any{} + for rows.Next() { + vals := make([]any, len(cols)) + ptrs := make([]any, len(cols)) + for i := range vals { + ptrs[i] = &vals[i] + } + if err := rows.Scan(ptrs...); err != nil { + return nil, err + } + m := map[string]any{} + for i, c := range cols { + if b, ok := vals[i].([]byte); ok { + vals[i] = string(b) + } + m[c] = vals[i] + } + out = append(out, m) + } + return out, rows.Err() +} + +func (a *App) sermonQuery(query string, args ...any) Resp { + if a.sermonDB == nil { + return jsonStatus(map[string]string{"error": "Sermon database not found"}, http.StatusServiceUnavailable) + } + rows, err := a.sermonDB.Query(query, args...) + if err != nil { + lg.Error("sermon query: %v", err) + return errJSON(http.StatusInternalServerError, "Sermon database error") + } + out, err := rowsToMaps(rows) + if err != nil { + return errJSON(http.StatusInternalServerError, "Sermon database error") + } + return jsonR(out) +} + +const sermonCols = "select distinct title,url,denomination,preacher,church_name from verse_refs where " + +// handleSermons serves everything under /sermon/ and /sermons/ ... by hand, +// since the paths overlap in ways the router would reject. +func (a *App) handleSermons(path string) Resp { + parts := strings.Split(strings.Trim(path, "/"), "/") + switch { + case len(parts) == 1 && parts[0] == "sermons": + return a.sermonQuery("select distinct title,url,denomination,preacher,church_name,filename from verse_refs") + case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "preacher": + return a.sermonQuery(sermonCols+"preacher=?", parts[2]) + case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "church": + return a.sermonQuery(sermonCols+"church_name=?", parts[2]) + case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "denomination": + return a.sermonQuery(sermonCols+"denomination=?", parts[2]) + case len(parts) == 3 && parts[0] == "sermon" && parts[1] == "search": + return a.sermonsByRef(parts[2]) + case len(parts) == 3 && parts[0] == "sermon" && parts[1] == "url": + return a.sermonQuery(sermonCols+"url=?", parts[2]) + case len(parts) == 3 && parts[0] == "sermon" && parts[2] == "verses": + return a.sermonQuery("select distinct verse_id,text from verse_refs where filename=?", parts[1]) + case len(parts) == 2 && parts[0] == "sermon": + return a.sermonQuery(sermonCols+"filename=?", parts[1]) + } + return a.notFound() +} + +func (a *App) sermonsByRef(ref string) Resp { + id, ok := a.refVerseID(ref) + if !ok { + return a.notFound() + } + if a.sermonDB == nil { + return jsonR(map[string]string{"error": "Sermon database not found"}) + } + rows, err := a.sermonDB.Query("select * from verse_refs where verse_id=?", id) + if err != nil { + return errJSON(http.StatusInternalServerError, "Sermon database error") + } + out, err := rowsToMaps(rows) + if err != nil { + return errJSON(http.StatusInternalServerError, "Sermon database error") + } + for _, m := range out { + if vid, ok := m["verse_id"].(int64); ok { + if v := a.store.Range(vid, 0); len(v) == 1 { + m["kjv"] = v[0].Text + } + } + } + return jsonR(out) +} + +// refVerseID: the verse id of the first reference in "1John5:7" style text. +func (a *App) refVerseID(ref string) (int64, bool) { + resolved := a.res.Resolve(ref, true) + if resolved == nil { + return 0, false + } + return resolved.Refs[0].Start, true +} + +func (a *App) handleRef(ref string) Resp { + id, ok := a.refVerseID(ref) + if !ok { + return a.notFound() + } + return htmlR(itoa64(id)) +} + +func (a *App) handleNaturalPost(body io.Reader) Resp { + var req struct { + Token *string `json:"token"` + Query string `json:"query"` + } + if err := json.NewDecoder(io.LimitReader(body, 1<<16)).Decode(&req); err != nil { + return errJSON(http.StatusBadRequest, "Invalid request format") + } + if req.Token == nil { + return errJSON(http.StatusUnauthorized, "Missing token in request body") + } + if a.secretKey == "" || subtle.ConstantTimeCompare([]byte(*req.Token), []byte(a.secretKey)) != 1 { + return errJSON(http.StatusUnauthorized, "Invalid Authorization") + } + conn, err := net.DialTimeout("tcp", a.semantic, 3*time.Second) + if err != nil { + return errJSON(http.StatusBadGateway, "Semantic search is not available") + } + defer conn.Close() + conn.SetDeadline(time.Now().Add(30 * time.Second)) + payload, _ := json.Marshal(map[string]string{"query": req.Query}) + if _, err := conn.Write(payload); err != nil { + return errJSON(http.StatusBadGateway, "Semantic search is not available") + } + data, err := io.ReadAll(io.LimitReader(conn, 8<<20)) + var out any + if err != nil || json.Unmarshal(data, &out) != nil { + return errJSON(http.StatusBadGateway, "Semantic search gave an invalid answer") + } + return jsonR(out) +} diff --git a/internal/kjv/server.go b/internal/kjv/server.go new file mode 100644 index 0000000..884f813 --- /dev/null +++ b/internal/kjv/server.go @@ -0,0 +1,313 @@ +package kjv + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "runtime" + "strconv" + "syscall" + "time" +) + +// localOnly insists on a loopback address: the program is meant to sit behind +// a reverse proxy on the same machine and never face the network itself. +func localOnly(addr string) (string, error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return "", fmt.Errorf("bad listen address %q: %w", addr, err) + } + switch host { + case "", "localhost": + host = "127.0.0.1" + default: + if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() { + return "", fmt.Errorf("refusing to listen on %q: only localhost addresses (127.0.0.1, ::1) are allowed; put a reverse proxy in front", host) + } + } + return net.JoinHostPort(host, port), nil +} + +// Run serves the application from the companion database until interrupted. +func Run(dbPath, addr string) error { + addr, err := localOnly(addr) + if err != nil { + return err + } + if _, err := os.Stat(dbPath); err != nil { + return fmt.Errorf("database not found: %s (use -db or KJV_DB)", dbPath) + } + store, err := OpenStore(dbPath) + if err != nil { + return err + } + defer store.Close() + if store.stateOK { + lg.Attach(store.db, 90) + } else { + fmt.Fprintln(os.Stderr, "WARN: database is read-only: logs go to stderr; reading plans and shortcodes are disabled") + } + defer lg.Close() + + app, err := NewApp(store) + if err != nil { + return err + } + app.secretKey = os.Getenv("KJV_SECRET_API_KEY") + app.semantic = envOr("KJV_SEMANTIC_ADDR", "localhost:1613") + if p := os.Getenv("KJV_SERMON_DB"); p != "" { + if st, err := os.Stat(p); err == nil && st.Size() > 0 { + if db, err := sql.Open("sqlite", "file:"+p+"?mode=ro&_pragma=busy_timeout(5000)"); err == nil { + app.sermonDB = db + } + } else { + lg.Warn("sermon database %s not available", p) + } + } + + srv := &http.Server{ + Addr: addr, + Handler: app.Routes(), + MaxHeaderBytes: 16 << 10, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + WriteTimeout: 60 * time.Second, + IdleTimeout: 120 * time.Second, + } + go app.maintain() + lg.Info("Started %s: db=%s addr=%s verses=%d state=%v sermons=%v", runtime.Version(), dbPath, addr, len(store.verses), store.stateOK, app.sermonDB != nil) + fmt.Printf("kjv-api listening on %s (database %s)\n", addr, dbPath) + + ln, err := net.Listen("tcp", addr) + if err != nil { + return err + } + errc := make(chan error, 1) + go func() { errc <- srv.Serve(ln) }() + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + select { + case err := <-errc: + return err + case s := <-sig: + lg.Info("Stopping on %v", s) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := srv.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) { + return err + } + } + return nil +} + +// --------------------------------------------------------------------------- +// routes + +type statusWriter struct { + http.ResponseWriter + status int +} + +func (w *statusWriter) WriteHeader(code int) { w.status = code; w.ResponseWriter.WriteHeader(code) } + +func (a *App) Routes() http.Handler { + mux := http.NewServeMux() + get := func(pattern string, h func(r *http.Request) Resp) { + mux.HandleFunc("GET "+pattern, func(w http.ResponseWriter, r *http.Request) { h(r).write(w) }) + } + + get("/{$}", func(r *http.Request) Resp { return htmlR(a.indexHTML) }) + get("/{shortcode}", func(r *http.Request) Resp { return a.handleShortcode(r.PathValue("shortcode")) }) + get("/fe", func(r *http.Request) Resp { return a.staticPage("kjv_fe.html") }) + get("/salvation", func(r *http.Request) Resp { return a.staticPage("html/heaven.html") }) + get("/saved", func(r *http.Request) Resp { return a.staticPage("html/saved.html") }) + get("/plan", func(r *http.Request) Resp { return a.staticPage("html/plan.html") }) + + get("/show", func(r *http.Request) Resp { return a.handleShow(r.URL.Query()) }) + get("/find", func(r *http.Request) Resp { return a.handleFind(r.URL.Query()) }) + get("/search", func(r *http.Request) Resp { + v := r.URL.Query() + return a.handleSearch(v.Get("kw"), viewOf(v, "rich")) + }) + get("/define", func(r *http.Request) Resp { return a.handleDefine(r.URL.Query()) }) + get("/defs", func(r *http.Request) Resp { return a.handleDefs(r.URL.Query()) }) + get("/random", func(r *http.Request) Resp { return a.handleRandom(r.URL.Query(), "json") }) + get("/t/random", func(r *http.Request) Resp { return a.handleRandom(r.URL.Query(), "plain") }) + get("/votd", func(r *http.Request) Resp { return a.handleVotd(r.URL.Query(), "json") }) + get("/t/votd", func(r *http.Request) Resp { return a.handleVotd(r.URL.Query(), "plain") }) + get("/seq", func(r *http.Request) Resp { + if r.URL.Query().Get("getuid") != "" && !creations.allow(clientAddr(r)) { + return tooMany() + } + return a.handleSeq(r.URL.Query()) + }) + get("/mksc/{query}", func(r *http.Request) Resp { + if !creations.allow(clientAddr(r)) { + return tooMany() + } + return a.handleMakeShortcode(r.PathValue("query")) + }) + get("/ref/{ref}", func(r *http.Request) Resp { return a.handleRef(r.PathValue("ref")) }) + get("/verse_id/{id}", func(r *http.Request) Resp { + id := cleanInput(r.PathValue("id"), "0123456789") + n, err := strconv.ParseInt(id, 10, 64) + if err != nil { + return errJSON(http.StatusBadRequest, "Invalid verse id") + } + v := a.store.Range(n, 0) + if len(v) == 0 { + return errJSON(http.StatusNotFound, "No such verse") + } + return jsonR([]any{v[0].ID, v[0].Book, v[0].Chapter, v[0].Verse, v[0].Text}) + }) + get("/xref/{id}", func(r *http.Request) Resp { return a.handleXref(r.PathValue("id")) }) + get("/browse", func(r *http.Request) Resp { + body := "

Books

" + for _, n := range a.store.bookNames { + body += fmt.Sprintf("", urlQuery(n), htmlEsc(n)) + } + return htmlR(a.staticHTMLDoc("Books", body+"
%s
")) + }) + get("/status", func(r *http.Request) Resp { + var m runtime.MemStats + runtime.ReadMemStats(&m) + return htmlR(a.staticHTMLDoc("Status", fmt.Sprintf("

Status

Sanity: %v

Uptime: %s

Memory: %d KB

", + a.store.stateOK, time.Since(a.started).Round(time.Second), m.Sys/1024))) + }) + get("/seqreport", func(r *http.Request) Resp { + if resp, ok := a.adminOnly(r); !ok { + return resp + } + return a.tableReport("Sequence Report", "seq_report", "SELECT uid, verse, last_used FROM reading_tracker", false) + }) + get("/screport", func(r *http.Request) Resp { + if resp, ok := a.adminOnly(r); !ok { + return resp + } + return a.tableReport("Shortcode Report", "shortcode_report", "SELECT shortcode, query, creation_date FROM shortcodes", true) + }) + get("/natural", func(r *http.Request) Resp { + return htmlR("Submit a POST request to this endpoint with the following params in the body:\n{\"token\": \"mytokenstring\", \"query\" : \"myquerystring\"}") + }) + mux.HandleFunc("POST /natural", func(w http.ResponseWriter, r *http.Request) { a.handleNaturalPost(r.Body).write(w) }) + mux.HandleFunc("POST /plan/new", func(w http.ResponseWriter, r *http.Request) { + if !creations.allow(clientAddr(r)) { + tooMany().write(w) + return + } + r.Body = http.MaxBytesReader(w, r.Body, 16<<10) + if r.ParseForm() != nil { + errJSON(http.StatusRequestEntityTooLarge, "Request too large").write(w) + return + } + a.handlePlanNew(r.PostForm).write(w) + }) + mux.HandleFunc("POST /plan/{uid}/move", func(w http.ResponseWriter, r *http.Request) { + r.Body = http.MaxBytesReader(w, r.Body, 16<<10) + if r.ParseForm() != nil { + errJSON(http.StatusRequestEntityTooLarge, "Request too large").write(w) + return + } + a.handlePlanMove(r.PathValue("uid"), r.PostForm).write(w) + }) + get("/plan/{uid}", func(r *http.Request) Resp { return a.handlePlanView(r.PathValue("uid"), r.URL.Query()) }) + + for _, p := range []string{"/sermons", "/sermons/{rest...}", "/sermon/{rest...}"} { + get(p, func(r *http.Request) Resp { return a.handleSermons(r.URL.Path) }) + } + for _, p := range []string{"/preachers", "/churches", "/denominations"} { + cols := map[string]string{"/preachers": "denomination,preacher,church_name", "/churches": "denomination,church_name", "/denominations": "denomination"}[p] + get(p, func(r *http.Request) Resp { return a.sermonQuery("select distinct " + cols + " from verse_refs") }) + } + + for u := range servedPaths { + u := u + mux.HandleFunc("GET "+u, func(w http.ResponseWriter, r *http.Request) { a.serveAsset(w, r, u) }) + } + favicon := func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "image/svg+xml") + w.Header().Set("Cache-Control", "public, max-age=86400") + w.Write(a.favicon) + } + mux.HandleFunc("GET /favicon.ico", favicon) + mux.HandleFunc("GET /favicon.svg", favicon) + mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { a.notFound().write(w) }) + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + sw := &statusWriter{ResponseWriter: w, status: 200} + defer func() { + if rec := recover(); rec != nil { + lg.Error("panic serving %s: %v", r.URL.String(), rec) + sw.status = 500 + errJSON(500, "Internal error").write(w) + } + lg.Info("%s %s %d %s", r.Method, r.URL.RequestURI(), sw.status, time.Since(start).Round(time.Microsecond)) + }() + h := w.Header() + h.Set("X-Content-Type-Options", "nosniff") + h.Set("X-Frame-Options", "DENY") + h.Set("Referrer-Policy", "same-origin") + mux.ServeHTTP(sw, r) + }) +} + +// serveAsset: a URL whose ?v= hash matches the file is immutable for a year; +// any other URL for it is revalidated every time. +func (a *App) serveAsset(w http.ResponseWriter, r *http.Request, u string) { + as := a.served[u] + h := w.Header() + h.Set("Content-Type", as.mime) + h.Set("ETag", as.etag) + if r.URL.Query().Get("v") == as.version { + h.Set("Cache-Control", "public, max-age=31536000, immutable") + } else { + h.Set("Cache-Control", "no-cache") + } + if r.Header.Get("If-None-Match") == as.etag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Write(as.body) +} + +func (a *App) handleXref(raw string) Resp { + id, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return a.notFound() + } + refs, err := a.store.CrossReferences(id) + if err != nil { + lg.Error("xref: %v", err) + return errJSON(http.StatusInternalServerError, "Database error") + } + out := []map[string]any{} + for _, c := range refs { + ref := a.store.FormatRef(c.Start) + var verses []Verse + if c.End > c.Start { + if c.Start/1000 == c.End/1000 { // "John 3:16-18" when the range stays in one chapter + ref += fmt.Sprintf("-%d", c.End%1000) + } else { + ref += "-" + a.store.FormatRef(c.End) + } + verses = a.store.Range(c.Start, c.End) + } else { + verses = a.store.Range(c.Start, 0) + } + text := "" + for i, v := range verses { + if i > 0 { + text += " " + } + text += v.Text + } + out = append(out, map[string]any{"ref": ref, "verse_id": c.Start, "text": text}) + } + return jsonR(out) +} diff --git a/internal/kjv/spell.go b/internal/kjv/spell.go new file mode 100644 index 0000000..c5ae434 --- /dev/null +++ b/internal/kjv/spell.go @@ -0,0 +1,253 @@ +package kjv + +import ( + "sort" + "strings" + "unicode" +) + +// Speller checks and suggests spellings from kjv.dic (a plain word list) and +// kjv.aff (which only adds a TRY order and REP replacement hints), so no C +// library is needed. +type Speller struct { + words map[string]bool // exact entries + byLower map[string][]string // lowercase -> entries + list []string // lowercase entries, dictionary order, unique + try []rune + rep [][2]string +} + +func NewSpeller(dic, aff []byte) *Speller { + s := &Speller{words: map[string]bool{}, byLower: map[string][]string{}} + lines := strings.Split(string(dic), "\n") + seen := map[string]bool{} + for i, line := range lines { + line = strings.TrimSpace(line) + if i == 0 || line == "" { // first line is the entry count + continue + } + if k := strings.IndexByte(line, '/'); k >= 0 { + line = line[:k] + } + s.words[line] = true + low := strings.ToLower(line) + s.byLower[low] = append(s.byLower[low], line) + if !seen[low] { + seen[low] = true + s.list = append(s.list, low) + } + } + for _, line := range strings.Split(string(aff), "\n") { + f := strings.Fields(line) + switch { + case len(f) == 2 && f[0] == "TRY": + s.try = []rune(f[1]) + case len(f) == 3 && f[0] == "REP": + s.rep = append(s.rep, [2]string{f[1], strings.ReplaceAll(f[2], "_", " ")}) + } + } + return s +} + +func stripSyntax(w string) string { + return strings.Map(func(r rune) rune { + if r == '+' || r == '^' || r == '*' { + return -1 + } + return r + }, w) +} + +func hasLetter(w string) bool { + for _, r := range w { + if unicode.IsLetter(r) { + return true + } + } + return false +} + +func (s *Speller) Spell(word string) bool { + word = stripSyntax(word) + if !hasLetter(word) || s.words[word] { + return true + } + low := strings.ToLower(word) + if low != word && (word == strings.ToUpper(word) || word == capitalize(low)) { + return len(s.byLower[low]) > 0 + } + return false +} + +func capitalize(w string) string { + r := []rune(w) + if len(r) == 0 { + return w + } + return strings.ToUpper(string(r[:1])) + strings.ToLower(string(r[1:])) +} + +func (s *Speller) known(low string) []string { return s.byLower[low] } + +// Suggest returns likely corrections, best first. +func (s *Speller) Suggest(word string) []string { + word = stripSyntax(word) + low := strings.ToLower(word) + if low == "" { + return nil + } + var out []string + seen := map[string]bool{} + add := func(cands ...string) { + for _, c := range cands { + if !seen[c] { + seen[c] = true + out = append(out, c) + } + } + } + addLower := func(l string) { // dictionary form(s) of a lowercase candidate + if l == low { + return + } + add(s.known(l)...) + } + if low != word { // wrong capitalization of a known word + add(s.known(low)...) + } + // replacement hints from the .aff file, applied at each occurrence + for _, r := range s.rep { + for at := 0; ; { + i := strings.Index(low[at:], r[0]) + if i < 0 { + break + } + i += at + cand := low[:i] + r[1] + low[i+len(r[0]):] + if strings.Contains(cand, " ") { + if p := strings.Fields(cand); len(p) == 2 && len(s.known(p[0])) > 0 && len(s.known(p[1])) > 0 { + add(p[0] + " " + p[1]) + } + } else { + addLower(cand) + } + at = i + 1 + } + } + rs := []rune(low) + // one edit away: swap, delete, replace, insert + for i := 0; i+1 < len(rs); i++ { + c := append([]rune(nil), rs...) + c[i], c[i+1] = c[i+1], c[i] + addLower(string(c)) + } + for i := range rs { + addLower(string(rs[:i]) + string(rs[i+1:])) + } + for i := range rs { + for _, t := range s.try { + addLower(string(rs[:i]) + string(t) + string(rs[i+1:])) + } + } + for i := 0; i <= len(rs); i++ { + for _, t := range s.try { + addLower(string(rs[:i]) + string(t) + string(rs[i:])) + } + } + // two words run together ("lordgod") + for i := 1; i < len(rs); i++ { + a, b := string(rs[:i]), string(rs[i:]) + if len(a) > 1 && len(b) > 1 && len(s.known(a)) > 0 && len(s.known(b)) > 0 { + add(a + " " + b) + } + } + // still little to offer: anything within two edits, closest first + if len(out) < 3 { + type cand struct { + w string + dist int + pre int + } + var cs []cand + for _, w := range s.list { + if abs(len(w)-len(low)) > 2 || w == low { + continue + } + if d := editDistance(low, w, 2); d <= 2 { + cs = append(cs, cand{w, d, commonPrefix(low, w)}) + } + } + sort.SliceStable(cs, func(i, j int) bool { + if cs[i].dist != cs[j].dist { + return cs[i].dist < cs[j].dist + } + return cs[i].pre > cs[j].pre + }) + for _, c := range cs { + addLower(c.w) + if len(out) >= 8 { + break + } + } + } + if len(out) > 10 { + out = out[:10] + } + if word == strings.ToUpper(word) && len(word) > 1 { + for i := range out { + out[i] = strings.ToUpper(out[i]) + } + } else if word == capitalize(low) && word != low { + for i := range out { + out[i] = capitalize(out[i]) + } + } + return out +} + +func abs(n int) int { + if n < 0 { + return -n + } + return n +} + +func commonPrefix(a, b string) int { + n := 0 + for n < len(a) && n < len(b) && a[n] == b[n] { + n++ + } + return n +} + +// editDistance is optimal-string-alignment distance, giving up above limit. +func editDistance(a, b string, limit int) int { + ra, rb := []rune(a), []rune(b) + prev2 := make([]int, len(rb)+1) + prev := make([]int, len(rb)+1) + cur := make([]int, len(rb)+1) + for j := range prev { + prev[j] = j + } + for i := 1; i <= len(ra); i++ { + cur[0] = i + best := cur[0] + for j := 1; j <= len(rb); j++ { + cost := 1 + if ra[i-1] == rb[j-1] { + cost = 0 + } + v := min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost) + if i > 1 && j > 1 && ra[i-1] == rb[j-2] && ra[i-2] == rb[j-1] { + v = min(v, prev2[j-2]+1) + } + cur[j] = v + best = min(best, v) + } + if best > limit { + return limit + 1 + } + prev2, prev, cur = prev, cur, prev2 + } + return prev[len(rb)] +} diff --git a/internal/kjv/state.go b/internal/kjv/state.go new file mode 100644 index 0000000..eb2512f --- /dev/null +++ b/internal/kjv/state.go @@ -0,0 +1,521 @@ +package kjv + +import ( + "crypto/rand" + "crypto/subtle" + "database/sql" + "fmt" + "html" + "math" + "math/big" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +// Reading sequences (/seq), reading plans (/plan) and shortcodes (/mksc) keep +// their rows in the companion database. + +const ( + idChars = "0123456789ACEFHJKLMNPRTUVWXY" + planMaxPerDay = 500 + seqPurgeAfter = 90 * 24 * time.Hour + stateTimeFormat = "2006-01-02 15:04:05.000000" +) + +func generateShortID() string { // 5 characters + n, err := rand.Int(rand.Reader, big.NewInt(17_210_367-614_656)) + if err != nil { + panic(err) + } + remain := 614_656 + int(n.Int64()) + out := "" + for remain > 0 { + out = string(idChars[remain%len(idChars)]) + out + remain /= len(idChars) + } + return out +} + +func stateNow() string { return time.Now().Format(stateTimeFormat) } + +var errNoState = fmt.Errorf("the database is read-only, so reading plans and shortcodes are unavailable") + +func (a *App) stateDB() (*sql.DB, error) { + if !a.store.stateOK { + return nil, errNoState + } + return a.store.db, nil +} + +// insertID stores a row under a fresh random id. +func insertID(db *sql.DB, query string, args ...any) (string, error) { + var err error + for i := 0; i < 5; i++ { // a clash on a random 5 character id is very unlikely + id := generateShortID() + if _, err = db.Exec(query, append([]any{id}, args...)...); err == nil { + return id, nil + } + } + return "", err +} + +// --------------------------------------------------------------------------- +// shortcodes + +func (a *App) handleMakeShortcode(query string) Resp { + query = cleanInput(query, searchChars+":-,") + if query == "" { + return htmlR("Empty query.") + } + db, err := a.stateDB() + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + var code string + if err := db.QueryRow("SELECT shortcode FROM shortcodes WHERE query = ?", query).Scan(&code); err == nil { + return htmlR(code) // the same query always gets the same code + } + code, err = insertID(db, "INSERT INTO shortcodes (shortcode, query, creation_date) VALUES (?, ?, ?)", query, stateNow()) + if err != nil { + lg.Error("shortcode insert: %v", err) + return errJSON(http.StatusInternalServerError, "Could not create a shortcode") + } + lg.Info("Generated shortcode %s for query '%s'", code, query) + return htmlR(code) +} + +func (a *App) handleShortcode(code string) Resp { + clean := cleanInput(code, idChars) + if clean == "" { + return htmlR("Empty shortcode.") + } + db, err := a.stateDB() + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + var query string + if err := db.QueryRow("SELECT query FROM shortcodes WHERE shortcode = ?", clean).Scan(&query); err != nil { + return a.notFound() + } + lg.Info("Resolved shortcode %s for query '%s'", clean, query) + return Resp{Status: http.StatusFound, Type: "text/html; charset=utf-8", + Header: map[string]string{"Location": "/find?kw=" + url.QueryEscape(query) + "&view=rich"}} +} + +func (a *App) notFound() Resp { + return jsonStatus(map[string]string{ + "error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."}, + http.StatusNotFound) +} + +func (a *App) tableReport(title, id, query string, header bool) Resp { + db, err := a.stateDB() + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + rows, err := db.Query(query) + if err != nil { + lg.Error("report: %v", err) + return errJSON(http.StatusInternalServerError, "Database error") + } + defer rows.Close() + var sb strings.Builder + fmt.Fprintf(&sb, "

%s

", title, id) + if header { + sb.WriteString("") + } + for rows.Next() { + var c1, c2, c3 sql.NullString + rows.Scan(&c1, &c2, &c3) + fmt.Fprintf(&sb, "", html.EscapeString(c1.String), html.EscapeString(c2.String), html.EscapeString(c3.String)) + } + sb.WriteString("
shortcodequerycreation_date
%s%s%s
") + return htmlR(a.staticHTMLDoc(title, sb.String())) +} + +// --------------------------------------------------------------------------- +// /seq : hand out scripture a few verses at a time + +func (a *App) purgeSequences(db *sql.DB) { + rows, err := db.Query("SELECT uid, last_used FROM reading_tracker LIMIT 10") + if err != nil { + return + } + var stale []string + for rows.Next() { + var uid, last string + rows.Scan(&uid, &last) + if t, err := time.ParseInLocation(stateTimeFormat, last, time.Local); err == nil && time.Since(t) > seqPurgeAfter { + stale = append(stale, uid) + } else if t, err := time.ParseInLocation("2006-01-02 15:04:05", last, time.Local); err == nil && time.Since(t) > seqPurgeAfter { + stale = append(stale, uid) + } + } + rows.Close() + for _, uid := range stale { + lg.Info("Purging UID: %s", uid) + db.Exec("DELETE FROM reading_tracker WHERE uid = ?", uid) + } +} + +func (a *App) startIndex(ref string) (int, bool) { + ref = strings.TrimSpace(ref) + if ref == "" { + return 0, true + } + resolved := a.res.Resolve(ref, true) + if resolved == nil { + return 0, false + } + i := a.store.VerseIndex(resolved.Refs[0].Start) + return i, i >= 0 +} + +func (a *App) handleSeq(q url.Values) Resp { + uid, getuid := q.Get("uid"), q.Get("getuid") != "" + num := 1 + if q.Has("num") { + n, err := strconv.Atoi(q.Get("num")) + if err != nil { + return htmlR("Invalid num argument") + } + num = n + } + if uid == "" && !getuid { + return a.staticPage("html/seq.html") + } + num-- // 0 will mean "next" + num = min(num, len(a.store.verseIDs)) + if num < 0 { + return jsonR(map[string]string{"error": "You may only increment forward."}) + } + db, err := a.stateDB() + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + if getuid { + start, ok := a.startIndex(q.Get("start")) + if !ok { + return jsonR(map[string]string{"error": "Could not find that starting verse."}) + } + id, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow()) + if err != nil { + lg.Error("UID database failed to insert: %v", err) + return jsonR(map[string]string{"error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"}) + } + lg.Info("New SEQ UID: %s", id) + return htmlR("Your new UID is: " + id) + } + uid = cleanInput(strings.ReplaceAll(uid, " ", ""), searchChars[:62]+":-,") + var next int + if err := db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&next); err != nil { + lg.Info("Invalid UID %s", uid) + return jsonR(map[string]string{"error": "UID Invalid or purged (90 days of inactivity)"}) + } + a.purgeSequences(db) + total := len(a.store.verseIDs) + if next >= total || next < 0 { // finished the Bible: start over + next = 0 + } + last := min(next+num, total-1) + verses := a.store.Range(a.store.verseIDs[next], a.store.verseIDs[last]) + if _, err := db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, last+1, stateNow()); err != nil { + lg.Error("seq update: %v", err) + } + lg.Info("Served %d verses to UID %s", last-next+1, uid) + return a.render(verses, viewOf(q, "json"), "").resp() +} + +// --------------------------------------------------------------------------- +// /plan : a /seq style tracker that can go backwards. The plan is its UID; the +// page URL (/plan/?n=) is what gets bookmarked. + +func planPerDay(raw string, def int) int { + n, err := strconv.Atoi(raw) + if err != nil { + return def + } + return max(1, min(planMaxPerDay, n)) +} + +func (a *App) planError(msg string) Resp { + r := htmlR(a.staticHTMLDoc("Reading Plan", + "

Reading Plan

"+html.EscapeString(msg)+"

Back to plan setup

")) + r.Status = http.StatusBadRequest + return r +} + +func redirect303(to string) Resp { + return Resp{Status: http.StatusSeeOther, Type: "text/html; charset=utf-8", Header: map[string]string{"Location": to}} +} + +func validPlanUID(uid string) bool { return uid != "" && cleanInput(uid, idChars) == uid } + +func (a *App) handlePlanNew(f url.Values) Resp { + db, err := a.stateDB() + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + total := len(a.store.verseIDs) + start, ok := a.startIndex(f.Get("start")) + if !ok { + return a.planError("We couldn't find that starting point. Try something like 'Matthew 1:1'.") + } + var perDay int + if f.Get("pace") == "verses" { + perDay = planPerDay(f.Get("verses"), 25) + } else { + days, err := strconv.Atoi(f.Get("days")) + if err != nil { + days = 365 + } + days = max(1, min(3650, days)) + perDay = planPerDay(strconv.Itoa(int(math.Ceil(float64(total-start)/float64(days)))), 25) + } + uid, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow()) + if err != nil { + return a.planError("Could not create a plan, please try again.") + } + lg.Info("New reading plan %s start=%d per_day=%d", uid, start, perDay) + return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay)) +} + +func (a *App) planPosition(uid string) (int, bool, error) { + db, err := a.stateDB() + if err != nil { + return 0, false, err + } + var pos int + err = db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&pos) + return pos, err == nil, nil +} + +func (a *App) handlePlanMove(uid string, f url.Values) Resp { + perDay := planPerDay(f.Get("n"), 25) + if !validPlanUID(uid) { + return a.planError("That reading plan id is not valid.") + } + pos, found, err := a.planPosition(uid) + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + if !found { + return a.planError("That reading plan was not found (plans unused for 90 days are removed).") + } + total := len(a.store.verseIDs) + switch f.Get("action") { + case "next": + pos = min(total, pos+perDay) + case "back": + pos = max(0, pos-perDay) + case "restart": + pos = 0 + case "goto": + target, ok := a.startIndex(f.Get("kw")) + if !ok { + return a.planError("We couldn't find that reference. Try something like 'John 3:1'.") + } + pos = target + } + a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, pos, stateNow()) + return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay)) +} + +func (a *App) handlePlanView(uid string, q url.Values) Resp { + if !validPlanUID(uid) { + return a.planError("That reading plan id is not valid.") + } + perDay := planPerDay(q.Get("n"), 25) + position, found, err := a.planPosition(uid) + if err != nil { + return errJSON(http.StatusServiceUnavailable, err.Error()) + } + if !found { + return a.planError("That reading plan was not found (plans unused for 90 days are removed).") + } + // keeps an active plan from being purged as unused + a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, position, stateNow()) + + ids := a.store.verseIDs + total := len(ids) + finished := position >= total + lastIndex := min(total, position+perDay) - 1 + percent := math.Round(1000*float64(min(position, total))/float64(total)) / 10 + esc := html.EscapeString + + controls := fmt.Sprintf("
\n", uid, perDay) + backDisabled := "" + if position <= 0 { + backDisabled = "disabled" + } + next := "" + if !finished { + next = "" + } + controls += fmt.Sprintf(`%s + +
+
+ + + +
+
+
`, next, backDisabled, uid, perDay) + + var summary, versesHTML string + if finished { + summary = fmt.Sprintf("
You've finished the plan!
You read all %s verses. Use Previous reading to look back, or start over.
", commas(total)) + controls += fmt.Sprintf("
\n\n
", uid, perDay) + } else { + first, last := a.store.FormatRef(ids[position]), a.store.FormatRef(ids[lastIndex]) + summary = fmt.Sprintf("
Today's reading
%s – %s
"+ + "
%d verses · %.1f%% of the Bible complete
"+ + "
", + esc(first), esc(last), lastIndex-position+1, percent, percent, percent) + var sb strings.Builder + for _, v := range a.store.Range(ids[position], ids[lastIndex]) { + sb.WriteString(a.responseRich(v)) + } + versesHTML = sb.String() + } + note := "
Bookmark this page — it is your reading plan. Come back each day, then press Done to move on.
" + store := fmt.Sprintf("", uid, perDay) + return htmlR(a.renderNavbar("", nil, "") + "
" + summary + versesHTML + + "
" + controls + "
" + note + store + "
") +} + +func commas(n int) string { + s := strconv.Itoa(n) + for i := len(s) - 3; i > 0; i -= 3 { + s = s[:i] + "," + s[i:] + } + return s +} + +// --------------------------------------------------------------------------- +// abuse limits + +// creationLimiter bounds how fast each visitor can add rows (shortcodes, +// plans, sequences), with a looser limit over everyone together as a backstop. +type creationLimiter struct { + mu sync.Mutex + clients map[string]*bucket + all bucket +} + +type bucket struct { + tokens float64 + last time.Time +} + +const ( + clientRate, clientBurst = 0.5, 10.0 // per visitor: a new row every 2 seconds, 10 at once + totalRate, totalBurst = 20.0, 100.0 + maxTrackedClients = 10000 +) + +var creations = &creationLimiter{clients: map[string]*bucket{}, all: bucket{totalBurst, time.Now()}} + +func (b *bucket) take(now time.Time, rate, burst float64) bool { + b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*rate) + b.last = now + if b.tokens < 1 { + return false + } + b.tokens-- + return true +} + +func (c *creationLimiter) allow(client string) bool { + c.mu.Lock() + defer c.mu.Unlock() + now := time.Now() + if len(c.clients) > maxTrackedClients { // forget visitors who have long since refilled + for k, b := range c.clients { + if now.Sub(b.last) > time.Minute { + delete(c.clients, k) + } + } + } + if isLoopback(client) { // the proxy did not say who this is: only the shared limit applies + return c.all.take(now, totalRate, totalBurst) + } + b := c.clients[client] + if b == nil { + b = &bucket{clientBurst, now} + c.clients[client] = b + } + if !b.take(now, clientRate, clientBurst) { + return false + } + return c.all.take(now, totalRate, totalBurst) +} + +// clientAddr is the visitor as the reverse proxy saw them. Only the proxy can +// reach this program (it listens on localhost), so the header it sets is +// trusted; the last X-Forwarded-For entry is the one the proxy itself added. +func clientAddr(r *http.Request) string { + if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); ip != "" { + return ip + } + if xff := r.Header.Get("X-Forwarded-For"); xff != "" { + parts := strings.Split(xff, ",") + return strings.TrimSpace(parts[len(parts)-1]) + } + host, _, _ := net.SplitHostPort(r.RemoteAddr) + noClientHeader.Do(func() { + lg.Warn("requests arrive without X-Real-IP or X-Forwarded-For: the reverse proxy should pass the visitor's address on, " + + "otherwise creating plans and shortcodes is limited for all visitors together, not for each") + }) + return host +} + +var noClientHeader sync.Once + +func isLoopback(addr string) bool { + ip := net.ParseIP(addr) + return addr == "" || (ip != nil && ip.IsLoopback()) +} + +func tooMany() Resp { + r := errJSON(http.StatusTooManyRequests, "Too many requests, try again in a moment") + r.Header = map[string]string{"Retry-After": "5"} + return r +} + +// maintain deletes reading plans nobody has used for 90 days (the pages promise this). +func (a *App) maintain() { + for { + if a.store.stateOK { + cutoff := time.Now().Add(-seqPurgeAfter).Format(stateTimeFormat) + if res, err := a.store.db.Exec("DELETE FROM reading_tracker WHERE last_used < ?", cutoff); err == nil { + if n, _ := res.RowsAffected(); n > 0 { + lg.Info("Purged %d unused reading plans", n) + } + } + } + time.Sleep(6 * time.Hour) + } +} + +// adminOnly protects the listings of every plan id and shortcode (a plan id is +// all that is needed to move that plan). They need the KJV_SECRET_API_KEY as a +// bearer token, and do not exist when no key is set. +func (a *App) adminOnly(r *http.Request) (Resp, bool) { + if a.secretKey == "" { + return a.notFound(), false + } + token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ") + if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(a.secretKey)) != 1 { + resp := errJSON(http.StatusUnauthorized, "Invalid Authorization") + resp.Header = map[string]string{"WWW-Authenticate": "Bearer"} + return resp, false + } + return Resp{}, true +} diff --git a/internal/kjv/store.go b/internal/kjv/store.go new file mode 100644 index 0000000..882a6f8 --- /dev/null +++ b/internal/kjv/store.go @@ -0,0 +1,227 @@ +package kjv + +import ( + "database/sql" + "fmt" + "sort" + + _ "modernc.org/sqlite" +) + +// Verse is one row of the fts_kjv table. ID is book*1_000_000 + chapter*1000 + verse. +type Verse struct { + ID int64 + Book int + Chapter int + Verse int + Text string +} + +type BookInfo struct { + Title string + Category string + OTNT string + Chapters int +} + +// Store holds the companion database plus everything read from it at startup. +// The whole text is only ~4MB, so verse lookups never touch SQLite; the FTS +// index is still queried for keyword search. +type Store struct { + db *sql.DB + + verses []Verse // sorted by ID + verseIDs []int64 + chapterList []int64 // chapter keys (ID/1000), sorted and unique + bookNames []string + bookName map[int]string + bookByName map[string]int + chapterLen map[int]int + bookInfo []BookInfo + abbrevs []abbrev + + stateOK bool // reading plans and shortcodes can be written +} + +type abbrev struct { + Name string + Book int +} + +func OpenStore(path string) (*Store, error) { + dsn := "file:" + path + "?_pragma=busy_timeout(5000)" + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, err + } + s := &Store{db: db, bookName: map[int]string{}, bookByName: map[string]int{}, chapterLen: map[int]int{}} + if err := s.load(); err != nil { + db.Close() + return nil, fmt.Errorf("%s: %w", path, err) + } + s.stateOK = s.initState() == nil + return s, nil +} + +func (s *Store) Close() error { return s.db.Close() } + +func (s *Store) load() error { + rows, err := s.db.Query("SELECT b, n FROM key_english ORDER BY b") + if err != nil { + return err + } + for rows.Next() { + var id int + var name string + if err := rows.Scan(&id, &name); err != nil { + rows.Close() + return err + } + s.bookNames = append(s.bookNames, name) + s.bookName[id] = name + s.bookByName[name] = id + } + rows.Close() + if len(s.bookNames) == 0 { + return fmt.Errorf("key_english is empty") + } + + rows, err = s.db.Query("SELECT id, b, c, v, t FROM fts_kjv ORDER BY id") + if err != nil { + return err + } + for rows.Next() { + var v Verse + if err := rows.Scan(&v.ID, &v.Book, &v.Chapter, &v.Verse, &v.Text); err != nil { + rows.Close() + return err + } + s.verses = append(s.verses, v) + } + rows.Close() + if len(s.verses) == 0 { + return fmt.Errorf("fts_kjv is empty") + } + for _, v := range s.verses { + s.verseIDs = append(s.verseIDs, v.ID) + if key := v.ID / 1000; len(s.chapterList) == 0 || s.chapterList[len(s.chapterList)-1] != key { + s.chapterList = append(s.chapterList, key) + } + if v.Chapter > s.chapterLen[v.Book] { + s.chapterLen[v.Book] = v.Chapter + } + } + + rows, err = s.db.Query(`SELECT title_short, category, otnt, chapters FROM book_info ORDER BY "order"`) + if err != nil { + return err + } + for rows.Next() { + var b BookInfo + if err := rows.Scan(&b.Title, &b.Category, &b.OTNT, &b.Chapters); err != nil { + rows.Close() + return err + } + s.bookInfo = append(s.bookInfo, b) + } + rows.Close() + + rows, err = s.db.Query("SELECT a, b FROM key_abbreviations_english ORDER BY id") + if err != nil { + return err + } + for rows.Next() { + var a abbrev + if err := rows.Scan(&a.Name, &a.Book); err != nil { + rows.Close() + return err + } + s.abbrevs = append(s.abbrevs, a) + } + rows.Close() + return nil +} + +// initState creates the tables that hold user-generated data (reading plans and +// shortcodes) inside the companion database. +func (s *Store) initState() error { + for _, stmt := range []string{ + "CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)", + "CREATE TABLE IF NOT EXISTS shortcodes (shortcode TEXT PRIMARY KEY, query TEXT UNIQUE, creation_date TEXT)", + } { + if _, err := s.db.Exec(stmt); err != nil { + return err + } + } + return nil +} + +func (s *Store) BookName(id int) string { return s.bookName[id] } + +func (s *Store) FormatRef(verseID int64) string { + return fmt.Sprintf("%s %d:%d", s.BookName(int(verseID/1_000_000)), verseID/1000%1000, verseID%1000) +} + +// Range returns the verses with start <= id <= end (end <= 0 means just start). +func (s *Store) Range(start, end int64) []Verse { + i := sort.Search(len(s.verses), func(i int) bool { return s.verses[i].ID >= start }) + if end <= 0 { + if i < len(s.verses) && s.verses[i].ID == start { + return s.verses[i : i+1] + } + return nil + } + j := sort.Search(len(s.verses), func(j int) bool { return s.verses[j].ID > end }) + if i >= j { + return nil + } + return s.verses[i:j] +} + +// VerseIndex is the position of a verse id in the whole Bible, or -1. +func (s *Store) VerseIndex(id int64) int { + i := sort.Search(len(s.verseIDs), func(i int) bool { return s.verseIDs[i] >= id }) + if i < len(s.verseIDs) && s.verseIDs[i] == id { + return i + } + return -1 +} + +// FTS runs a full text query and returns the matching verses in Bible order. +func (s *Store) FTS(query string) ([]Verse, error) { + rows, err := s.db.Query("SELECT id FROM fts_kjv(?)", query) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Verse + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + if v := s.Range(id, 0); len(v) == 1 { + out = append(out, v[0]) + } + } + return out, rows.Err() +} + +type crossRef struct{ Start, End int64 } + +func (s *Store) CrossReferences(verseID int64) ([]crossRef, error) { + rows, err := s.db.Query("SELECT sv, ev FROM cross_reference WHERE vid = ? ORDER BY r DESC LIMIT 15", verseID) + if err != nil { + return nil, err + } + defer rows.Close() + var out []crossRef + for rows.Next() { + var c crossRef + if err := rows.Scan(&c.Start, &c.End); err != nil { + return nil, err + } + out = append(out, c) + } + return out, rows.Err() +} diff --git a/kjv-api-test.sh b/kjv-api-test.sh deleted file mode 100755 index 4e0c2dd..0000000 --- a/kjv-api-test.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -# run from current directory only -FILE=app.py -if test -f "$FILE"; then - uwsgi_python3 --http-socket 0.0.0.0:1612 --wsgi-file app.py --callable app --pyargv "-vvv" --stats 0.0.0.0:9192 -else - cd /opt/kjv-api - echo "WARNING: RUNNING FROM /opt/kjv-api" - uwsgi_python3 --http-socket 0.0.0.0:1612 --wsgi-file /opt/kjv-api/app.py --callable app --pyargv "-vvv" -fi diff --git a/kjv-api.sh b/kjv-api.sh deleted file mode 100755 index f696fd4..0000000 --- a/kjv-api.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -cd /opt/kjv-api -uwsgi_python3 --http-socket :1611 --wsgi-file /opt/kjv-api/app.py --callable app --stats 127.0.0.1:9191 diff --git a/kjv_api.py b/kjv_api.py deleted file mode 100644 index bdf9f54..0000000 --- a/kjv_api.py +++ /dev/null @@ -1,1009 +0,0 @@ -# kjv_api.py -from flask import ( - Flask, - request, - jsonify, -) -import random -from datetime import date # used for daily verse -import sqlite3 -import string # for literals shortcut -import json # for dictionary -import os -import socket -import json -from tools import sanity_check -from dotenv import load_dotenv -# this is python-dotenv package in pip3, 'dotenv' with pip doesn't work! -load_dotenv() - - -kjv_api = Flask(__name__, template_folder="html") -design_goals = """ - GOALS GOALS GOALS GOALS GOALS - - ACTUAL SERMON ENDPOINTS LOCATIONS ARE NOT FINAL - -This rewrite should end up with a very flat and (ideally) stateless API... -EXCEPT for integration with the sermon database, which is a separate project. -AND the semantic search, which is also a separate project. - -Later, those projects may be integrated into this one (mostly via database merge), -but for now, they are separate and communicate via sqlite files and sockets. - -# RESPONSE TYPES - -All endpoints that return VERSES should return a consistent JSON format, -an array containing verse objects with the following fields: -- bookname: the name of the book - -- A string -- book_id: the internal book_id (1-66) - -- An integer -- chapter: the chapter number - -- An integer -- verse: the verse number - -- An integer -- text: the verse text - -- A string -- verse_id: the internal verse_id - -- A string -- full_ref: the full reference in the format "book chapter:verse" - -- A string -- chapter_length: the number of verses in the chapter - -- An integer -- book_length: the number of chapters in the book - -- An integer -- definitions: the definitions of the words in the verse (if available) - -- An array of words by the key of the word, the value containing the definitions of each word - -AND if the user has a valid token and a network connection, the following fields: -- similar_verses: an array of semantically similar verses (if available) - -- An array of verse objects that condain an additional 'score' field (a float) which is how similar the verse is -- sermons: an array of sermons that reference the verse (if available) - -- An array of sermon objects that contain an additional 'score' field (a float) which is how relevant the sermon is - (this should be filtered by the user's key, so they can only see sermons they have access to) - -An example response: -[ - { - "bookname": "Genesis", - "book_id": 1, - "chapter": 1, - "verse": 1, - "text": "In the beginning God created the heaven and the earth.", - "verse_id": "01001001", - "full_ref": "Genesis 1:1", - "chapter_length": 31, - "book_length": 50, - "definitions": { - "beginning": "The first part of time; the commencement; the first part of a series or sequence; the first part of a course.", - "God": "The Supreme Being; Jehovah; the eternal and infinite spirit, the creator, and the sovereign of the universe.", - "created": "Formed; made; produced; caused to exist; brought into being.", - "heaven": "The region of the air; the higher regions or the atmosphere; the place where the celestial orbs revolve; the firmament; the sky." - }, - "similar_verses": [ - { - "bookname": "Genesis", - "book_id": 1, - "chapter": 1, - "verse": 2, - "text": "And the earth was without form, and void; and darkness was upon the face of the deep. And the Spirit of God moved upon the face of the waters.", - "verse_id": "01001002", - "full_ref": "Genesis 1:2", - "chapter_length": 31, - "book_length": 50, - "score": 0.9 - }, - { - "bookname": "Genesis", - "book_id": 1, - "chapter": 1, - "verse": 3, - "text": "And God said, Let there be light: and there was light.", - "verse_id": "01001003", - "full_ref": "Genesis 1:3", - "chapter_length": 31, - "book_length": 50, - "score": 0.8 - } - ], - "sermons": [ - { - "title": "The Creation of the World", - "source_url": "https://example.com/sermon/1", - "url": "https://example.com/sermon/1", - "preacher": "John Doe", - "score": 0.9 - }, - { - "title": "The Beginning of All Things", - "source_url": "https://example.com/sermon/2", - "url": "https://example.com/sermon/2", - "preacher": "John Smith" - "score": 0.8 - } - ] - } -] - -Access to the sermon database for lists of sermons/simple lookups are free. -Access to the sermon database for full sermon text and other advanced features is restricted by a key. -Access to the semantic search database is restricted by a key. - - -Except for an individual sermon queries (by ID/URL), sermon endpoints should return a summary of the sermon information, -not the full text of the sermon and a complete list of referenced verses. -The full text of the sermon should only be returned in an individual sermon query. - -All endpoints that return an individual sermon should return a consistent JSON format, -an array containing sermon objects with the following fields: -Sermon List Information: -- title: the title of the sermon - -- A string -- source_url: the URL of the source sermon - -- A string -- url: the URL of the cached sermon in the API - -- A string -- preacher: the name of the preacher - -- A string -- church_name: the name of the church - -- A string -- denomination: the denomination of the church - -- A string -- sermon_id (currently 'filename'): the filename of the sermon (the filename variable name should be changed to sermon_id) - -- A string -- license: the license of the sermon - -- A string -- language: the language of the sermon - -- A string -- date: the date the sermon was given - -- A string -- duration: the duration of the sermon - -- A string -- views: the number of views the sermon has received on the API - -- An integer -- likes: the number of likes the sermon has received on the API - -- An integer -- notes: the notes of the sermon provided by the preacher or church - -- A string - -Individual Sermon Information (by ID/URL) will also include the following fields: -- verses: an array of verse objects that the sermon references - -- An array of verse objects -- similar_sermons: an array of semantically similar sermons (if available) - -- An array of sermon objects -- text: the full text of the sermon - -- An array of objects containing: - -- timestamp: the timestamp of when the text was spoken in the recording - -- text: the text of the sermon at that timestamp - -- matches: an array of verse objects that the text references at that timestamp -- comments: the comments on the sermon provided by users - -- An array of objects containing: - -- user: the username of the commenter - -- comment: the comment text - -- timestamp: the timestamp of the comment - - -An example sermon list response: -[ - { - "title": "Sermon 1", - "source_url": "https://example.com/sermon/1", - "url": "https://example.com/sermon/1", - "preacher": "John Doe", - "church_name": "Community Church", - "denomination": "Non-Denominational", - "sermon_id": "1", - "license": "CC BY-NC-ND 4.0", - "language": "English", - "date": "2022-01-01", - "duration": "00:30:00", - "views": 100, - "likes": 50, - "notes": "This sermon explores the power of love in our lives and how it can transform us." - }, - { - "title": "Sermon 2", - "source_url": "https://example.com/sermon/2", - "url": "https://example.com/sermon/2", - "preacher": "Mark Smith", - "church_name": "Grace Church", - "denomination": "Baptist", - "sermon_id": "2", - "license": "CC BY-NC-ND 4.0", - "language": "English", - "date": "2022-01-01", - "duration": "00:30:00", - "views": 100, - "likes": 50, - "notes": "This sermon explores the power of love in our lives and how it can transform us." - } -] - - -An example individual sermon response: -[ - { - "title": "The Power of Love", - "source_url": "https://example.com/sermon/3", - "url": "https://example.com/sermon/3", - "preacher": "John Smith", - "church_name": "Community Church", - "denomination": "Non-Denominational", - "sermon_id": "3", - "verses": [ - { - "bookname": "John", - "book_id": 43, - "chapter": 3, - "verse": 16, - "text": "For God so loved the world, that he gave his only begotten Son, that whosoever believeth in him should not perish, but have everlasting life.", - "verse_id": "43003016", - "full_ref": "John 3:16", - "chapter_length": 50, - "book_length": 21 - }, - { - "bookname": "1 Corinthians", - "book_id": 46, - "chapter": 13, - "verse": 4, - "text": "Charity suffereth long, and is kind; charity envieth not; charity vaunteth not itself, is not puffed up.", - "verse_id": "46013004", - "full_ref": "1 Corinthians 13:4", - "chapter_length": 16, - "book_length": 21 - } - ], - "similar_sermons": [ - { - "title": "The Love of God", - "source_url": "https://example.com/sermon/4", - "url": "https://example.com/sermon/4", - "preacher": "Jane Doe" - }, - { - "title": "Love Your Neighbor", - "source_url": "https://example.com/sermon/5", - "url": "https://example.com/sermon/5", - "preacher": "Mark Johnson" - } - ], - "text": [ - { - "timestamp": "00:00:00", - "text": "Welcome to today's sermon on the power of love.", - "matches": [ - { - "bookname": "John", - "book_id": 43, - "chapter": 3, - "verse": 16, - "text": "For God so loved the world, that he gave his only begotten Son, that whosoever believeth in him should not perish, but have everlasting life.", - "verse_id": "43003016", - "full_ref": "John 3:16", - "chapter_length": 50, - "book_length": 21 - }, - { - "bookname": "1 Corinthians", - "book_id": 46, - "chapter": 13, - "verse": 4, - "text": "Charity suffereth long, and is kind; charity envieth not; charity vaunteth not itself, is not puffed up.", - "verse_id": "46013004", - "full_ref": "1 Corinthians 13:4", - "chapter_length": 16, - "book_length": 21 - } - ] - } - ], - "license": "CC BY-NC-ND 4.0", - "language": "English", - "date": "2022-01-01", - "duration": "00:30:00", - "views": 100, - "likes": 50, - "notes": "This sermon explores the power of love in our lives and how it can transform us.", - "comments": [ - { - "user": "user1", - "comment": "Great sermon!", - "timestamp": "2022-01-01 12:00:00" - }, - { - "user": "user2", - "comment": "Thank you for sharing this message.", - "timestamp": "2022-01-01 12:30:00" - } - ] - } -] - - -Sermon Endpoints: -- Add endpoint for listing all sermons - -- Endpoint name: /sermons - -- HTTP Method: GET - -- Returns: JSON object of summary information for all sermons -- Add endpoint for listing all preachers - -- Endpoint name: /preachers - -- HTTP Method: GET - -- Returns: JSON object with list of preachers and their denominations and churches and sermons -- Add endpoint for listing all churches - -- Endpoint name: /churches - -- HTTP Method: GET - -- Returns: JSON object with list of churches and their denominations and sermons and preachers -- Add endpoint for listing all denominations - -- Endpoint name: /denominations - -- HTTP Method: GET -- Add endpoint for searching sermons by scripture reference - -- Endpoint name: /sermon/search/ - -- HTTP Method: GET -- Add endpoint for searching sermons by preacher - -- Endpoint name: /sermons/preacher/ - -- HTTP Method: GET -- Add endpoint for searching sermons by church - -- Endpoint name: /sermons/church/ - -- HTTP Method: GET -- Add endpoint for searching sermons by denomination - -- Endpoint name: /sermons/denomination/ - -- HTTP Method: GET -- Add endpoint for searching sermons by filename - -- Endpoint name: /sermon/ - -- HTTP Method: GET -- Add endpoint for searching sermons by URL - -- Endpoint name: /sermon/url/ - -- HTTP Method: GET -- Add endpoint for searching sermons by keyword - -- Endpoint name: /sermon/search/ - -- HTTP Method: GET -- Add endpoint for searching sermon content by semantic search - -- Endpoint name: /sermon/search/semantic - -- HTTP Method: POST -- Add endpoint for searching sermons by random sermon - -- Endpoint name: /sermon/random - -- HTTP Method: GET -- Add endpoint for searching sermons by sermon of the day - -- Endpoint name: /sermon/votd - -- HTTP Method: GET -- Add endpoint for searching sermons by similar sermons - -- Endpoint name: /sermon/similar/ - -- HTTP Method: GET -- Add endpoint for removing sermons - -- Endpoint name: /sermon/remove/ - -- HTTP Method: DELETE -- Add endpoint for adding sermons - -- Endpoint name: /sermon/add - -- HTTP Method: POST -- Add endpoint for updating sermons - -- Endpoint name: /sermon/update/ - -- HTTP Method: PUT - - -Semantic Search improvements: -# semantic search is slow and expensive, so we need to cache results and precompute common queries -- Add endpoint for general text semantic search - -- Endpoint name: /search/semantic - -- HTTP Method: POST - -- Request Body: {"query":"myquerystring"} - -- Result: JSON object with search results as verse objects -- Add endpoint for searching by similar verses - -- Endpoint name: /search/similar - -- HTTP Method: POST - -- Request Body: {"verse_id":"myverseidstring"} - -- Result: JSON object with search results as verse objects -- Cache semantic search results for faster response times (semantic search is EXPENSIVE and SLOW) - -- possibly use semantic search on cached results for faster response times, - and if the cache is empty, use the semantic search on the full text -- Use context windowing to improve semantic search results (later improvement) -- Precompute semantic search results (including individual keywords) for common queries (later improvement) - - -General API Improvements (see old app.py for more details): -(These should be implemented on the client and locally served but we still need to provide a standard) -(and can provide a fallback that just serves the json) -These can probaby call from the reference implementation we will make freely available. -- Add endpoint for serving the API documentation - -- Endpoint name: /docs - -- HTTP Method: GET - -- Result: HTML page with API documentation -- Add endpoint for searching by verse_id - -- Endpoint name: /verse_id/ - -- HTTP Method: GET -- Add endpoint for searching by verse reference -- Add endpoint for searching by keyword -- Add endpoint for searching by random verse -- Add endpoint for searching by verse of the day -- Add endpoint for searching by definition -- Add endpoint for searching by semantically -- Add endpoint for searching by similar verses - - -Key Control Implementation: -(All actions require a valid token, which is stored in the database and can be revoked at any time) -(Admins can generate new keys, revoke keys, and view usage metrics by giving a new user a unique single use token) -(Keys contain a prefix that identifies the key group subscriber, and a suffix that is a random string of characters per user of the key group) -(this is psuedo-secret and can be sniffed, but will rotate at a regualr intervals) -(This should allow for subscriber teirs, and rate limiting per key) -- Add endpoint for key generation - -- Endpoint name: /key - -- HTTP Method: POST - -- Reuqest Body: {"action":"generate", "token":"mytokenstring"} - -- Result: JSON object with new key - -- Result Example: {"key":"mykeystring"} -- Add endpoint for key revocation - -- Endpoint name: /key - -- HTTP Method: DELETE - -- Request Body: {"action":"revoke", "token":"mytokenstring"} - -- Result: JSON object with success message - -- Result Example: {"success":"Key revoked"} -- Add endpoint for key validation - -- Endpoint name: /key - -- HTTP Method: POST - -- Request Body: {"action":"validate", "token":"userkeystring"} - -- Result: JSON object with validation status - -- Result Example: {"valid":true} -- Add endpoint for listing active keys and their usage metrics - -- Endpoint name: /keys - -- HTTP Method: GET - -- Request Body: {"action":"list", "token":"mytokenstring"} - -- Result: JSON object with list of keys and their usage metrics (defined later) - -- Result Example: {"keys":[{"key":"mykeystring","usage":100},{"key":"myotherkeystring","usage":50}]} - - -Database Improvements: -- Merge all databases into one -- current databases are: - -- KJV Bible database - -- Sermon database - -- Semantic search database -- Evaluate using postgresql instead of sqlite - - -Sermon Analysis and Feature Improvements: -- Implement sentiment analysis on sermon text (is it positive, negative, encouraging, neutral?) -- Implement keyword extraction from sermon text (will be needed as database grows) -- Implement topic modeling on sermon text (what is the sermon about?) -- Implement named entity/event recognition on sermon text - - people, places, organizations, dates, etc. - - current events, historical events, etc. -- Implement sermon text summarization (MAYBE NOT! Could be accidentally heretical!) -- Implement sermon text translation (MAYBE NOT! Could be accidentally heretical!) -- Implement sermon text language detection -- Store all analysis results in the sermon database -- Store a copy of the sermon source file on local storage for user playback -- Store a copy of the sermon audio file on local storage for user playback (low bandwidth/offline mode) -- Quality control on sermon text and metadata - -- is it accurate to the audio? (transcription) - -- are there any misspellings? - -- are there any formatting issues? - -- are there any missing fields? - -- who is responsible for quality control? (the requester, the API host (us), the sermon provider?) - - -Abuse Prevention: -- Implement abuse detection for high volume requests -- Implement rate limiting -- Implement key-based access control for compute-intensive operations -- Implement notification system for abuse detection -- spam, DDoS, etc (keys should help with this) - - -Logging and Metrics: -- Implement logging for all errors -- Implement logging for all queries -- we want to know what users are doing, what they're searching for, and what they're not finding -- we want to know what errors are happening, and when they're happening -- we want to know how many users are using the service, and how often - - -General Code Improvements: -- Convert this outline to an OpenAPI schema for the API (Critical) - -""" - -openapi_yaml = """ ->:^) -""" - -SERMON_DB = os.getenv("KJV_SERMON_DB") -# currently it would require a differnt instance of the API to run with a different sermon database -# but these can be rolled into one database later - -SECRET_API_KEY = os.getenv("KJV_SECRET_API_KEY") # this should end up being in the database -# we want to be able to revoke keys, and have multiple keys for different users -# this should be moved to the database, and the database should be able to handle multiple keys -# and revoke them as needed. This is currently just for testing purposes. - -# makes data fetched from db easier to process -# might ditch this if we move to postgres -def dict_factory(cursor, row): - fields = [column[0] for column in cursor.description] - return {key: value for key, value in zip(fields, row)} - -# prevent editor complaints about these being undefined -kjv_cur = None -sermon_cur = None -# where we are in our sqlite db - -if __name__ == "__main__": - # single user mode, running without uwsgi, for testing - usermode: str = "Single" - kjv_cur = sqlite3.connect("data/kjv.db", check_same_thread=False).cursor() - # really as long as you aren't trying to write to the db, you can get away with this - if SERMON_DB: - sermon_con = sqlite3.connect(SERMON_DB, check_same_thread=False) - sermon_con.row_factory = dict_factory - sermon_cur = sermon_con.cursor() -else: - # you're running with uwsgi, as you should in a production environment - usermode: str = "Multi" - kjv_cur = sqlite3.connect("data/kjv.db").cursor() - if SERMON_DB: - sermon_con = sqlite3.connect(SERMON_DB) - sermon_con.row_factory = dict_factory - sermon_cur = sermon_con.cursor() - -if not kjv_cur: - exit("CRITICAL: Error connecting to KJV Database") -if not sermon_cur: - print("ERROR: Error connecting to Sermon Database") - -# this is a performance optimization, we don't want to have to query the database for this info every time -# we can just cache it in memory -cached: dict = { - "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()], - "book_names": [ - i[0] for i in kjv_cur.execute("SELECT n FROM key_english").fetchall() - ], - "chapter_list": [], - "books_chapter_lengths": {}, - "json_dictionary": json.load(open("data/1828_Webster_KJV.json", "r")), -} - -# Fill the chapter list with unique chapters for seeking back and forward by index. -# it is annoying to do it this way but makes it easier to seek back and forward by index -# to be honest I forgot why I did it this way, but it works. -[ - cached["chapter_list"].append(int(str(chap)[:-3])) - for chap in cached.get("verse_ids") - if int(str(chap)[:-3]) not in cached["chapter_list"] -] -cached["chapter_list"].sort() - -for book in range(1, 67): - cached["books_chapter_lengths"][book] = kjv_cur.execute( - "SELECT c FROM fts_kjv where b = ? ORDER BY id DESC LIMIT 1;", (book,) - ).fetchone()[0] - -# this was used for sharing verse groups with others, but will become a problem -# with multiple independent instances of the API by local users. -# It may be possible to share verse groups between users in the future by -# having it be a free service, but for now it's disabled and not in the API spec. -def generate_short_id(): - remain = random.randrange(614_656, 17_210_367) # resolves to length of 5 - chars = "0123456789ACEFHJKLMNPRTUVWXY" - length = len(chars) - result = "" - while remain > 0: - pos = remain % length - remain = remain // length - result = chars[pos] + result - return result - -# helper functions... -def lookup_bookname(book_id): - result = kjv_cur.execute( - "SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone() - if result: - result = result[0] - return result - - -def lookup_book_id(bookname) -> int: - result = kjv_cur.execute( - "SELECT b FROM key_english WHERE n = ?;",(bookname,)).fetchone() - if result: - result = result[0] - return result - -# fts is full text search, it's a virtual table that allows us to search the text of the bible quickly -# without having to scan the entire database, it's a performance optimization, but it's a feature -# of sqlite that we may have to find a workaround for if we move to postgres -def lookup_fts(kwds): - return kjv_cur.execute("SELECT * FROM fts_kjv(?)", (kwds,)).fetchall() - - -def lookup_by_verse_id(start_verse, end_verse=None) -> list: - # This is an internal function - we don't use placeholders because SQL injection should be impossible here - if not end_verse: - results = kjv_cur.execute( - f"SELECT * FROM fts_kjv WHERE id = {start_verse};" - ).fetchall() - else: - results = kjv_cur.execute( - f"SELECT * FROM fts_kjv WHERE id BETWEEN {start_verse} AND {end_verse};" - ).fetchall() - return results - -def get_sermons_by_verse_id(verse_id) -> list: - if not sermon_cur: - return - return sermon_cur.execute(f"select * from verse_refs where verse_id={verse_id}").fetchall() - -# this gives us a database key for a verse by book, chapter, and verse, because of the way the database is structured -def get_verse_id(book_id, chapter: str, verse: str) -> str: - # verse_id is 00000000 (00 000 000) (book + chapter + verse) - chapterid = chapter.zfill(3) - verseid = verse.zfill(3) - if verseid == "000": - verseid = "001" # Avoid verse_ids list IndexErrors on :- searches - return str(book_id) + chapterid + verseid - -# this gives us a human readable reference from a verse_id -def get_ref_from_verse_id(verse_id): - # verse_id is 00000000 (00 000 000) (book + chapter + verse) - verse_id = str(verse_id).zfill(8) - book_id = verse_id[:2] - chapter = verse_id[2:5] - verse = verse_id[5:8] - bookname = kjv_cur.execute("SELECT n FROM key_english WHERE b = ?;",(book_id,)).fetchone()[0] - return bookname + " " + chapter + ":" + verse - -# this is for parsing a reference from a user query, it is complex but catches a lot of edge cases -# and makes it easier to search the database, as well as improving security by cleaning the input -def ref_input_cleaning(ref): - valid_chars = (string.ascii_letters + string.digits + ":-,") - ref = ref.replace(" ", "") - if ref.count(":") != 1: - if len(ref) == 0: - return "Invalid Query" - if ref[-1] in string.digits: - ref += ":-" - else: - ref += "1:-" - if ref.endswith(":"): - ref += "-" - if ref.count("-") > 1: - return - if not ref[-1].isdigit() and ref[-1] != "-": - return - ref = list(ref) - # preserve book number if we can guess the first - # character is a number and 3nd is a letter - if ref[0].isdigit() and ref[1].isalpha(): - prefix = ref.pop(0) + " " - else: - prefix = "" - # clean the rest of the reference - cleaned_ref = [] - for character in ref: - if character in valid_chars: - cleaned_ref.append(character) - # Was there a problem? - if len(cleaned_ref) == 0: - return - else: - return "".join(cleaned_ref) - -# similar to ref_input_cleaning, but for search queries, it is more permissive because -# sqlite3 has search operators that can be used to search the fts table -def search_input_cleaning(query): - valid_chars = string.ascii_letters + string.digits + "+^* " - cleaned_query = [] - for character in query: - if character in valid_chars: - cleaned_query.append(character) - return "".join(cleaned_query) - -# this finds books if someone puts in a partial book name -def ref_parse_book(maybe_book): - for book in reversed(cached["book_names"]): - if book.startswith(maybe_book): - return book - return False - -# this is a helper function for the semantic search, it sends a query to the semantic search server -# and returns the results. This is a separate project that will be integrated into this one later. -def request_semantic_search(query): - HOST = 'localhost' - PORT = 1613 - data = {"query": query} - - data_string = json.dumps(data).encode() - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((HOST, PORT)) - s.sendall(data_string) - data = b'' - while True: - part = s.recv(1024) - if not part: - break - data += part - return json.loads(data.decode()) - -# this explains the semantic search endpoint for developers -@kjv_api.get("/natural") -def explain_natural(): - res = 'Submit a POST request to this endpoint with the following params in the body: ' - res += '{"token": "mytokenstring", "query" : "myquerystring"}' - return jsonify({'error':res}) - -# this actually does the semantic search, it is a POST request because it is more secure -# and we require a token to access it, this is a feature of the API that will be expanded -# especially when we integrate the semantic search project into this one. -@kjv_api.post("/natural") # needs valid token and query in request body -def do_semantic_search(): - request_data = request.get_json() - if request_data is None: - # Handle error: No JSON data sent - return jsonify({'error': 'Invalid request format'}), 400 - - token_string = request_data.get('token') - query_string = request_data.get('query') - if token_string is None: - # Handle error: Missing token in request - return jsonify({'error': 'Missing token in request body'}), 401 - - if token_string != SECRET_API_KEY: - return jsonify({'error': "Invalid Authorization"}) - - result = request_semantic_search(query=query_string) - return jsonify(result) - -# this returns an internal verse_id by english scripture reference (1John5:7) -@kjv_api.get("/ref/") -def get_single_reference_verse_id(ref:str): - ref = ref.split(",")[0] - ref = ref.split("-")[0] - # only want the first single ref, not a range, and not a group. - ref = ref_input_cleaning(ref) - maybe_chapter: str = ref.split(":")[0][3:] - ref_chapter: str = "" - for maybedigit in maybe_chapter: - if maybedigit.isdigit(): - ref_chapter += maybedigit - ref_verse = ref.split(":")[1] - book_str: str = ref_input_cleaning(ref, string.ascii_letters) - ref_bookname: str = ref_parse_book(book_str) - ref_book_id: int = lookup_book_id(ref_bookname) - if not ref_book_id: - return - return get_verse_id(ref_book_id, ref_chapter, ref_verse) - -# this finds sermons that contain an english scripture reference (1John5:7) -@kjv_api.get("/sermon/search/") -def find_sermon_by_ref(ref:str=None): - verse_id = get_single_reference_verse_id(ref=ref) - if not sermon_cur: - return jsonify({'error':"Sermon database not found"}) - sermons = get_sermons_by_verse_id(verse_id=verse_id) - for sermon in sermons: - sermon.update({'kjv':find_by_verseid(sermon.get("verse_id"))[4]}) - return jsonify(sermons) - -# there are a lot of endpoints here, almost all of which are just database queries. Once the individial -# endpoint names are decided, these can change of course. - -@kjv_api.get("/sermons") # lists all the sermons in our database -def list_sermons(): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name,filename from verse_refs").fetchall()) - -@kjv_api.get("/sermon//verses") # show all matched info on a sermon by its filename (hash) -def list_sermon_references(filename): - return jsonify(sermon_cur.execute("select distinct verse_id,text from verse_refs where filename=?", (filename,)).fetchall()) - -@kjv_api.get("/preachers") # return a list of preachers stored in database, their denomination and church name -def list_preachers(): - return jsonify(sermon_cur.execute("select distinct denomination,preacher,church_name from verse_refs").fetchall()) - -@kjv_api.get("/churches") # return a list of churches with sermons in the database -def list_churches(): - return jsonify(sermon_cur.execute("select distinct denomination,church_name from verse_refs").fetchall()) - -@kjv_api.get("/denominations") # return a list of denominations included in the database -def list_denominations(): - return jsonify(sermon_cur.execute("select distinct denomination from verse_refs").fetchall()) - -@kjv_api.get("/sermons/preacher/") # show all sermons by a paticular preacher -def find_sermons_by_preacher(preacher): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where preacher=?", (preacher,)).fetchall()) - -@kjv_api.get("/sermons/church/") # show all sermons by a paticular church -def find_sermons_by_church(church_name): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where church_name=?", (church_name,)).fetchall()) - -@kjv_api.get("/sermons/denomination/") # show all sermons by a paticular denomination -def find_sermons_by_denomination(denomination): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where denomination=?", (denomination,)).fetchall()) - -@kjv_api.get("/sermon/") # show general information about a sermon by filename (hash) -def find_sermons_by_filename(filename): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where filename=?", (filename,)).fetchall()) - -@kjv_api.get("/sermon/url/") # show general information about a sermon by url -def find_sermons_by_url(url): - return jsonify(sermon_cur.execute("select distinct title,url,denomination,preacher,church_name from verse_refs where url=?", (url,)).fetchall()) - -# this is in case someone searches for a verseid maliciously >:^) -def clean_verseid(verse_id): - verse_id = str(verse_id) - clean_verseid = '' - for char in verse_id: - if char in string.digits: - clean_verseid += char - return clean_verseid - - -# i do not remember why this is done this way, I was probably tired -def find_by_verseid(verse_id): - return lookup_by_verse_id(verse_id)[0] - -# I think this just returns a singular verse object -# as the API spec is changing, this will need to return an array of 1 verse object. -@kjv_api.get("/verse_id/") # return the database entry of an internal verse_id -def web_verseid(verse_id): - clean_verse_id = clean_verseid(verse_id=verse_id) - return jsonify(lookup_by_verse_id(clean_verse_id)[0]) - -# This used to be used for a direct search by word in the v1 API, but needs to change for v2 -def define(word=''): - # might have issues with case-matching - word = word.strip() - if not word: - return {'error':'No definition word provided'} - if not word in cached.get("json_dictionary").keys(): - return {'error':'Definition not found'} - return {word: cached.get("json_dictionary")[word]} - -# this checks for a valid request, and needs to check for a valid -# key in the database, it is a security feature -def is_valid_request(): - request_data = request.get_json() - if request_data is None: - return {'error': 'Invalid request format'} - token_string = request_data.get('token') - query_string = request_data.get('query') - mode_string = request_data.get('mode') - if token_string is None: - return {'error': 'Missing token in request body'} - if token_string != SECRET_API_KEY: - #TODO Replace SECRET_API_KEY with a list of valid keys in database - return {'error': "Invalid token"} - if query_string is None: - return {'error': 'Missing query parameter in request body'} - if mode_string is None: - return {'error': 'Missing mode parameter in request body'} - return {'success':"Authorized"} - - -# this is used for retrieving scripture by reference, it is a core feature of the API that -# is supposed to be implemented in the reference implementation, but is here for testing -# and development purposes. It is a complex function that handles a lot of edge cases. -# This will handle multiple references, and return an array of verse objects, as well -# as verse ranges and entire chapters. -# For the reference implementation, called functions should be simplified to -# make it easier to implement in other languages. -def get_kjv_verses_by_ref(ref=None, get_chapter=False): - if not ref: - return {'error':'Must provide reference query'} - if ref.endswith(","): - ref = ref[:-1] - refs: list = [] - for r in ref.split(","): - r_ = ref_input_cleaning(r.strip()) - if r_: - refs.append(r_) - if not refs: - return {'error':'Invalid Query'} - verse_results: list = [] - # check if the search is for multiple refs - for ref in refs: - if len(ref) < 2: - continue - maybe_chapter: str = ref.split(":")[0][3:] - ref_chapter: str = "" - for maybedigit in maybe_chapter: - if maybedigit.isdigit(): - ref_chapter += maybedigit - ref_verse = ref.split(":")[1] - if ref.__contains__("-") or get_chapter: - if ref.endswith("-") or get_chapter: - start_verse = ref_verse.split("-")[0] - end_verse = "500" # just get all the verses if it ends with a dash - else: - start_verse, end_verse = ( - ref_verse.split("-")[0], - ref_verse.split("-")[1], - ) - else: - start_verse = ref_verse - end_verse = None - end_verse_id = None - book_str: str = ref_input_cleaning(ref, string.ascii_letters) - ref_bookname: str = ref_parse_book(book_str) - ref_book_id: int = lookup_book_id(ref_bookname) - if not ref_book_id: - return {'error':'Invalid Query'} - maximum_chapters: int = cached.get("books_chapter_lengths").get(ref_book_id) - if int(ref_chapter) > maximum_chapters: - ref_chapter = str(maximum_chapters) - start_verse_id = get_verse_id(ref_book_id, ref_chapter, start_verse) - if end_verse: - end_verse_id = get_verse_id(ref_book_id, ref_chapter, end_verse) - results = lookup_by_verse_id(start_verse_id, end_verse_id) - for result in results: - verse_results.append(result) - response_list = scripture_response(verse_results) - return response_list - -# just packages the verse object into a dictionary for the API response -def response_dict(scripture): - verse = { - "bookname": lookup_bookname(scripture[1]), - "chapter": scripture[2], - "verse": scripture[3], - "text": scripture[4], - } - return verse - -# this packages a group of verse objects into an array of dictionaries for the API response -# this is mostly used for the get_kjv_verses_by_ref function and other reference implementation functions -def scripture_response(scriptures): - response = [] - for scripture in scriptures: - response.append(response_dict(scripture)) - return response - - -# might be good to implement detection of malicious endpoint 'testing' -# we don't currently have any secret endpoints, but may in the future -@kjv_api.errorhandler(404) -def page_not_found(e): - return jsonify({"error": "Invalid api endpoint"}) - -@kjv_api.get("/") # index -def idx(): - return jsonify({'error':'Nothing at this top level'}) - -# believe it or not, this is the most important endpoint in the API -# additionally, normal random.choice isn't random enough! -# This will be retained as a single verse object at this endpoint for backwards compatibility -@kjv_api.get("/random") # truely random Bible verse -def random_verse(): - rand_id = random.SystemRandom().choice(cached["verse_ids"]) - scripture = lookup_by_verse_id(rand_id) - response_list = scripture_response(scripture) - return jsonify(response_list) - -# same story as the random verse, but this is for the verse of the day and has a seed -@kjv_api.get("/votd") # today's Bible verse -def verse_of_the_day(): - today = int(str(date.today()).replace("-", "")) - random.seed(today) - scripture = lookup_by_verse_id(random.choice(cached["verse_ids"])) - return jsonify(scripture_response(scripture)) - - -def kjv_keyword_search(keywords): - if keywords: - cleaned_keywords = search_input_cleaning(keywords) - if not cleaned_keywords: - return {'error':'Cleaning of search parameters returned nothing'} - else: - return {'error':'No search parameters supplied'} - results = lookup_fts(cleaned_keywords) - response_list = scripture_response(results) - return response_list - -# if we run this in single user mode, we can test the API by running it locally -# but we will not have the security features of uwsgi, and we will not have the -# performance features of uwsgi. This is just for testing and development purposes. -if __name__ == "__main__": - print("Operating in single user debug mode.") - # port 1612 is our test port, in production we will use 1611 because that's - # the year the KJV was published! - kjv_api.run(host="0.0.0.0", port=1612, debug=True, threaded=True) diff --git a/logs/.gitkeep b/logs/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/logs/.include b/logs/.include deleted file mode 100644 index e69de29..0000000 diff --git a/scripts/kjv-api-test.sh b/scripts/kjv-api-test.sh new file mode 100755 index 0000000..d2f9999 --- /dev/null +++ b/scripts/kjv-api-test.sh @@ -0,0 +1,11 @@ +#!/bin/sh +# Builds the program and runs it against a throwaway copy of the database on port 1612. +# Reads page/script edits from assets/ each time (loads them into the copy). +set -e +cd "$(dirname "$0")/.." +make build +dir=$(mktemp -d "${TMPDIR:-/var/tmp}/kjv-test.XXXXXX") +cp db/kjv.db "$dir/kjv.db" +bin/kjv-api import-assets -db "$dir/kjv.db" assets +echo "test database: $dir/kjv.db" +exec bin/kjv-api -db "$dir/kjv.db" -addr 127.0.0.1:1612 diff --git a/scripts/kjv-api.sh b/scripts/kjv-api.sh new file mode 100755 index 0000000..21ecaf2 --- /dev/null +++ b/scripts/kjv-api.sh @@ -0,0 +1,6 @@ +#!/bin/sh +# Runs the built program on localhost port 1611, building dist/ first if needed. +# (The user service, dist/kjv-api.service, runs a copy in ~/kjv-api on port 1611 instead.) +cd "$(dirname "$0")/.." || exit 1 +[ -f dist/kjv-api ] && [ -f dist/kjv.db ] || make dist || exit 1 +exec dist/kjv-api -db dist/kjv.db -addr 127.0.0.1:1611 diff --git a/tools/fuzz_refs.py b/tools/fuzz_refs.py deleted file mode 100644 index 091f1c5..0000000 --- a/tools/fuzz_refs.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Fuzzes /find with many ways of typing a reference and checks the verses that come back. - -Run from the project root: python3 tools/fuzz_refs.py [sample_size] -Expected verse ids come straight from data/kjv.db, not from the app. -""" -import sys, re, random, sqlite3, itertools, json -sys.path.insert(0, ".") # run from the project root -import logging -import app as a -a.app.testing = True -client = a.app.test_client() - -db = sqlite3.connect("data/kjv.db") -BOOKS = [r[0] for r in db.execute("select n from key_english order by b")] -ids = {} # (book, chap) -> [(verse, id)] -for vid, b, c, v in db.execute("select id,b,c,v from fts_kjv"): - ids.setdefault((int(b), int(c)), []).append((int(v), int(vid))) -maxch = {b: max(c for (bb, c) in ids if bb == b) for b in range(1, 67)} - -NAME_VARIANTS = { - 1: ["Gen", "Ge", "Gn"], 2: ["Exod", "Ex", "Exo"], 3: ["Lev", "Lv"], 4: ["Num", "Nm", "Nu"], - 5: ["Deut", "Dt", "Deu"], 6: ["Josh", "Jos"], 7: ["Judg", "Jdg", "Jg"], 8: ["Ru"], - 9: ["1 Sam", "1 Sm", "1Sa", "I Samuel", "1st Samuel", "First Samuel"], - 10: ["2 Sam", "2 Sm", "2Sa", "II Samuel", "2nd Samuel", "Second Samuel"], - 11: ["1 Kgs", "1 Ki", "1Kin", "I Kings", "First Kings"], 12: ["2 Kgs", "2 Ki", "II Kings", "Second Kings"], - 13: ["1 Chron", "1 Chr", "1Ch", "I Chronicles", "First Chronicles"], 14: ["2 Chron", "2 Chr", "II Chronicles"], - 15: ["Ezr"], 16: ["Neh", "Ne"], 17: ["Est", "Esth"], 18: [], 19: ["Ps", "Psa", "Psalm", "Pslm"], - 20: ["Prov", "Pr", "Prv"], 21: ["Eccl", "Ecc", "Eccles", "Ecclesiasts"], - 22: ["Song", "Song of Songs", "Songs of Solomon", "Song of Sol", "SOS", "Canticles", "SongofSolomon"], - 23: ["Isa", "Is"], 24: ["Jer", "Je"], 25: ["Lam"], 26: ["Ezek", "Eze", "Ezk"], 27: ["Dan", "Dn"], - 28: ["Hos", "Ho"], 29: ["Joe", "Jl"], 30: ["Am"], 31: ["Obad", "Ob"], 32: ["Jon", "Jnh"], 33: ["Mic"], - 34: ["Nah", "Na"], 35: ["Hab"], 36: ["Zeph", "Zep"], 37: ["Hag", "Hg"], 38: ["Zech", "Zec"], 39: ["Mal"], - 40: ["Matt", "Mt", "Mat"], 41: ["Mk", "Mar"], 42: ["Lk", "Luk"], 43: ["Jn", "Joh"], 44: ["Act"], - 45: ["Rom", "Ro"], 46: ["1 Cor", "1Co", "I Corinthians", "First Corinthians", "1st Corinthians"], - 47: ["2 Cor", "2Co", "II Corinthians", "Second Corinthians"], 48: ["Gal", "Ga"], 49: ["Eph"], - 50: ["Phil", "Php", "Philip"], 51: ["Col"], 52: ["1 Thess", "1 Th", "1Thes", "I Thessalonians"], - 53: ["2 Thess", "2 Th", "II Thessalonians"], 54: ["1 Tim", "1 Ti", "I Timothy", "First Timothy"], - 55: ["2 Tim", "2 Ti", "II Timothy"], 56: ["Tit", "Ti"], 57: ["Phlm", "Phm", "Philem"], 58: ["Heb"], - 59: ["Jas", "Jam"], 60: ["1 Pet", "1 Pe", "1Pt", "I Peter", "First Peter"], 61: ["2 Pet", "2 Pe", "II Peter"], - 62: ["1 Jn", "1 Jo", "1Joh", "I John", "First John", "1st John"], 63: ["2 Jn", "2 John", "II John"], - 64: ["3 Jn", "3 John", "III John"], 65: ["Jud"], 66: ["Rev", "Re", "Revelations", "Revelation of John"], -} - -def name_forms(b): - full = BOOKS[b - 1] - forms = {full, full.lower(), full.upper(), full.replace(" ", ""), full.lower().replace(" ", "")} - forms |= set(NAME_VARIANTS.get(b, [])) - forms |= {f.lower() for f in NAME_VARIANTS.get(b, [])} - return sorted(forms) - -def cases(): - rnd = random.Random(1611) - for b in range(1, 67): - for nm in name_forms(b): - ch = rnd.randint(1, maxch[b]) - vs = ids[(b, ch)] - v1 = rnd.choice(vs)[0] - v2 = min(vs[-1][0], v1 + rnd.randint(1, 3)) - chap_ids = [i for _, i in vs] - single = [i for v, i in vs if v == v1] - rng = [i for v, i in vs if v1 <= v <= v2] - key = a.normalize_book_text(nm) - if key in a.BOOK_CANONICAL or key in a.BARE_BOOK_EXTRA: # abbreviations alone stay keyword searches - for sp in ("", " ", " "): - yield (f"{nm}{sp}", [i for _, i in ids[(b, 1)]], "bare") # book alone -> chapter 1 - for fmt, exp, kind in ( - (f"{nm} {ch}", chap_ids, "chapter"), - (f"{nm}{ch}", chap_ids, "chapter-nospace"), - (f"{nm} {ch}:{v1}", single, "verse"), - (f"{nm}{ch}:{v1}", single, "verse-nospace"), - (f" {nm} {ch} : {v1} ", single, "verse-spaced"), - (f"{nm} {ch}.{v1}", single, "verse-dot"), - (f"{nm} {ch} {v1}", single, "verse-space-sep"), - (f"{nm} {ch}:{v1}-{v2}", rng, "range"), - (f"{nm} {ch}:{v1} - {v2}", rng, "range-spaced"), - (f"{nm} {ch}:{v1}–{v2}", rng, "range-endash"), - (f"{nm.upper()} {ch}:{v1}", single, "verse-upper"), - (f"{nm} {ch}:{v1}-{v2}\t", rng, "range-trailing-tab"), - ): - if v2 > v1 or kind not in ("range", "range-spaced", "range-endash", "range-trailing-tab"): - yield fmt, exp, kind - -def verse_ids_in(html): - return [int(x) for x in re.findall(r'data-verse-id="(\d+)"', html)] - -def run(limit=None, show=25): - allcases = list(cases()) - if limit: - random.Random(7).shuffle(allcases); allcases = allcases[:limit] - fails = {} - total = 0 - for q, exp, kind in allcases: - total += 1 - try: - r = client.get("/find", query_string={"kw": q, "view": "rich"}) - got = verse_ids_in(r.get_data(as_text=True)) if r.status_code == 200 else None - except Exception as e: - got = None - ok = got is not None and got[:len(exp)] == exp and (kind.startswith("range") or kind.startswith("verse") or kind == "chapter" or True) and (len(got) == len(exp) if kind != "bare" else got[:len(exp)] == exp) - if not ok: - fails.setdefault(kind, []).append((q, len(exp), None if got is None else len(got))) - nfail = sum(len(v) for v in fails.values()) - print(f"{total} cases, {nfail} failed ({100*nfail//max(total,1)}%)") - for k, v in sorted(fails.items(), key=lambda kv: -len(kv[1])): - print(f" {k}: {len(v)} e.g. {v[:3]}") - return fails - -if __name__ == "__main__": - logging.disable(logging.CRITICAL) - run(int(sys.argv[1]) if len(sys.argv) > 1 else None) diff --git a/uninstall.sh b/uninstall.sh deleted file mode 100755 index c8b9765..0000000 --- a/uninstall.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# Check for root privileges -if [[ $EUID -ne 0 ]]; then - echo "This script requires root privileges. Please run with sudo." - exit 1 -fi - -echo "Stopping and disabling kjv-api service..." -systemctl stop kjv-api.service -systemctl disable kjv-api.service - -echo "Removing kjv-api systemd service file..." -rm /etc/systemd/system/kjv-api.service - -echo "Removing kjv-api directory and user..." -rm -rf /opt/kjv-api -userdel kjv - -echo "Uninstall complete." -echo "Be sure to remove your webserver configuration file."