claude go reimplement
This commit is contained in:
tyler committed
1 parent
4bcb924158
commit
f523c79618
83 files changed
+3714
-3454
No files matched your search
@@ -1,3 +0,0 @@
|
||||
KJV_SERMON_DB="./data/sermons.db"
|
||||
# this does not do anything unless sermon_analyzer.py is running in tcp socket mode as a separate process
|
||||
KJV_SECRET_API_KEY="change-me"
|
||||
+14
-5
@@ -1,6 +1,15 @@
|
||||
logs/*.log
|
||||
*__pycache__*
|
||||
data/shortcodes.db
|
||||
data/seq.db
|
||||
data/sermons.db
|
||||
# build output: dist/ holds what gets deployed, the unit and proxy configs are
|
||||
# tracked, the two built files (and the database journal of a local run) are not
|
||||
/bin/
|
||||
/dist/kjv-api
|
||||
/dist/kjv.db
|
||||
/dist/kjv.db-*
|
||||
|
||||
# local settings, may hold KJV_SECRET_API_KEY
|
||||
.env
|
||||
*.env
|
||||
|
||||
# tools/
|
||||
__pycache__/
|
||||
|
||||
*.swp
|
||||
-55
@@ -1,55 +0,0 @@
|
||||
|
||||
Tested with Dependancy:Version
|
||||
=======================
|
||||
python3:3.10.6-1~22.04
|
||||
python3-flask:2.0.1-2ubuntu1
|
||||
sqlite3:3.37.2-2ubuntu0.1
|
||||
libhunspell-dev:1.7.0-4build1
|
||||
python3-hunspell:0.5.5-2build6
|
||||
uwsgi-core:2.0.20-4
|
||||
uwsgi-plugin-python3:2.0.20-4
|
||||
apache2:2.4.52-1ubuntu4.2
|
||||
python3-requests:2.25.1+dfsg-2
|
||||
|
||||
However, most up to date packages should be fine.
|
||||
|
||||
Installation
|
||||
======
|
||||
This software expects to be installed at /opt/kjv-api/ (for uwsgi use on a reverse proxy)
|
||||
If you change this installation directoy, make sure kjv-api.sh has its path updated.
|
||||
This installation method is intended to play nice among other Virtual Hosts
|
||||
|
||||
0. Install dependancies listed at the head of this document.
|
||||
- I used ubuntu repositories, but many can be installed via pip in a venv. it
|
||||
would probably be a better way than with system repositories. uwsgi however is
|
||||
a system package, I'm pretty sure.
|
||||
- If you use pip/venv, you will need to modify the kjv-api.sh script to activate
|
||||
the venv before executing it, I'm pretty sure.
|
||||
- You may substitute the webserver for one of your choice, as long as it's capable
|
||||
of reverse proxy. Do not run it with the flask development server or serve directly
|
||||
as a uswgi thing.
|
||||
|
||||
1. For security, create a new user with the name 'kjv'
|
||||
- sudo useradd -r -s /bin/false kjv
|
||||
|
||||
2. Change ownership of the /opt/kjv-api/ folder to the kjv user
|
||||
- sudo chmod -R kjv:kjv /opt/kjv-api/
|
||||
|
||||
3. Test the software runs by running kjv-api.sh as a non-root user (preferably as kjv user)
|
||||
and that you are able to query it locally.
|
||||
- term1. sudo -Hu kjv ./kjv-api.sh
|
||||
- term2. curl -XGET 127.0.0.1:1611/random
|
||||
|
||||
4. If you intend to set up the software as a persistant service, copy kjv-api.service from
|
||||
the install folder to /etc/systemd/system/kjv-api.service if you're using systemd.
|
||||
- sudo cp /opt/kjv-api/install/kjv-api.service /etc/systemd/system/
|
||||
- sudo systemctl enable kjv-api.service
|
||||
- sudo systemctl start kjv-api.service
|
||||
|
||||
5. Configure your webserver as a reverse proxy to point to 127.0.0.1:1611 for where you
|
||||
wish to have the api hosted.
|
||||
- A sample configuration for apache is provided in the install folder.
|
||||
- It is up to you to use https or not, but if you know nginx or haproxy you should be able
|
||||
to set up local proxying to the WSGI server that runs the flask.
|
||||
- There are WSGI modules for apache avaliable, which may be better for larger scaled
|
||||
deployments, but http proxying to the local uwsgi served flask works fine for smaller uses.
|
||||
@@ -0,0 +1,28 @@
|
||||
# Builds the two-file distribution: dist/kjv-api (static binary) and dist/kjv.db
|
||||
# (the Bible, every page/script/style/data file, plus runtime tables and logs).
|
||||
# dist/ also holds the user service and the nginx/apache configs, which are kept in git.
|
||||
|
||||
GO ?= go
|
||||
SOURCES := $(shell find cmd internal -name '*.go') go.mod go.sum
|
||||
|
||||
.PHONY: all build dist test clean
|
||||
all: dist
|
||||
|
||||
build: bin/kjv-api
|
||||
|
||||
bin/kjv-api: $(SOURCES)
|
||||
CGO_ENABLED=0 $(GO) build -trimpath -ldflags="-s -w" -o $@ ./cmd/kjv-api
|
||||
|
||||
# kjv.db is a fresh copy of db/kjv.db with everything under assets/ loaded into it
|
||||
dist: bin/kjv-api db/kjv.db $(shell find assets -type f)
|
||||
mkdir -p dist
|
||||
cp bin/kjv-api dist/kjv-api
|
||||
cp db/kjv.db dist/kjv.db
|
||||
bin/kjv-api import-assets -db dist/kjv.db assets
|
||||
@ls -l dist
|
||||
|
||||
test:
|
||||
$(GO) vet ./... && $(GO) test ./...
|
||||
|
||||
clean:
|
||||
rm -rf bin dist/kjv-api dist/kjv.db
|
||||
+190
-59
@@ -4,6 +4,9 @@ King James Version web API is a machine and human callable service to
|
||||
query the King James Bible and provides useful features, such as random
|
||||
verses and a quote of the day.
|
||||
|
||||
It is a single dependancyless program (kjv-api) and its companion sqlite3
|
||||
database (kjv.db). Those two files are the entire distribution.
|
||||
|
||||
|
||||
EXTERNAL DATA
|
||||
=============
|
||||
@@ -11,79 +14,207 @@ The software uses a sqlite3 database as external data dependancy, saved locally
|
||||
and modified to remove false Bible translations (included)
|
||||
https://github.com/scrollmapper/bible_databases
|
||||
|
||||
For software dependancies see INSTALL.txt
|
||||
kjv.db also holds everything else the program serves, so there is nothing else
|
||||
to copy next to it: the pages, scripts and styles (the assets table), the 1828
|
||||
Webster dictionary, the spellchecking word list, users' reading plans and
|
||||
shortcodes, and the logs.
|
||||
|
||||
|
||||
INSTALLATION
|
||||
============
|
||||
See INSTALL.TXT
|
||||
Nothing needs to be installed to run it: no runtime, no packages, no root.
|
||||
There are two ways to run it, pick one. Both start from the dist folder:
|
||||
kjv-api - the program
|
||||
kjv.db - its database
|
||||
kjv-api.service - systemd user service, for way 2
|
||||
kjv-api.nginx.conf - nginx reverse proxy site, for way 2
|
||||
kjv-api.apache.conf - the same for apache, if you use that instead
|
||||
Get the two files in it by building them ($ make dist, see BUILDING) or from
|
||||
whoever gave them to you.
|
||||
|
||||
1. Run it locally, just to use it or try it
|
||||
-------------------------------------------
|
||||
$ dist/kjv-api
|
||||
Then browse to http://127.0.0.1:1611 , Ctrl-C stops it. That's all.
|
||||
It uses the kjv.db next to it, which keeps your reading plans, shortcodes and logs
|
||||
(make dist replaces dist/kjv.db with a fresh one). On another port, for example
|
||||
when the service (way 2) already has 1611 on this machine:
|
||||
$ dist/kjv-api -addr 127.0.0.1:1612
|
||||
|
||||
2. Install it as a persistent service behind nginx
|
||||
--------------------------------------------------
|
||||
It runs as a systemd user service under your own account on 127.0.0.1:1611,
|
||||
and nginx forwards your domain to it. There is no root service and no special user.
|
||||
|
||||
a. Put the program and database in ~/kjv-api, and start the service:
|
||||
$ mkdir -p ~/kjv-api ~/.config/systemd/user
|
||||
$ cp dist/kjv-api dist/kjv.db ~/kjv-api/
|
||||
$ cp dist/kjv-api.service ~/.config/systemd/user/
|
||||
$ systemctl --user daemon-reload
|
||||
$ systemctl --user enable --now kjv-api
|
||||
$ sudo loginctl enable-linger $USER # keeps it running when you log out, starts it at boot
|
||||
$ curl 127.0.0.1:1611/random # a Bible verse in json means it works
|
||||
|
||||
b. Point nginx at it (change example.com in the file to your domain first):
|
||||
$ sudo cp dist/kjv-api.nginx.conf /etc/nginx/sites-available/kjv-api
|
||||
$ sudo ln -s /etc/nginx/sites-available/kjv-api /etc/nginx/sites-enabled/
|
||||
$ sudo nginx -t && sudo systemctl reload nginx
|
||||
$ sudo certbot --nginx -d example.com # https, optional but recommended
|
||||
Already have a site on that domain? Copy just the location block from the file into it.
|
||||
|
||||
Settings: optional, see the environment variables below. Put them in
|
||||
~/kjv-api/kjv-api.env (one NAME=value per line), chmod 600 it, and
|
||||
systemctl --user restart kjv-api.
|
||||
|
||||
Updating: replace only the program, never kjv.db (it holds your users' reading
|
||||
plans and shortcodes, and your logs). From the source folder:
|
||||
$ make dist
|
||||
$ install -m 755 dist/kjv-api ~/kjv-api/kjv-api
|
||||
$ ~/kjv-api/kjv-api import-assets -db ~/kjv-api/kjv.db assets # only if pages changed
|
||||
$ systemctl --user restart kjv-api
|
||||
|
||||
Logs: journalctl --user -u kjv-api -f
|
||||
or: ~/kjv-api/kjv-api logs -db ~/kjv-api/kjv.db -n 100
|
||||
|
||||
Uninstall: systemctl --user disable --now kjv-api, then delete ~/kjv-api,
|
||||
~/.config/systemd/user/kjv-api.service and the nginx site.
|
||||
|
||||
USAGE
|
||||
=====
|
||||
The program may be invoked by running kjv-api.sh in a non-root account, or via
|
||||
systemd daemon process in a non-root account.
|
||||
Alternately for testing, the program may be run on another port by running:
|
||||
$ uwsgi_python3 --http-socket <listen address>:<port> --wsgi-file /opt/kjv-api/app.py --callable app
|
||||
or by executing kjv-api-test.sh which will listen on 0.0.0.0:1612
|
||||
It only ever listens on localhost (127.0.0.1:1611 unless told otherwise), it is
|
||||
meant to sit behind a reverse proxy. Asking it to listen on any other address is refused.
|
||||
For development, scripts/kjv-api-test.sh builds it, copies the database, and
|
||||
listens on 127.0.0.1:1612
|
||||
|
||||
Visiting the base directory will show the user-facing documentation via web browser or cURL.
|
||||
|
||||
When testing or doing a deployment, the data directory must be owned by whatever user is
|
||||
running the program in order to prevent a sqlite3.OperationalError for writing to a "readonly"
|
||||
database.
|
||||
Changing a page, script, style or data file means editing it in the source tree
|
||||
and loading it into the database again, then restarting the program:
|
||||
$ ./kjv-api import-assets -db kjv.db assets
|
||||
|
||||
The logs are in the database, there is no log rotation but the program deletes
|
||||
lines older than 90 days when it starts:
|
||||
$ ./kjv-api logs -db kjv.db -n 100
|
||||
$ sqlite3 kjv.db "select * from logs where level = 'ERROR'"
|
||||
When run by systemd the same lines also go to the journal (journalctl --user -u kjv-api),
|
||||
and KJV_LOG_STDERR=1 does that outside of systemd.
|
||||
|
||||
When testing or doing a deployment, the directory holding kjv.db must be owned by
|
||||
whatever user is running the program (sqlite3 writes its journal beside the
|
||||
database), otherwise reading plans, shortcodes and logs will not be saved.
|
||||
|
||||
Optional environment variables
|
||||
- KJV_DB - path to kjv.db instead of the -db flag
|
||||
- KJV_ADDR - localhost address to listen on instead of the -addr flag
|
||||
- KJV_SECRET_API_KEY - the token POST /natural expects, and the bearer token the /seqreport and
|
||||
/screport listings need (without it those two do not exist).
|
||||
- KJV_LOG_STDERR - set to 1 to also print the log lines to stderr (automatic under systemd)
|
||||
- KJV_SEMANTIC_ADDR - where the semantic search service listens, default localhost:1613
|
||||
- KJV_SERMON_DB - path to the separate sermon database. Without it the sermon
|
||||
endpoints answer 503.
|
||||
|
||||
BUILDING
|
||||
========
|
||||
Only the machine that compiles needs anything: go 1.26 or newer, and make (built
|
||||
and tested with go 1.27.3, get it from https://go.dev/dl/ or your package manager).
|
||||
The sqlite3 driver (modernc.org/sqlite) is pure Go, no C compiler needed, and go
|
||||
downloads it the first time it builds.
|
||||
$ make build builds bin/kjv-api
|
||||
$ make dist builds dist/kjv-api and dist/kjv.db, the two files to distribute
|
||||
$ make test go vet, and any tests
|
||||
Optional: sqlite3 (the command line tool) for looking at the logs and database.
|
||||
|
||||
CONTENTS
|
||||
========
|
||||
.
|
||||
├── app.py - main program
|
||||
├── data
|
||||
│ ├── 1828_Webster_KJV.json - Dictionary
|
||||
│ ├── kjv.aff - KJV Spellchecking Description file (hunspell)
|
||||
│ ├── kjv.db - KJV sqlite3 database
|
||||
│ └── kjv.dic - KJV Spellchecking dictionary file (hunspell)
|
||||
│ └── seq.db - Sqlite3 Database with user's ids for sequenced scripture
|
||||
│ └── shortcodes.db - Sqlite3 Database with saved shortcodes for scripture groups.
|
||||
├── favicon.ico
|
||||
├── html
|
||||
│ ├── dictionary.html - dictionary search landing page /define
|
||||
│ ├── find.html - unified search bar landing page /find
|
||||
│ ├── navbar.html - navigation bar added programatically to rich results
|
||||
│ ├── heaven.html - Bible way to heaven, static page /salvation
|
||||
│ ├── on_reading_bible.html - Introduction on good reading practices
|
||||
│ ├── index.html - / landing page with unified search bar
|
||||
│ ├── reader.html - not yet implemented Bible browser end point
|
||||
│ ├── reading_plan.html - not yet implemented Bible reading plan generator
|
||||
│ ├── response.html - Template for rich verse responses
|
||||
│ ├── search.html - Bible keyword search landing page /search
|
||||
│ ├── search_bar.html - Unused
|
||||
│ ├── show.html - Bible verse search landing page /show
|
||||
│ ├── seq.html - Bible sequential verse landing page /seq
|
||||
│ └── votd.html - Matty's test VOTD rich landing page
|
||||
├── install
|
||||
│ ├── apache.site.conf.example - example installation configuration
|
||||
│ └── kjv-api.service - example systemd service file
|
||||
├── INSTALL.txt - installation instructions
|
||||
├── logs - log folder (populated at runtime - there is NO log rotation)
|
||||
├── js
|
||||
│ └── collapsible.js - js for the definition drop-down menu used in unified search
|
||||
├── kjv-api.sh - runs uwsgi flask app on port 1611
|
||||
├── kjv-api-test.sh - runs uwsgi flask app on port 1612
|
||||
├── README.txt - This document
|
||||
├── styles
|
||||
│ ├── collapsible.css
|
||||
│ ├── header.css
|
||||
│ ├── fonts
|
||||
│ │ ├── Arizonia-Regular.ttf
|
||||
│ │ └── Parisienne-Regular.ttf
|
||||
│ ├── human_readable.css
|
||||
│ ├── search_rich.css
|
||||
│ └── votd.css
|
||||
└── tools
|
||||
└── av1611_dictionary_scraper.py - builds a dictionary json for the 1828 dictionary
|
||||
└── sanity_check.py - Integrated during runtime and also as endpoint testing when run alone.
|
||||
└── search_keywords.txt - endpoint testing queries
|
||||
└── sanity_results.txt - results of running sanity_check.py as a standalone
|
||||
├── cmd
|
||||
│ └── kjv-api
|
||||
│ └── main.go - command line: run the server, import-assets, logs
|
||||
├── internal
|
||||
│ └── kjv - the program, one package
|
||||
│ ├── server.go - startup, localhost listener, routes
|
||||
│ ├── app.go - loads the assets from the database, builds the rich page pieces and verse rendering
|
||||
│ ├── pages.go - /show /find /search /define /random /votd
|
||||
│ ├── refs.go - reference resolver ("1 jn 3.16", "II Kings 2:1-4", "song of songs 2")
|
||||
│ ├── state.go - /seq /plan and shortcodes, saved in the database
|
||||
│ ├── sermons.go - sermon database and semantic search endpoints, optional
|
||||
│ ├── store.go - the kjv.db queries and the Bible held in memory
|
||||
│ ├── assets.go - import-assets, putting the files in assets/ into the database
|
||||
│ ├── logdb.go - logs saved in the database
|
||||
│ ├── spell.go - spellchecking suggestions from the word list
|
||||
│ ├── difflib.go - fuzzy book name matching
|
||||
│ └── mt.go - the random number generator that picks the verse of the day
|
||||
├── db
|
||||
│ └── kjv.db - KJV sqlite3 database, the start of the companion database
|
||||
├── assets - everything that gets loaded into kjv.db by import-assets
|
||||
│ ├── data
|
||||
│ │ ├── 1828_Webster_KJV.json - Dictionary
|
||||
│ │ ├── kjv.aff - KJV Spellchecking Description file
|
||||
│ │ ├── kjv.dic - KJV Spellchecking dictionary file
|
||||
│ │ └── *.json - book descriptions, chapter summaries, search synonyms and variants
|
||||
│ ├── html
|
||||
│ │ ├── dictionary.html - dictionary search landing page /define
|
||||
│ │ ├── find.html - unified search bar landing page /find
|
||||
│ │ ├── navbar.html - navigation bar added programatically to rich results
|
||||
│ │ ├── chapter_nav.html - previous/next chapter bar
|
||||
│ │ ├── heaven.html - Bible way to heaven, static page /salvation
|
||||
│ │ ├── on_reading_bible.html - Introduction on good reading practices
|
||||
│ │ ├── index.html - / landing page with unified search bar, the program fills in the book lists
|
||||
│ │ ├── plan.html - Bible reading plan setup /plan
|
||||
│ │ ├── saved.html - favorites, definitions and cross references kept in the browser /saved
|
||||
│ │ ├── response.html - Template for rich verse responses
|
||||
│ │ ├── search.html - Bible keyword search landing page /search
|
||||
│ │ ├── search_bar.html - Unused
|
||||
│ │ ├── show.html - Bible verse search landing page /show
|
||||
│ │ ├── seq.html - Bible sequential verse landing page /seq
|
||||
│ │ └── votd.html - Matty's test VOTD rich landing page
|
||||
│ ├── js
|
||||
│ │ ├── collapsible.js - js for the definition drop-down menu used in unified search
|
||||
│ │ └── book_suggest.js, shortcode.js, verse_menu.js
|
||||
│ ├── static
|
||||
│ │ └── favicon.svg, favicon.ico
|
||||
│ ├── styles
|
||||
│ │ ├── collapsible.css
|
||||
│ │ ├── header.css
|
||||
│ │ ├── fonts
|
||||
│ │ │ ├── Arizonia-Regular.ttf
|
||||
│ │ │ └── Parisienne-Regular.ttf
|
||||
│ │ ├── human_readable.css
|
||||
│ │ ├── search_rich.css
|
||||
│ │ └── votd.css
|
||||
│ └── kjv_fe.html - /fe frontend page
|
||||
├── dist - what gets deployed, see INSTALLATION
|
||||
│ ├── kjv-api, kjv.db - built by make dist (not kept in git)
|
||||
│ ├── kjv-api.service - systemd user service, port 1611
|
||||
│ ├── kjv-api.nginx.conf - nginx reverse proxy site
|
||||
│ └── kjv-api.apache.conf - apache reverse proxy site
|
||||
├── scripts
|
||||
│ ├── kjv-api.sh - builds if needed and runs the program on localhost port 1611
|
||||
│ └── kjv-api-test.sh - builds and runs the program on a copy of the database on port 1612
|
||||
├── tools
|
||||
│ ├── av1611_dictionary_scraper.py - builds a dictionary json for the 1828 dictionary
|
||||
│ ├── sanity_check.py - Endpoint testing, run alone against a running instance (needs python3-requests)
|
||||
│ └── search_keywords.txt - endpoint testing queries
|
||||
├── doc
|
||||
│ ├── openapi.yaml
|
||||
│ └── notes.txt
|
||||
├── Makefile - make, make dist, make test
|
||||
├── go.mod, go.sum
|
||||
└── README.txt - This document
|
||||
|
||||
AUTHOR
|
||||
======
|
||||
Copyright Tyler R Dinsmoor & Matty Sneederson 2023
|
||||
Copyright Tyler R Dinsmoor & Matty Sneederson 2022-2026
|
||||
Reimplemented in Go by demon machine claude.
|
||||
|
||||
|
||||
SECURITY
|
||||
========
|
||||
- Listens on localhost only, and refuses any other address.
|
||||
- The user service runs as your account, so it can read and write whatever you can. For more
|
||||
separation, make an account just for it (sudo useradd -m kjv; sudo loginctl enable-linger kjv)
|
||||
and do step 2a logged in as that account.
|
||||
- Reading plan ids and shortcodes are random but they are the only secret; anyone holding a plan id can
|
||||
move that plan. Making new ones is rate limited per visitor (by the X-Real-IP the proxy sends,
|
||||
the nginx and apache files do). Plans unused for 90 days are deleted.
|
||||
- Please report anything you find to the author.
|
||||
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
|
Before Width: | Height: | Size: 15 KiB After Width: | Height: | Size: 15 KiB |
File renamed without changes.
|
Before Width: | Height: | Size: 346 B After Width: | Height: | Size: 346 B |
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,93 @@
|
||||
// kjv-api: the King James Bible web API as one program plus one SQLite
|
||||
// database (kjv.db) that holds the text, every page and asset, user data and
|
||||
// the logs.
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"kjv-api/internal/kjv"
|
||||
)
|
||||
|
||||
const usage = `usage:
|
||||
kjv-api [-db kjv.db] [-addr 127.0.0.1:1611] run the server
|
||||
kjv-api import-assets [-db kjv.db] DIR load the pages, scripts, styles and data files from the assets folder into the database
|
||||
kjv-api logs [-db kjv.db] [-n 100] print the newest log lines stored in the database
|
||||
`
|
||||
|
||||
func findDB(flagValue string) string {
|
||||
if flagValue != "" {
|
||||
return flagValue
|
||||
}
|
||||
if v := os.Getenv("KJV_DB"); v != "" {
|
||||
return v
|
||||
}
|
||||
candidates := []string{"kjv.db"}
|
||||
if exe, err := os.Executable(); err == nil {
|
||||
candidates = append(candidates, filepath.Join(filepath.Dir(exe), "kjv.db"))
|
||||
}
|
||||
for _, c := range candidates {
|
||||
if _, err := os.Stat(c); err == nil {
|
||||
return c
|
||||
}
|
||||
}
|
||||
return "kjv.db"
|
||||
}
|
||||
|
||||
func main() {
|
||||
args := os.Args[1:]
|
||||
if len(args) > 0 {
|
||||
switch args[0] {
|
||||
case "import-assets":
|
||||
fs := flag.NewFlagSet("import-assets", flag.ExitOnError)
|
||||
db := fs.String("db", "", "path to kjv.db")
|
||||
fs.Parse(args[1:])
|
||||
if fs.NArg() != 1 {
|
||||
fmt.Fprint(os.Stderr, usage)
|
||||
os.Exit(2)
|
||||
}
|
||||
dbPath := findDB(*db)
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
if err := kjv.ImportAssets(dbPath, fs.Arg(0)); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
return
|
||||
case "logs":
|
||||
fs := flag.NewFlagSet("logs", flag.ExitOnError)
|
||||
db := fs.String("db", "", "path to kjv.db")
|
||||
n := fs.Int("n", 100, "number of lines")
|
||||
fs.Parse(args[1:])
|
||||
if err := kjv.ShowLogs(findDB(*db), *n); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
return
|
||||
case "help", "-h", "--help":
|
||||
fmt.Print(usage)
|
||||
return
|
||||
}
|
||||
}
|
||||
fs := flag.NewFlagSet("kjv-api", flag.ExitOnError)
|
||||
dbFlag := fs.String("db", "", "path to kjv.db (default: $KJV_DB, ./kjv.db, next to the program)")
|
||||
addr := fs.String("addr", envOr("KJV_ADDR", "127.0.0.1:1611"), "listen address (localhost only)")
|
||||
fs.Parse(args)
|
||||
if err := kjv.Run(findDB(*dbFlag), *addr); err != nil {
|
||||
fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func fatal(err error) {
|
||||
fmt.Fprintln(os.Stderr, "kjv-api:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
Whitespace-only changes.
File renamed without changes.
Vendored
+16
@@ -0,0 +1,16 @@
|
||||
# apache site for kjv-api (needs mod_proxy, mod_proxy_http and mod_headers:
|
||||
# sudo a2enmod proxy proxy_http headers)
|
||||
# Change domain.tld to your domain, copy to /etc/apache2/sites-available/ and a2ensite it.
|
||||
# For https: sudo certbot --apache -d domain.tld
|
||||
|
||||
<VirtualHost *:80>
|
||||
ServerName domain.tld
|
||||
|
||||
ProxyPreserveHost On
|
||||
ProxyRequests Off
|
||||
ProxyVia Off
|
||||
# the program limits how fast each visitor can make plans and shortcodes by this address
|
||||
RequestHeader set X-Real-IP "%{REMOTE_ADDR}s"
|
||||
ProxyPass / http://127.0.0.1:1611/
|
||||
ProxyPassReverse / http://127.0.0.1:1611/
|
||||
</VirtualHost>
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
# nginx site for kjv-api. Change example.com to your domain, then:
|
||||
# sudo cp kjv-api.nginx.conf /etc/nginx/sites-available/kjv-api
|
||||
# sudo ln -s /etc/nginx/sites-available/kjv-api /etc/nginx/sites-enabled/
|
||||
# sudo nginx -t && sudo systemctl reload nginx
|
||||
# sudo certbot --nginx -d example.com (adds https to this file for you)
|
||||
#
|
||||
# Already have a site? Copy just the location block into it.
|
||||
|
||||
server {
|
||||
listen 80;
|
||||
server_name example.com;
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:1611;
|
||||
proxy_set_header Host $host;
|
||||
# the program limits how fast each visitor can make plans and shortcodes by this address
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
}
|
||||
Vendored
+19
@@ -0,0 +1,19 @@
|
||||
[Unit]
|
||||
Description=KJV API endpoint at localhost port 1611
|
||||
|
||||
# A systemd user service, no root needed. It runs ~/kjv-api/kjv-api with the
|
||||
# database next to it. Install with:
|
||||
# mkdir -p ~/.config/systemd/user && cp kjv-api.service ~/.config/systemd/user/
|
||||
# systemctl --user daemon-reload && systemctl --user enable --now kjv-api
|
||||
# sudo loginctl enable-linger $USER (keeps it running while logged out, and starts it at boot)
|
||||
|
||||
[Service]
|
||||
ExecStart=%h/kjv-api/kjv-api -db %h/kjv-api/kjv.db -addr 127.0.0.1:1611
|
||||
# Optional settings (KJV_SECRET_API_KEY, KJV_SERMON_DB, KJV_SEMANTIC_ADDR), one NAME=value per line
|
||||
EnvironmentFile=-%h/kjv-api/kjv-api.env
|
||||
Restart=on-failure
|
||||
NoNewPrivileges=yes
|
||||
UMask=0077
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
File renamed without changes.
@@ -0,0 +1,17 @@
|
||||
module kjv-api
|
||||
|
||||
go 1.26.0
|
||||
|
||||
require modernc.org/sqlite v1.61.0
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.1.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/mattn/go-isatty v0.0.24 // indirect
|
||||
github.com/ncruces/go-strftime v1.1.0 // indirect
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
|
||||
golang.org/x/sys v0.49.0 // indirect
|
||||
modernc.org/libc v1.79.0 // indirect
|
||||
modernc.org/mathutil v1.7.1 // indirect
|
||||
modernc.org/memory v1.12.1 // indirect
|
||||
)
|
||||
@@ -0,0 +1,50 @@
|
||||
github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg=
|
||||
github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI=
|
||||
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50 h1:PlNWJRxu5APSV0pWqfIsZ0qt9OOGHSWfZQ4zvx+WgWs=
|
||||
github.com/google/pprof v0.0.0-20261016150608-1facf8f91b50/go.mod h1:yUo1AMMpsPIDFGuwDK8nLcf8+vvAd5l2O6JDZ6hpwrM=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
|
||||
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk=
|
||||
github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
|
||||
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
|
||||
golang.org/x/mod v0.42.0 h1:ICeL3t2L30ljSPHHxDnR4fqVcBt/zR2zJAb9juavmQg=
|
||||
golang.org/x/mod v0.42.0/go.mod h1:u5fzIrRInkL8MaO1cbRkUHT64v303YIT5KEqpNr7jRs=
|
||||
golang.org/x/sync v0.24.0 h1:iqiDrGoDgyuvAKrhLQthUEzU/R378N78gXbBjxed7oA=
|
||||
golang.org/x/sync v0.24.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
|
||||
golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA=
|
||||
golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
||||
golang.org/x/tools v0.52.0 h1:ryxg4U63oaUYk7uOLTF37n43f4qgtGvqV4Ioit1BaEs=
|
||||
golang.org/x/tools v0.52.0/go.mod h1:ItcP5UNaVGG+Yc4eIYmaFLNJ4soEopkx5Y2d4lcEurs=
|
||||
modernc.org/cc/v4 v4.30.3 h1:aDq8gIX1X7+VFgO/JYYji6pLWL2H4tv3g8WatT3KCys=
|
||||
modernc.org/cc/v4 v4.30.3/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
|
||||
modernc.org/ccgo/v4 v4.37.0 h1:BAuJCUtIHIxFIYaFhqXIw5YaoMBlNCUlFovejHfMip4=
|
||||
modernc.org/ccgo/v4 v4.37.0/go.mod h1:/8627VhwWtAbqUXmPORxFQU5SgLrAx2jl9seLw9GBhY=
|
||||
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
|
||||
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
|
||||
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
|
||||
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
|
||||
modernc.org/gc/v3 v3.2.0 h1:/Y3LGDMQaoYf55PebP1qsZ5FDGgAZyG1RYKEHeQzh1o=
|
||||
modernc.org/gc/v3 v3.2.0/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
|
||||
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
|
||||
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
|
||||
modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8=
|
||||
modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M=
|
||||
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
|
||||
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
|
||||
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
|
||||
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
|
||||
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
|
||||
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
|
||||
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
|
||||
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
|
||||
modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY=
|
||||
modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0=
|
||||
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
|
||||
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
|
||||
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
|
||||
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
|
||||
@@ -1,24 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Books - KJV API</title>
|
||||
<link rel="stylesheet" href="{{ asset_url('/styles/page.css') }}">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<a href="/">Home</a> <a href="/find">Quick Search</a>
|
||||
</header>
|
||||
<h2>Books</h2>
|
||||
<div class="table-wrap">
|
||||
<table id="Books">
|
||||
{% for book in books %}
|
||||
<tr>
|
||||
<td><a href="/find?kw={{ book[0] }}&view=rich">{{ book[0] }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,26 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Sequence Report - KJV API</title>
|
||||
<link rel="stylesheet" href="{{ asset_url('/styles/page.css') }}">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<a href="/">Home</a> <a href="/find">Quick Search</a>
|
||||
</header>
|
||||
<h2>Sequence Report</h2>
|
||||
<div class="table-wrap">
|
||||
<table id="seq_report">
|
||||
{% for uid in uids %}
|
||||
<tr>
|
||||
<td>{{ uid[0] }}</td>
|
||||
<td>{{ uid[1] }}</td>
|
||||
<td>{{ uid[2] }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -1,31 +0,0 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
<title>Shortcode Report - KJV API</title>
|
||||
<link rel="stylesheet" href="{{ asset_url('/styles/page.css') }}">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<a href="/">Home</a> <a href="/find">Quick Search</a>
|
||||
</header>
|
||||
<h2>Shortcode Report</h2>
|
||||
<div class="table-wrap">
|
||||
<table id="shortcode_report">
|
||||
<tr>
|
||||
<td>shortcode</td>
|
||||
<td>query</td>
|
||||
<td>creation_date</td>
|
||||
</tr>
|
||||
{% for code in shortcodes %}
|
||||
<tr>
|
||||
<td>{{ code[0] }}</td>
|
||||
<td>{{ code[1] }}</td>
|
||||
<td>{{ code[2] }}</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</table>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
-119
@@ -1,119 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check for root privileges
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This install script requires root privileges. Please run with sudo."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
dest=/opt/kjv-api
|
||||
|
||||
echo "Attempting to install dependancies"
|
||||
apt install python3-flask sqlite3 libhunspell-dev python3-hunspell uwsgi-core uwsgi-plugin-python3 python3-requests python3-dotenv curl rsync
|
||||
|
||||
if id kjv >/dev/null 2>&1; then
|
||||
echo "User kjv already exists, leaving it alone"
|
||||
else
|
||||
echo "Creating new user kjv with no login shell"
|
||||
useradd -r -s /bin/false kjv
|
||||
fi
|
||||
|
||||
# Stop a running install so files are not swapped underneath it
|
||||
if systemctl is-active --quiet kjv-api.service; then
|
||||
echo "Stopping running kjv-api service for the update"
|
||||
systemctl stop kjv-api.service
|
||||
fi
|
||||
|
||||
mkdir -p "$dest"
|
||||
|
||||
# Never overwrite local state on a re-run: the .env, the runtime databases
|
||||
# (seq/shortcodes/sermons, including any -wal/-shm files), logs, and dev cruft.
|
||||
# Files are only added or updated, never deleted.
|
||||
if [[ "$script_dir" == "$dest" ]]; then
|
||||
echo "Running from $dest, nothing to copy"
|
||||
else
|
||||
echo "Copying server files to $dest (existing .env, databases and logs are kept)"
|
||||
rsync -rlt \
|
||||
--exclude='.git/' --exclude='__pycache__/' --exclude='*.pyc' \
|
||||
--exclude='/.env' \
|
||||
--exclude='/data/seq.db*' --exclude='/data/shortcodes.db*' --exclude='/data/sermons.db*' \
|
||||
--exclude='/logs/*' \
|
||||
"${script_dir}"/ "$dest"/
|
||||
fi
|
||||
|
||||
mkdir -p "$dest/logs" "$dest/data"
|
||||
|
||||
if [[ ! -f "$dest/.env" ]]; then
|
||||
echo "Creating $dest/.env from .env.example (edit it to set real values)"
|
||||
cp "$dest/.env.example" "$dest/.env"
|
||||
else
|
||||
echo "Keeping existing $dest/.env"
|
||||
fi
|
||||
|
||||
# Admins manage the install through the kjv group (the service's own group).
|
||||
admin_user="${KJV_ADMIN_USER:-$SUDO_USER}"
|
||||
if [[ -n "$admin_user" && "$admin_user" != root ]] && id "$admin_user" >/dev/null 2>&1; then
|
||||
echo "Adding $admin_user to group kjv (log out and back in to apply)"
|
||||
usermod -aG kjv "$admin_user"
|
||||
else
|
||||
echo "No admin user detected; set KJV_ADMIN_USER=name to add one to the kjv group"
|
||||
fi
|
||||
|
||||
echo "Setting permissions (code group-writable by kjv; .env, logs and data limited)"
|
||||
chown -R root:kjv "$dest"
|
||||
find "$dest" -type d -exec chmod 2775 {} +
|
||||
find "$dest" -type f -exec chmod 664 {} +
|
||||
chmod 775 "$dest"/*.sh
|
||||
chmod 640 "$dest/.env"
|
||||
# SQLite needs a writable directory for journal files
|
||||
chown kjv:kjv "$dest/logs" "$dest/data"
|
||||
chmod 750 "$dest/logs" "$dest/data"
|
||||
chown -R kjv:kjv "$dest/logs"
|
||||
chmod -R u+rwX,g+rX,o-rwx "$dest/logs"
|
||||
# Runtime-written databases belong to kjv; shipped read-only databases stay root-owned
|
||||
for db in "$dest"/data/seq.db* "$dest"/data/shortcodes.db*; do
|
||||
if [[ -f "$db" ]]; then
|
||||
chown kjv:kjv "$db"
|
||||
chmod 640 "$db"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "Installing systemd service task"
|
||||
unit=/etc/systemd/system/kjv-api.service
|
||||
if [[ -f "$unit" ]] && ! cmp -s "$dest/install/kjv-api.service" "$unit"; then
|
||||
echo "Existing service file differs, saving it as $unit.bak"
|
||||
cp "$unit" "$unit.bak"
|
||||
fi
|
||||
cp "$dest/install/kjv-api.service" "$unit"
|
||||
systemctl daemon-reload
|
||||
|
||||
echo "Enabling auto-start and starting kjv-api"
|
||||
systemctl enable kjv-api.service
|
||||
systemctl restart kjv-api.service
|
||||
|
||||
echo "Waiting for the service to initialize"
|
||||
sleep 3
|
||||
|
||||
echo "If you see a random Bible verse below in json format, your installation was successful:"
|
||||
echo
|
||||
echo
|
||||
curl -XGET 127.0.0.1:1611/random
|
||||
echo
|
||||
echo
|
||||
|
||||
YEL='\033[0;33m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo -e "${YEL}You must now set your webserver as a reverse proxy for your domain and point it to 127.0.0.1:1611 ${NC}"
|
||||
echo "See the install folder for example configuration files."
|
||||
echo
|
||||
echo "Notes:"
|
||||
echo " - Thank you for running this software to bring the Bible to people."
|
||||
echo " - You can change the pages (it's not fancy) in /opt/kjv-api/html/. There's no fancy libraries."
|
||||
echo " - Logs are located in two places (application logs and uwsgi logs)."
|
||||
echo " - /opt/kjv-api/logs/"
|
||||
echo " - journalctl -e -u kjv-api.service"
|
||||
echo " - If this install failed (you didn't get a Bible verse above) - try running /opt/kjv-api/kjv-api-test.sh"
|
||||
echo " after stopping the kjv-api.service with systemctl, it should give you a traceback in your console."
|
||||
@@ -1,23 +0,0 @@
|
||||
# Example proxy configuration
|
||||
|
||||
|
||||
<VirtualHost *:80>
|
||||
ServerName domain.tld
|
||||
RewriteEngine On
|
||||
RewriteCond %{HTTPS} off
|
||||
RewriteRule (.*) https://%{SERVER_NAME}/$1 [R,L]
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName domain.tld
|
||||
|
||||
SSLEngine on
|
||||
SSLCertificateFile /etc/letsencrypt/live/domain.tld/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/domain.tld/privkey.pem
|
||||
|
||||
ProxyPreserveHost On
|
||||
ProxyRequests Off
|
||||
ProxyVia Off
|
||||
ProxyPass / http://127.0.0.1:1611/
|
||||
ProxyPassReverse / http://127.0.0.1:1611/
|
||||
</VirtualHost>
|
||||
@@ -1,18 +0,0 @@
|
||||
[Unit]
|
||||
Description=KJV API endpoint at localhost port 1611
|
||||
|
||||
[Service]
|
||||
#ExecStart=uwsgi_python3 --http-socket :1611 --wsgi-file /opt/kjv-api/app.py --callable app --stats 127.0.0.1:9191
|
||||
ExecStart=/opt/kjv-api/kjv-api.sh
|
||||
User=kjv
|
||||
EnvironmentFile=/etc/environment
|
||||
KillMode=mixed
|
||||
# Hardening: the app may only write to its databases and logs, never its own code
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ReadWritePaths=/opt/kjv-api/data /opt/kjv-api/logs
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,24 +0,0 @@
|
||||
server {
|
||||
listen 80;
|
||||
server_name domain.tld;
|
||||
|
||||
return 301 https://$host$request_uri;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 443 ssl;
|
||||
server_name domain.tld;
|
||||
|
||||
ssl_certificate /etc/letsencrypt/live/domain.tld/fullchain.pem;
|
||||
ssl_certificate_key /etc/letsencrypt/live/domain.tld/privkey.pem;
|
||||
|
||||
|
||||
location / {
|
||||
proxy_pass http://127.0.0.1:1611;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,430 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"html"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// App is the whole web application. Everything it serves comes from the
|
||||
// companion database (see assets.go), loaded once at startup.
|
||||
type App struct {
|
||||
store *Store
|
||||
res *Resolver
|
||||
speller *Speller
|
||||
started time.Time
|
||||
|
||||
dictionary map[string][]string // 1828 Webster
|
||||
variants map[string][]string
|
||||
synonyms map[string][]string
|
||||
concepts map[string]concept
|
||||
bookDescs map[string]string
|
||||
chapSums map[string][]string
|
||||
|
||||
served map[string]servedAsset // versioned static files: /styles/page.css ...
|
||||
pages map[string]string // simple html pages, asset urls versioned
|
||||
|
||||
htmlResponse, htmlNavbar, htmlChapterNav string
|
||||
cssNavbar, cssCollapsible string
|
||||
favicon []byte
|
||||
indexHTML string
|
||||
|
||||
sermonDB *sql.DB
|
||||
secretKey string
|
||||
semantic string // host:port of the optional semantic search service
|
||||
}
|
||||
|
||||
type concept struct {
|
||||
Query string `json:"query"`
|
||||
Explanation string `json:"explanation"`
|
||||
SearchTerms []string `json:"search_terms"`
|
||||
}
|
||||
|
||||
type servedAsset struct {
|
||||
body []byte
|
||||
mime string
|
||||
version string
|
||||
etag string
|
||||
}
|
||||
|
||||
var servedPaths = map[string]string{ // url -> asset path
|
||||
"/styles/page.css": "styles/page.css",
|
||||
"/js/collapsible.js": "js/collapsible.js",
|
||||
"/js/shortcode.js": "js/shortcode.js",
|
||||
"/js/verse_menu.js": "js/verse_menu.js",
|
||||
"/js/book_suggest.js": "js/book_suggest.js",
|
||||
}
|
||||
|
||||
func NewApp(store *Store) (*App, error) {
|
||||
assets, err := store.LoadAssets()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
need := func(p string) ([]byte, error) {
|
||||
b, ok := assets[p]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("asset %s is missing from the database (run: kjv-api import-assets <source dir>)", p)
|
||||
}
|
||||
return b, nil
|
||||
}
|
||||
a := &App{store: store, res: NewResolver(store), started: time.Now(), pages: map[string]string{}, served: map[string]servedAsset{}}
|
||||
|
||||
loadJSON := func(p string, into any) error {
|
||||
b, err := need(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := json.Unmarshal(b, into); err != nil {
|
||||
return fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := loadJSON("data/1828_Webster_KJV.json", &a.dictionary); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadJSON("data/book_descriptions.json", &a.bookDescs); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := loadJSON("data/chapter_summaries.json", &a.chapSums); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var groups [][]string
|
||||
if err := loadJSON("data/grammatical_variants.json", &groups); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.variants = groupMap(groups)
|
||||
if err := loadJSON("data/theological_synonyms.json", &groups); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.synonyms = groupMap(groups)
|
||||
var concepts []concept
|
||||
if err := loadJSON("data/concept_mappings.json", &concepts); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.concepts = map[string]concept{}
|
||||
for _, c := range concepts {
|
||||
a.concepts[strings.ToLower(c.Query)] = c
|
||||
}
|
||||
dic, err := need("data/kjv.dic")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
aff, err := need("data/kjv.aff")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.speller = NewSpeller(dic, aff)
|
||||
|
||||
// static assets, each URL carrying a hash of its own contents
|
||||
for u, p := range servedPaths {
|
||||
b, err := need(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if p == "js/book_suggest.js" {
|
||||
names, _ := json.Marshal(store.bookNames)
|
||||
b = bytes.ReplaceAll(b, []byte("__BOOKS__"), names)
|
||||
}
|
||||
sum := sha256.Sum256(b)
|
||||
mime := assetMIME[p[strings.LastIndexByte(p, '.'):]]
|
||||
a.served[u] = servedAsset{b, mime, fmt.Sprintf("%x", sum[:4]), fmt.Sprintf(`"%x"`, sum[:8])}
|
||||
}
|
||||
text := func(p string) (string, error) { b, err := need(p); return string(b), err }
|
||||
if a.htmlResponse, err = text("html/response.html"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.htmlChapterNav, err = text("html/chapter_nav.html"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.cssNavbar, err = text("styles/header.css"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if a.cssCollapsible, err = text("styles/collapsible.css"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
nav, err := text("html/navbar.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.htmlNavbar = a.loadHTML(nav)
|
||||
if a.favicon, err = need("static/favicon.svg"); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, p := range []string{"html/find.html", "html/show.html", "html/search.html", "html/seq.html", "html/dictionary.html",
|
||||
"html/saved.html", "html/plan.html", "html/heaven.html", "kjv_fe.html"} {
|
||||
t, err := text(p)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.pages[p] = a.loadHTML(t)
|
||||
}
|
||||
index, err := text("html/index.html")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.indexHTML = a.renderIndex(a.loadHTML(index))
|
||||
return a, nil
|
||||
}
|
||||
|
||||
func groupMap(groups [][]string) map[string][]string {
|
||||
m := map[string][]string{}
|
||||
for _, g := range groups {
|
||||
for _, w := range g {
|
||||
var others []string
|
||||
for _, o := range g {
|
||||
if o != w {
|
||||
others = append(others, o)
|
||||
}
|
||||
}
|
||||
m[w] = others
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
var assetRefRE = regexp.MustCompile(`(/(?:js|styles)/[\w.\-]+)\?v=__V__`)
|
||||
|
||||
// loadHTML fills each '/js/x.js?v=__V__' with that file's own version.
|
||||
func (a *App) loadHTML(s string) string {
|
||||
return assetRefRE.ReplaceAllStringFunc(s, func(m string) string {
|
||||
u := assetRefRE.FindStringSubmatch(m)[1]
|
||||
if as, ok := a.served[u]; ok {
|
||||
return u + "?v=" + as.version
|
||||
}
|
||||
return m
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) assetURL(u string) string { return u + "?v=" + a.served[u].version }
|
||||
|
||||
var indexLoopRE = regexp.MustCompile(`(?s)\{% for category, books in testament_data\.(OT|NT)\.items\(\) %\}.*?\n \{% endfor %\}`)
|
||||
|
||||
// renderIndex fills the book lists of the home page.
|
||||
func (a *App) renderIndex(tmpl string) string {
|
||||
esc := html.EscapeString
|
||||
return indexLoopRE.ReplaceAllStringFunc(tmpl, func(m string) string {
|
||||
testament := indexLoopRE.FindStringSubmatch(m)[1]
|
||||
var order []string
|
||||
byCat := map[string][]BookInfo{}
|
||||
for _, b := range a.store.bookInfo {
|
||||
if b.OTNT != testament {
|
||||
continue
|
||||
}
|
||||
if _, ok := byCat[b.Category]; !ok {
|
||||
order = append(order, b.Category)
|
||||
}
|
||||
byCat[b.Category] = append(byCat[b.Category], b)
|
||||
}
|
||||
var sb strings.Builder
|
||||
for _, cat := range order {
|
||||
fmt.Fprintf(&sb, "\n <h3 class=\"category-header\">%s</h3>\n <div class=\"book-grid\">", esc(cat))
|
||||
for _, b := range byCat[cat] {
|
||||
fmt.Fprintf(&sb, "\n<div><button class=\"book-button collapsible\" data-book=\"%s\">%s</button>\n<div class=\"chapter-grid content\">", esc(b.Title), esc(b.Title))
|
||||
if about := a.bookDescs[b.Title]; about != "" {
|
||||
fmt.Fprintf(&sb, "<p class=\"book-about\">%s</p>", esc(about))
|
||||
}
|
||||
sums := a.chapSums[b.Title]
|
||||
for c := 1; c <= b.Chapters; c++ {
|
||||
fmt.Fprintf(&sb, "\n<a href=\"/show?kw=%s %d&view=rich\" class=\"chapter-link\"><span class=\"chapter-num\">%d</span>", esc(b.Title), c, c)
|
||||
if len(sums) >= c {
|
||||
fmt.Fprintf(&sb, "<span class=\"chapter-sum\">%s</span>", esc(sums[c-1]))
|
||||
}
|
||||
sb.WriteString("</a>")
|
||||
}
|
||||
sb.WriteString("\n</div></div>")
|
||||
}
|
||||
sb.WriteString("\n </div>")
|
||||
}
|
||||
return sb.String()
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// responses
|
||||
|
||||
type Resp struct {
|
||||
Body string
|
||||
Type string
|
||||
Status int
|
||||
Header map[string]string
|
||||
}
|
||||
|
||||
func htmlR(s string) Resp { return Resp{Body: s, Type: "text/html; charset=utf-8"} }
|
||||
|
||||
func jsonR(v any) Resp { return jsonStatus(v, 0) }
|
||||
|
||||
func jsonStatus(v any, status int) Resp {
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
enc.SetEscapeHTML(false)
|
||||
enc.Encode(v)
|
||||
return Resp{Body: buf.String(), Type: "application/json", Status: status}
|
||||
}
|
||||
|
||||
func errJSON(status int, msg string) Resp { return jsonStatus(map[string]string{"error": msg}, status) }
|
||||
|
||||
func (r Resp) write(w http.ResponseWriter) {
|
||||
h := w.Header()
|
||||
h.Set("Content-Type", r.Type)
|
||||
if strings.HasPrefix(r.Type, "text/html") && r.Header["Cache-Control"] == "" {
|
||||
h.Set("Cache-Control", "no-cache")
|
||||
}
|
||||
for k, v := range r.Header {
|
||||
h.Set(k, v)
|
||||
}
|
||||
if r.Status != 0 {
|
||||
w.WriteHeader(r.Status)
|
||||
}
|
||||
w.Write([]byte(r.Body))
|
||||
}
|
||||
|
||||
func (a *App) staticPage(p string) Resp { return htmlR(a.pages[p]) }
|
||||
|
||||
// staticHTMLDoc wraps a small html snippet in the standard page shell.
|
||||
func (a *App) staticHTMLDoc(title, body string) string {
|
||||
return "<!DOCTYPE html><html lang='en'><head><meta charset='UTF-8'>" +
|
||||
"<meta name='viewport' content='width=device-width, initial-scale=1.0'>" +
|
||||
"<title>" + title + " - KJV API</title>" +
|
||||
"<link rel='stylesheet' href='" + a.assetURL("/styles/page.css") + "'></head><body>" +
|
||||
"<header><a href='/'>Home</a> <a href='/find'>Quick Search</a></header>" +
|
||||
body + "</body></html>"
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// verse rendering
|
||||
|
||||
func (a *App) responseRich(v Verse) string {
|
||||
passage := fmt.Sprintf("%s %d:%d", a.store.BookName(v.Book), v.Chapter, v.Verse)
|
||||
return strings.NewReplacer(
|
||||
"{chapter}", passage[:strings.LastIndexByte(passage, ':')],
|
||||
"{passage}", passage,
|
||||
"{text}", v.Text,
|
||||
"{verse_id}", fmt.Sprint(v.ID),
|
||||
).Replace(a.htmlResponse)
|
||||
}
|
||||
|
||||
func chapterFindLink(name string) string { return "/find?kw=" + url.PathEscape(name) + "&view=rich" }
|
||||
|
||||
type chapterNav struct{ current, prev, next string }
|
||||
|
||||
func (a *App) chapterName(key int64) string {
|
||||
return fmt.Sprintf("%s %d", a.store.BookName(int(key/1000)), key%1000)
|
||||
}
|
||||
|
||||
func (a *App) nextPrevChapter(verseID int64) chapterNav {
|
||||
key := verseID / 1000
|
||||
n := len(a.store.chapterList)
|
||||
i := sort.Search(n, func(i int) bool { return a.store.chapterList[i] >= key })
|
||||
return chapterNav{
|
||||
current: a.chapterName(key),
|
||||
prev: a.chapterName(a.store.chapterList[(i-1+n)%n]),
|
||||
next: a.chapterName(a.store.chapterList[(i+1)%n]),
|
||||
}
|
||||
}
|
||||
|
||||
// renderNavbar opens the rich html document: <head>, site header and the
|
||||
// chapter bar. The caller adds the body content after it.
|
||||
func (a *App) renderNavbar(query string, nav *chapterNav, barHTML string) string {
|
||||
esc := html.EscapeString
|
||||
title := strings.TrimSpace(query)
|
||||
if title == "" && nav != nil {
|
||||
title = nav.current
|
||||
}
|
||||
if title == "" {
|
||||
title = "Scripture"
|
||||
}
|
||||
chapterHTML := barHTML
|
||||
if nav != nil {
|
||||
chapterHTML = strings.NewReplacer(
|
||||
"{current_chapter_name}", esc(nav.current),
|
||||
"{prev_chapter_name}", esc(nav.prev),
|
||||
"{prev_chapter_link}", esc("/show?kw="+nav.prev+"&view=rich"),
|
||||
"{next_chapter_name}", esc(nav.next),
|
||||
"{next_chapter_link}", esc("/show?kw="+nav.next+"&view=rich"),
|
||||
"{current_chapter_link}", esc(chapterFindLink(nav.current)),
|
||||
).Replace(a.htmlChapterNav)
|
||||
}
|
||||
return strings.NewReplacer(
|
||||
"{title}", esc(title), "{css}", a.cssNavbar, "{query}", esc(query), "{chapter_nav}", chapterHTML,
|
||||
).Replace(a.htmlNavbar)
|
||||
}
|
||||
|
||||
type rendered struct {
|
||||
text string // plain and rich
|
||||
list []map[string]any
|
||||
isJSON bool
|
||||
}
|
||||
|
||||
func (r rendered) resp() Resp {
|
||||
if r.isJSON {
|
||||
return jsonR(r.list)
|
||||
}
|
||||
return htmlR(r.text)
|
||||
}
|
||||
|
||||
// render lays out verses as view=plain, view=rich or (anything else) json.
|
||||
func (a *App) render(verses []Verse, view, kw string) rendered {
|
||||
switch view {
|
||||
case "plain":
|
||||
parts := make([]string, len(verses))
|
||||
for i, v := range verses {
|
||||
parts[i] = fmt.Sprintf("%s %d:%d %s\n", a.store.BookName(v.Book), v.Chapter, v.Verse, v.Text)
|
||||
}
|
||||
return rendered{text: strings.Join(parts, "<br>")}
|
||||
case "rich":
|
||||
if len(verses) == 0 {
|
||||
lg.Error("Malformed query: %s", kw)
|
||||
return rendered{text: "Malformed Query - if you believe this to be in error (your query should resolve), email tyler@dinsmoor.us"}
|
||||
}
|
||||
// Prev/next chapter links only make sense when reading a single chapter,
|
||||
// not for a keyword search that spans books.
|
||||
var nav *chapterNav
|
||||
first := verses[0].ID / 1000
|
||||
single := true
|
||||
for _, v := range verses {
|
||||
if v.ID/1000 != first {
|
||||
single = false
|
||||
break
|
||||
}
|
||||
}
|
||||
if single {
|
||||
n := a.nextPrevChapter(verses[0].ID)
|
||||
nav = &n
|
||||
}
|
||||
var sb strings.Builder
|
||||
sb.WriteString(a.renderNavbar(kw, nav, ""))
|
||||
sb.WriteString("<div class='responses'>")
|
||||
fmt.Fprintf(&sb, "<div class='results-count'>%d Results</div>", len(verses))
|
||||
for _, v := range verses {
|
||||
sb.WriteString(a.responseRich(v))
|
||||
}
|
||||
sb.WriteString("</div>")
|
||||
return rendered{text: sb.String()}
|
||||
}
|
||||
list := make([]map[string]any, len(verses))
|
||||
for i, v := range verses {
|
||||
list[i] = map[string]any{"bookname": a.store.BookName(v.Book), "chapter": v.Chapter, "verse": v.Verse, "text": v.Text}
|
||||
}
|
||||
return rendered{list: list, isJSON: true}
|
||||
}
|
||||
|
||||
func envOr(key, def string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func itoa64(n int64) string { return fmt.Sprint(n) }
|
||||
func urlQuery(s string) string { return url.QueryEscape(s) }
|
||||
func htmlEsc(s string) string { return html.EscapeString(s) }
|
||||
@@ -0,0 +1,113 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Everything the program serves or reads besides the Bible itself (pages,
|
||||
// scripts, styles, the 1828 dictionary, word lists) lives in the assets table
|
||||
// of the companion database, so the program and the database are the only two
|
||||
// files that need to be distributed.
|
||||
|
||||
var assetMIME = map[string]string{
|
||||
".html": "text/html; charset=utf-8",
|
||||
".css": "text/css; charset=utf-8",
|
||||
".js": "application/javascript; charset=utf-8",
|
||||
".json": "application/json",
|
||||
".svg": "image/svg+xml",
|
||||
".dic": "text/plain; charset=utf-8",
|
||||
".aff": "text/plain; charset=utf-8",
|
||||
}
|
||||
|
||||
func isAssetFile(rel string) bool {
|
||||
dir := path.Dir(rel)
|
||||
switch {
|
||||
case rel == "kjv_fe.html":
|
||||
return true
|
||||
case dir == "html", dir == "js", dir == "static":
|
||||
return true
|
||||
case dir == "styles":
|
||||
return strings.HasSuffix(rel, ".css")
|
||||
case dir == "data":
|
||||
ext := path.Ext(rel)
|
||||
return ext == ".json" || ext == ".dic" || ext == ".aff"
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ImportAssets replaces the assets table with the files of a source checkout.
|
||||
func ImportAssets(dbPath, srcDir string) error {
|
||||
db, err := sql.Open("sqlite", "file:"+dbPath+"?_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
tx, err := db.Begin()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer tx.Rollback()
|
||||
for _, stmt := range []string{
|
||||
"CREATE TABLE IF NOT EXISTS assets (path TEXT PRIMARY KEY, mime TEXT NOT NULL, body BLOB NOT NULL)",
|
||||
"DELETE FROM assets",
|
||||
} {
|
||||
if _, err := tx.Exec(stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
n := 0
|
||||
err = filepath.WalkDir(srcDir, func(p string, d fs.DirEntry, err error) error {
|
||||
if err != nil || !d.Type().IsRegular() { // no directories, and never follow a symlink out of the tree
|
||||
return err
|
||||
}
|
||||
rel, _ := filepath.Rel(srcDir, p)
|
||||
rel = filepath.ToSlash(rel)
|
||||
if !isAssetFile(rel) {
|
||||
return nil
|
||||
}
|
||||
body, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mime := assetMIME[path.Ext(rel)]
|
||||
if mime == "" {
|
||||
mime = "application/octet-stream"
|
||||
}
|
||||
n++
|
||||
_, err = tx.Exec("INSERT INTO assets (path, mime, body) VALUES (?, ?, ?)", rel, mime, body)
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n == 0 {
|
||||
return fmt.Errorf("no assets found under %s (expected html/, js/, styles/, data/*.json ...)", srcDir)
|
||||
}
|
||||
fmt.Printf("imported %d assets into %s\n", n, dbPath)
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// LoadAssets reads every asset into memory.
|
||||
func (s *Store) LoadAssets() (map[string][]byte, error) {
|
||||
rows, err := s.db.Query("SELECT path, body FROM assets")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no assets table (run: kjv-api import-assets <source dir>): %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string][]byte{}
|
||||
for rows.Next() {
|
||||
var p string
|
||||
var b []byte
|
||||
if err := rows.Scan(&p, &b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[p] = b
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
package kjv
|
||||
|
||||
// Ratcliff/Obershelp string similarity (the ratio of matching characters) and
|
||||
// a closest match lookup, which the fuzzy book name matching relies on.
|
||||
|
||||
type matchBlock struct{ a, b, size int }
|
||||
|
||||
func longestMatch(a, b []rune, alo, ahi, blo, bhi int) matchBlock {
|
||||
besti, bestj, bestsize := alo, blo, 0
|
||||
prev := make([]int, len(b)+1)
|
||||
cur := make([]int, len(b)+1)
|
||||
for i := alo; i < ahi; i++ {
|
||||
for j := blo; j < bhi; j++ {
|
||||
if a[i] != b[j] {
|
||||
cur[j+1] = 0
|
||||
continue
|
||||
}
|
||||
k := prev[j] + 1
|
||||
cur[j+1] = k
|
||||
if k > bestsize {
|
||||
besti, bestj, bestsize = i-k+1, j-k+1, k
|
||||
}
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
for j := range cur {
|
||||
cur[j] = 0
|
||||
}
|
||||
}
|
||||
return matchBlock{besti, bestj, bestsize}
|
||||
}
|
||||
|
||||
func matchedChars(a, b []rune) int {
|
||||
type span struct{ alo, ahi, blo, bhi int }
|
||||
queue := []span{{0, len(a), 0, len(b)}}
|
||||
total := 0
|
||||
for len(queue) > 0 {
|
||||
q := queue[len(queue)-1]
|
||||
queue = queue[:len(queue)-1]
|
||||
m := longestMatch(a, b, q.alo, q.ahi, q.blo, q.bhi)
|
||||
if m.size == 0 {
|
||||
continue
|
||||
}
|
||||
total += m.size
|
||||
if q.alo < m.a && q.blo < m.b {
|
||||
queue = append(queue, span{q.alo, m.a, q.blo, m.b})
|
||||
}
|
||||
if m.a+m.size < q.ahi && m.b+m.size < q.bhi {
|
||||
queue = append(queue, span{m.a + m.size, q.ahi, m.b + m.size, q.bhi})
|
||||
}
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func similarity(a, b string) float64 {
|
||||
ra, rb := []rune(a), []rune(b)
|
||||
if len(ra)+len(rb) == 0 {
|
||||
return 1
|
||||
}
|
||||
return 2.0 * float64(matchedChars(ra, rb)) / float64(len(ra)+len(rb))
|
||||
}
|
||||
|
||||
// closestMatch is the best scoring possibility with a ratio of at least cutoff.
|
||||
// Ties on score go to the lexicographically larger possibility.
|
||||
func closestMatch(word string, possibilities []string, cutoff float64) (string, bool) {
|
||||
best, bestScore, found := "", 0.0, false
|
||||
for _, p := range possibilities {
|
||||
score := similarity(p, word) // difflib compares possibility (a) against word (b)
|
||||
if score < cutoff {
|
||||
continue
|
||||
}
|
||||
if !found || score > bestScore || (score == bestScore && p > best) {
|
||||
best, bestScore, found = p, score, true
|
||||
}
|
||||
}
|
||||
return best, found
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// DBLog writes log lines into the logs table of the companion database. Lines
|
||||
// are queued and inserted in batches so requests never wait on a write. Until
|
||||
// a database is attached (and if it turns out to be read-only) lines go to stderr.
|
||||
type DBLog struct {
|
||||
mu sync.Mutex
|
||||
db *sql.DB
|
||||
ch chan logLine
|
||||
done chan struct{}
|
||||
started bool
|
||||
retain int
|
||||
}
|
||||
|
||||
type logLine struct {
|
||||
ts, level, msg string
|
||||
}
|
||||
|
||||
var lg = &DBLog{ch: make(chan logLine, 4096), done: make(chan struct{})}
|
||||
|
||||
const (
|
||||
logTimeFmt = "2006-01-02 15:04:05.000"
|
||||
maxLogLine = 400 // request URLs and queries come from strangers
|
||||
)
|
||||
|
||||
func truncate(s string, n int) string {
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
for n > 0 && !utf8.RuneStart(s[n]) { // don't cut a character in half
|
||||
n--
|
||||
}
|
||||
return s[:n] + "..."
|
||||
}
|
||||
|
||||
func (l *DBLog) Attach(db *sql.DB, retainDays int) {
|
||||
if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS logs (id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT NOT NULL, level TEXT NOT NULL, msg TEXT NOT NULL)`); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "WARN: cannot write logs to the database, logging to stderr:", err)
|
||||
return
|
||||
}
|
||||
l.retain = retainDays
|
||||
l.prune(db)
|
||||
l.mu.Lock()
|
||||
l.db, l.started = db, true
|
||||
l.mu.Unlock()
|
||||
go l.run()
|
||||
}
|
||||
|
||||
func (l *DBLog) prune(db *sql.DB) {
|
||||
db.Exec("DELETE FROM logs WHERE ts < ?", time.Now().AddDate(0, 0, -l.retain).Format(logTimeFmt))
|
||||
}
|
||||
|
||||
func (l *DBLog) run() {
|
||||
tick := time.NewTicker(time.Second)
|
||||
defer tick.Stop()
|
||||
daily := time.NewTicker(6 * time.Hour)
|
||||
defer daily.Stop()
|
||||
var batch []logLine
|
||||
flush := func() {
|
||||
if len(batch) == 0 {
|
||||
return
|
||||
}
|
||||
tx, err := l.db.Begin()
|
||||
if err == nil {
|
||||
for _, ln := range batch {
|
||||
if _, err = tx.Exec("INSERT INTO logs (ts, level, msg) VALUES (?, ?, ?)", ln.ts, ln.level, ln.msg); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
err = tx.Commit()
|
||||
} else {
|
||||
tx.Rollback()
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
for _, ln := range batch {
|
||||
fmt.Fprintf(os.Stderr, "%s %s %s\n", ln.ts, ln.level, ln.msg)
|
||||
}
|
||||
}
|
||||
batch = batch[:0]
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case ln := <-l.ch:
|
||||
batch = append(batch, ln)
|
||||
if len(batch) >= 200 {
|
||||
flush()
|
||||
}
|
||||
case <-tick.C:
|
||||
flush()
|
||||
case <-daily.C:
|
||||
l.prune(l.db)
|
||||
case <-l.done:
|
||||
for {
|
||||
select {
|
||||
case ln := <-l.ch:
|
||||
batch = append(batch, ln)
|
||||
default:
|
||||
flush()
|
||||
l.done <- struct{}{}
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Close flushes queued lines.
|
||||
func (l *DBLog) Close() {
|
||||
l.mu.Lock()
|
||||
started := l.started
|
||||
l.mu.Unlock()
|
||||
if !started {
|
||||
return
|
||||
}
|
||||
l.done <- struct{}{}
|
||||
<-l.done
|
||||
}
|
||||
|
||||
// Under systemd (stderr is the journal) or with KJV_LOG_STDERR=1 every line is
|
||||
// also written to stderr, where the journal collects it. The "<n>" prefix is
|
||||
// the syslog priority journald understands; it adds its own timestamps.
|
||||
var teeStderr = stderrIsJournal() || os.Getenv("KJV_LOG_STDERR") == "1"
|
||||
var journalPriority = map[string]int{"ERROR": 3, "WARN": 4, "INFO": 6}
|
||||
|
||||
// stderrIsJournal: JOURNAL_STREAM is inherited by everything a systemd
|
||||
// started desktop session runs, terminals included, so it only counts when it
|
||||
// names the device:inode stderr really is (as systemd.exec(5) advises).
|
||||
func stderrIsJournal() bool {
|
||||
want := os.Getenv("JOURNAL_STREAM")
|
||||
if want == "" {
|
||||
return false
|
||||
}
|
||||
fi, err := os.Stderr.Stat()
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
st, ok := fi.Sys().(*syscall.Stat_t)
|
||||
return ok && fmt.Sprintf("%d:%d", st.Dev, st.Ino) == want
|
||||
}
|
||||
|
||||
func (l *DBLog) log(level, format string, args ...any) {
|
||||
ln := logLine{time.Now().Format(logTimeFmt), level, truncate(fmt.Sprintf(format, args...), maxLogLine)}
|
||||
if teeStderr {
|
||||
fmt.Fprintf(os.Stderr, "<%d>%s\n", journalPriority[level], strings.ReplaceAll(ln.msg, "\n", " "))
|
||||
}
|
||||
l.mu.Lock()
|
||||
started := l.started
|
||||
l.mu.Unlock()
|
||||
if started {
|
||||
select {
|
||||
case l.ch <- ln:
|
||||
return
|
||||
default: // queue full: don't block a request
|
||||
}
|
||||
}
|
||||
if !teeStderr { // not in the database and not already on stderr
|
||||
fmt.Fprintf(os.Stderr, "%s %s %s\n", ln.ts, ln.level, ln.msg)
|
||||
}
|
||||
}
|
||||
|
||||
func (l *DBLog) Info(format string, args ...any) { l.log("INFO", format, args...) }
|
||||
func (l *DBLog) Warn(format string, args ...any) { l.log("WARN", format, args...) }
|
||||
func (l *DBLog) Error(format string, args ...any) { l.log("ERROR", format, args...) }
|
||||
|
||||
// ShowLogs prints the newest n log lines (oldest first).
|
||||
func ShowLogs(dbPath string, n int) error {
|
||||
db, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro&_pragma=busy_timeout(5000)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer db.Close()
|
||||
rows, err := db.Query("SELECT ts, level, msg FROM (SELECT * FROM logs ORDER BY id DESC LIMIT ?) ORDER BY id", n)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var ts, level, msg string
|
||||
if err := rows.Scan(&ts, &level, &msg); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s %-5s %s\n", ts, level, msg)
|
||||
}
|
||||
return rows.Err()
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package kjv
|
||||
|
||||
// MT19937 seeded with init_by_array from one integer, and a choice made by
|
||||
// rejection sampling over its bits, so the verse of the day for any given date
|
||||
// never changes.
|
||||
|
||||
type mtRandom struct {
|
||||
mt [624]uint32
|
||||
idx int
|
||||
}
|
||||
|
||||
func newMTRandom(seed uint32) *mtRandom {
|
||||
r := &mtRandom{}
|
||||
r.initGenrand(19650218)
|
||||
key := []uint32{seed}
|
||||
i, j := 1, 0
|
||||
k := max(624, len(key))
|
||||
for ; k > 0; k-- {
|
||||
r.mt[i] = (r.mt[i] ^ ((r.mt[i-1] ^ (r.mt[i-1] >> 30)) * 1664525)) + key[j] + uint32(j)
|
||||
i++
|
||||
j++
|
||||
if i >= 624 {
|
||||
r.mt[0] = r.mt[623]
|
||||
i = 1
|
||||
}
|
||||
if j >= len(key) {
|
||||
j = 0
|
||||
}
|
||||
}
|
||||
for k = 623; k > 0; k-- {
|
||||
r.mt[i] = (r.mt[i] ^ ((r.mt[i-1] ^ (r.mt[i-1] >> 30)) * 1566083941)) - uint32(i)
|
||||
i++
|
||||
if i >= 624 {
|
||||
r.mt[0] = r.mt[623]
|
||||
i = 1
|
||||
}
|
||||
}
|
||||
r.mt[0] = 0x80000000
|
||||
r.idx = 624
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *mtRandom) initGenrand(s uint32) {
|
||||
r.mt[0] = s
|
||||
for i := 1; i < 624; i++ {
|
||||
r.mt[i] = 1812433253*(r.mt[i-1]^(r.mt[i-1]>>30)) + uint32(i)
|
||||
}
|
||||
r.idx = 624
|
||||
}
|
||||
|
||||
func (r *mtRandom) next() uint32 {
|
||||
if r.idx >= 624 {
|
||||
for k := 0; k < 624; k++ {
|
||||
y := (r.mt[k] & 0x80000000) | (r.mt[(k+1)%624] & 0x7fffffff)
|
||||
v := r.mt[(k+397)%624] ^ (y >> 1)
|
||||
if y&1 != 0 {
|
||||
v ^= 0x9908b0df
|
||||
}
|
||||
r.mt[k] = v
|
||||
}
|
||||
r.idx = 0
|
||||
}
|
||||
y := r.mt[r.idx]
|
||||
r.idx++
|
||||
y ^= y >> 11
|
||||
y ^= (y << 7) & 0x9d2c5680
|
||||
y ^= (y << 15) & 0xefc60000
|
||||
y ^= y >> 18
|
||||
return y
|
||||
}
|
||||
|
||||
// choiceIndex is random.choice's _randbelow(n) for n < 2**32.
|
||||
func (r *mtRandom) choiceIndex(n int) int {
|
||||
if n <= 0 {
|
||||
return 0
|
||||
}
|
||||
k := 0
|
||||
for v := n; v > 0; v >>= 1 {
|
||||
k++
|
||||
}
|
||||
for {
|
||||
if x := int(r.next() >> (32 - k)); x < n {
|
||||
return x
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,512 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html"
|
||||
"math/rand/v2"
|
||||
"net/url"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
const (
|
||||
maxSearchWords = 20
|
||||
maxSearchLen = 200
|
||||
)
|
||||
|
||||
const searchChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+^* "
|
||||
|
||||
// cleanInput keeps only the allowed characters.
|
||||
func cleanInput(s, valid string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if strings.ContainsRune(valid, r) {
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, s)
|
||||
}
|
||||
|
||||
func searchClean(s string) string { return cleanInput(s, searchChars) }
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /show
|
||||
|
||||
func (a *App) show(kw, view string, all bool) (Resp, bool) {
|
||||
resolved := a.res.Resolve(kw, true)
|
||||
if resolved == nil {
|
||||
return htmlR("Invalid Query"), false
|
||||
}
|
||||
var verses []Verse
|
||||
for _, r := range resolved.Refs {
|
||||
end := r.End
|
||||
if all { // ?all= : from the given verse to the end of its chapter
|
||||
end = r.Start/1000*1000 + maxChapterIDSpan
|
||||
}
|
||||
verses = append(verses, a.store.Range(r.Start, end)...)
|
||||
}
|
||||
return a.render(verses, view, kw).resp(), true
|
||||
}
|
||||
|
||||
func viewOf(q url.Values, def string) string {
|
||||
if q.Has("view") {
|
||||
return q.Get("view")
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
func (a *App) handleShow(q url.Values) Resp {
|
||||
kw := q.Get("kw")
|
||||
if kw == "" {
|
||||
return a.staticPage("html/show.html")
|
||||
}
|
||||
resp, _ := a.show(kw, viewOf(q, "json"), q.Get("all") != "")
|
||||
return resp
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /random and /votd
|
||||
|
||||
func (a *App) singleVerse(id int64, q url.Values, defView string) Resp {
|
||||
return a.render(a.store.Range(id, 0), viewOf(q, defView), "").resp()
|
||||
}
|
||||
|
||||
func (a *App) handleRandom(q url.Values, defView string) Resp {
|
||||
id := a.store.verseIDs[rand.IntN(len(a.store.verseIDs))]
|
||||
lg.Info("Random served: %d", id)
|
||||
return a.singleVerse(id, q, defView)
|
||||
}
|
||||
|
||||
// votdID is deterministic: the date seeds the generator, so everyone gets the same verse that day.
|
||||
func (a *App) votdID(t time.Time) int64 {
|
||||
seed, _ := strconv.Atoi(t.Format("20060102"))
|
||||
r := newMTRandom(uint32(seed))
|
||||
return a.store.verseIDs[r.choiceIndex(len(a.store.verseIDs))]
|
||||
}
|
||||
|
||||
func (a *App) handleVotd(q url.Values, defView string) Resp {
|
||||
lg.Info("VOTD served")
|
||||
return a.singleVerse(a.votdID(time.Now()), q, defView)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// dictionary
|
||||
|
||||
func (a *App) defineRich(ref string, defs []string, simple bool) string {
|
||||
head := ""
|
||||
if !simple {
|
||||
head = "<!DOCTYPE html><html lang='en'><head><meta charset='UTF-8'>" +
|
||||
"<meta name='viewport' content='width=device-width, initial-scale=1'>" +
|
||||
"<meta name='color-scheme' content='dark light'>" +
|
||||
"<title>" + html.EscapeString(capitalize(ref)) + " - KJV API</title>" +
|
||||
a.cssNavbar + "</head><body><div class='responses'>"
|
||||
}
|
||||
text := ""
|
||||
for _, d := range defs {
|
||||
text += d + "<br><br>"
|
||||
}
|
||||
out := head + strings.NewReplacer(
|
||||
"{chapter}", strings.SplitN(ref, ":", 2)[0], "{passage}", capitalize(ref), "{text}", text, "{verse_id}", "",
|
||||
).Replace(a.htmlResponse)
|
||||
if !simple {
|
||||
out += "</div></body></html>"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) handleDefine(q url.Values) Resp {
|
||||
ref := strings.ToLower(strings.TrimSpace(q.Get("kw")))
|
||||
if ref == "" {
|
||||
return a.staticPage("html/dictionary.html")
|
||||
}
|
||||
defs, ok := a.dictionary[ref]
|
||||
if !ok {
|
||||
lg.Warn("WEBSTER MISSING: %s", ref)
|
||||
return htmlR(html.EscapeString(ref) + " not found.")
|
||||
}
|
||||
switch q.Get("view") {
|
||||
case "plain":
|
||||
text := ""
|
||||
for _, d := range defs {
|
||||
text += d + "<br><br>"
|
||||
}
|
||||
return htmlR(capitalize(ref) + "<br><br>" + text + "\n")
|
||||
case "rich":
|
||||
return htmlR(a.defineRich(ref, defs, false))
|
||||
}
|
||||
return jsonR(map[string][]string{ref: defs})
|
||||
}
|
||||
|
||||
var dictStopwords = map[string]bool{}
|
||||
|
||||
func init() {
|
||||
for _, w := range strings.Fields(`the and that for him his her but not with are was were unto thou thee thy ye you all from
|
||||
they them this have had shall will which into upon then`) {
|
||||
dictStopwords[w] = true
|
||||
}
|
||||
}
|
||||
|
||||
// /defs?words=grace,loved : several definitions at once
|
||||
func (a *App) handleDefs(q url.Values) Resp {
|
||||
words := strings.Fields(strings.ReplaceAll(strings.ToLower(q.Get("words")), ",", " "))
|
||||
if len(words) > 40 {
|
||||
words = words[:40]
|
||||
}
|
||||
found := map[string][]string{}
|
||||
for _, w := range words {
|
||||
w = strings.Map(func(r rune) rune {
|
||||
if unicode.IsLetter(r) {
|
||||
return r
|
||||
}
|
||||
return -1
|
||||
}, w)
|
||||
if d, ok := a.dictionary[w]; ok && w != "" && !dictStopwords[w] {
|
||||
found[w] = d
|
||||
}
|
||||
}
|
||||
return jsonR(found)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /search
|
||||
|
||||
// proximityScore: higher when the query words appear closer together.
|
||||
func proximityScore(text string, words []string) float64 {
|
||||
lower := strings.ToLower(text)
|
||||
type hit struct{ pos, list int }
|
||||
var hits []hit
|
||||
seen := map[string]int{}
|
||||
for _, w := range words {
|
||||
if _, dup := seen[w]; dup {
|
||||
continue
|
||||
}
|
||||
li := len(seen)
|
||||
seen[w] = li
|
||||
w = strings.ToLower(w)
|
||||
n := 0
|
||||
for at := 0; ; {
|
||||
i := strings.Index(lower[at:], w)
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
hits = append(hits, hit{at + i, li})
|
||||
n++
|
||||
at += i + 1
|
||||
}
|
||||
if n == 0 {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
lists := len(seen)
|
||||
sort.SliceStable(hits, func(i, j int) bool { return hits[i].pos < hits[j].pos })
|
||||
// smallest window holding at least one position of every word
|
||||
counts := make([]int, lists)
|
||||
have, left, best := 0, 0, -1
|
||||
for right := range hits {
|
||||
if counts[hits[right].list]++; counts[hits[right].list] == 1 {
|
||||
have++
|
||||
}
|
||||
for have == lists {
|
||||
if span := hits[right].pos - hits[left].pos; best < 0 || span < best {
|
||||
best = span
|
||||
}
|
||||
if counts[hits[left].list]--; counts[hits[left].list] == 0 {
|
||||
have--
|
||||
}
|
||||
left++
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return 0
|
||||
}
|
||||
return 1000.0 / float64(best+1)
|
||||
}
|
||||
|
||||
func sortVariants(original string, variants []string) []string {
|
||||
type sv struct {
|
||||
score int
|
||||
v string
|
||||
}
|
||||
var scored []sv
|
||||
orig := strings.ToLower(original)
|
||||
for _, v := range variants {
|
||||
lv := strings.ToLower(v)
|
||||
prefix := 0
|
||||
for prefix < len(orig) && prefix < len(lv) && orig[prefix] == lv[prefix] {
|
||||
prefix++
|
||||
}
|
||||
if prefix < min(2, len(orig)/2) { // doesn't share the stem
|
||||
continue
|
||||
}
|
||||
scored = append(scored, sv{prefix*1000 + len(v), v})
|
||||
}
|
||||
sort.SliceStable(scored, func(i, j int) bool { return scored[i].score > scored[j].score })
|
||||
out := make([]string, len(scored))
|
||||
for i, s := range scored {
|
||||
out[i] = s.v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) fts(q string) []Verse {
|
||||
v, err := a.store.FTS(q)
|
||||
if err != nil { // FTS5 syntax the user typed by accident: just no results
|
||||
lg.Info("fts %q: %v", q, err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func insertAfterResponsesOpen(page, notice string) string {
|
||||
return strings.Replace(page, "<div class='responses'>", "<div class='responses'>"+notice, 1)
|
||||
}
|
||||
|
||||
// fallbackResp shows the results of a rewritten query with an explanation.
|
||||
func (a *App) fallbackResp(results []Verse, view, kw, plainNote, richNotice string, jsonObj map[string]any) Resp {
|
||||
r := a.render(results, view, kw)
|
||||
switch view {
|
||||
case "plain":
|
||||
return htmlR(plainNote + "\n<br>" + r.text)
|
||||
case "rich":
|
||||
return htmlR(insertAfterResponsesOpen(r.text, richNotice))
|
||||
}
|
||||
jsonObj["results"] = r.list
|
||||
return jsonR(jsonObj)
|
||||
}
|
||||
|
||||
func (a *App) handleSearch(kw, view string) Resp {
|
||||
cleaned := searchClean(kw)
|
||||
if words := strings.Fields(cleaned); len(words) > maxSearchWords || len(cleaned) > maxSearchLen {
|
||||
return htmlR("Your query is too long. 20 words or less.")
|
||||
}
|
||||
if cleaned == "" {
|
||||
return a.staticPage("html/search.html")
|
||||
}
|
||||
esc := html.EscapeString
|
||||
|
||||
// a common theological concept that is not in the KJV?
|
||||
conceptNotice := ""
|
||||
if c, ok := a.concepts[strings.ToLower(cleaned)]; ok {
|
||||
var links []string
|
||||
for _, t := range c.SearchTerms {
|
||||
links = append(links, fmt.Sprintf("<a href='/search?kw=%s&view=rich'><strong>%s</strong></a>", strings.ReplaceAll(t, " ", "+"), t))
|
||||
}
|
||||
conceptNotice = fmt.Sprintf("<div class='correction-notice'><strong>%s</strong>: %s<br>You may find what you are looking for with these search terms: %s</div>",
|
||||
capitalize(cleaned), c.Explanation, strings.Join(links, ", "))
|
||||
cleaned = c.SearchTerms[0]
|
||||
}
|
||||
|
||||
results := a.fts(cleaned)
|
||||
words := strings.Fields(cleaned)
|
||||
if len(results) > 0 && len(words) > 1 {
|
||||
scores := make(map[int64]float64, len(results))
|
||||
for _, v := range results {
|
||||
scores[v.ID] = proximityScore(v.Text, words)
|
||||
}
|
||||
sort.SliceStable(results, func(i, j int) bool { return scores[results[i].ID] > scores[results[j].ID] })
|
||||
}
|
||||
|
||||
if len(results) == 0 {
|
||||
// 1. grammatical variants
|
||||
for _, word := range words {
|
||||
for _, variant := range sortVariants(word, a.variants[strings.ToLower(word)]) {
|
||||
vq := strings.Replace(cleaned, word, variant, 1)
|
||||
if r := a.fts(vq); len(r) > 0 {
|
||||
return a.fallbackResp(r, view, kw,
|
||||
fmt.Sprintf("Showing results for '%s' (grammatical variant of '%s')", vq, cleaned),
|
||||
fmt.Sprintf("<div class='variant-notice'>Showing results for '<strong>%s</strong>' (grammatical variant of '%s')</div>", vq, cleaned),
|
||||
map[string]any{"original": cleaned, "variant": vq})
|
||||
}
|
||||
}
|
||||
}
|
||||
// 2. spell check
|
||||
corrected := make([]string, len(words))
|
||||
changed := false
|
||||
for i, w := range words {
|
||||
corrected[i] = w
|
||||
if !a.speller.Spell(w) {
|
||||
if sug := a.speller.Suggest(w); len(sug) > 0 {
|
||||
corrected[i], changed = sug[0], true
|
||||
}
|
||||
}
|
||||
}
|
||||
if changed {
|
||||
ck := strings.Join(corrected, " ")
|
||||
if r := a.fts(ck); len(r) > 0 {
|
||||
return a.fallbackResp(r, view, kw,
|
||||
fmt.Sprintf("Showing results for '%s' (corrected from '%s')", ck, cleaned),
|
||||
fmt.Sprintf("<div class='correction-notice'>Showing results for '<strong>%s</strong>' (corrected from '%s')</div>", esc(ck), cleaned),
|
||||
map[string]any{"corrected_from": cleaned, "corrected_to": ck})
|
||||
}
|
||||
}
|
||||
// 3. theological equivalents
|
||||
for _, word := range words {
|
||||
for _, syn := range a.synonyms[strings.ToLower(word)] {
|
||||
sq := strings.Replace(cleaned, word, syn, 1)
|
||||
if r := a.fts(sq); len(r) > 0 {
|
||||
explanation := fmt.Sprintf("In the KJV, '%s' and '%s' often refer to similar theological concepts. Consider also searching for related verses.", word, syn)
|
||||
return a.fallbackResp(r, view, kw,
|
||||
fmt.Sprintf("Showing results for '%s' (theological equivalent of '%s')\n%s", sq, cleaned, explanation),
|
||||
fmt.Sprintf("<div class='theological-notice'>Showing results for '<strong>%s</strong>' (theological equivalent of '%s')<br><em>%s</em></div>", sq, cleaned, explanation),
|
||||
map[string]any{"original": cleaned, "theological_equivalent": sq, "explanation": explanation})
|
||||
}
|
||||
}
|
||||
}
|
||||
// 4. nothing: offer spelling suggestions
|
||||
var suggestions []string
|
||||
for i, w := range words {
|
||||
if a.speller.Spell(w) {
|
||||
continue
|
||||
}
|
||||
for _, s := range a.speller.Suggest(w) {
|
||||
if len(suggestions) >= 5 {
|
||||
break
|
||||
}
|
||||
alt := append([]string(nil), words...)
|
||||
alt[i] = s
|
||||
suggestions = append(suggestions, strings.Join(alt, " "))
|
||||
}
|
||||
}
|
||||
if view != "rich" {
|
||||
list := "No suggestions available"
|
||||
if len(suggestions) > 0 {
|
||||
list = strings.Join(suggestions, ", ")
|
||||
}
|
||||
return htmlR(fmt.Sprintf("%s not found, try one of the following: %s", cleaned, list))
|
||||
}
|
||||
var sb strings.Builder
|
||||
sb.WriteString(a.renderNavbar(cleaned, nil, ""))
|
||||
sb.WriteString("<div class='responses'>")
|
||||
fmt.Fprintf(&sb, "<div class='correction-notice'><strong>No results found</strong> for '%s'.</div>", cleaned)
|
||||
if len(suggestions) > 0 {
|
||||
sb.WriteString("<div class='theological-suggestions'><h3>Did you mean?</h3>")
|
||||
sb.WriteString("<p>We couldn't find any verses matching your search. Here are some spelling suggestions:</p>")
|
||||
for _, s := range suggestions {
|
||||
fmt.Fprintf(&sb, "<div class='suggestion-item'><a href='/search?kw=%s&view=rich'><strong>%s</strong></a></div>", url.QueryEscape(s), esc(s))
|
||||
}
|
||||
sb.WriteString("</div>")
|
||||
} else {
|
||||
sb.WriteString("<div class='theological-suggestions'><h3>Search Tips</h3>")
|
||||
sb.WriteString("<p>We couldn't find any verses matching your search. Try:</p>")
|
||||
sb.WriteString("<ul style='font-family: var(--font-ui); color: var(--color-text-secondary);'>")
|
||||
sb.WriteString("<li>Checking your spelling</li><li>Using different keywords</li>")
|
||||
sb.WriteString("<li>Searching for <a href='/search?kw=love&view=rich'>love</a>, <a href='/search?kw=faith&view=rich'>faith</a>, or <a href='/search?kw=hope&view=rich'>hope</a></li>")
|
||||
sb.WriteString("</ul></div>")
|
||||
}
|
||||
sb.WriteString("</div>")
|
||||
return htmlR(sb.String())
|
||||
}
|
||||
|
||||
r := a.render(results, view, kw)
|
||||
var related []string
|
||||
for _, w := range words {
|
||||
related = append(related, a.synonyms[strings.ToLower(w)]...)
|
||||
}
|
||||
switch view {
|
||||
case "plain":
|
||||
out := r.text
|
||||
if conceptNotice != "" {
|
||||
out = conceptNotice + "<br>" + out
|
||||
}
|
||||
if len(related) > 0 {
|
||||
out += "<br><br>Also consider searching: "
|
||||
for _, s := range related {
|
||||
out += s + "<br>"
|
||||
}
|
||||
}
|
||||
return htmlR(out)
|
||||
case "rich":
|
||||
out := r.text
|
||||
if conceptNotice != "" {
|
||||
if start := strings.Index(out, "<div class='responses'>"); start != -1 {
|
||||
if end := strings.Index(out[start+len("<div class='responses'>"):], "</div>"); end != -1 {
|
||||
at := start + len("<div class='responses'>") + end + len("</div>")
|
||||
out = out[:at] + conceptNotice + out[at:]
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(related) > 0 {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("<div class='theological-suggestions'><h3>Related Theological Concepts</h3>")
|
||||
sb.WriteString("<p>In the KJV, these terms often refer to similar theological concepts. Consider also searching for related verses.</p>")
|
||||
for _, s := range related {
|
||||
fmt.Fprintf(&sb, "<div class='suggestion-item'><a href='/search?kw=%s&view=rich'><strong>%s</strong></a></div>", url.QueryEscape(s), capitalize(s))
|
||||
}
|
||||
sb.WriteString("</div>")
|
||||
if at := strings.LastIndex(out, "</div>"); at != -1 {
|
||||
out = out[:at] + sb.String() + out[at:]
|
||||
}
|
||||
}
|
||||
return htmlR(out)
|
||||
}
|
||||
return r.resp()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /find : scripture if it reads as scripture, otherwise a keyword search with definitions
|
||||
|
||||
func (a *App) handleFind(q url.Values) Resp {
|
||||
query := strings.TrimSpace(q.Get("kw"))
|
||||
if viewOf(q, "rich") != "rich" {
|
||||
return jsonR(map[string]string{"Error": "/find endpoint only supports rich responses"})
|
||||
}
|
||||
lg.Info("Query: %s", query)
|
||||
if query == "" {
|
||||
return a.staticPage("html/find.html")
|
||||
}
|
||||
query = strings.TrimSuffix(query, ",")
|
||||
|
||||
// A query that reads as scripture references ("Exodus", "1 jn 3.16") is shown
|
||||
// as scripture; anything else is a keyword search.
|
||||
if resolved := a.res.Resolve(query, false); resolved != nil {
|
||||
if resp, ok := a.show(query, "rich", false); ok {
|
||||
notices := ""
|
||||
for _, n := range resolved.Notes {
|
||||
notices += fmt.Sprintf("<div class='correction-notice'>Showing <strong>%s</strong> (interpreted from '%s')</div>",
|
||||
html.EscapeString(n.Name), html.EscapeString(n.Original))
|
||||
}
|
||||
if resolved.Bare {
|
||||
keyword := url.QueryEscape(searchClean(query))
|
||||
notices += fmt.Sprintf("<div class='correction-notice'>Showing <strong>%s</strong>, chapter 1. Looking for the word instead? "+
|
||||
"<a href='/search?kw=%s&view=rich'>Search the text for '%s'</a></div>",
|
||||
html.EscapeString(resolved.Label), keyword, html.EscapeString(query))
|
||||
}
|
||||
if notices != "" {
|
||||
resp.Body = insertAfterResponsesOpen(resp.Body, notices)
|
||||
}
|
||||
return resp
|
||||
}
|
||||
}
|
||||
if strings.TrimSpace(searchClean(query)) == "" {
|
||||
return a.staticPage("html/find.html") // nothing searchable left (punctuation, accents)
|
||||
}
|
||||
var tokens []string
|
||||
for _, w := range strings.Split(strings.ReplaceAll(query, ",", " "), " ") {
|
||||
if w != "" {
|
||||
tokens = append(tokens, w)
|
||||
}
|
||||
}
|
||||
if len(tokens) > 20 {
|
||||
return htmlR("Your Query is too long. 20 Words or less.")
|
||||
}
|
||||
result := a.handleSearch(q.Get("kw"), "rich")
|
||||
body := strings.TrimSuffix(result.Body, "</div>") // reopen .responses to add the dictionary inside it
|
||||
|
||||
var defs strings.Builder
|
||||
for _, word := range tokens {
|
||||
entry, ok := a.dictionary[strings.ToLower(word)]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
defs.WriteString("<button type='button' class='collapsible'>" + html.EscapeString(capitalize(word)) + "</button>")
|
||||
defs.WriteString("<div class='definition'>" + a.defineRich(strings.ToLower(word), entry, true) + "</div>")
|
||||
}
|
||||
if defs.Len() > 0 {
|
||||
body += "<style>" + a.cssCollapsible + "</style>"
|
||||
body += "<div class='dictionary-section'><h3>1828 Webster Definitions</h3><p>Click on the word to get the definition.</p></div>"
|
||||
body += defs.String()
|
||||
body += "<script src='" + a.assetURL("/js/collapsible.js") + "'></script>"
|
||||
}
|
||||
return htmlR(body + "</div>")
|
||||
}
|
||||
@@ -0,0 +1,357 @@
|
||||
package kjv
|
||||
|
||||
// Scripture reference resolver. Understands the many ways people type a
|
||||
// reference ("1 jn 3.16", "II Kings 2:1-4", "song of songs 2", "Exodus ") and
|
||||
// turns it into ranges of verse ids (book*1000000 + chapter*1000 + verse).
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Extra spellings people use that the abbreviations table does not know.
|
||||
var extraBookAliases = map[string][]string{
|
||||
"Song of Solomon": {"song of songs", "songs of solomon", "songs of songs", "song of sol",
|
||||
"song of sol", "canticles", "canticle of canticles", "sos", "sng", "ss"},
|
||||
"Psalms": {"psalm", "psalter", "pslm", "psa", "pss", "psm", "ps"},
|
||||
"Revelation": {"revelations", "revelation of john", "revelation of st john", "the revelation", "apocalypse"},
|
||||
"Ecclesiastes": {"ecclesiasts", "eccles", "ecc", "qoheleth"},
|
||||
"Philemon": {"phlm", "phm", "philem"},
|
||||
"Judges": {"jdg", "jdgs", "jg"},
|
||||
"Mark": {"mrk", "mk", "mr"},
|
||||
"John": {"jhn", "jn"},
|
||||
"Acts": {"acts of the apostles", "act"},
|
||||
"James": {"jas", "jm"},
|
||||
}
|
||||
|
||||
// Words that are books but also ordinary search words: a bare "numbers" or
|
||||
// "mark" still shows the book, but /find offers a keyword search as well.
|
||||
var bareBookExtra = map[string]bool{
|
||||
"psalm": true, "songofsongs": true, "songsofsolomon": true, "songofsol": true, "canticles": true,
|
||||
"canticleofcanticles": true, "revelations": true, "revelationofjohn": true, "therevelation": true,
|
||||
}
|
||||
|
||||
const maxChapterIDSpan = 999
|
||||
|
||||
// A reference is never this long; longer text goes to the keyword search. Bounds the fuzzy matching work.
|
||||
const maxRefQueryLen = 300
|
||||
|
||||
// Go's regexp has no lookaheads, so they are written as a consumed [\s.] here, which matches the same text.
|
||||
var (
|
||||
bookPieceRE = regexp.MustCompile(`(?i)^\s*(?P<book>(?:(?:[1-3]|(?:iii|ii|i)[\s.]|1st|2nd|3rd|first|second|third)\.?\s*)?[a-z][a-z.\s]*?)` +
|
||||
`\s*(?P<rest>\d[\d\s:.\-–—]*)?\s*$`)
|
||||
continuationRE = regexp.MustCompile(`^\s*\d[\d\s:.\-–—]*$`)
|
||||
|
||||
reBookOf = regexp.MustCompile(`^(?:the\s+)?(?:book\s+of\s+)`)
|
||||
reFirst = regexp.MustCompile(`^(?:1st|first)([\s.a-z]|$)`)
|
||||
reSecond = regexp.MustCompile(`^(?:2nd|second)([\s.a-z]|$)`)
|
||||
reThird = regexp.MustCompile(`^(?:3rd|third)([\s.a-z]|$)`)
|
||||
reRoman3 = regexp.MustCompile(`^iii([\s.]|$)`)
|
||||
reRoman2 = regexp.MustCompile(`^ii([\s.]|$)`)
|
||||
reRoman1 = regexp.MustCompile(`^i([\s.])`)
|
||||
reNotAlnum = regexp.MustCompile(`[^a-z0-9]`)
|
||||
reSepSpace = regexp.MustCompile(`\s*([:\-])\s*`)
|
||||
|
||||
reChapter = regexp.MustCompile(`^(\d+)$`)
|
||||
reChapterAll = regexp.MustCompile(`^(\d+)(?::-|-)$`)
|
||||
reChapterRange = regexp.MustCompile(`^(\d+)-(\d+)$`)
|
||||
reVerse = regexp.MustCompile(`^(\d+)[:.\s](\d+)$`)
|
||||
reVerseOn = regexp.MustCompile(`^(\d+)[:.\s](\d+)-$`)
|
||||
reVerseRange = regexp.MustCompile(`^(\d+)[:.\s](\d+)-(\d+)$`)
|
||||
reVerseAcrossCh = regexp.MustCompile(`^(\d+)[:.\s](\d+)-(\d+)[:.](\d+)$`)
|
||||
)
|
||||
|
||||
// normalizeBookText: 'I Cor.' / '1st Corinthians' / '1 corinthians' -> '1corinthians'
|
||||
func normalizeBookText(text string) string {
|
||||
text = strings.ToLower(strings.TrimSpace(text))
|
||||
text = reBookOf.ReplaceAllString(text, "")
|
||||
text = reFirst.ReplaceAllString(text, "1${1}")
|
||||
text = reSecond.ReplaceAllString(text, "2${1}")
|
||||
text = reThird.ReplaceAllString(text, "3${1}")
|
||||
text = reRoman3.ReplaceAllString(text, "3${1}")
|
||||
text = reRoman2.ReplaceAllString(text, "2${1}")
|
||||
text = reRoman1.ReplaceAllString(text, "1${1}")
|
||||
return reNotAlnum.ReplaceAllString(text, "")
|
||||
}
|
||||
|
||||
type Resolver struct {
|
||||
store *Store
|
||||
canonical map[string]int
|
||||
aliases map[string]int
|
||||
aliasKeys []string
|
||||
canonKeys []string
|
||||
}
|
||||
|
||||
func NewResolver(s *Store) *Resolver {
|
||||
r := &Resolver{store: s, canonical: map[string]int{}, aliases: map[string]int{}}
|
||||
for id, name := range s.bookName {
|
||||
r.canonical[normalizeBookText(name)] = id
|
||||
}
|
||||
for k, v := range r.canonical {
|
||||
r.aliases[k] = v
|
||||
}
|
||||
for _, a := range s.abbrevs {
|
||||
if k := normalizeBookText(a.Name); r.aliases[k] == 0 {
|
||||
r.aliases[k] = a.Book
|
||||
}
|
||||
}
|
||||
for name, extras := range extraBookAliases {
|
||||
id := s.bookByName[name]
|
||||
for _, extra := range extras {
|
||||
r.aliases[normalizeBookText(extra)] = id
|
||||
}
|
||||
}
|
||||
for k := range r.aliases {
|
||||
r.aliasKeys = append(r.aliasKeys, k)
|
||||
}
|
||||
sort.Strings(r.aliasKeys)
|
||||
for k := range r.canonical {
|
||||
r.canonKeys = append(r.canonKeys, k)
|
||||
}
|
||||
sort.Strings(r.canonKeys)
|
||||
return r
|
||||
}
|
||||
|
||||
type bookMatch struct {
|
||||
ID int
|
||||
How string // exact | prefix | fuzzy
|
||||
}
|
||||
|
||||
// resolveBook finds a book from a typed name. loose=false only accepts full
|
||||
// names and spellings that are unmistakably a book (used for a bare word with
|
||||
// no chapter number).
|
||||
func (r *Resolver) resolveBook(token string, loose bool) (bookMatch, bool) {
|
||||
key := normalizeBookText(token)
|
||||
if key == "" {
|
||||
return bookMatch{}, false
|
||||
}
|
||||
if !loose {
|
||||
if _, ok := r.canonical[key]; ok || bareBookExtra[key] {
|
||||
return bookMatch{r.aliases[key], "exact"}, true
|
||||
}
|
||||
// a clear misspelling of a long full name ("genisis")
|
||||
if len(key) >= 6 && len(key) <= 40 && !(key[0] >= '0' && key[0] <= '9') {
|
||||
if close, ok := closestMatch(key, r.canonKeys, 0.85); ok {
|
||||
return bookMatch{r.canonical[close], "fuzzy"}, true
|
||||
}
|
||||
}
|
||||
return bookMatch{}, false
|
||||
}
|
||||
if id, ok := r.aliases[key]; ok {
|
||||
return bookMatch{id, "exact"}, true
|
||||
}
|
||||
// unique (or best) prefix: "deut", "1cor", "matth"
|
||||
books := map[int]bool{}
|
||||
for _, k := range r.aliasKeys {
|
||||
if strings.HasPrefix(k, key) {
|
||||
books[r.aliases[k]] = true
|
||||
}
|
||||
}
|
||||
if len(books) > 0 {
|
||||
canonHits := map[int]bool{}
|
||||
for _, k := range r.canonKeys {
|
||||
if strings.HasPrefix(k, key) {
|
||||
canonHits[r.canonical[k]] = true
|
||||
}
|
||||
}
|
||||
if len(canonHits) == 1 {
|
||||
return bookMatch{anyKey(canonHits), "prefix"}, true
|
||||
}
|
||||
if len(books) == 1 {
|
||||
return bookMatch{anyKey(books), "prefix"}, true
|
||||
}
|
||||
pool := books
|
||||
if len(canonHits) > 0 {
|
||||
pool = canonHits
|
||||
}
|
||||
best := 0
|
||||
for id := range pool {
|
||||
if id > best {
|
||||
best = id // ambiguous: the later book, as before
|
||||
}
|
||||
}
|
||||
return bookMatch{best, "prefix"}, true
|
||||
}
|
||||
if len(key) >= 4 && len(key) <= 40 {
|
||||
if close, ok := closestMatch(key, r.aliasKeys, 0.78); ok {
|
||||
return bookMatch{r.aliases[close], "fuzzy"}, true
|
||||
}
|
||||
}
|
||||
return bookMatch{}, false
|
||||
}
|
||||
|
||||
func anyKey(m map[int]bool) int {
|
||||
for k := range m {
|
||||
return k
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// clampChapter: 0 -> 1, past the end -> the last chapter.
|
||||
func (r *Resolver) clampChapter(book, chapter int) int {
|
||||
last := r.store.chapterLen[book]
|
||||
if chapter > last {
|
||||
chapter = last
|
||||
}
|
||||
if chapter < 1 {
|
||||
chapter = 1
|
||||
}
|
||||
return chapter
|
||||
}
|
||||
|
||||
// atoi never fails: absurdly long numbers saturate.
|
||||
func atoi(s string) int {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return 1_000_000
|
||||
}
|
||||
if n > 1_000_000 {
|
||||
return 1_000_000
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
type refPos struct{ book, chapter, verse int }
|
||||
|
||||
// parseRefRest: '3:16-18', '3 16', '3.16', '3-4', '3:16-4:2', '3', '3:-' -> (start, end or 0, position)
|
||||
func (r *Resolver) parseRefRest(rest string, book int, previous *refPos) (start, end int64, pos refPos, ok bool) {
|
||||
rest = strings.NewReplacer("–", "-", "—", "-").Replace(rest)
|
||||
rest = reSepSpace.ReplaceAllString(strings.TrimSpace(rest), "$1")
|
||||
base := int64(book) * 1_000_000
|
||||
at := func(chapter, verse int) int64 { return base + int64(chapter)*1000 + int64(verse) }
|
||||
pos.book = book
|
||||
if m := reChapter.FindStringSubmatch(rest); m != nil {
|
||||
n := atoi(m[1])
|
||||
if previous != nil && previous.verse != 0 { // "John 3:16, 18": a verse of the same chapter
|
||||
pos.chapter, pos.verse = previous.chapter, n
|
||||
return at(previous.chapter, n), 0, pos, true
|
||||
}
|
||||
ch := r.clampChapter(book, n)
|
||||
pos.chapter = ch
|
||||
return at(ch, 1), at(ch, maxChapterIDSpan), pos, true
|
||||
}
|
||||
if m := reChapterAll.FindStringSubmatch(rest); m != nil { // "3:-" or "3-": the whole chapter
|
||||
ch := r.clampChapter(book, atoi(m[1]))
|
||||
pos.chapter = ch
|
||||
return at(ch, 1), at(ch, maxChapterIDSpan), pos, true
|
||||
}
|
||||
if m := reChapterRange.FindStringSubmatch(rest); m != nil { // chapters 3 to 4
|
||||
first, last := r.clampChapter(book, atoi(m[1])), r.clampChapter(book, atoi(m[2]))
|
||||
if last < first {
|
||||
first, last = last, first
|
||||
}
|
||||
pos.chapter = first
|
||||
return at(first, 1), at(last, maxChapterIDSpan), pos, true
|
||||
}
|
||||
if m := reVerse.FindStringSubmatch(rest); m != nil {
|
||||
ch := r.clampChapter(book, atoi(m[1]))
|
||||
verse := max(1, atoi(m[2]))
|
||||
pos.chapter, pos.verse = ch, verse
|
||||
return at(ch, verse), 0, pos, true
|
||||
}
|
||||
if m := reVerseOn.FindStringSubmatch(rest); m != nil { // "3:16-": from verse 16 to the end
|
||||
ch := r.clampChapter(book, atoi(m[1]))
|
||||
pos.chapter, pos.verse = ch, atoi(m[2])
|
||||
return at(ch, max(1, atoi(m[2]))), at(ch, maxChapterIDSpan), pos, true
|
||||
}
|
||||
if m := reVerseRange.FindStringSubmatch(rest); m != nil {
|
||||
ch := r.clampChapter(book, atoi(m[1]))
|
||||
first, last := max(1, atoi(m[2])), max(1, atoi(m[3]))
|
||||
if last < first {
|
||||
first, last = last, first
|
||||
}
|
||||
pos.chapter, pos.verse = ch, first
|
||||
return at(ch, first), at(ch, last), pos, true
|
||||
}
|
||||
if m := reVerseAcrossCh.FindStringSubmatch(rest); m != nil { // 3:16-4:2 across chapters
|
||||
c1, c2 := r.clampChapter(book, atoi(m[1])), r.clampChapter(book, atoi(m[3]))
|
||||
s, e := at(c1, max(1, atoi(m[2]))), at(c2, max(1, atoi(m[4])))
|
||||
if e < s {
|
||||
return 0, 0, pos, false
|
||||
}
|
||||
pos.chapter, pos.verse = c1, atoi(m[2])
|
||||
return s, e, pos, true
|
||||
}
|
||||
return 0, 0, pos, false
|
||||
}
|
||||
|
||||
type RefRange struct{ Start, End int64 } // End == 0: a single verse
|
||||
|
||||
type Note struct{ Original, Name string }
|
||||
|
||||
// Resolved is a whole parsed query.
|
||||
type Resolved struct {
|
||||
Refs []RefRange
|
||||
Notes []Note // short explanations of guesses that were made
|
||||
Bare bool // the query was just a book name (shows its first chapter)
|
||||
Label string // the name of the first book, for messages
|
||||
}
|
||||
|
||||
// Resolve parses a whole query ('John 3:16, Romans 5:8') into verse id ranges.
|
||||
// It returns nil when the text does not read as scripture references, so the
|
||||
// caller can treat it as a keyword search.
|
||||
func (r *Resolver) Resolve(text string, looseBare bool) *Resolved {
|
||||
if strings.TrimSpace(text) == "" || len(text) > maxRefQueryLen {
|
||||
return nil
|
||||
}
|
||||
var pieces []string
|
||||
for _, p := range regexp.MustCompile(`[,;\n]+`).Split(text, -1) {
|
||||
if strings.TrimSpace(p) != "" {
|
||||
pieces = append(pieces, p)
|
||||
}
|
||||
}
|
||||
if len(pieces) == 0 || len(pieces) > 20 {
|
||||
return nil
|
||||
}
|
||||
out := &Resolved{}
|
||||
var previous *refPos
|
||||
for _, piece := range pieces {
|
||||
var start, end int64
|
||||
var pos refPos
|
||||
var name string
|
||||
if m := bookPieceRE.FindStringSubmatchIndex(piece); m != nil {
|
||||
bi, ri := bookPieceRE.SubexpIndex("book"), bookPieceRE.SubexpIndex("rest")
|
||||
token := piece[m[2*bi]:m[2*bi+1]]
|
||||
hasRest := m[2*ri] >= 0
|
||||
found, ok := r.resolveBook(token, hasRest || looseBare)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
name = r.store.BookName(found.ID)
|
||||
if found.How == "fuzzy" {
|
||||
out.Notes = append(out.Notes, Note{strings.TrimSpace(token), name})
|
||||
}
|
||||
if !hasRest {
|
||||
if len(pieces) == 1 {
|
||||
out.Bare = true
|
||||
}
|
||||
base := int64(found.ID) * 1_000_000
|
||||
start, end, pos = base+1001, base+1999, refPos{found.ID, 1, 0}
|
||||
} else {
|
||||
var ok bool
|
||||
start, end, pos, ok = r.parseRefRest(piece[m[2*ri]:m[2*ri+1]], found.ID, nil)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
} else if continuationRE.MatchString(piece) && previous != nil {
|
||||
name = r.store.BookName(previous.book)
|
||||
var ok bool
|
||||
start, end, pos, ok = r.parseRefRest(piece, previous.book, previous)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
p := pos
|
||||
previous = &p
|
||||
if out.Label == "" {
|
||||
out.Label = name
|
||||
}
|
||||
out.Refs = append(out.Refs, RefRange{start, end})
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,163 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Optional endpoints backed by the separate sermon database (KJV_SERMON_DB)
|
||||
// and the separate semantic search service (KJV_SEMANTIC_ADDR, default
|
||||
// localhost:1613). Neither is part of the distribution; without them these
|
||||
// endpoints answer with an error.
|
||||
|
||||
func rowsToMaps(rows *sql.Rows) ([]map[string]any, error) {
|
||||
defer rows.Close()
|
||||
cols, err := rows.Columns()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []map[string]any{}
|
||||
for rows.Next() {
|
||||
vals := make([]any, len(cols))
|
||||
ptrs := make([]any, len(cols))
|
||||
for i := range vals {
|
||||
ptrs[i] = &vals[i]
|
||||
}
|
||||
if err := rows.Scan(ptrs...); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := map[string]any{}
|
||||
for i, c := range cols {
|
||||
if b, ok := vals[i].([]byte); ok {
|
||||
vals[i] = string(b)
|
||||
}
|
||||
m[c] = vals[i]
|
||||
}
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (a *App) sermonQuery(query string, args ...any) Resp {
|
||||
if a.sermonDB == nil {
|
||||
return jsonStatus(map[string]string{"error": "Sermon database not found"}, http.StatusServiceUnavailable)
|
||||
}
|
||||
rows, err := a.sermonDB.Query(query, args...)
|
||||
if err != nil {
|
||||
lg.Error("sermon query: %v", err)
|
||||
return errJSON(http.StatusInternalServerError, "Sermon database error")
|
||||
}
|
||||
out, err := rowsToMaps(rows)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusInternalServerError, "Sermon database error")
|
||||
}
|
||||
return jsonR(out)
|
||||
}
|
||||
|
||||
const sermonCols = "select distinct title,url,denomination,preacher,church_name from verse_refs where "
|
||||
|
||||
// handleSermons serves everything under /sermon/ and /sermons/ ... by hand,
|
||||
// since the paths overlap in ways the router would reject.
|
||||
func (a *App) handleSermons(path string) Resp {
|
||||
parts := strings.Split(strings.Trim(path, "/"), "/")
|
||||
switch {
|
||||
case len(parts) == 1 && parts[0] == "sermons":
|
||||
return a.sermonQuery("select distinct title,url,denomination,preacher,church_name,filename from verse_refs")
|
||||
case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "preacher":
|
||||
return a.sermonQuery(sermonCols+"preacher=?", parts[2])
|
||||
case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "church":
|
||||
return a.sermonQuery(sermonCols+"church_name=?", parts[2])
|
||||
case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "denomination":
|
||||
return a.sermonQuery(sermonCols+"denomination=?", parts[2])
|
||||
case len(parts) == 3 && parts[0] == "sermon" && parts[1] == "search":
|
||||
return a.sermonsByRef(parts[2])
|
||||
case len(parts) == 3 && parts[0] == "sermon" && parts[1] == "url":
|
||||
return a.sermonQuery(sermonCols+"url=?", parts[2])
|
||||
case len(parts) == 3 && parts[0] == "sermon" && parts[2] == "verses":
|
||||
return a.sermonQuery("select distinct verse_id,text from verse_refs where filename=?", parts[1])
|
||||
case len(parts) == 2 && parts[0] == "sermon":
|
||||
return a.sermonQuery(sermonCols+"filename=?", parts[1])
|
||||
}
|
||||
return a.notFound()
|
||||
}
|
||||
|
||||
func (a *App) sermonsByRef(ref string) Resp {
|
||||
id, ok := a.refVerseID(ref)
|
||||
if !ok {
|
||||
return a.notFound()
|
||||
}
|
||||
if a.sermonDB == nil {
|
||||
return jsonR(map[string]string{"error": "Sermon database not found"})
|
||||
}
|
||||
rows, err := a.sermonDB.Query("select * from verse_refs where verse_id=?", id)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusInternalServerError, "Sermon database error")
|
||||
}
|
||||
out, err := rowsToMaps(rows)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusInternalServerError, "Sermon database error")
|
||||
}
|
||||
for _, m := range out {
|
||||
if vid, ok := m["verse_id"].(int64); ok {
|
||||
if v := a.store.Range(vid, 0); len(v) == 1 {
|
||||
m["kjv"] = v[0].Text
|
||||
}
|
||||
}
|
||||
}
|
||||
return jsonR(out)
|
||||
}
|
||||
|
||||
// refVerseID: the verse id of the first reference in "1John5:7" style text.
|
||||
func (a *App) refVerseID(ref string) (int64, bool) {
|
||||
resolved := a.res.Resolve(ref, true)
|
||||
if resolved == nil {
|
||||
return 0, false
|
||||
}
|
||||
return resolved.Refs[0].Start, true
|
||||
}
|
||||
|
||||
func (a *App) handleRef(ref string) Resp {
|
||||
id, ok := a.refVerseID(ref)
|
||||
if !ok {
|
||||
return a.notFound()
|
||||
}
|
||||
return htmlR(itoa64(id))
|
||||
}
|
||||
|
||||
func (a *App) handleNaturalPost(body io.Reader) Resp {
|
||||
var req struct {
|
||||
Token *string `json:"token"`
|
||||
Query string `json:"query"`
|
||||
}
|
||||
if err := json.NewDecoder(io.LimitReader(body, 1<<16)).Decode(&req); err != nil {
|
||||
return errJSON(http.StatusBadRequest, "Invalid request format")
|
||||
}
|
||||
if req.Token == nil {
|
||||
return errJSON(http.StatusUnauthorized, "Missing token in request body")
|
||||
}
|
||||
if a.secretKey == "" || subtle.ConstantTimeCompare([]byte(*req.Token), []byte(a.secretKey)) != 1 {
|
||||
return errJSON(http.StatusUnauthorized, "Invalid Authorization")
|
||||
}
|
||||
conn, err := net.DialTimeout("tcp", a.semantic, 3*time.Second)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusBadGateway, "Semantic search is not available")
|
||||
}
|
||||
defer conn.Close()
|
||||
conn.SetDeadline(time.Now().Add(30 * time.Second))
|
||||
payload, _ := json.Marshal(map[string]string{"query": req.Query})
|
||||
if _, err := conn.Write(payload); err != nil {
|
||||
return errJSON(http.StatusBadGateway, "Semantic search is not available")
|
||||
}
|
||||
data, err := io.ReadAll(io.LimitReader(conn, 8<<20))
|
||||
var out any
|
||||
if err != nil || json.Unmarshal(data, &out) != nil {
|
||||
return errJSON(http.StatusBadGateway, "Semantic search gave an invalid answer")
|
||||
}
|
||||
return jsonR(out)
|
||||
}
|
||||
@@ -0,0 +1,313 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/signal"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// localOnly insists on a loopback address: the program is meant to sit behind
|
||||
// a reverse proxy on the same machine and never face the network itself.
|
||||
func localOnly(addr string) (string, error) {
|
||||
host, port, err := net.SplitHostPort(addr)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("bad listen address %q: %w", addr, err)
|
||||
}
|
||||
switch host {
|
||||
case "", "localhost":
|
||||
host = "127.0.0.1"
|
||||
default:
|
||||
if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
|
||||
return "", fmt.Errorf("refusing to listen on %q: only localhost addresses (127.0.0.1, ::1) are allowed; put a reverse proxy in front", host)
|
||||
}
|
||||
}
|
||||
return net.JoinHostPort(host, port), nil
|
||||
}
|
||||
|
||||
// Run serves the application from the companion database until interrupted.
|
||||
func Run(dbPath, addr string) error {
|
||||
addr, err := localOnly(addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := os.Stat(dbPath); err != nil {
|
||||
return fmt.Errorf("database not found: %s (use -db or KJV_DB)", dbPath)
|
||||
}
|
||||
store, err := OpenStore(dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer store.Close()
|
||||
if store.stateOK {
|
||||
lg.Attach(store.db, 90)
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "WARN: database is read-only: logs go to stderr; reading plans and shortcodes are disabled")
|
||||
}
|
||||
defer lg.Close()
|
||||
|
||||
app, err := NewApp(store)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
app.secretKey = os.Getenv("KJV_SECRET_API_KEY")
|
||||
app.semantic = envOr("KJV_SEMANTIC_ADDR", "localhost:1613")
|
||||
if p := os.Getenv("KJV_SERMON_DB"); p != "" {
|
||||
if st, err := os.Stat(p); err == nil && st.Size() > 0 {
|
||||
if db, err := sql.Open("sqlite", "file:"+p+"?mode=ro&_pragma=busy_timeout(5000)"); err == nil {
|
||||
app.sermonDB = db
|
||||
}
|
||||
} else {
|
||||
lg.Warn("sermon database %s not available", p)
|
||||
}
|
||||
}
|
||||
|
||||
srv := &http.Server{
|
||||
Addr: addr,
|
||||
Handler: app.Routes(),
|
||||
MaxHeaderBytes: 16 << 10,
|
||||
ReadHeaderTimeout: 10 * time.Second,
|
||||
ReadTimeout: 30 * time.Second,
|
||||
WriteTimeout: 60 * time.Second,
|
||||
IdleTimeout: 120 * time.Second,
|
||||
}
|
||||
go app.maintain()
|
||||
lg.Info("Started %s: db=%s addr=%s verses=%d state=%v sermons=%v", runtime.Version(), dbPath, addr, len(store.verses), store.stateOK, app.sermonDB != nil)
|
||||
fmt.Printf("kjv-api listening on %s (database %s)\n", addr, dbPath)
|
||||
|
||||
ln, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- srv.Serve(ln) }()
|
||||
sig := make(chan os.Signal, 1)
|
||||
signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
|
||||
select {
|
||||
case err := <-errc:
|
||||
return err
|
||||
case s := <-sig:
|
||||
lg.Info("Stopping on %v", s)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
if err := srv.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// routes
|
||||
|
||||
type statusWriter struct {
|
||||
http.ResponseWriter
|
||||
status int
|
||||
}
|
||||
|
||||
func (w *statusWriter) WriteHeader(code int) { w.status = code; w.ResponseWriter.WriteHeader(code) }
|
||||
|
||||
func (a *App) Routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
get := func(pattern string, h func(r *http.Request) Resp) {
|
||||
mux.HandleFunc("GET "+pattern, func(w http.ResponseWriter, r *http.Request) { h(r).write(w) })
|
||||
}
|
||||
|
||||
get("/{$}", func(r *http.Request) Resp { return htmlR(a.indexHTML) })
|
||||
get("/{shortcode}", func(r *http.Request) Resp { return a.handleShortcode(r.PathValue("shortcode")) })
|
||||
get("/fe", func(r *http.Request) Resp { return a.staticPage("kjv_fe.html") })
|
||||
get("/salvation", func(r *http.Request) Resp { return a.staticPage("html/heaven.html") })
|
||||
get("/saved", func(r *http.Request) Resp { return a.staticPage("html/saved.html") })
|
||||
get("/plan", func(r *http.Request) Resp { return a.staticPage("html/plan.html") })
|
||||
|
||||
get("/show", func(r *http.Request) Resp { return a.handleShow(r.URL.Query()) })
|
||||
get("/find", func(r *http.Request) Resp { return a.handleFind(r.URL.Query()) })
|
||||
get("/search", func(r *http.Request) Resp {
|
||||
v := r.URL.Query()
|
||||
return a.handleSearch(v.Get("kw"), viewOf(v, "rich"))
|
||||
})
|
||||
get("/define", func(r *http.Request) Resp { return a.handleDefine(r.URL.Query()) })
|
||||
get("/defs", func(r *http.Request) Resp { return a.handleDefs(r.URL.Query()) })
|
||||
get("/random", func(r *http.Request) Resp { return a.handleRandom(r.URL.Query(), "json") })
|
||||
get("/t/random", func(r *http.Request) Resp { return a.handleRandom(r.URL.Query(), "plain") })
|
||||
get("/votd", func(r *http.Request) Resp { return a.handleVotd(r.URL.Query(), "json") })
|
||||
get("/t/votd", func(r *http.Request) Resp { return a.handleVotd(r.URL.Query(), "plain") })
|
||||
get("/seq", func(r *http.Request) Resp {
|
||||
if r.URL.Query().Get("getuid") != "" && !creations.allow(clientAddr(r)) {
|
||||
return tooMany()
|
||||
}
|
||||
return a.handleSeq(r.URL.Query())
|
||||
})
|
||||
get("/mksc/{query}", func(r *http.Request) Resp {
|
||||
if !creations.allow(clientAddr(r)) {
|
||||
return tooMany()
|
||||
}
|
||||
return a.handleMakeShortcode(r.PathValue("query"))
|
||||
})
|
||||
get("/ref/{ref}", func(r *http.Request) Resp { return a.handleRef(r.PathValue("ref")) })
|
||||
get("/verse_id/{id}", func(r *http.Request) Resp {
|
||||
id := cleanInput(r.PathValue("id"), "0123456789")
|
||||
n, err := strconv.ParseInt(id, 10, 64)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusBadRequest, "Invalid verse id")
|
||||
}
|
||||
v := a.store.Range(n, 0)
|
||||
if len(v) == 0 {
|
||||
return errJSON(http.StatusNotFound, "No such verse")
|
||||
}
|
||||
return jsonR([]any{v[0].ID, v[0].Book, v[0].Chapter, v[0].Verse, v[0].Text})
|
||||
})
|
||||
get("/xref/{id}", func(r *http.Request) Resp { return a.handleXref(r.PathValue("id")) })
|
||||
get("/browse", func(r *http.Request) Resp {
|
||||
body := "<h2>Books</h2><div class=\"table-wrap\"><table id=\"Books\">"
|
||||
for _, n := range a.store.bookNames {
|
||||
body += fmt.Sprintf("<tr><td><a href=\"/find?kw=%s&view=rich\">%s</a></td></tr>", urlQuery(n), htmlEsc(n))
|
||||
}
|
||||
return htmlR(a.staticHTMLDoc("Books", body+"</table></div>"))
|
||||
})
|
||||
get("/status", func(r *http.Request) Resp {
|
||||
var m runtime.MemStats
|
||||
runtime.ReadMemStats(&m)
|
||||
return htmlR(a.staticHTMLDoc("Status", fmt.Sprintf("<h2>Status</h2><p>Sanity: %v</p><p>Uptime: %s</p><p>Memory: %d KB</p>",
|
||||
a.store.stateOK, time.Since(a.started).Round(time.Second), m.Sys/1024)))
|
||||
})
|
||||
get("/seqreport", func(r *http.Request) Resp {
|
||||
if resp, ok := a.adminOnly(r); !ok {
|
||||
return resp
|
||||
}
|
||||
return a.tableReport("Sequence Report", "seq_report", "SELECT uid, verse, last_used FROM reading_tracker", false)
|
||||
})
|
||||
get("/screport", func(r *http.Request) Resp {
|
||||
if resp, ok := a.adminOnly(r); !ok {
|
||||
return resp
|
||||
}
|
||||
return a.tableReport("Shortcode Report", "shortcode_report", "SELECT shortcode, query, creation_date FROM shortcodes", true)
|
||||
})
|
||||
get("/natural", func(r *http.Request) Resp {
|
||||
return htmlR("Submit a POST request to this endpoint with the following params in the body:\n{\"token\": \"mytokenstring\", \"query\" : \"myquerystring\"}")
|
||||
})
|
||||
mux.HandleFunc("POST /natural", func(w http.ResponseWriter, r *http.Request) { a.handleNaturalPost(r.Body).write(w) })
|
||||
mux.HandleFunc("POST /plan/new", func(w http.ResponseWriter, r *http.Request) {
|
||||
if !creations.allow(clientAddr(r)) {
|
||||
tooMany().write(w)
|
||||
return
|
||||
}
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 16<<10)
|
||||
if r.ParseForm() != nil {
|
||||
errJSON(http.StatusRequestEntityTooLarge, "Request too large").write(w)
|
||||
return
|
||||
}
|
||||
a.handlePlanNew(r.PostForm).write(w)
|
||||
})
|
||||
mux.HandleFunc("POST /plan/{uid}/move", func(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, 16<<10)
|
||||
if r.ParseForm() != nil {
|
||||
errJSON(http.StatusRequestEntityTooLarge, "Request too large").write(w)
|
||||
return
|
||||
}
|
||||
a.handlePlanMove(r.PathValue("uid"), r.PostForm).write(w)
|
||||
})
|
||||
get("/plan/{uid}", func(r *http.Request) Resp { return a.handlePlanView(r.PathValue("uid"), r.URL.Query()) })
|
||||
|
||||
for _, p := range []string{"/sermons", "/sermons/{rest...}", "/sermon/{rest...}"} {
|
||||
get(p, func(r *http.Request) Resp { return a.handleSermons(r.URL.Path) })
|
||||
}
|
||||
for _, p := range []string{"/preachers", "/churches", "/denominations"} {
|
||||
cols := map[string]string{"/preachers": "denomination,preacher,church_name", "/churches": "denomination,church_name", "/denominations": "denomination"}[p]
|
||||
get(p, func(r *http.Request) Resp { return a.sermonQuery("select distinct " + cols + " from verse_refs") })
|
||||
}
|
||||
|
||||
for u := range servedPaths {
|
||||
u := u
|
||||
mux.HandleFunc("GET "+u, func(w http.ResponseWriter, r *http.Request) { a.serveAsset(w, r, u) })
|
||||
}
|
||||
favicon := func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "image/svg+xml")
|
||||
w.Header().Set("Cache-Control", "public, max-age=86400")
|
||||
w.Write(a.favicon)
|
||||
}
|
||||
mux.HandleFunc("GET /favicon.ico", favicon)
|
||||
mux.HandleFunc("GET /favicon.svg", favicon)
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { a.notFound().write(w) })
|
||||
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
sw := &statusWriter{ResponseWriter: w, status: 200}
|
||||
defer func() {
|
||||
if rec := recover(); rec != nil {
|
||||
lg.Error("panic serving %s: %v", r.URL.String(), rec)
|
||||
sw.status = 500
|
||||
errJSON(500, "Internal error").write(w)
|
||||
}
|
||||
lg.Info("%s %s %d %s", r.Method, r.URL.RequestURI(), sw.status, time.Since(start).Round(time.Microsecond))
|
||||
}()
|
||||
h := w.Header()
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("X-Frame-Options", "DENY")
|
||||
h.Set("Referrer-Policy", "same-origin")
|
||||
mux.ServeHTTP(sw, r)
|
||||
})
|
||||
}
|
||||
|
||||
// serveAsset: a URL whose ?v= hash matches the file is immutable for a year;
|
||||
// any other URL for it is revalidated every time.
|
||||
func (a *App) serveAsset(w http.ResponseWriter, r *http.Request, u string) {
|
||||
as := a.served[u]
|
||||
h := w.Header()
|
||||
h.Set("Content-Type", as.mime)
|
||||
h.Set("ETag", as.etag)
|
||||
if r.URL.Query().Get("v") == as.version {
|
||||
h.Set("Cache-Control", "public, max-age=31536000, immutable")
|
||||
} else {
|
||||
h.Set("Cache-Control", "no-cache")
|
||||
}
|
||||
if r.Header.Get("If-None-Match") == as.etag {
|
||||
w.WriteHeader(http.StatusNotModified)
|
||||
return
|
||||
}
|
||||
w.Write(as.body)
|
||||
}
|
||||
|
||||
func (a *App) handleXref(raw string) Resp {
|
||||
id, err := strconv.ParseInt(raw, 10, 64)
|
||||
if err != nil {
|
||||
return a.notFound()
|
||||
}
|
||||
refs, err := a.store.CrossReferences(id)
|
||||
if err != nil {
|
||||
lg.Error("xref: %v", err)
|
||||
return errJSON(http.StatusInternalServerError, "Database error")
|
||||
}
|
||||
out := []map[string]any{}
|
||||
for _, c := range refs {
|
||||
ref := a.store.FormatRef(c.Start)
|
||||
var verses []Verse
|
||||
if c.End > c.Start {
|
||||
if c.Start/1000 == c.End/1000 { // "John 3:16-18" when the range stays in one chapter
|
||||
ref += fmt.Sprintf("-%d", c.End%1000)
|
||||
} else {
|
||||
ref += "-" + a.store.FormatRef(c.End)
|
||||
}
|
||||
verses = a.store.Range(c.Start, c.End)
|
||||
} else {
|
||||
verses = a.store.Range(c.Start, 0)
|
||||
}
|
||||
text := ""
|
||||
for i, v := range verses {
|
||||
if i > 0 {
|
||||
text += " "
|
||||
}
|
||||
text += v.Text
|
||||
}
|
||||
out = append(out, map[string]any{"ref": ref, "verse_id": c.Start, "text": text})
|
||||
}
|
||||
return jsonR(out)
|
||||
}
|
||||
@@ -0,0 +1,253 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Speller checks and suggests spellings from kjv.dic (a plain word list) and
|
||||
// kjv.aff (which only adds a TRY order and REP replacement hints), so no C
|
||||
// library is needed.
|
||||
type Speller struct {
|
||||
words map[string]bool // exact entries
|
||||
byLower map[string][]string // lowercase -> entries
|
||||
list []string // lowercase entries, dictionary order, unique
|
||||
try []rune
|
||||
rep [][2]string
|
||||
}
|
||||
|
||||
func NewSpeller(dic, aff []byte) *Speller {
|
||||
s := &Speller{words: map[string]bool{}, byLower: map[string][]string{}}
|
||||
lines := strings.Split(string(dic), "\n")
|
||||
seen := map[string]bool{}
|
||||
for i, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if i == 0 || line == "" { // first line is the entry count
|
||||
continue
|
||||
}
|
||||
if k := strings.IndexByte(line, '/'); k >= 0 {
|
||||
line = line[:k]
|
||||
}
|
||||
s.words[line] = true
|
||||
low := strings.ToLower(line)
|
||||
s.byLower[low] = append(s.byLower[low], line)
|
||||
if !seen[low] {
|
||||
seen[low] = true
|
||||
s.list = append(s.list, low)
|
||||
}
|
||||
}
|
||||
for _, line := range strings.Split(string(aff), "\n") {
|
||||
f := strings.Fields(line)
|
||||
switch {
|
||||
case len(f) == 2 && f[0] == "TRY":
|
||||
s.try = []rune(f[1])
|
||||
case len(f) == 3 && f[0] == "REP":
|
||||
s.rep = append(s.rep, [2]string{f[1], strings.ReplaceAll(f[2], "_", " ")})
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func stripSyntax(w string) string {
|
||||
return strings.Map(func(r rune) rune {
|
||||
if r == '+' || r == '^' || r == '*' {
|
||||
return -1
|
||||
}
|
||||
return r
|
||||
}, w)
|
||||
}
|
||||
|
||||
func hasLetter(w string) bool {
|
||||
for _, r := range w {
|
||||
if unicode.IsLetter(r) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (s *Speller) Spell(word string) bool {
|
||||
word = stripSyntax(word)
|
||||
if !hasLetter(word) || s.words[word] {
|
||||
return true
|
||||
}
|
||||
low := strings.ToLower(word)
|
||||
if low != word && (word == strings.ToUpper(word) || word == capitalize(low)) {
|
||||
return len(s.byLower[low]) > 0
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func capitalize(w string) string {
|
||||
r := []rune(w)
|
||||
if len(r) == 0 {
|
||||
return w
|
||||
}
|
||||
return strings.ToUpper(string(r[:1])) + strings.ToLower(string(r[1:]))
|
||||
}
|
||||
|
||||
func (s *Speller) known(low string) []string { return s.byLower[low] }
|
||||
|
||||
// Suggest returns likely corrections, best first.
|
||||
func (s *Speller) Suggest(word string) []string {
|
||||
word = stripSyntax(word)
|
||||
low := strings.ToLower(word)
|
||||
if low == "" {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
add := func(cands ...string) {
|
||||
for _, c := range cands {
|
||||
if !seen[c] {
|
||||
seen[c] = true
|
||||
out = append(out, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
addLower := func(l string) { // dictionary form(s) of a lowercase candidate
|
||||
if l == low {
|
||||
return
|
||||
}
|
||||
add(s.known(l)...)
|
||||
}
|
||||
if low != word { // wrong capitalization of a known word
|
||||
add(s.known(low)...)
|
||||
}
|
||||
// replacement hints from the .aff file, applied at each occurrence
|
||||
for _, r := range s.rep {
|
||||
for at := 0; ; {
|
||||
i := strings.Index(low[at:], r[0])
|
||||
if i < 0 {
|
||||
break
|
||||
}
|
||||
i += at
|
||||
cand := low[:i] + r[1] + low[i+len(r[0]):]
|
||||
if strings.Contains(cand, " ") {
|
||||
if p := strings.Fields(cand); len(p) == 2 && len(s.known(p[0])) > 0 && len(s.known(p[1])) > 0 {
|
||||
add(p[0] + " " + p[1])
|
||||
}
|
||||
} else {
|
||||
addLower(cand)
|
||||
}
|
||||
at = i + 1
|
||||
}
|
||||
}
|
||||
rs := []rune(low)
|
||||
// one edit away: swap, delete, replace, insert
|
||||
for i := 0; i+1 < len(rs); i++ {
|
||||
c := append([]rune(nil), rs...)
|
||||
c[i], c[i+1] = c[i+1], c[i]
|
||||
addLower(string(c))
|
||||
}
|
||||
for i := range rs {
|
||||
addLower(string(rs[:i]) + string(rs[i+1:]))
|
||||
}
|
||||
for i := range rs {
|
||||
for _, t := range s.try {
|
||||
addLower(string(rs[:i]) + string(t) + string(rs[i+1:]))
|
||||
}
|
||||
}
|
||||
for i := 0; i <= len(rs); i++ {
|
||||
for _, t := range s.try {
|
||||
addLower(string(rs[:i]) + string(t) + string(rs[i:]))
|
||||
}
|
||||
}
|
||||
// two words run together ("lordgod")
|
||||
for i := 1; i < len(rs); i++ {
|
||||
a, b := string(rs[:i]), string(rs[i:])
|
||||
if len(a) > 1 && len(b) > 1 && len(s.known(a)) > 0 && len(s.known(b)) > 0 {
|
||||
add(a + " " + b)
|
||||
}
|
||||
}
|
||||
// still little to offer: anything within two edits, closest first
|
||||
if len(out) < 3 {
|
||||
type cand struct {
|
||||
w string
|
||||
dist int
|
||||
pre int
|
||||
}
|
||||
var cs []cand
|
||||
for _, w := range s.list {
|
||||
if abs(len(w)-len(low)) > 2 || w == low {
|
||||
continue
|
||||
}
|
||||
if d := editDistance(low, w, 2); d <= 2 {
|
||||
cs = append(cs, cand{w, d, commonPrefix(low, w)})
|
||||
}
|
||||
}
|
||||
sort.SliceStable(cs, func(i, j int) bool {
|
||||
if cs[i].dist != cs[j].dist {
|
||||
return cs[i].dist < cs[j].dist
|
||||
}
|
||||
return cs[i].pre > cs[j].pre
|
||||
})
|
||||
for _, c := range cs {
|
||||
addLower(c.w)
|
||||
if len(out) >= 8 {
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(out) > 10 {
|
||||
out = out[:10]
|
||||
}
|
||||
if word == strings.ToUpper(word) && len(word) > 1 {
|
||||
for i := range out {
|
||||
out[i] = strings.ToUpper(out[i])
|
||||
}
|
||||
} else if word == capitalize(low) && word != low {
|
||||
for i := range out {
|
||||
out[i] = capitalize(out[i])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func abs(n int) int {
|
||||
if n < 0 {
|
||||
return -n
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
func commonPrefix(a, b string) int {
|
||||
n := 0
|
||||
for n < len(a) && n < len(b) && a[n] == b[n] {
|
||||
n++
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// editDistance is optimal-string-alignment distance, giving up above limit.
|
||||
func editDistance(a, b string, limit int) int {
|
||||
ra, rb := []rune(a), []rune(b)
|
||||
prev2 := make([]int, len(rb)+1)
|
||||
prev := make([]int, len(rb)+1)
|
||||
cur := make([]int, len(rb)+1)
|
||||
for j := range prev {
|
||||
prev[j] = j
|
||||
}
|
||||
for i := 1; i <= len(ra); i++ {
|
||||
cur[0] = i
|
||||
best := cur[0]
|
||||
for j := 1; j <= len(rb); j++ {
|
||||
cost := 1
|
||||
if ra[i-1] == rb[j-1] {
|
||||
cost = 0
|
||||
}
|
||||
v := min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
|
||||
if i > 1 && j > 1 && ra[i-1] == rb[j-2] && ra[i-2] == rb[j-1] {
|
||||
v = min(v, prev2[j-2]+1)
|
||||
}
|
||||
cur[j] = v
|
||||
best = min(best, v)
|
||||
}
|
||||
if best > limit {
|
||||
return limit + 1
|
||||
}
|
||||
prev2, prev, cur = prev, cur, prev2
|
||||
}
|
||||
return prev[len(rb)]
|
||||
}
|
||||
@@ -0,0 +1,521 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/subtle"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"html"
|
||||
"math"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Reading sequences (/seq), reading plans (/plan) and shortcodes (/mksc) keep
|
||||
// their rows in the companion database.
|
||||
|
||||
const (
|
||||
idChars = "0123456789ACEFHJKLMNPRTUVWXY"
|
||||
planMaxPerDay = 500
|
||||
seqPurgeAfter = 90 * 24 * time.Hour
|
||||
stateTimeFormat = "2006-01-02 15:04:05.000000"
|
||||
)
|
||||
|
||||
func generateShortID() string { // 5 characters
|
||||
n, err := rand.Int(rand.Reader, big.NewInt(17_210_367-614_656))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
remain := 614_656 + int(n.Int64())
|
||||
out := ""
|
||||
for remain > 0 {
|
||||
out = string(idChars[remain%len(idChars)]) + out
|
||||
remain /= len(idChars)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func stateNow() string { return time.Now().Format(stateTimeFormat) }
|
||||
|
||||
var errNoState = fmt.Errorf("the database is read-only, so reading plans and shortcodes are unavailable")
|
||||
|
||||
func (a *App) stateDB() (*sql.DB, error) {
|
||||
if !a.store.stateOK {
|
||||
return nil, errNoState
|
||||
}
|
||||
return a.store.db, nil
|
||||
}
|
||||
|
||||
// insertID stores a row under a fresh random id.
|
||||
func insertID(db *sql.DB, query string, args ...any) (string, error) {
|
||||
var err error
|
||||
for i := 0; i < 5; i++ { // a clash on a random 5 character id is very unlikely
|
||||
id := generateShortID()
|
||||
if _, err = db.Exec(query, append([]any{id}, args...)...); err == nil {
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
return "", err
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// shortcodes
|
||||
|
||||
func (a *App) handleMakeShortcode(query string) Resp {
|
||||
query = cleanInput(query, searchChars+":-,")
|
||||
if query == "" {
|
||||
return htmlR("Empty query.")
|
||||
}
|
||||
db, err := a.stateDB()
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
var code string
|
||||
if err := db.QueryRow("SELECT shortcode FROM shortcodes WHERE query = ?", query).Scan(&code); err == nil {
|
||||
return htmlR(code) // the same query always gets the same code
|
||||
}
|
||||
code, err = insertID(db, "INSERT INTO shortcodes (shortcode, query, creation_date) VALUES (?, ?, ?)", query, stateNow())
|
||||
if err != nil {
|
||||
lg.Error("shortcode insert: %v", err)
|
||||
return errJSON(http.StatusInternalServerError, "Could not create a shortcode")
|
||||
}
|
||||
lg.Info("Generated shortcode %s for query '%s'", code, query)
|
||||
return htmlR(code)
|
||||
}
|
||||
|
||||
func (a *App) handleShortcode(code string) Resp {
|
||||
clean := cleanInput(code, idChars)
|
||||
if clean == "" {
|
||||
return htmlR("Empty shortcode.")
|
||||
}
|
||||
db, err := a.stateDB()
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
var query string
|
||||
if err := db.QueryRow("SELECT query FROM shortcodes WHERE shortcode = ?", clean).Scan(&query); err != nil {
|
||||
return a.notFound()
|
||||
}
|
||||
lg.Info("Resolved shortcode %s for query '%s'", clean, query)
|
||||
return Resp{Status: http.StatusFound, Type: "text/html; charset=utf-8",
|
||||
Header: map[string]string{"Location": "/find?kw=" + url.QueryEscape(query) + "&view=rich"}}
|
||||
}
|
||||
|
||||
func (a *App) notFound() Resp {
|
||||
return jsonStatus(map[string]string{
|
||||
"error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."},
|
||||
http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (a *App) tableReport(title, id, query string, header bool) Resp {
|
||||
db, err := a.stateDB()
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
rows, err := db.Query(query)
|
||||
if err != nil {
|
||||
lg.Error("report: %v", err)
|
||||
return errJSON(http.StatusInternalServerError, "Database error")
|
||||
}
|
||||
defer rows.Close()
|
||||
var sb strings.Builder
|
||||
fmt.Fprintf(&sb, "<h2>%s</h2><div class=\"table-wrap\"><table id=\"%s\">", title, id)
|
||||
if header {
|
||||
sb.WriteString("<tr><td>shortcode</td><td>query</td><td>creation_date</td></tr>")
|
||||
}
|
||||
for rows.Next() {
|
||||
var c1, c2, c3 sql.NullString
|
||||
rows.Scan(&c1, &c2, &c3)
|
||||
fmt.Fprintf(&sb, "<tr><td>%s</td><td>%s</td><td>%s</td></tr>", html.EscapeString(c1.String), html.EscapeString(c2.String), html.EscapeString(c3.String))
|
||||
}
|
||||
sb.WriteString("</table></div>")
|
||||
return htmlR(a.staticHTMLDoc(title, sb.String()))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /seq : hand out scripture a few verses at a time
|
||||
|
||||
func (a *App) purgeSequences(db *sql.DB) {
|
||||
rows, err := db.Query("SELECT uid, last_used FROM reading_tracker LIMIT 10")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var stale []string
|
||||
for rows.Next() {
|
||||
var uid, last string
|
||||
rows.Scan(&uid, &last)
|
||||
if t, err := time.ParseInLocation(stateTimeFormat, last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
|
||||
stale = append(stale, uid)
|
||||
} else if t, err := time.ParseInLocation("2006-01-02 15:04:05", last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
|
||||
stale = append(stale, uid)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
for _, uid := range stale {
|
||||
lg.Info("Purging UID: %s", uid)
|
||||
db.Exec("DELETE FROM reading_tracker WHERE uid = ?", uid)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) startIndex(ref string) (int, bool) {
|
||||
ref = strings.TrimSpace(ref)
|
||||
if ref == "" {
|
||||
return 0, true
|
||||
}
|
||||
resolved := a.res.Resolve(ref, true)
|
||||
if resolved == nil {
|
||||
return 0, false
|
||||
}
|
||||
i := a.store.VerseIndex(resolved.Refs[0].Start)
|
||||
return i, i >= 0
|
||||
}
|
||||
|
||||
func (a *App) handleSeq(q url.Values) Resp {
|
||||
uid, getuid := q.Get("uid"), q.Get("getuid") != ""
|
||||
num := 1
|
||||
if q.Has("num") {
|
||||
n, err := strconv.Atoi(q.Get("num"))
|
||||
if err != nil {
|
||||
return htmlR("Invalid num argument")
|
||||
}
|
||||
num = n
|
||||
}
|
||||
if uid == "" && !getuid {
|
||||
return a.staticPage("html/seq.html")
|
||||
}
|
||||
num-- // 0 will mean "next"
|
||||
num = min(num, len(a.store.verseIDs))
|
||||
if num < 0 {
|
||||
return jsonR(map[string]string{"error": "You may only increment forward."})
|
||||
}
|
||||
db, err := a.stateDB()
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
if getuid {
|
||||
start, ok := a.startIndex(q.Get("start"))
|
||||
if !ok {
|
||||
return jsonR(map[string]string{"error": "Could not find that starting verse."})
|
||||
}
|
||||
id, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
|
||||
if err != nil {
|
||||
lg.Error("UID database failed to insert: %v", err)
|
||||
return jsonR(map[string]string{"error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"})
|
||||
}
|
||||
lg.Info("New SEQ UID: %s", id)
|
||||
return htmlR("Your new UID is: " + id)
|
||||
}
|
||||
uid = cleanInput(strings.ReplaceAll(uid, " ", ""), searchChars[:62]+":-,")
|
||||
var next int
|
||||
if err := db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&next); err != nil {
|
||||
lg.Info("Invalid UID %s", uid)
|
||||
return jsonR(map[string]string{"error": "UID Invalid or purged (90 days of inactivity)"})
|
||||
}
|
||||
a.purgeSequences(db)
|
||||
total := len(a.store.verseIDs)
|
||||
if next >= total || next < 0 { // finished the Bible: start over
|
||||
next = 0
|
||||
}
|
||||
last := min(next+num, total-1)
|
||||
verses := a.store.Range(a.store.verseIDs[next], a.store.verseIDs[last])
|
||||
if _, err := db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, last+1, stateNow()); err != nil {
|
||||
lg.Error("seq update: %v", err)
|
||||
}
|
||||
lg.Info("Served %d verses to UID %s", last-next+1, uid)
|
||||
return a.render(verses, viewOf(q, "json"), "").resp()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// /plan : a /seq style tracker that can go backwards. The plan is its UID; the
|
||||
// page URL (/plan/<uid>?n=<verses per day>) is what gets bookmarked.
|
||||
|
||||
func planPerDay(raw string, def int) int {
|
||||
n, err := strconv.Atoi(raw)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
return max(1, min(planMaxPerDay, n))
|
||||
}
|
||||
|
||||
func (a *App) planError(msg string) Resp {
|
||||
r := htmlR(a.staticHTMLDoc("Reading Plan",
|
||||
"<h2>Reading Plan</h2><p>"+html.EscapeString(msg)+"</p><p><a href='/plan'>Back to plan setup</a></p>"))
|
||||
r.Status = http.StatusBadRequest
|
||||
return r
|
||||
}
|
||||
|
||||
func redirect303(to string) Resp {
|
||||
return Resp{Status: http.StatusSeeOther, Type: "text/html; charset=utf-8", Header: map[string]string{"Location": to}}
|
||||
}
|
||||
|
||||
func validPlanUID(uid string) bool { return uid != "" && cleanInput(uid, idChars) == uid }
|
||||
|
||||
func (a *App) handlePlanNew(f url.Values) Resp {
|
||||
db, err := a.stateDB()
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
total := len(a.store.verseIDs)
|
||||
start, ok := a.startIndex(f.Get("start"))
|
||||
if !ok {
|
||||
return a.planError("We couldn't find that starting point. Try something like 'Matthew 1:1'.")
|
||||
}
|
||||
var perDay int
|
||||
if f.Get("pace") == "verses" {
|
||||
perDay = planPerDay(f.Get("verses"), 25)
|
||||
} else {
|
||||
days, err := strconv.Atoi(f.Get("days"))
|
||||
if err != nil {
|
||||
days = 365
|
||||
}
|
||||
days = max(1, min(3650, days))
|
||||
perDay = planPerDay(strconv.Itoa(int(math.Ceil(float64(total-start)/float64(days)))), 25)
|
||||
}
|
||||
uid, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
|
||||
if err != nil {
|
||||
return a.planError("Could not create a plan, please try again.")
|
||||
}
|
||||
lg.Info("New reading plan %s start=%d per_day=%d", uid, start, perDay)
|
||||
return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
|
||||
}
|
||||
|
||||
func (a *App) planPosition(uid string) (int, bool, error) {
|
||||
db, err := a.stateDB()
|
||||
if err != nil {
|
||||
return 0, false, err
|
||||
}
|
||||
var pos int
|
||||
err = db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&pos)
|
||||
return pos, err == nil, nil
|
||||
}
|
||||
|
||||
func (a *App) handlePlanMove(uid string, f url.Values) Resp {
|
||||
perDay := planPerDay(f.Get("n"), 25)
|
||||
if !validPlanUID(uid) {
|
||||
return a.planError("That reading plan id is not valid.")
|
||||
}
|
||||
pos, found, err := a.planPosition(uid)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
if !found {
|
||||
return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
|
||||
}
|
||||
total := len(a.store.verseIDs)
|
||||
switch f.Get("action") {
|
||||
case "next":
|
||||
pos = min(total, pos+perDay)
|
||||
case "back":
|
||||
pos = max(0, pos-perDay)
|
||||
case "restart":
|
||||
pos = 0
|
||||
case "goto":
|
||||
target, ok := a.startIndex(f.Get("kw"))
|
||||
if !ok {
|
||||
return a.planError("We couldn't find that reference. Try something like 'John 3:1'.")
|
||||
}
|
||||
pos = target
|
||||
}
|
||||
a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, pos, stateNow())
|
||||
return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
|
||||
}
|
||||
|
||||
func (a *App) handlePlanView(uid string, q url.Values) Resp {
|
||||
if !validPlanUID(uid) {
|
||||
return a.planError("That reading plan id is not valid.")
|
||||
}
|
||||
perDay := planPerDay(q.Get("n"), 25)
|
||||
position, found, err := a.planPosition(uid)
|
||||
if err != nil {
|
||||
return errJSON(http.StatusServiceUnavailable, err.Error())
|
||||
}
|
||||
if !found {
|
||||
return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
|
||||
}
|
||||
// keeps an active plan from being purged as unused
|
||||
a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, position, stateNow())
|
||||
|
||||
ids := a.store.verseIDs
|
||||
total := len(ids)
|
||||
finished := position >= total
|
||||
lastIndex := min(total, position+perDay) - 1
|
||||
percent := math.Round(1000*float64(min(position, total))/float64(total)) / 10
|
||||
esc := html.EscapeString
|
||||
|
||||
controls := fmt.Sprintf("<form method='post' action='/plan/%s/move' class='plan-form'>\n<input type='hidden' name='n' value='%d'>", uid, perDay)
|
||||
backDisabled := ""
|
||||
if position <= 0 {
|
||||
backDisabled = "disabled"
|
||||
}
|
||||
next := ""
|
||||
if !finished {
|
||||
next = "<button type='submit' name='action' value='next' class='plan-primary'>Done — next reading →</button>"
|
||||
}
|
||||
controls += fmt.Sprintf(`%s
|
||||
<button type='submit' name='action' value='back' class='plan-secondary' %s>← Previous reading</button>
|
||||
</form>
|
||||
<form method='post' action='/plan/%s/move' class='plan-form plan-goto'>
|
||||
<input type='hidden' name='n' value='%d'>
|
||||
<input type='hidden' name='action' value='goto'>
|
||||
<label for='goto-kw'>Jump to</label>
|
||||
<div class='plan-goto-row'><input type='text' id='goto-kw' name='kw' data-book-suggest placeholder='e.g. Psalms 23:1' autocapitalize='off' autocomplete='off'>
|
||||
<button type='submit' class='plan-secondary'>Go</button></div>
|
||||
</form>`, next, backDisabled, uid, perDay)
|
||||
|
||||
var summary, versesHTML string
|
||||
if finished {
|
||||
summary = fmt.Sprintf("<div class='plan-summary'><strong>You've finished the plan!</strong><div>You read all %s verses. Use Previous reading to look back, or start over.</div></div>", commas(total))
|
||||
controls += fmt.Sprintf("<form method='post' action='/plan/%s/move' class='plan-form'>\n<input type='hidden' name='n' value='%d'>\n<button type='submit' name='action' value='restart' class='plan-secondary'>Start over from Genesis 1:1</button></form>", uid, perDay)
|
||||
} else {
|
||||
first, last := a.store.FormatRef(ids[position]), a.store.FormatRef(ids[lastIndex])
|
||||
summary = fmt.Sprintf("<div class='plan-summary'><strong>Today's reading</strong><div class='plan-range'>%s – %s</div>"+
|
||||
"<div class='plan-meta'>%d verses · %.1f%% of the Bible complete</div>"+
|
||||
"<div class='plan-progress' role='img' aria-label='%.1f%% complete'><span style='width:%.1f%%'></span></div></div>",
|
||||
esc(first), esc(last), lastIndex-position+1, percent, percent, percent)
|
||||
var sb strings.Builder
|
||||
for _, v := range a.store.Range(ids[position], ids[lastIndex]) {
|
||||
sb.WriteString(a.responseRich(v))
|
||||
}
|
||||
versesHTML = sb.String()
|
||||
}
|
||||
note := "<div class='plan-note'>Bookmark this page — it is your reading plan. Come back each day, then press <em>Done</em> to move on.</div>"
|
||||
store := fmt.Sprintf("<script>try{localStorage.setItem('kjv.plan',JSON.stringify({uid:'%s',n:%d,t:Date.now()}))}catch(e){}</script>", uid, perDay)
|
||||
return htmlR(a.renderNavbar("", nil, "") + "<div class='responses'>" + summary + versesHTML +
|
||||
"<div class='plan-controls'>" + controls + "</div>" + note + store + "</div>")
|
||||
}
|
||||
|
||||
func commas(n int) string {
|
||||
s := strconv.Itoa(n)
|
||||
for i := len(s) - 3; i > 0; i -= 3 {
|
||||
s = s[:i] + "," + s[i:]
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// abuse limits
|
||||
|
||||
// creationLimiter bounds how fast each visitor can add rows (shortcodes,
|
||||
// plans, sequences), with a looser limit over everyone together as a backstop.
|
||||
type creationLimiter struct {
|
||||
mu sync.Mutex
|
||||
clients map[string]*bucket
|
||||
all bucket
|
||||
}
|
||||
|
||||
type bucket struct {
|
||||
tokens float64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
const (
|
||||
clientRate, clientBurst = 0.5, 10.0 // per visitor: a new row every 2 seconds, 10 at once
|
||||
totalRate, totalBurst = 20.0, 100.0
|
||||
maxTrackedClients = 10000
|
||||
)
|
||||
|
||||
var creations = &creationLimiter{clients: map[string]*bucket{}, all: bucket{totalBurst, time.Now()}}
|
||||
|
||||
func (b *bucket) take(now time.Time, rate, burst float64) bool {
|
||||
b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*rate)
|
||||
b.last = now
|
||||
if b.tokens < 1 {
|
||||
return false
|
||||
}
|
||||
b.tokens--
|
||||
return true
|
||||
}
|
||||
|
||||
func (c *creationLimiter) allow(client string) bool {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
now := time.Now()
|
||||
if len(c.clients) > maxTrackedClients { // forget visitors who have long since refilled
|
||||
for k, b := range c.clients {
|
||||
if now.Sub(b.last) > time.Minute {
|
||||
delete(c.clients, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
if isLoopback(client) { // the proxy did not say who this is: only the shared limit applies
|
||||
return c.all.take(now, totalRate, totalBurst)
|
||||
}
|
||||
b := c.clients[client]
|
||||
if b == nil {
|
||||
b = &bucket{clientBurst, now}
|
||||
c.clients[client] = b
|
||||
}
|
||||
if !b.take(now, clientRate, clientBurst) {
|
||||
return false
|
||||
}
|
||||
return c.all.take(now, totalRate, totalBurst)
|
||||
}
|
||||
|
||||
// clientAddr is the visitor as the reverse proxy saw them. Only the proxy can
|
||||
// reach this program (it listens on localhost), so the header it sets is
|
||||
// trusted; the last X-Forwarded-For entry is the one the proxy itself added.
|
||||
func clientAddr(r *http.Request) string {
|
||||
if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); ip != "" {
|
||||
return ip
|
||||
}
|
||||
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||
parts := strings.Split(xff, ",")
|
||||
return strings.TrimSpace(parts[len(parts)-1])
|
||||
}
|
||||
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
noClientHeader.Do(func() {
|
||||
lg.Warn("requests arrive without X-Real-IP or X-Forwarded-For: the reverse proxy should pass the visitor's address on, " +
|
||||
"otherwise creating plans and shortcodes is limited for all visitors together, not for each")
|
||||
})
|
||||
return host
|
||||
}
|
||||
|
||||
var noClientHeader sync.Once
|
||||
|
||||
func isLoopback(addr string) bool {
|
||||
ip := net.ParseIP(addr)
|
||||
return addr == "" || (ip != nil && ip.IsLoopback())
|
||||
}
|
||||
|
||||
func tooMany() Resp {
|
||||
r := errJSON(http.StatusTooManyRequests, "Too many requests, try again in a moment")
|
||||
r.Header = map[string]string{"Retry-After": "5"}
|
||||
return r
|
||||
}
|
||||
|
||||
// maintain deletes reading plans nobody has used for 90 days (the pages promise this).
|
||||
func (a *App) maintain() {
|
||||
for {
|
||||
if a.store.stateOK {
|
||||
cutoff := time.Now().Add(-seqPurgeAfter).Format(stateTimeFormat)
|
||||
if res, err := a.store.db.Exec("DELETE FROM reading_tracker WHERE last_used < ?", cutoff); err == nil {
|
||||
if n, _ := res.RowsAffected(); n > 0 {
|
||||
lg.Info("Purged %d unused reading plans", n)
|
||||
}
|
||||
}
|
||||
}
|
||||
time.Sleep(6 * time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
// adminOnly protects the listings of every plan id and shortcode (a plan id is
|
||||
// all that is needed to move that plan). They need the KJV_SECRET_API_KEY as a
|
||||
// bearer token, and do not exist when no key is set.
|
||||
func (a *App) adminOnly(r *http.Request) (Resp, bool) {
|
||||
if a.secretKey == "" {
|
||||
return a.notFound(), false
|
||||
}
|
||||
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
|
||||
if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(a.secretKey)) != 1 {
|
||||
resp := errJSON(http.StatusUnauthorized, "Invalid Authorization")
|
||||
resp.Header = map[string]string{"WWW-Authenticate": "Bearer"}
|
||||
return resp, false
|
||||
}
|
||||
return Resp{}, true
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
package kjv
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"sort"
|
||||
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
// Verse is one row of the fts_kjv table. ID is book*1_000_000 + chapter*1000 + verse.
|
||||
type Verse struct {
|
||||
ID int64
|
||||
Book int
|
||||
Chapter int
|
||||
Verse int
|
||||
Text string
|
||||
}
|
||||
|
||||
type BookInfo struct {
|
||||
Title string
|
||||
Category string
|
||||
OTNT string
|
||||
Chapters int
|
||||
}
|
||||
|
||||
// Store holds the companion database plus everything read from it at startup.
|
||||
// The whole text is only ~4MB, so verse lookups never touch SQLite; the FTS
|
||||
// index is still queried for keyword search.
|
||||
type Store struct {
|
||||
db *sql.DB
|
||||
|
||||
verses []Verse // sorted by ID
|
||||
verseIDs []int64
|
||||
chapterList []int64 // chapter keys (ID/1000), sorted and unique
|
||||
bookNames []string
|
||||
bookName map[int]string
|
||||
bookByName map[string]int
|
||||
chapterLen map[int]int
|
||||
bookInfo []BookInfo
|
||||
abbrevs []abbrev
|
||||
|
||||
stateOK bool // reading plans and shortcodes can be written
|
||||
}
|
||||
|
||||
type abbrev struct {
|
||||
Name string
|
||||
Book int
|
||||
}
|
||||
|
||||
func OpenStore(path string) (*Store, error) {
|
||||
dsn := "file:" + path + "?_pragma=busy_timeout(5000)"
|
||||
db, err := sql.Open("sqlite", dsn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s := &Store{db: db, bookName: map[int]string{}, bookByName: map[string]int{}, chapterLen: map[int]int{}}
|
||||
if err := s.load(); err != nil {
|
||||
db.Close()
|
||||
return nil, fmt.Errorf("%s: %w", path, err)
|
||||
}
|
||||
s.stateOK = s.initState() == nil
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Store) Close() error { return s.db.Close() }
|
||||
|
||||
func (s *Store) load() error {
|
||||
rows, err := s.db.Query("SELECT b, n FROM key_english ORDER BY b")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var id int
|
||||
var name string
|
||||
if err := rows.Scan(&id, &name); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
s.bookNames = append(s.bookNames, name)
|
||||
s.bookName[id] = name
|
||||
s.bookByName[name] = id
|
||||
}
|
||||
rows.Close()
|
||||
if len(s.bookNames) == 0 {
|
||||
return fmt.Errorf("key_english is empty")
|
||||
}
|
||||
|
||||
rows, err = s.db.Query("SELECT id, b, c, v, t FROM fts_kjv ORDER BY id")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var v Verse
|
||||
if err := rows.Scan(&v.ID, &v.Book, &v.Chapter, &v.Verse, &v.Text); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
s.verses = append(s.verses, v)
|
||||
}
|
||||
rows.Close()
|
||||
if len(s.verses) == 0 {
|
||||
return fmt.Errorf("fts_kjv is empty")
|
||||
}
|
||||
for _, v := range s.verses {
|
||||
s.verseIDs = append(s.verseIDs, v.ID)
|
||||
if key := v.ID / 1000; len(s.chapterList) == 0 || s.chapterList[len(s.chapterList)-1] != key {
|
||||
s.chapterList = append(s.chapterList, key)
|
||||
}
|
||||
if v.Chapter > s.chapterLen[v.Book] {
|
||||
s.chapterLen[v.Book] = v.Chapter
|
||||
}
|
||||
}
|
||||
|
||||
rows, err = s.db.Query(`SELECT title_short, category, otnt, chapters FROM book_info ORDER BY "order"`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var b BookInfo
|
||||
if err := rows.Scan(&b.Title, &b.Category, &b.OTNT, &b.Chapters); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
s.bookInfo = append(s.bookInfo, b)
|
||||
}
|
||||
rows.Close()
|
||||
|
||||
rows, err = s.db.Query("SELECT a, b FROM key_abbreviations_english ORDER BY id")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for rows.Next() {
|
||||
var a abbrev
|
||||
if err := rows.Scan(&a.Name, &a.Book); err != nil {
|
||||
rows.Close()
|
||||
return err
|
||||
}
|
||||
s.abbrevs = append(s.abbrevs, a)
|
||||
}
|
||||
rows.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// initState creates the tables that hold user-generated data (reading plans and
|
||||
// shortcodes) inside the companion database.
|
||||
func (s *Store) initState() error {
|
||||
for _, stmt := range []string{
|
||||
"CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)",
|
||||
"CREATE TABLE IF NOT EXISTS shortcodes (shortcode TEXT PRIMARY KEY, query TEXT UNIQUE, creation_date TEXT)",
|
||||
} {
|
||||
if _, err := s.db.Exec(stmt); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) BookName(id int) string { return s.bookName[id] }
|
||||
|
||||
func (s *Store) FormatRef(verseID int64) string {
|
||||
return fmt.Sprintf("%s %d:%d", s.BookName(int(verseID/1_000_000)), verseID/1000%1000, verseID%1000)
|
||||
}
|
||||
|
||||
// Range returns the verses with start <= id <= end (end <= 0 means just start).
|
||||
func (s *Store) Range(start, end int64) []Verse {
|
||||
i := sort.Search(len(s.verses), func(i int) bool { return s.verses[i].ID >= start })
|
||||
if end <= 0 {
|
||||
if i < len(s.verses) && s.verses[i].ID == start {
|
||||
return s.verses[i : i+1]
|
||||
}
|
||||
return nil
|
||||
}
|
||||
j := sort.Search(len(s.verses), func(j int) bool { return s.verses[j].ID > end })
|
||||
if i >= j {
|
||||
return nil
|
||||
}
|
||||
return s.verses[i:j]
|
||||
}
|
||||
|
||||
// VerseIndex is the position of a verse id in the whole Bible, or -1.
|
||||
func (s *Store) VerseIndex(id int64) int {
|
||||
i := sort.Search(len(s.verseIDs), func(i int) bool { return s.verseIDs[i] >= id })
|
||||
if i < len(s.verseIDs) && s.verseIDs[i] == id {
|
||||
return i
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// FTS runs a full text query and returns the matching verses in Bible order.
|
||||
func (s *Store) FTS(query string) ([]Verse, error) {
|
||||
rows, err := s.db.Query("SELECT id FROM fts_kjv(?)", query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Verse
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if v := s.Range(id, 0); len(v) == 1 {
|
||||
out = append(out, v[0])
|
||||
}
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
type crossRef struct{ Start, End int64 }
|
||||
|
||||
func (s *Store) CrossReferences(verseID int64) ([]crossRef, error) {
|
||||
rows, err := s.db.Query("SELECT sv, ev FROM cross_reference WHERE vid = ? ORDER BY r DESC LIMIT 15", verseID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []crossRef
|
||||
for rows.Next() {
|
||||
var c crossRef
|
||||
if err := rows.Scan(&c.Start, &c.End); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
#!/bin/sh
|
||||
# run from current directory only
|
||||
FILE=app.py
|
||||
if test -f "$FILE"; then
|
||||
uwsgi_python3 --http-socket 0.0.0.0:1612 --wsgi-file app.py --callable app --pyargv "-vvv" --stats 0.0.0.0:9192
|
||||
else
|
||||
cd /opt/kjv-api
|
||||
echo "WARNING: RUNNING FROM /opt/kjv-api"
|
||||
uwsgi_python3 --http-socket 0.0.0.0:1612 --wsgi-file /opt/kjv-api/app.py --callable app --pyargv "-vvv"
|
||||
fi
|
||||
@@ -1,3 +0,0 @@
|
||||
#!/bin/sh
|
||||
cd /opt/kjv-api
|
||||
uwsgi_python3 --http-socket :1611 --wsgi-file /opt/kjv-api/app.py --callable app --stats 127.0.0.1:9191
|
||||
-1009
File diff suppressed because it is too large.
Load diff
Whitespace-only changes.
Whitespace-only changes.
Executable
+11
@@ -0,0 +1,11 @@
|
||||
#!/bin/sh
|
||||
# Builds the program and runs it against a throwaway copy of the database on port 1612.
|
||||
# Reads page/script edits from assets/ each time (loads them into the copy).
|
||||
set -e
|
||||
cd "$(dirname "$0")/.."
|
||||
make build
|
||||
dir=$(mktemp -d "${TMPDIR:-/var/tmp}/kjv-test.XXXXXX")
|
||||
cp db/kjv.db "$dir/kjv.db"
|
||||
bin/kjv-api import-assets -db "$dir/kjv.db" assets
|
||||
echo "test database: $dir/kjv.db"
|
||||
exec bin/kjv-api -db "$dir/kjv.db" -addr 127.0.0.1:1612
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/bin/sh
|
||||
# Runs the built program on localhost port 1611, building dist/ first if needed.
|
||||
# (The user service, dist/kjv-api.service, runs a copy in ~/kjv-api on port 1611 instead.)
|
||||
cd "$(dirname "$0")/.." || exit 1
|
||||
[ -f dist/kjv-api ] && [ -f dist/kjv.db ] || make dist || exit 1
|
||||
exec dist/kjv-api -db dist/kjv.db -addr 127.0.0.1:1611
|
||||
@@ -1,110 +0,0 @@
|
||||
"""Fuzzes /find with many ways of typing a reference and checks the verses that come back.
|
||||
|
||||
Run from the project root: python3 tools/fuzz_refs.py [sample_size]
|
||||
Expected verse ids come straight from data/kjv.db, not from the app.
|
||||
"""
|
||||
import sys, re, random, sqlite3, itertools, json
|
||||
sys.path.insert(0, ".") # run from the project root
|
||||
import logging
|
||||
import app as a
|
||||
a.app.testing = True
|
||||
client = a.app.test_client()
|
||||
|
||||
db = sqlite3.connect("data/kjv.db")
|
||||
BOOKS = [r[0] for r in db.execute("select n from key_english order by b")]
|
||||
ids = {} # (book, chap) -> [(verse, id)]
|
||||
for vid, b, c, v in db.execute("select id,b,c,v from fts_kjv"):
|
||||
ids.setdefault((int(b), int(c)), []).append((int(v), int(vid)))
|
||||
maxch = {b: max(c for (bb, c) in ids if bb == b) for b in range(1, 67)}
|
||||
|
||||
NAME_VARIANTS = {
|
||||
1: ["Gen", "Ge", "Gn"], 2: ["Exod", "Ex", "Exo"], 3: ["Lev", "Lv"], 4: ["Num", "Nm", "Nu"],
|
||||
5: ["Deut", "Dt", "Deu"], 6: ["Josh", "Jos"], 7: ["Judg", "Jdg", "Jg"], 8: ["Ru"],
|
||||
9: ["1 Sam", "1 Sm", "1Sa", "I Samuel", "1st Samuel", "First Samuel"],
|
||||
10: ["2 Sam", "2 Sm", "2Sa", "II Samuel", "2nd Samuel", "Second Samuel"],
|
||||
11: ["1 Kgs", "1 Ki", "1Kin", "I Kings", "First Kings"], 12: ["2 Kgs", "2 Ki", "II Kings", "Second Kings"],
|
||||
13: ["1 Chron", "1 Chr", "1Ch", "I Chronicles", "First Chronicles"], 14: ["2 Chron", "2 Chr", "II Chronicles"],
|
||||
15: ["Ezr"], 16: ["Neh", "Ne"], 17: ["Est", "Esth"], 18: [], 19: ["Ps", "Psa", "Psalm", "Pslm"],
|
||||
20: ["Prov", "Pr", "Prv"], 21: ["Eccl", "Ecc", "Eccles", "Ecclesiasts"],
|
||||
22: ["Song", "Song of Songs", "Songs of Solomon", "Song of Sol", "SOS", "Canticles", "SongofSolomon"],
|
||||
23: ["Isa", "Is"], 24: ["Jer", "Je"], 25: ["Lam"], 26: ["Ezek", "Eze", "Ezk"], 27: ["Dan", "Dn"],
|
||||
28: ["Hos", "Ho"], 29: ["Joe", "Jl"], 30: ["Am"], 31: ["Obad", "Ob"], 32: ["Jon", "Jnh"], 33: ["Mic"],
|
||||
34: ["Nah", "Na"], 35: ["Hab"], 36: ["Zeph", "Zep"], 37: ["Hag", "Hg"], 38: ["Zech", "Zec"], 39: ["Mal"],
|
||||
40: ["Matt", "Mt", "Mat"], 41: ["Mk", "Mar"], 42: ["Lk", "Luk"], 43: ["Jn", "Joh"], 44: ["Act"],
|
||||
45: ["Rom", "Ro"], 46: ["1 Cor", "1Co", "I Corinthians", "First Corinthians", "1st Corinthians"],
|
||||
47: ["2 Cor", "2Co", "II Corinthians", "Second Corinthians"], 48: ["Gal", "Ga"], 49: ["Eph"],
|
||||
50: ["Phil", "Php", "Philip"], 51: ["Col"], 52: ["1 Thess", "1 Th", "1Thes", "I Thessalonians"],
|
||||
53: ["2 Thess", "2 Th", "II Thessalonians"], 54: ["1 Tim", "1 Ti", "I Timothy", "First Timothy"],
|
||||
55: ["2 Tim", "2 Ti", "II Timothy"], 56: ["Tit", "Ti"], 57: ["Phlm", "Phm", "Philem"], 58: ["Heb"],
|
||||
59: ["Jas", "Jam"], 60: ["1 Pet", "1 Pe", "1Pt", "I Peter", "First Peter"], 61: ["2 Pet", "2 Pe", "II Peter"],
|
||||
62: ["1 Jn", "1 Jo", "1Joh", "I John", "First John", "1st John"], 63: ["2 Jn", "2 John", "II John"],
|
||||
64: ["3 Jn", "3 John", "III John"], 65: ["Jud"], 66: ["Rev", "Re", "Revelations", "Revelation of John"],
|
||||
}
|
||||
|
||||
def name_forms(b):
|
||||
full = BOOKS[b - 1]
|
||||
forms = {full, full.lower(), full.upper(), full.replace(" ", ""), full.lower().replace(" ", "")}
|
||||
forms |= set(NAME_VARIANTS.get(b, []))
|
||||
forms |= {f.lower() for f in NAME_VARIANTS.get(b, [])}
|
||||
return sorted(forms)
|
||||
|
||||
def cases():
|
||||
rnd = random.Random(1611)
|
||||
for b in range(1, 67):
|
||||
for nm in name_forms(b):
|
||||
ch = rnd.randint(1, maxch[b])
|
||||
vs = ids[(b, ch)]
|
||||
v1 = rnd.choice(vs)[0]
|
||||
v2 = min(vs[-1][0], v1 + rnd.randint(1, 3))
|
||||
chap_ids = [i for _, i in vs]
|
||||
single = [i for v, i in vs if v == v1]
|
||||
rng = [i for v, i in vs if v1 <= v <= v2]
|
||||
key = a.normalize_book_text(nm)
|
||||
if key in a.BOOK_CANONICAL or key in a.BARE_BOOK_EXTRA: # abbreviations alone stay keyword searches
|
||||
for sp in ("", " ", " "):
|
||||
yield (f"{nm}{sp}", [i for _, i in ids[(b, 1)]], "bare") # book alone -> chapter 1
|
||||
for fmt, exp, kind in (
|
||||
(f"{nm} {ch}", chap_ids, "chapter"),
|
||||
(f"{nm}{ch}", chap_ids, "chapter-nospace"),
|
||||
(f"{nm} {ch}:{v1}", single, "verse"),
|
||||
(f"{nm}{ch}:{v1}", single, "verse-nospace"),
|
||||
(f" {nm} {ch} : {v1} ", single, "verse-spaced"),
|
||||
(f"{nm} {ch}.{v1}", single, "verse-dot"),
|
||||
(f"{nm} {ch} {v1}", single, "verse-space-sep"),
|
||||
(f"{nm} {ch}:{v1}-{v2}", rng, "range"),
|
||||
(f"{nm} {ch}:{v1} - {v2}", rng, "range-spaced"),
|
||||
(f"{nm} {ch}:{v1}–{v2}", rng, "range-endash"),
|
||||
(f"{nm.upper()} {ch}:{v1}", single, "verse-upper"),
|
||||
(f"{nm} {ch}:{v1}-{v2}\t", rng, "range-trailing-tab"),
|
||||
):
|
||||
if v2 > v1 or kind not in ("range", "range-spaced", "range-endash", "range-trailing-tab"):
|
||||
yield fmt, exp, kind
|
||||
|
||||
def verse_ids_in(html):
|
||||
return [int(x) for x in re.findall(r'data-verse-id="(\d+)"', html)]
|
||||
|
||||
def run(limit=None, show=25):
|
||||
allcases = list(cases())
|
||||
if limit:
|
||||
random.Random(7).shuffle(allcases); allcases = allcases[:limit]
|
||||
fails = {}
|
||||
total = 0
|
||||
for q, exp, kind in allcases:
|
||||
total += 1
|
||||
try:
|
||||
r = client.get("/find", query_string={"kw": q, "view": "rich"})
|
||||
got = verse_ids_in(r.get_data(as_text=True)) if r.status_code == 200 else None
|
||||
except Exception as e:
|
||||
got = None
|
||||
ok = got is not None and got[:len(exp)] == exp and (kind.startswith("range") or kind.startswith("verse") or kind == "chapter" or True) and (len(got) == len(exp) if kind != "bare" else got[:len(exp)] == exp)
|
||||
if not ok:
|
||||
fails.setdefault(kind, []).append((q, len(exp), None if got is None else len(got)))
|
||||
nfail = sum(len(v) for v in fails.values())
|
||||
print(f"{total} cases, {nfail} failed ({100*nfail//max(total,1)}%)")
|
||||
for k, v in sorted(fails.items(), key=lambda kv: -len(kv[1])):
|
||||
print(f" {k}: {len(v)} e.g. {v[:3]}")
|
||||
return fails
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.disable(logging.CRITICAL)
|
||||
run(int(sys.argv[1]) if len(sys.argv) > 1 else None)
|
||||
@@ -1,21 +0,0 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Check for root privileges
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This script requires root privileges. Please run with sudo."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Stopping and disabling kjv-api service..."
|
||||
systemctl stop kjv-api.service
|
||||
systemctl disable kjv-api.service
|
||||
|
||||
echo "Removing kjv-api systemd service file..."
|
||||
rm /etc/systemd/system/kjv-api.service
|
||||
|
||||
echo "Removing kjv-api directory and user..."
|
||||
rm -rf /opt/kjv-api
|
||||
userdel kjv
|
||||
|
||||
echo "Uninstall complete."
|
||||
echo "Be sure to remove your webserver configuration file."
|
||||
Reference in new issue
Block a user