claude go reimplement

internal/kjv/app.go

@@ -0,0 +1,430 @@
+package kjv
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"database/sql"
+	"encoding/json"
+	"fmt"
+	"html"
+	"net/http"
+	"net/url"
+	"os"
+	"regexp"
+	"sort"
+	"strings"
+	"time"
+)
+
+// App is the whole web application. Everything it serves comes from the
+// companion database (see assets.go), loaded once at startup.
+type App struct {
+	store   *Store
+	res     *Resolver
+	speller *Speller
+	started time.Time
+
+	dictionary map[string][]string // 1828 Webster
+	variants   map[string][]string
+	synonyms   map[string][]string
+	concepts   map[string]concept
+	bookDescs  map[string]string
+	chapSums   map[string][]string
+
+	served map[string]servedAsset // versioned static files: /styles/page.css ...
+	pages  map[string]string      // simple html pages, asset urls versioned
+
+	htmlResponse, htmlNavbar, htmlChapterNav string
+	cssNavbar, cssCollapsible                string
+	favicon                                  []byte
+	indexHTML                                string
+
+	sermonDB  *sql.DB
+	secretKey string
+	semantic  string // host:port of the optional semantic search service
+}
+
+type concept struct {
+	Query       string   `json:"query"`
+	Explanation string   `json:"explanation"`
+	SearchTerms []string `json:"search_terms"`
+}
+
+type servedAsset struct {
+	body    []byte
+	mime    string
+	version string
+	etag    string
+}
+
+var servedPaths = map[string]string{ // url -> asset path
+	"/styles/page.css":    "styles/page.css",
+	"/js/collapsible.js":  "js/collapsible.js",
+	"/js/shortcode.js":    "js/shortcode.js",
+	"/js/verse_menu.js":   "js/verse_menu.js",
+	"/js/book_suggest.js": "js/book_suggest.js",
+}
+
+func NewApp(store *Store) (*App, error) {
+	assets, err := store.LoadAssets()
+	if err != nil {
+		return nil, err
+	}
+	need := func(p string) ([]byte, error) {
+		b, ok := assets[p]
+		if !ok {
+			return nil, fmt.Errorf("asset %s is missing from the database (run: kjv-api import-assets <source dir>)", p)
+		}
+		return b, nil
+	}
+	a := &App{store: store, res: NewResolver(store), started: time.Now(), pages: map[string]string{}, served: map[string]servedAsset{}}
+
+	loadJSON := func(p string, into any) error {
+		b, err := need(p)
+		if err != nil {
+			return err
+		}
+		if err := json.Unmarshal(b, into); err != nil {
+			return fmt.Errorf("%s: %w", p, err)
+		}
+		return nil
+	}
+	if err := loadJSON("data/1828_Webster_KJV.json", &a.dictionary); err != nil {
+		return nil, err
+	}
+	if err := loadJSON("data/book_descriptions.json", &a.bookDescs); err != nil {
+		return nil, err
+	}
+	if err := loadJSON("data/chapter_summaries.json", &a.chapSums); err != nil {
+		return nil, err
+	}
+	var groups [][]string
+	if err := loadJSON("data/grammatical_variants.json", &groups); err != nil {
+		return nil, err
+	}
+	a.variants = groupMap(groups)
+	if err := loadJSON("data/theological_synonyms.json", &groups); err != nil {
+		return nil, err
+	}
+	a.synonyms = groupMap(groups)
+	var concepts []concept
+	if err := loadJSON("data/concept_mappings.json", &concepts); err != nil {
+		return nil, err
+	}
+	a.concepts = map[string]concept{}
+	for _, c := range concepts {
+		a.concepts[strings.ToLower(c.Query)] = c
+	}
+	dic, err := need("data/kjv.dic")
+	if err != nil {
+		return nil, err
+	}
+	aff, err := need("data/kjv.aff")
+	if err != nil {
+		return nil, err
+	}
+	a.speller = NewSpeller(dic, aff)
+
+	// static assets, each URL carrying a hash of its own contents
+	for u, p := range servedPaths {
+		b, err := need(p)
+		if err != nil {
+			return nil, err
+		}
+		if p == "js/book_suggest.js" {
+			names, _ := json.Marshal(store.bookNames)
+			b = bytes.ReplaceAll(b, []byte("__BOOKS__"), names)
+		}
+		sum := sha256.Sum256(b)
+		mime := assetMIME[p[strings.LastIndexByte(p, '.'):]]
+		a.served[u] = servedAsset{b, mime, fmt.Sprintf("%x", sum[:4]), fmt.Sprintf(`"%x"`, sum[:8])}
+	}
+	text := func(p string) (string, error) { b, err := need(p); return string(b), err }
+	if a.htmlResponse, err = text("html/response.html"); err != nil {
+		return nil, err
+	}
+	if a.htmlChapterNav, err = text("html/chapter_nav.html"); err != nil {
+		return nil, err
+	}
+	if a.cssNavbar, err = text("styles/header.css"); err != nil {
+		return nil, err
+	}
+	if a.cssCollapsible, err = text("styles/collapsible.css"); err != nil {
+		return nil, err
+	}
+	nav, err := text("html/navbar.html")
+	if err != nil {
+		return nil, err
+	}
+	a.htmlNavbar = a.loadHTML(nav)
+	if a.favicon, err = need("static/favicon.svg"); err != nil {
+		return nil, err
+	}
+	for _, p := range []string{"html/find.html", "html/show.html", "html/search.html", "html/seq.html", "html/dictionary.html",
+		"html/saved.html", "html/plan.html", "html/heaven.html", "kjv_fe.html"} {
+		t, err := text(p)
+		if err != nil {
+			return nil, err
+		}
+		a.pages[p] = a.loadHTML(t)
+	}
+	index, err := text("html/index.html")
+	if err != nil {
+		return nil, err
+	}
+	a.indexHTML = a.renderIndex(a.loadHTML(index))
+	return a, nil
+}
+
+func groupMap(groups [][]string) map[string][]string {
+	m := map[string][]string{}
+	for _, g := range groups {
+		for _, w := range g {
+			var others []string
+			for _, o := range g {
+				if o != w {
+					others = append(others, o)
+				}
+			}
+			m[w] = others
+		}
+	}
+	return m
+}
+
+var assetRefRE = regexp.MustCompile(`(/(?:js|styles)/[\w.\-]+)\?v=__V__`)
+
+// loadHTML fills each '/js/x.js?v=__V__' with that file's own version.
+func (a *App) loadHTML(s string) string {
+	return assetRefRE.ReplaceAllStringFunc(s, func(m string) string {
+		u := assetRefRE.FindStringSubmatch(m)[1]
+		if as, ok := a.served[u]; ok {
+			return u + "?v=" + as.version
+		}
+		return m
+	})
+}
+
+func (a *App) assetURL(u string) string { return u + "?v=" + a.served[u].version }
+
+var indexLoopRE = regexp.MustCompile(`(?s)\{% for category, books in testament_data\.(OT|NT)\.items\(\) %\}.*?\n    \{% endfor %\}`)
+
+// renderIndex fills the book lists of the home page.
+func (a *App) renderIndex(tmpl string) string {
+	esc := html.EscapeString
+	return indexLoopRE.ReplaceAllStringFunc(tmpl, func(m string) string {
+		testament := indexLoopRE.FindStringSubmatch(m)[1]
+		var order []string
+		byCat := map[string][]BookInfo{}
+		for _, b := range a.store.bookInfo {
+			if b.OTNT != testament {
+				continue
+			}
+			if _, ok := byCat[b.Category]; !ok {
+				order = append(order, b.Category)
+			}
+			byCat[b.Category] = append(byCat[b.Category], b)
+		}
+		var sb strings.Builder
+		for _, cat := range order {
+			fmt.Fprintf(&sb, "\n        <h3 class=\"category-header\">%s</h3>\n        <div class=\"book-grid\">", esc(cat))
+			for _, b := range byCat[cat] {
+				fmt.Fprintf(&sb, "\n<div><button class=\"book-button collapsible\" data-book=\"%s\">%s</button>\n<div class=\"chapter-grid content\">", esc(b.Title), esc(b.Title))
+				if about := a.bookDescs[b.Title]; about != "" {
+					fmt.Fprintf(&sb, "<p class=\"book-about\">%s</p>", esc(about))
+				}
+				sums := a.chapSums[b.Title]
+				for c := 1; c <= b.Chapters; c++ {
+					fmt.Fprintf(&sb, "\n<a href=\"/show?kw=%s %d&view=rich\" class=\"chapter-link\"><span class=\"chapter-num\">%d</span>", esc(b.Title), c, c)
+					if len(sums) >= c {
+						fmt.Fprintf(&sb, "<span class=\"chapter-sum\">%s</span>", esc(sums[c-1]))
+					}
+					sb.WriteString("</a>")
+				}
+				sb.WriteString("\n</div></div>")
+			}
+			sb.WriteString("\n        </div>")
+		}
+		return sb.String()
+	})
+}
+
+// ---------------------------------------------------------------------------
+// responses
+
+type Resp struct {
+	Body   string
+	Type   string
+	Status int
+	Header map[string]string
+}
+
+func htmlR(s string) Resp { return Resp{Body: s, Type: "text/html; charset=utf-8"} }
+
+func jsonR(v any) Resp { return jsonStatus(v, 0) }
+
+func jsonStatus(v any, status int) Resp {
+	var buf bytes.Buffer
+	enc := json.NewEncoder(&buf)
+	enc.SetEscapeHTML(false)
+	enc.Encode(v)
+	return Resp{Body: buf.String(), Type: "application/json", Status: status}
+}
+
+func errJSON(status int, msg string) Resp { return jsonStatus(map[string]string{"error": msg}, status) }
+
+func (r Resp) write(w http.ResponseWriter) {
+	h := w.Header()
+	h.Set("Content-Type", r.Type)
+	if strings.HasPrefix(r.Type, "text/html") && r.Header["Cache-Control"] == "" {
+		h.Set("Cache-Control", "no-cache")
+	}
+	for k, v := range r.Header {
+		h.Set(k, v)
+	}
+	if r.Status != 0 {
+		w.WriteHeader(r.Status)
+	}
+	w.Write([]byte(r.Body))
+}
+
+func (a *App) staticPage(p string) Resp { return htmlR(a.pages[p]) }
+
+// staticHTMLDoc wraps a small html snippet in the standard page shell.
+func (a *App) staticHTMLDoc(title, body string) string {
+	return "<!DOCTYPE html><html lang='en'><head><meta charset='UTF-8'>" +
+		"<meta name='viewport' content='width=device-width, initial-scale=1.0'>" +
+		"<title>" + title + " - KJV API</title>" +
+		"<link rel='stylesheet' href='" + a.assetURL("/styles/page.css") + "'></head><body>" +
+		"<header><a href='/'>Home</a> <a href='/find'>Quick Search</a></header>" +
+		body + "</body></html>"
+}
+
+// ---------------------------------------------------------------------------
+// verse rendering
+
+func (a *App) responseRich(v Verse) string {
+	passage := fmt.Sprintf("%s %d:%d", a.store.BookName(v.Book), v.Chapter, v.Verse)
+	return strings.NewReplacer(
+		"{chapter}", passage[:strings.LastIndexByte(passage, ':')],
+		"{passage}", passage,
+		"{text}", v.Text,
+		"{verse_id}", fmt.Sprint(v.ID),
+	).Replace(a.htmlResponse)
+}
+
+func chapterFindLink(name string) string { return "/find?kw=" + url.PathEscape(name) + "&view=rich" }
+
+type chapterNav struct{ current, prev, next string }
+
+func (a *App) chapterName(key int64) string {
+	return fmt.Sprintf("%s %d", a.store.BookName(int(key/1000)), key%1000)
+}
+
+func (a *App) nextPrevChapter(verseID int64) chapterNav {
+	key := verseID / 1000
+	n := len(a.store.chapterList)
+	i := sort.Search(n, func(i int) bool { return a.store.chapterList[i] >= key })
+	return chapterNav{
+		current: a.chapterName(key),
+		prev:    a.chapterName(a.store.chapterList[(i-1+n)%n]),
+		next:    a.chapterName(a.store.chapterList[(i+1)%n]),
+	}
+}
+
+// renderNavbar opens the rich html document: <head>, site header and the
+// chapter bar. The caller adds the body content after it.
+func (a *App) renderNavbar(query string, nav *chapterNav, barHTML string) string {
+	esc := html.EscapeString
+	title := strings.TrimSpace(query)
+	if title == "" && nav != nil {
+		title = nav.current
+	}
+	if title == "" {
+		title = "Scripture"
+	}
+	chapterHTML := barHTML
+	if nav != nil {
+		chapterHTML = strings.NewReplacer(
+			"{current_chapter_name}", esc(nav.current),
+			"{prev_chapter_name}", esc(nav.prev),
+			"{prev_chapter_link}", esc("/show?kw="+nav.prev+"&view=rich"),
+			"{next_chapter_name}", esc(nav.next),
+			"{next_chapter_link}", esc("/show?kw="+nav.next+"&view=rich"),
+			"{current_chapter_link}", esc(chapterFindLink(nav.current)),
+		).Replace(a.htmlChapterNav)
+	}
+	return strings.NewReplacer(
+		"{title}", esc(title), "{css}", a.cssNavbar, "{query}", esc(query), "{chapter_nav}", chapterHTML,
+	).Replace(a.htmlNavbar)
+}
+
+type rendered struct {
+	text   string // plain and rich
+	list   []map[string]any
+	isJSON bool
+}
+
+func (r rendered) resp() Resp {
+	if r.isJSON {
+		return jsonR(r.list)
+	}
+	return htmlR(r.text)
+}
+
+// render lays out verses as view=plain, view=rich or (anything else) json.
+func (a *App) render(verses []Verse, view, kw string) rendered {
+	switch view {
+	case "plain":
+		parts := make([]string, len(verses))
+		for i, v := range verses {
+			parts[i] = fmt.Sprintf("%s %d:%d %s\n", a.store.BookName(v.Book), v.Chapter, v.Verse, v.Text)
+		}
+		return rendered{text: strings.Join(parts, "<br>")}
+	case "rich":
+		if len(verses) == 0 {
+			lg.Error("Malformed query: %s", kw)
+			return rendered{text: "Malformed Query - if you believe this to be in error (your query should resolve), email tyler@dinsmoor.us"}
+		}
+		// Prev/next chapter links only make sense when reading a single chapter,
+		// not for a keyword search that spans books.
+		var nav *chapterNav
+		first := verses[0].ID / 1000
+		single := true
+		for _, v := range verses {
+			if v.ID/1000 != first {
+				single = false
+				break
+			}
+		}
+		if single {
+			n := a.nextPrevChapter(verses[0].ID)
+			nav = &n
+		}
+		var sb strings.Builder
+		sb.WriteString(a.renderNavbar(kw, nav, ""))
+		sb.WriteString("<div class='responses'>")
+		fmt.Fprintf(&sb, "<div class='results-count'>%d Results</div>", len(verses))
+		for _, v := range verses {
+			sb.WriteString(a.responseRich(v))
+		}
+		sb.WriteString("</div>")
+		return rendered{text: sb.String()}
+	}
+	list := make([]map[string]any, len(verses))
+	for i, v := range verses {
+		list[i] = map[string]any{"bookname": a.store.BookName(v.Book), "chapter": v.Chapter, "verse": v.Verse, "text": v.Text}
+	}
+	return rendered{list: list, isJSON: true}
+}
+
+func envOr(key, def string) string {
+	if v := os.Getenv(key); v != "" {
+		return v
+	}
+	return def
+}
+
+func itoa64(n int64) string    { return fmt.Sprint(n) }
+func urlQuery(s string) string { return url.QueryEscape(s) }
+func htmlEsc(s string) string  { return html.EscapeString(s) }

internal/kjv/assets.go

@@ -0,0 +1,113 @@
+package kjv
+
+import (
+	"database/sql"
+	"fmt"
+	"io/fs"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+)
+
+// Everything the program serves or reads besides the Bible itself (pages,
+// scripts, styles, the 1828 dictionary, word lists) lives in the assets table
+// of the companion database, so the program and the database are the only two
+// files that need to be distributed.
+
+var assetMIME = map[string]string{
+	".html": "text/html; charset=utf-8",
+	".css":  "text/css; charset=utf-8",
+	".js":   "application/javascript; charset=utf-8",
+	".json": "application/json",
+	".svg":  "image/svg+xml",
+	".dic":  "text/plain; charset=utf-8",
+	".aff":  "text/plain; charset=utf-8",
+}
+
+func isAssetFile(rel string) bool {
+	dir := path.Dir(rel)
+	switch {
+	case rel == "kjv_fe.html":
+		return true
+	case dir == "html", dir == "js", dir == "static":
+		return true
+	case dir == "styles":
+		return strings.HasSuffix(rel, ".css")
+	case dir == "data":
+		ext := path.Ext(rel)
+		return ext == ".json" || ext == ".dic" || ext == ".aff"
+	}
+	return false
+}
+
+// ImportAssets replaces the assets table with the files of a source checkout.
+func ImportAssets(dbPath, srcDir string) error {
+	db, err := sql.Open("sqlite", "file:"+dbPath+"?_pragma=busy_timeout(5000)")
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	tx, err := db.Begin()
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	for _, stmt := range []string{
+		"CREATE TABLE IF NOT EXISTS assets (path TEXT PRIMARY KEY, mime TEXT NOT NULL, body BLOB NOT NULL)",
+		"DELETE FROM assets",
+	} {
+		if _, err := tx.Exec(stmt); err != nil {
+			return err
+		}
+	}
+	n := 0
+	err = filepath.WalkDir(srcDir, func(p string, d fs.DirEntry, err error) error {
+		if err != nil || !d.Type().IsRegular() { // no directories, and never follow a symlink out of the tree
+			return err
+		}
+		rel, _ := filepath.Rel(srcDir, p)
+		rel = filepath.ToSlash(rel)
+		if !isAssetFile(rel) {
+			return nil
+		}
+		body, err := os.ReadFile(p)
+		if err != nil {
+			return err
+		}
+		mime := assetMIME[path.Ext(rel)]
+		if mime == "" {
+			mime = "application/octet-stream"
+		}
+		n++
+		_, err = tx.Exec("INSERT INTO assets (path, mime, body) VALUES (?, ?, ?)", rel, mime, body)
+		return err
+	})
+	if err != nil {
+		return err
+	}
+	if n == 0 {
+		return fmt.Errorf("no assets found under %s (expected html/, js/, styles/, data/*.json ...)", srcDir)
+	}
+	fmt.Printf("imported %d assets into %s\n", n, dbPath)
+	return tx.Commit()
+}
+
+// LoadAssets reads every asset into memory.
+func (s *Store) LoadAssets() (map[string][]byte, error) {
+	rows, err := s.db.Query("SELECT path, body FROM assets")
+	if err != nil {
+		return nil, fmt.Errorf("no assets table (run: kjv-api import-assets <source dir>): %w", err)
+	}
+	defer rows.Close()
+	out := map[string][]byte{}
+	for rows.Next() {
+		var p string
+		var b []byte
+		if err := rows.Scan(&p, &b); err != nil {
+			return nil, err
+		}
+		out[p] = b
+	}
+	return out, rows.Err()
+}

internal/kjv/difflib.go

@@ -0,0 +1,76 @@
+package kjv
+
+// Ratcliff/Obershelp string similarity (the ratio of matching characters) and
+// a closest match lookup, which the fuzzy book name matching relies on.
+
+type matchBlock struct{ a, b, size int }
+
+func longestMatch(a, b []rune, alo, ahi, blo, bhi int) matchBlock {
+	besti, bestj, bestsize := alo, blo, 0
+	prev := make([]int, len(b)+1)
+	cur := make([]int, len(b)+1)
+	for i := alo; i < ahi; i++ {
+		for j := blo; j < bhi; j++ {
+			if a[i] != b[j] {
+				cur[j+1] = 0
+				continue
+			}
+			k := prev[j] + 1
+			cur[j+1] = k
+			if k > bestsize {
+				besti, bestj, bestsize = i-k+1, j-k+1, k
+			}
+		}
+		prev, cur = cur, prev
+		for j := range cur {
+			cur[j] = 0
+		}
+	}
+	return matchBlock{besti, bestj, bestsize}
+}
+
+func matchedChars(a, b []rune) int {
+	type span struct{ alo, ahi, blo, bhi int }
+	queue := []span{{0, len(a), 0, len(b)}}
+	total := 0
+	for len(queue) > 0 {
+		q := queue[len(queue)-1]
+		queue = queue[:len(queue)-1]
+		m := longestMatch(a, b, q.alo, q.ahi, q.blo, q.bhi)
+		if m.size == 0 {
+			continue
+		}
+		total += m.size
+		if q.alo < m.a && q.blo < m.b {
+			queue = append(queue, span{q.alo, m.a, q.blo, m.b})
+		}
+		if m.a+m.size < q.ahi && m.b+m.size < q.bhi {
+			queue = append(queue, span{m.a + m.size, q.ahi, m.b + m.size, q.bhi})
+		}
+	}
+	return total
+}
+
+func similarity(a, b string) float64 {
+	ra, rb := []rune(a), []rune(b)
+	if len(ra)+len(rb) == 0 {
+		return 1
+	}
+	return 2.0 * float64(matchedChars(ra, rb)) / float64(len(ra)+len(rb))
+}
+
+// closestMatch is the best scoring possibility with a ratio of at least cutoff.
+// Ties on score go to the lexicographically larger possibility.
+func closestMatch(word string, possibilities []string, cutoff float64) (string, bool) {
+	best, bestScore, found := "", 0.0, false
+	for _, p := range possibilities {
+		score := similarity(p, word) // difflib compares possibility (a) against word (b)
+		if score < cutoff {
+			continue
+		}
+		if !found || score > bestScore || (score == bestScore && p > best) {
+			best, bestScore, found = p, score, true
+		}
+	}
+	return best, found
+}

internal/kjv/logdb.go

@@ -0,0 +1,198 @@
+package kjv
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+	"unicode/utf8"
+)
+
+// DBLog writes log lines into the logs table of the companion database. Lines
+// are queued and inserted in batches so requests never wait on a write. Until
+// a database is attached (and if it turns out to be read-only) lines go to stderr.
+type DBLog struct {
+	mu      sync.Mutex
+	db      *sql.DB
+	ch      chan logLine
+	done    chan struct{}
+	started bool
+	retain  int
+}
+
+type logLine struct {
+	ts, level, msg string
+}
+
+var lg = &DBLog{ch: make(chan logLine, 4096), done: make(chan struct{})}
+
+const (
+	logTimeFmt = "2006-01-02 15:04:05.000"
+	maxLogLine = 400 // request URLs and queries come from strangers
+)
+
+func truncate(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	for n > 0 && !utf8.RuneStart(s[n]) { // don't cut a character in half
+		n--
+	}
+	return s[:n] + "..."
+}
+
+func (l *DBLog) Attach(db *sql.DB, retainDays int) {
+	if _, err := db.Exec(`CREATE TABLE IF NOT EXISTS logs (id INTEGER PRIMARY KEY AUTOINCREMENT, ts TEXT NOT NULL, level TEXT NOT NULL, msg TEXT NOT NULL)`); err != nil {
+		fmt.Fprintln(os.Stderr, "WARN: cannot write logs to the database, logging to stderr:", err)
+		return
+	}
+	l.retain = retainDays
+	l.prune(db)
+	l.mu.Lock()
+	l.db, l.started = db, true
+	l.mu.Unlock()
+	go l.run()
+}
+
+func (l *DBLog) prune(db *sql.DB) {
+	db.Exec("DELETE FROM logs WHERE ts < ?", time.Now().AddDate(0, 0, -l.retain).Format(logTimeFmt))
+}
+
+func (l *DBLog) run() {
+	tick := time.NewTicker(time.Second)
+	defer tick.Stop()
+	daily := time.NewTicker(6 * time.Hour)
+	defer daily.Stop()
+	var batch []logLine
+	flush := func() {
+		if len(batch) == 0 {
+			return
+		}
+		tx, err := l.db.Begin()
+		if err == nil {
+			for _, ln := range batch {
+				if _, err = tx.Exec("INSERT INTO logs (ts, level, msg) VALUES (?, ?, ?)", ln.ts, ln.level, ln.msg); err != nil {
+					break
+				}
+			}
+			if err == nil {
+				err = tx.Commit()
+			} else {
+				tx.Rollback()
+			}
+		}
+		if err != nil {
+			for _, ln := range batch {
+				fmt.Fprintf(os.Stderr, "%s %s %s\n", ln.ts, ln.level, ln.msg)
+			}
+		}
+		batch = batch[:0]
+	}
+	for {
+		select {
+		case ln := <-l.ch:
+			batch = append(batch, ln)
+			if len(batch) >= 200 {
+				flush()
+			}
+		case <-tick.C:
+			flush()
+		case <-daily.C:
+			l.prune(l.db)
+		case <-l.done:
+			for {
+				select {
+				case ln := <-l.ch:
+					batch = append(batch, ln)
+				default:
+					flush()
+					l.done <- struct{}{}
+					return
+				}
+			}
+		}
+	}
+}
+
+// Close flushes queued lines.
+func (l *DBLog) Close() {
+	l.mu.Lock()
+	started := l.started
+	l.mu.Unlock()
+	if !started {
+		return
+	}
+	l.done <- struct{}{}
+	<-l.done
+}
+
+// Under systemd (stderr is the journal) or with KJV_LOG_STDERR=1 every line is
+// also written to stderr, where the journal collects it. The "<n>" prefix is
+// the syslog priority journald understands; it adds its own timestamps.
+var teeStderr = stderrIsJournal() || os.Getenv("KJV_LOG_STDERR") == "1"
+var journalPriority = map[string]int{"ERROR": 3, "WARN": 4, "INFO": 6}
+
+// stderrIsJournal: JOURNAL_STREAM is inherited by everything a systemd
+// started desktop session runs, terminals included, so it only counts when it
+// names the device:inode stderr really is (as systemd.exec(5) advises).
+func stderrIsJournal() bool {
+	want := os.Getenv("JOURNAL_STREAM")
+	if want == "" {
+		return false
+	}
+	fi, err := os.Stderr.Stat()
+	if err != nil {
+		return false
+	}
+	st, ok := fi.Sys().(*syscall.Stat_t)
+	return ok && fmt.Sprintf("%d:%d", st.Dev, st.Ino) == want
+}
+
+func (l *DBLog) log(level, format string, args ...any) {
+	ln := logLine{time.Now().Format(logTimeFmt), level, truncate(fmt.Sprintf(format, args...), maxLogLine)}
+	if teeStderr {
+		fmt.Fprintf(os.Stderr, "<%d>%s\n", journalPriority[level], strings.ReplaceAll(ln.msg, "\n", " "))
+	}
+	l.mu.Lock()
+	started := l.started
+	l.mu.Unlock()
+	if started {
+		select {
+		case l.ch <- ln:
+			return
+		default: // queue full: don't block a request
+		}
+	}
+	if !teeStderr { // not in the database and not already on stderr
+		fmt.Fprintf(os.Stderr, "%s %s %s\n", ln.ts, ln.level, ln.msg)
+	}
+}
+
+func (l *DBLog) Info(format string, args ...any)  { l.log("INFO", format, args...) }
+func (l *DBLog) Warn(format string, args ...any)  { l.log("WARN", format, args...) }
+func (l *DBLog) Error(format string, args ...any) { l.log("ERROR", format, args...) }
+
+// ShowLogs prints the newest n log lines (oldest first).
+func ShowLogs(dbPath string, n int) error {
+	db, err := sql.Open("sqlite", "file:"+dbPath+"?mode=ro&_pragma=busy_timeout(5000)")
+	if err != nil {
+		return err
+	}
+	defer db.Close()
+	rows, err := db.Query("SELECT ts, level, msg FROM (SELECT * FROM logs ORDER BY id DESC LIMIT ?) ORDER BY id", n)
+	if err != nil {
+		return err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var ts, level, msg string
+		if err := rows.Scan(&ts, &level, &msg); err != nil {
+			return err
+		}
+		fmt.Printf("%s %-5s %s\n", ts, level, msg)
+	}
+	return rows.Err()
+}

internal/kjv/mt.go

@@ -0,0 +1,86 @@
+package kjv
+
+// MT19937 seeded with init_by_array from one integer, and a choice made by
+// rejection sampling over its bits, so the verse of the day for any given date
+// never changes.
+
+type mtRandom struct {
+	mt  [624]uint32
+	idx int
+}
+
+func newMTRandom(seed uint32) *mtRandom {
+	r := &mtRandom{}
+	r.initGenrand(19650218)
+	key := []uint32{seed}
+	i, j := 1, 0
+	k := max(624, len(key))
+	for ; k > 0; k-- {
+		r.mt[i] = (r.mt[i] ^ ((r.mt[i-1] ^ (r.mt[i-1] >> 30)) * 1664525)) + key[j] + uint32(j)
+		i++
+		j++
+		if i >= 624 {
+			r.mt[0] = r.mt[623]
+			i = 1
+		}
+		if j >= len(key) {
+			j = 0
+		}
+	}
+	for k = 623; k > 0; k-- {
+		r.mt[i] = (r.mt[i] ^ ((r.mt[i-1] ^ (r.mt[i-1] >> 30)) * 1566083941)) - uint32(i)
+		i++
+		if i >= 624 {
+			r.mt[0] = r.mt[623]
+			i = 1
+		}
+	}
+	r.mt[0] = 0x80000000
+	r.idx = 624
+	return r
+}
+
+func (r *mtRandom) initGenrand(s uint32) {
+	r.mt[0] = s
+	for i := 1; i < 624; i++ {
+		r.mt[i] = 1812433253*(r.mt[i-1]^(r.mt[i-1]>>30)) + uint32(i)
+	}
+	r.idx = 624
+}
+
+func (r *mtRandom) next() uint32 {
+	if r.idx >= 624 {
+		for k := 0; k < 624; k++ {
+			y := (r.mt[k] & 0x80000000) | (r.mt[(k+1)%624] & 0x7fffffff)
+			v := r.mt[(k+397)%624] ^ (y >> 1)
+			if y&1 != 0 {
+				v ^= 0x9908b0df
+			}
+			r.mt[k] = v
+		}
+		r.idx = 0
+	}
+	y := r.mt[r.idx]
+	r.idx++
+	y ^= y >> 11
+	y ^= (y << 7) & 0x9d2c5680
+	y ^= (y << 15) & 0xefc60000
+	y ^= y >> 18
+	return y
+}
+
+// choiceIndex is random.choice's _randbelow(n) for n < 2**32.
+func (r *mtRandom) choiceIndex(n int) int {
+	if n <= 0 {
+		return 0
+	}
+	k := 0
+	for v := n; v > 0; v >>= 1 {
+		k++
+	}
+	for {
+		if x := int(r.next() >> (32 - k)); x < n {
+			return x
+		}
+	}
+}

internal/kjv/pages.go

@@ -0,0 +1,512 @@
+package kjv
+
+import (
+	"fmt"
+	"html"
+	"math/rand/v2"
+	"net/url"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+	"unicode"
+)
+
+const (
+	maxSearchWords = 20
+	maxSearchLen   = 200
+)
+
+const searchChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789+^* "
+
+// cleanInput keeps only the allowed characters.
+func cleanInput(s, valid string) string {
+	return strings.Map(func(r rune) rune {
+		if strings.ContainsRune(valid, r) {
+			return r
+		}
+		return -1
+	}, s)
+}
+
+func searchClean(s string) string { return cleanInput(s, searchChars) }
+
+// ---------------------------------------------------------------------------
+// /show
+
+func (a *App) show(kw, view string, all bool) (Resp, bool) {
+	resolved := a.res.Resolve(kw, true)
+	if resolved == nil {
+		return htmlR("Invalid Query"), false
+	}
+	var verses []Verse
+	for _, r := range resolved.Refs {
+		end := r.End
+		if all { // ?all= : from the given verse to the end of its chapter
+			end = r.Start/1000*1000 + maxChapterIDSpan
+		}
+		verses = append(verses, a.store.Range(r.Start, end)...)
+	}
+	return a.render(verses, view, kw).resp(), true
+}
+
+func viewOf(q url.Values, def string) string {
+	if q.Has("view") {
+		return q.Get("view")
+	}
+	return def
+}
+
+func (a *App) handleShow(q url.Values) Resp {
+	kw := q.Get("kw")
+	if kw == "" {
+		return a.staticPage("html/show.html")
+	}
+	resp, _ := a.show(kw, viewOf(q, "json"), q.Get("all") != "")
+	return resp
+}
+
+// ---------------------------------------------------------------------------
+// /random and /votd
+
+func (a *App) singleVerse(id int64, q url.Values, defView string) Resp {
+	return a.render(a.store.Range(id, 0), viewOf(q, defView), "").resp()
+}
+
+func (a *App) handleRandom(q url.Values, defView string) Resp {
+	id := a.store.verseIDs[rand.IntN(len(a.store.verseIDs))]
+	lg.Info("Random served: %d", id)
+	return a.singleVerse(id, q, defView)
+}
+
+// votdID is deterministic: the date seeds the generator, so everyone gets the same verse that day.
+func (a *App) votdID(t time.Time) int64 {
+	seed, _ := strconv.Atoi(t.Format("20060102"))
+	r := newMTRandom(uint32(seed))
+	return a.store.verseIDs[r.choiceIndex(len(a.store.verseIDs))]
+}
+
+func (a *App) handleVotd(q url.Values, defView string) Resp {
+	lg.Info("VOTD served")
+	return a.singleVerse(a.votdID(time.Now()), q, defView)
+}
+
+// ---------------------------------------------------------------------------
+// dictionary
+
+func (a *App) defineRich(ref string, defs []string, simple bool) string {
+	head := ""
+	if !simple {
+		head = "<!DOCTYPE html><html lang='en'><head><meta charset='UTF-8'>" +
+			"<meta name='viewport' content='width=device-width, initial-scale=1'>" +
+			"<meta name='color-scheme' content='dark light'>" +
+			"<title>" + html.EscapeString(capitalize(ref)) + " - KJV API</title>" +
+			a.cssNavbar + "</head><body><div class='responses'>"
+	}
+	text := ""
+	for _, d := range defs {
+		text += d + "<br><br>"
+	}
+	out := head + strings.NewReplacer(
+		"{chapter}", strings.SplitN(ref, ":", 2)[0], "{passage}", capitalize(ref), "{text}", text, "{verse_id}", "",
+	).Replace(a.htmlResponse)
+	if !simple {
+		out += "</div></body></html>"
+	}
+	return out
+}
+
+func (a *App) handleDefine(q url.Values) Resp {
+	ref := strings.ToLower(strings.TrimSpace(q.Get("kw")))
+	if ref == "" {
+		return a.staticPage("html/dictionary.html")
+	}
+	defs, ok := a.dictionary[ref]
+	if !ok {
+		lg.Warn("WEBSTER MISSING: %s", ref)
+		return htmlR(html.EscapeString(ref) + " not found.")
+	}
+	switch q.Get("view") {
+	case "plain":
+		text := ""
+		for _, d := range defs {
+			text += d + "<br><br>"
+		}
+		return htmlR(capitalize(ref) + "<br><br>" + text + "\n")
+	case "rich":
+		return htmlR(a.defineRich(ref, defs, false))
+	}
+	return jsonR(map[string][]string{ref: defs})
+}
+
+var dictStopwords = map[string]bool{}
+
+func init() {
+	for _, w := range strings.Fields(`the and that for him his her but not with are was were unto thou thee thy ye you all from
+		they them this have had shall will which into upon then`) {
+		dictStopwords[w] = true
+	}
+}
+
+// /defs?words=grace,loved : several definitions at once
+func (a *App) handleDefs(q url.Values) Resp {
+	words := strings.Fields(strings.ReplaceAll(strings.ToLower(q.Get("words")), ",", " "))
+	if len(words) > 40 {
+		words = words[:40]
+	}
+	found := map[string][]string{}
+	for _, w := range words {
+		w = strings.Map(func(r rune) rune {
+			if unicode.IsLetter(r) {
+				return r
+			}
+			return -1
+		}, w)
+		if d, ok := a.dictionary[w]; ok && w != "" && !dictStopwords[w] {
+			found[w] = d
+		}
+	}
+	return jsonR(found)
+}
+
+// ---------------------------------------------------------------------------
+// /search
+
+// proximityScore: higher when the query words appear closer together.
+func proximityScore(text string, words []string) float64 {
+	lower := strings.ToLower(text)
+	type hit struct{ pos, list int }
+	var hits []hit
+	seen := map[string]int{}
+	for _, w := range words {
+		if _, dup := seen[w]; dup {
+			continue
+		}
+		li := len(seen)
+		seen[w] = li
+		w = strings.ToLower(w)
+		n := 0
+		for at := 0; ; {
+			i := strings.Index(lower[at:], w)
+			if i < 0 {
+				break
+			}
+			hits = append(hits, hit{at + i, li})
+			n++
+			at += i + 1
+		}
+		if n == 0 {
+			return 0
+		}
+	}
+	lists := len(seen)
+	sort.SliceStable(hits, func(i, j int) bool { return hits[i].pos < hits[j].pos })
+	// smallest window holding at least one position of every word
+	counts := make([]int, lists)
+	have, left, best := 0, 0, -1
+	for right := range hits {
+		if counts[hits[right].list]++; counts[hits[right].list] == 1 {
+			have++
+		}
+		for have == lists {
+			if span := hits[right].pos - hits[left].pos; best < 0 || span < best {
+				best = span
+			}
+			if counts[hits[left].list]--; counts[hits[left].list] == 0 {
+				have--
+			}
+			left++
+		}
+	}
+	if best < 0 {
+		return 0
+	}
+	return 1000.0 / float64(best+1)
+}
+
+func sortVariants(original string, variants []string) []string {
+	type sv struct {
+		score int
+		v     string
+	}
+	var scored []sv
+	orig := strings.ToLower(original)
+	for _, v := range variants {
+		lv := strings.ToLower(v)
+		prefix := 0
+		for prefix < len(orig) && prefix < len(lv) && orig[prefix] == lv[prefix] {
+			prefix++
+		}
+		if prefix < min(2, len(orig)/2) { // doesn't share the stem
+			continue
+		}
+		scored = append(scored, sv{prefix*1000 + len(v), v})
+	}
+	sort.SliceStable(scored, func(i, j int) bool { return scored[i].score > scored[j].score })
+	out := make([]string, len(scored))
+	for i, s := range scored {
+		out[i] = s.v
+	}
+	return out
+}
+
+func (a *App) fts(q string) []Verse {
+	v, err := a.store.FTS(q)
+	if err != nil { // FTS5 syntax the user typed by accident: just no results
+		lg.Info("fts %q: %v", q, err)
+	}
+	return v
+}
+
+func insertAfterResponsesOpen(page, notice string) string {
+	return strings.Replace(page, "<div class='responses'>", "<div class='responses'>"+notice, 1)
+}
+
+// fallbackResp shows the results of a rewritten query with an explanation.
+func (a *App) fallbackResp(results []Verse, view, kw, plainNote, richNotice string, jsonObj map[string]any) Resp {
+	r := a.render(results, view, kw)
+	switch view {
+	case "plain":
+		return htmlR(plainNote + "\n<br>" + r.text)
+	case "rich":
+		return htmlR(insertAfterResponsesOpen(r.text, richNotice))
+	}
+	jsonObj["results"] = r.list
+	return jsonR(jsonObj)
+}
+
+func (a *App) handleSearch(kw, view string) Resp {
+	cleaned := searchClean(kw)
+	if words := strings.Fields(cleaned); len(words) > maxSearchWords || len(cleaned) > maxSearchLen {
+		return htmlR("Your query is too long. 20 words or less.")
+	}
+	if cleaned == "" {
+		return a.staticPage("html/search.html")
+	}
+	esc := html.EscapeString
+
+	// a common theological concept that is not in the KJV?
+	conceptNotice := ""
+	if c, ok := a.concepts[strings.ToLower(cleaned)]; ok {
+		var links []string
+		for _, t := range c.SearchTerms {
+			links = append(links, fmt.Sprintf("<a href='/search?kw=%s&view=rich'><strong>%s</strong></a>", strings.ReplaceAll(t, " ", "+"), t))
+		}
+		conceptNotice = fmt.Sprintf("<div class='correction-notice'><strong>%s</strong>: %s<br>You may find what you are looking for with these search terms: %s</div>",
+			capitalize(cleaned), c.Explanation, strings.Join(links, ", "))
+		cleaned = c.SearchTerms[0]
+	}
+
+	results := a.fts(cleaned)
+	words := strings.Fields(cleaned)
+	if len(results) > 0 && len(words) > 1 {
+		scores := make(map[int64]float64, len(results))
+		for _, v := range results {
+			scores[v.ID] = proximityScore(v.Text, words)
+		}
+		sort.SliceStable(results, func(i, j int) bool { return scores[results[i].ID] > scores[results[j].ID] })
+	}
+
+	if len(results) == 0 {
+		// 1. grammatical variants
+		for _, word := range words {
+			for _, variant := range sortVariants(word, a.variants[strings.ToLower(word)]) {
+				vq := strings.Replace(cleaned, word, variant, 1)
+				if r := a.fts(vq); len(r) > 0 {
+					return a.fallbackResp(r, view, kw,
+						fmt.Sprintf("Showing results for '%s' (grammatical variant of '%s')", vq, cleaned),
+						fmt.Sprintf("<div class='variant-notice'>Showing results for '<strong>%s</strong>' (grammatical variant of '%s')</div>", vq, cleaned),
+						map[string]any{"original": cleaned, "variant": vq})
+				}
+			}
+		}
+		// 2. spell check
+		corrected := make([]string, len(words))
+		changed := false
+		for i, w := range words {
+			corrected[i] = w
+			if !a.speller.Spell(w) {
+				if sug := a.speller.Suggest(w); len(sug) > 0 {
+					corrected[i], changed = sug[0], true
+				}
+			}
+		}
+		if changed {
+			ck := strings.Join(corrected, " ")
+			if r := a.fts(ck); len(r) > 0 {
+				return a.fallbackResp(r, view, kw,
+					fmt.Sprintf("Showing results for '%s' (corrected from '%s')", ck, cleaned),
+					fmt.Sprintf("<div class='correction-notice'>Showing results for '<strong>%s</strong>' (corrected from '%s')</div>", esc(ck), cleaned),
+					map[string]any{"corrected_from": cleaned, "corrected_to": ck})
+			}
+		}
+		// 3. theological equivalents
+		for _, word := range words {
+			for _, syn := range a.synonyms[strings.ToLower(word)] {
+				sq := strings.Replace(cleaned, word, syn, 1)
+				if r := a.fts(sq); len(r) > 0 {
+					explanation := fmt.Sprintf("In the KJV, '%s' and '%s' often refer to similar theological concepts. Consider also searching for related verses.", word, syn)
+					return a.fallbackResp(r, view, kw,
+						fmt.Sprintf("Showing results for '%s' (theological equivalent of '%s')\n%s", sq, cleaned, explanation),
+						fmt.Sprintf("<div class='theological-notice'>Showing results for '<strong>%s</strong>' (theological equivalent of '%s')<br><em>%s</em></div>", sq, cleaned, explanation),
+						map[string]any{"original": cleaned, "theological_equivalent": sq, "explanation": explanation})
+				}
+			}
+		}
+		// 4. nothing: offer spelling suggestions
+		var suggestions []string
+		for i, w := range words {
+			if a.speller.Spell(w) {
+				continue
+			}
+			for _, s := range a.speller.Suggest(w) {
+				if len(suggestions) >= 5 {
+					break
+				}
+				alt := append([]string(nil), words...)
+				alt[i] = s
+				suggestions = append(suggestions, strings.Join(alt, " "))
+			}
+		}
+		if view != "rich" {
+			list := "No suggestions available"
+			if len(suggestions) > 0 {
+				list = strings.Join(suggestions, ", ")
+			}
+			return htmlR(fmt.Sprintf("%s not found, try one of the following: %s", cleaned, list))
+		}
+		var sb strings.Builder
+		sb.WriteString(a.renderNavbar(cleaned, nil, ""))
+		sb.WriteString("<div class='responses'>")
+		fmt.Fprintf(&sb, "<div class='correction-notice'><strong>No results found</strong> for '%s'.</div>", cleaned)
+		if len(suggestions) > 0 {
+			sb.WriteString("<div class='theological-suggestions'><h3>Did you mean?</h3>")
+			sb.WriteString("<p>We couldn't find any verses matching your search. Here are some spelling suggestions:</p>")
+			for _, s := range suggestions {
+				fmt.Fprintf(&sb, "<div class='suggestion-item'><a href='/search?kw=%s&view=rich'><strong>%s</strong></a></div>", url.QueryEscape(s), esc(s))
+			}
+			sb.WriteString("</div>")
+		} else {
+			sb.WriteString("<div class='theological-suggestions'><h3>Search Tips</h3>")
+			sb.WriteString("<p>We couldn't find any verses matching your search. Try:</p>")
+			sb.WriteString("<ul style='font-family: var(--font-ui); color: var(--color-text-secondary);'>")
+			sb.WriteString("<li>Checking your spelling</li><li>Using different keywords</li>")
+			sb.WriteString("<li>Searching for <a href='/search?kw=love&view=rich'>love</a>, <a href='/search?kw=faith&view=rich'>faith</a>, or <a href='/search?kw=hope&view=rich'>hope</a></li>")
+			sb.WriteString("</ul></div>")
+		}
+		sb.WriteString("</div>")
+		return htmlR(sb.String())
+	}
+
+	r := a.render(results, view, kw)
+	var related []string
+	for _, w := range words {
+		related = append(related, a.synonyms[strings.ToLower(w)]...)
+	}
+	switch view {
+	case "plain":
+		out := r.text
+		if conceptNotice != "" {
+			out = conceptNotice + "<br>" + out
+		}
+		if len(related) > 0 {
+			out += "<br><br>Also consider searching: "
+			for _, s := range related {
+				out += s + "<br>"
+			}
+		}
+		return htmlR(out)
+	case "rich":
+		out := r.text
+		if conceptNotice != "" {
+			if start := strings.Index(out, "<div class='responses'>"); start != -1 {
+				if end := strings.Index(out[start+len("<div class='responses'>"):], "</div>"); end != -1 {
+					at := start + len("<div class='responses'>") + end + len("</div>")
+					out = out[:at] + conceptNotice + out[at:]
+				}
+			}
+		}
+		if len(related) > 0 {
+			var sb strings.Builder
+			sb.WriteString("<div class='theological-suggestions'><h3>Related Theological Concepts</h3>")
+			sb.WriteString("<p>In the KJV, these terms often refer to similar theological concepts. Consider also searching for related verses.</p>")
+			for _, s := range related {
+				fmt.Fprintf(&sb, "<div class='suggestion-item'><a href='/search?kw=%s&view=rich'><strong>%s</strong></a></div>", url.QueryEscape(s), capitalize(s))
+			}
+			sb.WriteString("</div>")
+			if at := strings.LastIndex(out, "</div>"); at != -1 {
+				out = out[:at] + sb.String() + out[at:]
+			}
+		}
+		return htmlR(out)
+	}
+	return r.resp()
+}
+
+// ---------------------------------------------------------------------------
+// /find : scripture if it reads as scripture, otherwise a keyword search with definitions
+
+func (a *App) handleFind(q url.Values) Resp {
+	query := strings.TrimSpace(q.Get("kw"))
+	if viewOf(q, "rich") != "rich" {
+		return jsonR(map[string]string{"Error": "/find endpoint only supports rich responses"})
+	}
+	lg.Info("Query: %s", query)
+	if query == "" {
+		return a.staticPage("html/find.html")
+	}
+	query = strings.TrimSuffix(query, ",")
+
+	// A query that reads as scripture references ("Exodus", "1 jn 3.16") is shown
+	// as scripture; anything else is a keyword search.
+	if resolved := a.res.Resolve(query, false); resolved != nil {
+		if resp, ok := a.show(query, "rich", false); ok {
+			notices := ""
+			for _, n := range resolved.Notes {
+				notices += fmt.Sprintf("<div class='correction-notice'>Showing <strong>%s</strong> (interpreted from '%s')</div>",
+					html.EscapeString(n.Name), html.EscapeString(n.Original))
+			}
+			if resolved.Bare {
+				keyword := url.QueryEscape(searchClean(query))
+				notices += fmt.Sprintf("<div class='correction-notice'>Showing <strong>%s</strong>, chapter 1. Looking for the word instead? "+
+					"<a href='/search?kw=%s&view=rich'>Search the text for '%s'</a></div>",
+					html.EscapeString(resolved.Label), keyword, html.EscapeString(query))
+			}
+			if notices != "" {
+				resp.Body = insertAfterResponsesOpen(resp.Body, notices)
+			}
+			return resp
+		}
+	}
+	if strings.TrimSpace(searchClean(query)) == "" {
+		return a.staticPage("html/find.html") // nothing searchable left (punctuation, accents)
+	}
+	var tokens []string
+	for _, w := range strings.Split(strings.ReplaceAll(query, ",", " "), " ") {
+		if w != "" {
+			tokens = append(tokens, w)
+		}
+	}
+	if len(tokens) > 20 {
+		return htmlR("Your Query is too long. 20 Words or less.")
+	}
+	result := a.handleSearch(q.Get("kw"), "rich")
+	body := strings.TrimSuffix(result.Body, "</div>") // reopen .responses to add the dictionary inside it
+
+	var defs strings.Builder
+	for _, word := range tokens {
+		entry, ok := a.dictionary[strings.ToLower(word)]
+		if !ok {
+			continue
+		}
+		defs.WriteString("<button type='button' class='collapsible'>" + html.EscapeString(capitalize(word)) + "</button>")
+		defs.WriteString("<div class='definition'>" + a.defineRich(strings.ToLower(word), entry, true) + "</div>")
+	}
+	if defs.Len() > 0 {
+		body += "<style>" + a.cssCollapsible + "</style>"
+		body += "<div class='dictionary-section'><h3>1828 Webster Definitions</h3><p>Click on the word to get the definition.</p></div>"
+		body += defs.String()
+		body += "<script src='" + a.assetURL("/js/collapsible.js") + "'></script>"
+	}
+	return htmlR(body + "</div>")
+}

internal/kjv/refs.go

@@ -0,0 +1,357 @@
+package kjv
+
+// Scripture reference resolver. Understands the many ways people type a
+// reference ("1 jn 3.16", "II Kings 2:1-4", "song of songs 2", "Exodus ") and
+// turns it into ranges of verse ids (book*1000000 + chapter*1000 + verse).
+
+import (
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+)
+
+// Extra spellings people use that the abbreviations table does not know.
+var extraBookAliases = map[string][]string{
+	"Song of Solomon": {"song of songs", "songs of solomon", "songs of songs", "song of sol",
+		"song of sol", "canticles", "canticle of canticles", "sos", "sng", "ss"},
+	"Psalms":       {"psalm", "psalter", "pslm", "psa", "pss", "psm", "ps"},
+	"Revelation":   {"revelations", "revelation of john", "revelation of st john", "the revelation", "apocalypse"},
+	"Ecclesiastes": {"ecclesiasts", "eccles", "ecc", "qoheleth"},
+	"Philemon":     {"phlm", "phm", "philem"},
+	"Judges":       {"jdg", "jdgs", "jg"},
+	"Mark":         {"mrk", "mk", "mr"},
+	"John":         {"jhn", "jn"},
+	"Acts":         {"acts of the apostles", "act"},
+	"James":        {"jas", "jm"},
+}
+
+// Words that are books but also ordinary search words: a bare "numbers" or
+// "mark" still shows the book, but /find offers a keyword search as well.
+var bareBookExtra = map[string]bool{
+	"psalm": true, "songofsongs": true, "songsofsolomon": true, "songofsol": true, "canticles": true,
+	"canticleofcanticles": true, "revelations": true, "revelationofjohn": true, "therevelation": true,
+}
+
+const maxChapterIDSpan = 999
+
+// A reference is never this long; longer text goes to the keyword search. Bounds the fuzzy matching work.
+const maxRefQueryLen = 300
+
+// Go's regexp has no lookaheads, so they are written as a consumed [\s.] here, which matches the same text.
+var (
+	bookPieceRE = regexp.MustCompile(`(?i)^\s*(?P<book>(?:(?:[1-3]|(?:iii|ii|i)[\s.]|1st|2nd|3rd|first|second|third)\.?\s*)?[a-z][a-z.\s]*?)` +
+		`\s*(?P<rest>\d[\d\s:.\-–—]*)?\s*$`)
+	continuationRE = regexp.MustCompile(`^\s*\d[\d\s:.\-–—]*$`)
+
+	reBookOf   = regexp.MustCompile(`^(?:the\s+)?(?:book\s+of\s+)`)
+	reFirst    = regexp.MustCompile(`^(?:1st|first)([\s.a-z]|$)`)
+	reSecond   = regexp.MustCompile(`^(?:2nd|second)([\s.a-z]|$)`)
+	reThird    = regexp.MustCompile(`^(?:3rd|third)([\s.a-z]|$)`)
+	reRoman3   = regexp.MustCompile(`^iii([\s.]|$)`)
+	reRoman2   = regexp.MustCompile(`^ii([\s.]|$)`)
+	reRoman1   = regexp.MustCompile(`^i([\s.])`)
+	reNotAlnum = regexp.MustCompile(`[^a-z0-9]`)
+	reSepSpace = regexp.MustCompile(`\s*([:\-])\s*`)
+
+	reChapter       = regexp.MustCompile(`^(\d+)$`)
+	reChapterAll    = regexp.MustCompile(`^(\d+)(?::-|-)$`)
+	reChapterRange  = regexp.MustCompile(`^(\d+)-(\d+)$`)
+	reVerse         = regexp.MustCompile(`^(\d+)[:.\s](\d+)$`)
+	reVerseOn       = regexp.MustCompile(`^(\d+)[:.\s](\d+)-$`)
+	reVerseRange    = regexp.MustCompile(`^(\d+)[:.\s](\d+)-(\d+)$`)
+	reVerseAcrossCh = regexp.MustCompile(`^(\d+)[:.\s](\d+)-(\d+)[:.](\d+)$`)
+)
+
+// normalizeBookText: 'I Cor.' / '1st Corinthians' / '1 corinthians' -> '1corinthians'
+func normalizeBookText(text string) string {
+	text = strings.ToLower(strings.TrimSpace(text))
+	text = reBookOf.ReplaceAllString(text, "")
+	text = reFirst.ReplaceAllString(text, "1${1}")
+	text = reSecond.ReplaceAllString(text, "2${1}")
+	text = reThird.ReplaceAllString(text, "3${1}")
+	text = reRoman3.ReplaceAllString(text, "3${1}")
+	text = reRoman2.ReplaceAllString(text, "2${1}")
+	text = reRoman1.ReplaceAllString(text, "1${1}")
+	return reNotAlnum.ReplaceAllString(text, "")
+}
+
+type Resolver struct {
+	store     *Store
+	canonical map[string]int
+	aliases   map[string]int
+	aliasKeys []string
+	canonKeys []string
+}
+
+func NewResolver(s *Store) *Resolver {
+	r := &Resolver{store: s, canonical: map[string]int{}, aliases: map[string]int{}}
+	for id, name := range s.bookName {
+		r.canonical[normalizeBookText(name)] = id
+	}
+	for k, v := range r.canonical {
+		r.aliases[k] = v
+	}
+	for _, a := range s.abbrevs {
+		if k := normalizeBookText(a.Name); r.aliases[k] == 0 {
+			r.aliases[k] = a.Book
+		}
+	}
+	for name, extras := range extraBookAliases {
+		id := s.bookByName[name]
+		for _, extra := range extras {
+			r.aliases[normalizeBookText(extra)] = id
+		}
+	}
+	for k := range r.aliases {
+		r.aliasKeys = append(r.aliasKeys, k)
+	}
+	sort.Strings(r.aliasKeys)
+	for k := range r.canonical {
+		r.canonKeys = append(r.canonKeys, k)
+	}
+	sort.Strings(r.canonKeys)
+	return r
+}
+
+type bookMatch struct {
+	ID  int
+	How string // exact | prefix | fuzzy
+}
+
+// resolveBook finds a book from a typed name. loose=false only accepts full
+// names and spellings that are unmistakably a book (used for a bare word with
+// no chapter number).
+func (r *Resolver) resolveBook(token string, loose bool) (bookMatch, bool) {
+	key := normalizeBookText(token)
+	if key == "" {
+		return bookMatch{}, false
+	}
+	if !loose {
+		if _, ok := r.canonical[key]; ok || bareBookExtra[key] {
+			return bookMatch{r.aliases[key], "exact"}, true
+		}
+		// a clear misspelling of a long full name ("genisis")
+		if len(key) >= 6 && len(key) <= 40 && !(key[0] >= '0' && key[0] <= '9') {
+			if close, ok := closestMatch(key, r.canonKeys, 0.85); ok {
+				return bookMatch{r.canonical[close], "fuzzy"}, true
+			}
+		}
+		return bookMatch{}, false
+	}
+	if id, ok := r.aliases[key]; ok {
+		return bookMatch{id, "exact"}, true
+	}
+	// unique (or best) prefix: "deut", "1cor", "matth"
+	books := map[int]bool{}
+	for _, k := range r.aliasKeys {
+		if strings.HasPrefix(k, key) {
+			books[r.aliases[k]] = true
+		}
+	}
+	if len(books) > 0 {
+		canonHits := map[int]bool{}
+		for _, k := range r.canonKeys {
+			if strings.HasPrefix(k, key) {
+				canonHits[r.canonical[k]] = true
+			}
+		}
+		if len(canonHits) == 1 {
+			return bookMatch{anyKey(canonHits), "prefix"}, true
+		}
+		if len(books) == 1 {
+			return bookMatch{anyKey(books), "prefix"}, true
+		}
+		pool := books
+		if len(canonHits) > 0 {
+			pool = canonHits
+		}
+		best := 0
+		for id := range pool {
+			if id > best {
+				best = id // ambiguous: the later book, as before
+			}
+		}
+		return bookMatch{best, "prefix"}, true
+	}
+	if len(key) >= 4 && len(key) <= 40 {
+		if close, ok := closestMatch(key, r.aliasKeys, 0.78); ok {
+			return bookMatch{r.aliases[close], "fuzzy"}, true
+		}
+	}
+	return bookMatch{}, false
+}
+
+func anyKey(m map[int]bool) int {
+	for k := range m {
+		return k
+	}
+	return 0
+}
+
+// clampChapter: 0 -> 1, past the end -> the last chapter.
+func (r *Resolver) clampChapter(book, chapter int) int {
+	last := r.store.chapterLen[book]
+	if chapter > last {
+		chapter = last
+	}
+	if chapter < 1 {
+		chapter = 1
+	}
+	return chapter
+}
+
+// atoi never fails: absurdly long numbers saturate.
+func atoi(s string) int {
+	n, err := strconv.Atoi(s)
+	if err != nil {
+		return 1_000_000
+	}
+	if n > 1_000_000 {
+		return 1_000_000
+	}
+	return n
+}
+
+type refPos struct{ book, chapter, verse int }
+
+// parseRefRest: '3:16-18', '3 16', '3.16', '3-4', '3:16-4:2', '3', '3:-' -> (start, end or 0, position)
+func (r *Resolver) parseRefRest(rest string, book int, previous *refPos) (start, end int64, pos refPos, ok bool) {
+	rest = strings.NewReplacer("–", "-", "—", "-").Replace(rest)
+	rest = reSepSpace.ReplaceAllString(strings.TrimSpace(rest), "$1")
+	base := int64(book) * 1_000_000
+	at := func(chapter, verse int) int64 { return base + int64(chapter)*1000 + int64(verse) }
+	pos.book = book
+	if m := reChapter.FindStringSubmatch(rest); m != nil {
+		n := atoi(m[1])
+		if previous != nil && previous.verse != 0 { // "John 3:16, 18": a verse of the same chapter
+			pos.chapter, pos.verse = previous.chapter, n
+			return at(previous.chapter, n), 0, pos, true
+		}
+		ch := r.clampChapter(book, n)
+		pos.chapter = ch
+		return at(ch, 1), at(ch, maxChapterIDSpan), pos, true
+	}
+	if m := reChapterAll.FindStringSubmatch(rest); m != nil { // "3:-" or "3-": the whole chapter
+		ch := r.clampChapter(book, atoi(m[1]))
+		pos.chapter = ch
+		return at(ch, 1), at(ch, maxChapterIDSpan), pos, true
+	}
+	if m := reChapterRange.FindStringSubmatch(rest); m != nil { // chapters 3 to 4
+		first, last := r.clampChapter(book, atoi(m[1])), r.clampChapter(book, atoi(m[2]))
+		if last < first {
+			first, last = last, first
+		}
+		pos.chapter = first
+		return at(first, 1), at(last, maxChapterIDSpan), pos, true
+	}
+	if m := reVerse.FindStringSubmatch(rest); m != nil {
+		ch := r.clampChapter(book, atoi(m[1]))
+		verse := max(1, atoi(m[2]))
+		pos.chapter, pos.verse = ch, verse
+		return at(ch, verse), 0, pos, true
+	}
+	if m := reVerseOn.FindStringSubmatch(rest); m != nil { // "3:16-": from verse 16 to the end
+		ch := r.clampChapter(book, atoi(m[1]))
+		pos.chapter, pos.verse = ch, atoi(m[2])
+		return at(ch, max(1, atoi(m[2]))), at(ch, maxChapterIDSpan), pos, true
+	}
+	if m := reVerseRange.FindStringSubmatch(rest); m != nil {
+		ch := r.clampChapter(book, atoi(m[1]))
+		first, last := max(1, atoi(m[2])), max(1, atoi(m[3]))
+		if last < first {
+			first, last = last, first
+		}
+		pos.chapter, pos.verse = ch, first
+		return at(ch, first), at(ch, last), pos, true
+	}
+	if m := reVerseAcrossCh.FindStringSubmatch(rest); m != nil { // 3:16-4:2 across chapters
+		c1, c2 := r.clampChapter(book, atoi(m[1])), r.clampChapter(book, atoi(m[3]))
+		s, e := at(c1, max(1, atoi(m[2]))), at(c2, max(1, atoi(m[4])))
+		if e < s {
+			return 0, 0, pos, false
+		}
+		pos.chapter, pos.verse = c1, atoi(m[2])
+		return s, e, pos, true
+	}
+	return 0, 0, pos, false
+}
+
+type RefRange struct{ Start, End int64 } // End == 0: a single verse
+
+type Note struct{ Original, Name string }
+
+// Resolved is a whole parsed query.
+type Resolved struct {
+	Refs  []RefRange
+	Notes []Note // short explanations of guesses that were made
+	Bare  bool   // the query was just a book name (shows its first chapter)
+	Label string // the name of the first book, for messages
+}
+
+// Resolve parses a whole query ('John 3:16, Romans 5:8') into verse id ranges.
+// It returns nil when the text does not read as scripture references, so the
+// caller can treat it as a keyword search.
+func (r *Resolver) Resolve(text string, looseBare bool) *Resolved {
+	if strings.TrimSpace(text) == "" || len(text) > maxRefQueryLen {
+		return nil
+	}
+	var pieces []string
+	for _, p := range regexp.MustCompile(`[,;\n]+`).Split(text, -1) {
+		if strings.TrimSpace(p) != "" {
+			pieces = append(pieces, p)
+		}
+	}
+	if len(pieces) == 0 || len(pieces) > 20 {
+		return nil
+	}
+	out := &Resolved{}
+	var previous *refPos
+	for _, piece := range pieces {
+		var start, end int64
+		var pos refPos
+		var name string
+		if m := bookPieceRE.FindStringSubmatchIndex(piece); m != nil {
+			bi, ri := bookPieceRE.SubexpIndex("book"), bookPieceRE.SubexpIndex("rest")
+			token := piece[m[2*bi]:m[2*bi+1]]
+			hasRest := m[2*ri] >= 0
+			found, ok := r.resolveBook(token, hasRest || looseBare)
+			if !ok {
+				return nil
+			}
+			name = r.store.BookName(found.ID)
+			if found.How == "fuzzy" {
+				out.Notes = append(out.Notes, Note{strings.TrimSpace(token), name})
+			}
+			if !hasRest {
+				if len(pieces) == 1 {
+					out.Bare = true
+				}
+				base := int64(found.ID) * 1_000_000
+				start, end, pos = base+1001, base+1999, refPos{found.ID, 1, 0}
+			} else {
+				var ok bool
+				start, end, pos, ok = r.parseRefRest(piece[m[2*ri]:m[2*ri+1]], found.ID, nil)
+				if !ok {
+					return nil
+				}
+			}
+		} else if continuationRE.MatchString(piece) && previous != nil {
+			name = r.store.BookName(previous.book)
+			var ok bool
+			start, end, pos, ok = r.parseRefRest(piece, previous.book, previous)
+			if !ok {
+				return nil
+			}
+		} else {
+			return nil
+		}
+		p := pos
+		previous = &p
+		if out.Label == "" {
+			out.Label = name
+		}
+		out.Refs = append(out.Refs, RefRange{start, end})
+	}
+	return out
+}

internal/kjv/sermons.go

@@ -0,0 +1,163 @@
+package kjv
+
+import (
+	"crypto/subtle"
+	"database/sql"
+	"encoding/json"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+)
+
+// Optional endpoints backed by the separate sermon database (KJV_SERMON_DB)
+// and the separate semantic search service (KJV_SEMANTIC_ADDR, default
+// localhost:1613). Neither is part of the distribution; without them these
+// endpoints answer with an error.
+
+func rowsToMaps(rows *sql.Rows) ([]map[string]any, error) {
+	defer rows.Close()
+	cols, err := rows.Columns()
+	if err != nil {
+		return nil, err
+	}
+	out := []map[string]any{}
+	for rows.Next() {
+		vals := make([]any, len(cols))
+		ptrs := make([]any, len(cols))
+		for i := range vals {
+			ptrs[i] = &vals[i]
+		}
+		if err := rows.Scan(ptrs...); err != nil {
+			return nil, err
+		}
+		m := map[string]any{}
+		for i, c := range cols {
+			if b, ok := vals[i].([]byte); ok {
+				vals[i] = string(b)
+			}
+			m[c] = vals[i]
+		}
+		out = append(out, m)
+	}
+	return out, rows.Err()
+}
+
+func (a *App) sermonQuery(query string, args ...any) Resp {
+	if a.sermonDB == nil {
+		return jsonStatus(map[string]string{"error": "Sermon database not found"}, http.StatusServiceUnavailable)
+	}
+	rows, err := a.sermonDB.Query(query, args...)
+	if err != nil {
+		lg.Error("sermon query: %v", err)
+		return errJSON(http.StatusInternalServerError, "Sermon database error")
+	}
+	out, err := rowsToMaps(rows)
+	if err != nil {
+		return errJSON(http.StatusInternalServerError, "Sermon database error")
+	}
+	return jsonR(out)
+}
+
+const sermonCols = "select distinct title,url,denomination,preacher,church_name from verse_refs where "
+
+// handleSermons serves everything under /sermon/ and /sermons/ ... by hand,
+// since the paths overlap in ways the router would reject.
+func (a *App) handleSermons(path string) Resp {
+	parts := strings.Split(strings.Trim(path, "/"), "/")
+	switch {
+	case len(parts) == 1 && parts[0] == "sermons":
+		return a.sermonQuery("select distinct title,url,denomination,preacher,church_name,filename from verse_refs")
+	case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "preacher":
+		return a.sermonQuery(sermonCols+"preacher=?", parts[2])
+	case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "church":
+		return a.sermonQuery(sermonCols+"church_name=?", parts[2])
+	case len(parts) == 3 && parts[0] == "sermons" && parts[1] == "denomination":
+		return a.sermonQuery(sermonCols+"denomination=?", parts[2])
+	case len(parts) == 3 && parts[0] == "sermon" && parts[1] == "search":
+		return a.sermonsByRef(parts[2])
+	case len(parts) == 3 && parts[0] == "sermon" && parts[1] == "url":
+		return a.sermonQuery(sermonCols+"url=?", parts[2])
+	case len(parts) == 3 && parts[0] == "sermon" && parts[2] == "verses":
+		return a.sermonQuery("select distinct verse_id,text from verse_refs where filename=?", parts[1])
+	case len(parts) == 2 && parts[0] == "sermon":
+		return a.sermonQuery(sermonCols+"filename=?", parts[1])
+	}
+	return a.notFound()
+}
+
+func (a *App) sermonsByRef(ref string) Resp {
+	id, ok := a.refVerseID(ref)
+	if !ok {
+		return a.notFound()
+	}
+	if a.sermonDB == nil {
+		return jsonR(map[string]string{"error": "Sermon database not found"})
+	}
+	rows, err := a.sermonDB.Query("select * from verse_refs where verse_id=?", id)
+	if err != nil {
+		return errJSON(http.StatusInternalServerError, "Sermon database error")
+	}
+	out, err := rowsToMaps(rows)
+	if err != nil {
+		return errJSON(http.StatusInternalServerError, "Sermon database error")
+	}
+	for _, m := range out {
+		if vid, ok := m["verse_id"].(int64); ok {
+			if v := a.store.Range(vid, 0); len(v) == 1 {
+				m["kjv"] = v[0].Text
+			}
+		}
+	}
+	return jsonR(out)
+}
+
+// refVerseID: the verse id of the first reference in "1John5:7" style text.
+func (a *App) refVerseID(ref string) (int64, bool) {
+	resolved := a.res.Resolve(ref, true)
+	if resolved == nil {
+		return 0, false
+	}
+	return resolved.Refs[0].Start, true
+}
+
+func (a *App) handleRef(ref string) Resp {
+	id, ok := a.refVerseID(ref)
+	if !ok {
+		return a.notFound()
+	}
+	return htmlR(itoa64(id))
+}
+
+func (a *App) handleNaturalPost(body io.Reader) Resp {
+	var req struct {
+		Token *string `json:"token"`
+		Query string  `json:"query"`
+	}
+	if err := json.NewDecoder(io.LimitReader(body, 1<<16)).Decode(&req); err != nil {
+		return errJSON(http.StatusBadRequest, "Invalid request format")
+	}
+	if req.Token == nil {
+		return errJSON(http.StatusUnauthorized, "Missing token in request body")
+	}
+	if a.secretKey == "" || subtle.ConstantTimeCompare([]byte(*req.Token), []byte(a.secretKey)) != 1 {
+		return errJSON(http.StatusUnauthorized, "Invalid Authorization")
+	}
+	conn, err := net.DialTimeout("tcp", a.semantic, 3*time.Second)
+	if err != nil {
+		return errJSON(http.StatusBadGateway, "Semantic search is not available")
+	}
+	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(30 * time.Second))
+	payload, _ := json.Marshal(map[string]string{"query": req.Query})
+	if _, err := conn.Write(payload); err != nil {
+		return errJSON(http.StatusBadGateway, "Semantic search is not available")
+	}
+	data, err := io.ReadAll(io.LimitReader(conn, 8<<20))
+	var out any
+	if err != nil || json.Unmarshal(data, &out) != nil {
+		return errJSON(http.StatusBadGateway, "Semantic search gave an invalid answer")
+	}
+	return jsonR(out)
+}

internal/kjv/server.go

@@ -0,0 +1,313 @@
+package kjv
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"os/signal"
+	"runtime"
+	"strconv"
+	"syscall"
+	"time"
+)
+
+// localOnly insists on a loopback address: the program is meant to sit behind
+// a reverse proxy on the same machine and never face the network itself.
+func localOnly(addr string) (string, error) {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return "", fmt.Errorf("bad listen address %q: %w", addr, err)
+	}
+	switch host {
+	case "", "localhost":
+		host = "127.0.0.1"
+	default:
+		if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() {
+			return "", fmt.Errorf("refusing to listen on %q: only localhost addresses (127.0.0.1, ::1) are allowed; put a reverse proxy in front", host)
+		}
+	}
+	return net.JoinHostPort(host, port), nil
+}
+
+// Run serves the application from the companion database until interrupted.
+func Run(dbPath, addr string) error {
+	addr, err := localOnly(addr)
+	if err != nil {
+		return err
+	}
+	if _, err := os.Stat(dbPath); err != nil {
+		return fmt.Errorf("database not found: %s (use -db or KJV_DB)", dbPath)
+	}
+	store, err := OpenStore(dbPath)
+	if err != nil {
+		return err
+	}
+	defer store.Close()
+	if store.stateOK {
+		lg.Attach(store.db, 90)
+	} else {
+		fmt.Fprintln(os.Stderr, "WARN: database is read-only: logs go to stderr; reading plans and shortcodes are disabled")
+	}
+	defer lg.Close()
+
+	app, err := NewApp(store)
+	if err != nil {
+		return err
+	}
+	app.secretKey = os.Getenv("KJV_SECRET_API_KEY")
+	app.semantic = envOr("KJV_SEMANTIC_ADDR", "localhost:1613")
+	if p := os.Getenv("KJV_SERMON_DB"); p != "" {
+		if st, err := os.Stat(p); err == nil && st.Size() > 0 {
+			if db, err := sql.Open("sqlite", "file:"+p+"?mode=ro&_pragma=busy_timeout(5000)"); err == nil {
+				app.sermonDB = db
+			}
+		} else {
+			lg.Warn("sermon database %s not available", p)
+		}
+	}
+
+	srv := &http.Server{
+		Addr:              addr,
+		Handler:           app.Routes(),
+		MaxHeaderBytes:    16 << 10,
+		ReadHeaderTimeout: 10 * time.Second,
+		ReadTimeout:       30 * time.Second,
+		WriteTimeout:      60 * time.Second,
+		IdleTimeout:       120 * time.Second,
+	}
+	go app.maintain()
+	lg.Info("Started %s: db=%s addr=%s verses=%d state=%v sermons=%v", runtime.Version(), dbPath, addr, len(store.verses), store.stateOK, app.sermonDB != nil)
+	fmt.Printf("kjv-api listening on %s (database %s)\n", addr, dbPath)
+
+	ln, err := net.Listen("tcp", addr)
+	if err != nil {
+		return err
+	}
+	errc := make(chan error, 1)
+	go func() { errc <- srv.Serve(ln) }()
+	sig := make(chan os.Signal, 1)
+	signal.Notify(sig, os.Interrupt, syscall.SIGTERM)
+	select {
+	case err := <-errc:
+		return err
+	case s := <-sig:
+		lg.Info("Stopping on %v", s)
+		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		if err := srv.Shutdown(ctx); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			return err
+		}
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------------------
+// routes
+
+type statusWriter struct {
+	http.ResponseWriter
+	status int
+}
+
+func (w *statusWriter) WriteHeader(code int) { w.status = code; w.ResponseWriter.WriteHeader(code) }
+
+func (a *App) Routes() http.Handler {
+	mux := http.NewServeMux()
+	get := func(pattern string, h func(r *http.Request) Resp) {
+		mux.HandleFunc("GET "+pattern, func(w http.ResponseWriter, r *http.Request) { h(r).write(w) })
+	}
+
+	get("/{$}", func(r *http.Request) Resp { return htmlR(a.indexHTML) })
+	get("/{shortcode}", func(r *http.Request) Resp { return a.handleShortcode(r.PathValue("shortcode")) })
+	get("/fe", func(r *http.Request) Resp { return a.staticPage("kjv_fe.html") })
+	get("/salvation", func(r *http.Request) Resp { return a.staticPage("html/heaven.html") })
+	get("/saved", func(r *http.Request) Resp { return a.staticPage("html/saved.html") })
+	get("/plan", func(r *http.Request) Resp { return a.staticPage("html/plan.html") })
+
+	get("/show", func(r *http.Request) Resp { return a.handleShow(r.URL.Query()) })
+	get("/find", func(r *http.Request) Resp { return a.handleFind(r.URL.Query()) })
+	get("/search", func(r *http.Request) Resp {
+		v := r.URL.Query()
+		return a.handleSearch(v.Get("kw"), viewOf(v, "rich"))
+	})
+	get("/define", func(r *http.Request) Resp { return a.handleDefine(r.URL.Query()) })
+	get("/defs", func(r *http.Request) Resp { return a.handleDefs(r.URL.Query()) })
+	get("/random", func(r *http.Request) Resp { return a.handleRandom(r.URL.Query(), "json") })
+	get("/t/random", func(r *http.Request) Resp { return a.handleRandom(r.URL.Query(), "plain") })
+	get("/votd", func(r *http.Request) Resp { return a.handleVotd(r.URL.Query(), "json") })
+	get("/t/votd", func(r *http.Request) Resp { return a.handleVotd(r.URL.Query(), "plain") })
+	get("/seq", func(r *http.Request) Resp {
+		if r.URL.Query().Get("getuid") != "" && !creations.allow(clientAddr(r)) {
+			return tooMany()
+		}
+		return a.handleSeq(r.URL.Query())
+	})
+	get("/mksc/{query}", func(r *http.Request) Resp {
+		if !creations.allow(clientAddr(r)) {
+			return tooMany()
+		}
+		return a.handleMakeShortcode(r.PathValue("query"))
+	})
+	get("/ref/{ref}", func(r *http.Request) Resp { return a.handleRef(r.PathValue("ref")) })
+	get("/verse_id/{id}", func(r *http.Request) Resp {
+		id := cleanInput(r.PathValue("id"), "0123456789")
+		n, err := strconv.ParseInt(id, 10, 64)
+		if err != nil {
+			return errJSON(http.StatusBadRequest, "Invalid verse id")
+		}
+		v := a.store.Range(n, 0)
+		if len(v) == 0 {
+			return errJSON(http.StatusNotFound, "No such verse")
+		}
+		return jsonR([]any{v[0].ID, v[0].Book, v[0].Chapter, v[0].Verse, v[0].Text})
+	})
+	get("/xref/{id}", func(r *http.Request) Resp { return a.handleXref(r.PathValue("id")) })
+	get("/browse", func(r *http.Request) Resp {
+		body := "<h2>Books</h2><div class=\"table-wrap\"><table id=\"Books\">"
+		for _, n := range a.store.bookNames {
+			body += fmt.Sprintf("<tr><td><a href=\"/find?kw=%s&view=rich\">%s</a></td></tr>", urlQuery(n), htmlEsc(n))
+		}
+		return htmlR(a.staticHTMLDoc("Books", body+"</table></div>"))
+	})
+	get("/status", func(r *http.Request) Resp {
+		var m runtime.MemStats
+		runtime.ReadMemStats(&m)
+		return htmlR(a.staticHTMLDoc("Status", fmt.Sprintf("<h2>Status</h2><p>Sanity: %v</p><p>Uptime: %s</p><p>Memory: %d KB</p>",
+			a.store.stateOK, time.Since(a.started).Round(time.Second), m.Sys/1024)))
+	})
+	get("/seqreport", func(r *http.Request) Resp {
+		if resp, ok := a.adminOnly(r); !ok {
+			return resp
+		}
+		return a.tableReport("Sequence Report", "seq_report", "SELECT uid, verse, last_used FROM reading_tracker", false)
+	})
+	get("/screport", func(r *http.Request) Resp {
+		if resp, ok := a.adminOnly(r); !ok {
+			return resp
+		}
+		return a.tableReport("Shortcode Report", "shortcode_report", "SELECT shortcode, query, creation_date FROM shortcodes", true)
+	})
+	get("/natural", func(r *http.Request) Resp {
+		return htmlR("Submit a POST request to this endpoint with the following params in the body:\n{\"token\": \"mytokenstring\", \"query\" : \"myquerystring\"}")
+	})
+	mux.HandleFunc("POST /natural", func(w http.ResponseWriter, r *http.Request) { a.handleNaturalPost(r.Body).write(w) })
+	mux.HandleFunc("POST /plan/new", func(w http.ResponseWriter, r *http.Request) {
+		if !creations.allow(clientAddr(r)) {
+			tooMany().write(w)
+			return
+		}
+		r.Body = http.MaxBytesReader(w, r.Body, 16<<10)
+		if r.ParseForm() != nil {
+			errJSON(http.StatusRequestEntityTooLarge, "Request too large").write(w)
+			return
+		}
+		a.handlePlanNew(r.PostForm).write(w)
+	})
+	mux.HandleFunc("POST /plan/{uid}/move", func(w http.ResponseWriter, r *http.Request) {
+		r.Body = http.MaxBytesReader(w, r.Body, 16<<10)
+		if r.ParseForm() != nil {
+			errJSON(http.StatusRequestEntityTooLarge, "Request too large").write(w)
+			return
+		}
+		a.handlePlanMove(r.PathValue("uid"), r.PostForm).write(w)
+	})
+	get("/plan/{uid}", func(r *http.Request) Resp { return a.handlePlanView(r.PathValue("uid"), r.URL.Query()) })
+
+	for _, p := range []string{"/sermons", "/sermons/{rest...}", "/sermon/{rest...}"} {
+		get(p, func(r *http.Request) Resp { return a.handleSermons(r.URL.Path) })
+	}
+	for _, p := range []string{"/preachers", "/churches", "/denominations"} {
+		cols := map[string]string{"/preachers": "denomination,preacher,church_name", "/churches": "denomination,church_name", "/denominations": "denomination"}[p]
+		get(p, func(r *http.Request) Resp { return a.sermonQuery("select distinct " + cols + " from verse_refs") })
+	}
+
+	for u := range servedPaths {
+		u := u
+		mux.HandleFunc("GET "+u, func(w http.ResponseWriter, r *http.Request) { a.serveAsset(w, r, u) })
+	}
+	favicon := func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "image/svg+xml")
+		w.Header().Set("Cache-Control", "public, max-age=86400")
+		w.Write(a.favicon)
+	}
+	mux.HandleFunc("GET /favicon.ico", favicon)
+	mux.HandleFunc("GET /favicon.svg", favicon)
+	mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { a.notFound().write(w) })
+
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		start := time.Now()
+		sw := &statusWriter{ResponseWriter: w, status: 200}
+		defer func() {
+			if rec := recover(); rec != nil {
+				lg.Error("panic serving %s: %v", r.URL.String(), rec)
+				sw.status = 500
+				errJSON(500, "Internal error").write(w)
+			}
+			lg.Info("%s %s %d %s", r.Method, r.URL.RequestURI(), sw.status, time.Since(start).Round(time.Microsecond))
+		}()
+		h := w.Header()
+		h.Set("X-Content-Type-Options", "nosniff")
+		h.Set("X-Frame-Options", "DENY")
+		h.Set("Referrer-Policy", "same-origin")
+		mux.ServeHTTP(sw, r)
+	})
+}
+
+// serveAsset: a URL whose ?v= hash matches the file is immutable for a year;
+// any other URL for it is revalidated every time.
+func (a *App) serveAsset(w http.ResponseWriter, r *http.Request, u string) {
+	as := a.served[u]
+	h := w.Header()
+	h.Set("Content-Type", as.mime)
+	h.Set("ETag", as.etag)
+	if r.URL.Query().Get("v") == as.version {
+		h.Set("Cache-Control", "public, max-age=31536000, immutable")
+	} else {
+		h.Set("Cache-Control", "no-cache")
+	}
+	if r.Header.Get("If-None-Match") == as.etag {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+	w.Write(as.body)
+}
+
+func (a *App) handleXref(raw string) Resp {
+	id, err := strconv.ParseInt(raw, 10, 64)
+	if err != nil {
+		return a.notFound()
+	}
+	refs, err := a.store.CrossReferences(id)
+	if err != nil {
+		lg.Error("xref: %v", err)
+		return errJSON(http.StatusInternalServerError, "Database error")
+	}
+	out := []map[string]any{}
+	for _, c := range refs {
+		ref := a.store.FormatRef(c.Start)
+		var verses []Verse
+		if c.End > c.Start {
+			if c.Start/1000 == c.End/1000 { // "John 3:16-18" when the range stays in one chapter
+				ref += fmt.Sprintf("-%d", c.End%1000)
+			} else {
+				ref += "-" + a.store.FormatRef(c.End)
+			}
+			verses = a.store.Range(c.Start, c.End)
+		} else {
+			verses = a.store.Range(c.Start, 0)
+		}
+		text := ""
+		for i, v := range verses {
+			if i > 0 {
+				text += " "
+			}
+			text += v.Text
+		}
+		out = append(out, map[string]any{"ref": ref, "verse_id": c.Start, "text": text})
+	}
+	return jsonR(out)
+}

internal/kjv/spell.go

@@ -0,0 +1,253 @@
+package kjv
+
+import (
+	"sort"
+	"strings"
+	"unicode"
+)
+
+// Speller checks and suggests spellings from kjv.dic (a plain word list) and
+// kjv.aff (which only adds a TRY order and REP replacement hints), so no C
+// library is needed.
+type Speller struct {
+	words   map[string]bool     // exact entries
+	byLower map[string][]string // lowercase -> entries
+	list    []string            // lowercase entries, dictionary order, unique
+	try     []rune
+	rep     [][2]string
+}
+
+func NewSpeller(dic, aff []byte) *Speller {
+	s := &Speller{words: map[string]bool{}, byLower: map[string][]string{}}
+	lines := strings.Split(string(dic), "\n")
+	seen := map[string]bool{}
+	for i, line := range lines {
+		line = strings.TrimSpace(line)
+		if i == 0 || line == "" { // first line is the entry count
+			continue
+		}
+		if k := strings.IndexByte(line, '/'); k >= 0 {
+			line = line[:k]
+		}
+		s.words[line] = true
+		low := strings.ToLower(line)
+		s.byLower[low] = append(s.byLower[low], line)
+		if !seen[low] {
+			seen[low] = true
+			s.list = append(s.list, low)
+		}
+	}
+	for _, line := range strings.Split(string(aff), "\n") {
+		f := strings.Fields(line)
+		switch {
+		case len(f) == 2 && f[0] == "TRY":
+			s.try = []rune(f[1])
+		case len(f) == 3 && f[0] == "REP":
+			s.rep = append(s.rep, [2]string{f[1], strings.ReplaceAll(f[2], "_", " ")})
+		}
+	}
+	return s
+}
+
+func stripSyntax(w string) string {
+	return strings.Map(func(r rune) rune {
+		if r == '+' || r == '^' || r == '*' {
+			return -1
+		}
+		return r
+	}, w)
+}
+
+func hasLetter(w string) bool {
+	for _, r := range w {
+		if unicode.IsLetter(r) {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *Speller) Spell(word string) bool {
+	word = stripSyntax(word)
+	if !hasLetter(word) || s.words[word] {
+		return true
+	}
+	low := strings.ToLower(word)
+	if low != word && (word == strings.ToUpper(word) || word == capitalize(low)) {
+		return len(s.byLower[low]) > 0
+	}
+	return false
+}
+
+func capitalize(w string) string {
+	r := []rune(w)
+	if len(r) == 0 {
+		return w
+	}
+	return strings.ToUpper(string(r[:1])) + strings.ToLower(string(r[1:]))
+}
+
+func (s *Speller) known(low string) []string { return s.byLower[low] }
+
+// Suggest returns likely corrections, best first.
+func (s *Speller) Suggest(word string) []string {
+	word = stripSyntax(word)
+	low := strings.ToLower(word)
+	if low == "" {
+		return nil
+	}
+	var out []string
+	seen := map[string]bool{}
+	add := func(cands ...string) {
+		for _, c := range cands {
+			if !seen[c] {
+				seen[c] = true
+				out = append(out, c)
+			}
+		}
+	}
+	addLower := func(l string) { // dictionary form(s) of a lowercase candidate
+		if l == low {
+			return
+		}
+		add(s.known(l)...)
+	}
+	if low != word { // wrong capitalization of a known word
+		add(s.known(low)...)
+	}
+	// replacement hints from the .aff file, applied at each occurrence
+	for _, r := range s.rep {
+		for at := 0; ; {
+			i := strings.Index(low[at:], r[0])
+			if i < 0 {
+				break
+			}
+			i += at
+			cand := low[:i] + r[1] + low[i+len(r[0]):]
+			if strings.Contains(cand, " ") {
+				if p := strings.Fields(cand); len(p) == 2 && len(s.known(p[0])) > 0 && len(s.known(p[1])) > 0 {
+					add(p[0] + " " + p[1])
+				}
+			} else {
+				addLower(cand)
+			}
+			at = i + 1
+		}
+	}
+	rs := []rune(low)
+	// one edit away: swap, delete, replace, insert
+	for i := 0; i+1 < len(rs); i++ {
+		c := append([]rune(nil), rs...)
+		c[i], c[i+1] = c[i+1], c[i]
+		addLower(string(c))
+	}
+	for i := range rs {
+		addLower(string(rs[:i]) + string(rs[i+1:]))
+	}
+	for i := range rs {
+		for _, t := range s.try {
+			addLower(string(rs[:i]) + string(t) + string(rs[i+1:]))
+		}
+	}
+	for i := 0; i <= len(rs); i++ {
+		for _, t := range s.try {
+			addLower(string(rs[:i]) + string(t) + string(rs[i:]))
+		}
+	}
+	// two words run together ("lordgod")
+	for i := 1; i < len(rs); i++ {
+		a, b := string(rs[:i]), string(rs[i:])
+		if len(a) > 1 && len(b) > 1 && len(s.known(a)) > 0 && len(s.known(b)) > 0 {
+			add(a + " " + b)
+		}
+	}
+	// still little to offer: anything within two edits, closest first
+	if len(out) < 3 {
+		type cand struct {
+			w    string
+			dist int
+			pre  int
+		}
+		var cs []cand
+		for _, w := range s.list {
+			if abs(len(w)-len(low)) > 2 || w == low {
+				continue
+			}
+			if d := editDistance(low, w, 2); d <= 2 {
+				cs = append(cs, cand{w, d, commonPrefix(low, w)})
+			}
+		}
+		sort.SliceStable(cs, func(i, j int) bool {
+			if cs[i].dist != cs[j].dist {
+				return cs[i].dist < cs[j].dist
+			}
+			return cs[i].pre > cs[j].pre
+		})
+		for _, c := range cs {
+			addLower(c.w)
+			if len(out) >= 8 {
+				break
+			}
+		}
+	}
+	if len(out) > 10 {
+		out = out[:10]
+	}
+	if word == strings.ToUpper(word) && len(word) > 1 {
+		for i := range out {
+			out[i] = strings.ToUpper(out[i])
+		}
+	} else if word == capitalize(low) && word != low {
+		for i := range out {
+			out[i] = capitalize(out[i])
+		}
+	}
+	return out
+}
+
+func abs(n int) int {
+	if n < 0 {
+		return -n
+	}
+	return n
+}
+
+func commonPrefix(a, b string) int {
+	n := 0
+	for n < len(a) && n < len(b) && a[n] == b[n] {
+		n++
+	}
+	return n
+}
+
+// editDistance is optimal-string-alignment distance, giving up above limit.
+func editDistance(a, b string, limit int) int {
+	ra, rb := []rune(a), []rune(b)
+	prev2 := make([]int, len(rb)+1)
+	prev := make([]int, len(rb)+1)
+	cur := make([]int, len(rb)+1)
+	for j := range prev {
+		prev[j] = j
+	}
+	for i := 1; i <= len(ra); i++ {
+		cur[0] = i
+		best := cur[0]
+		for j := 1; j <= len(rb); j++ {
+			cost := 1
+			if ra[i-1] == rb[j-1] {
+				cost = 0
+			}
+			v := min(prev[j]+1, cur[j-1]+1, prev[j-1]+cost)
+			if i > 1 && j > 1 && ra[i-1] == rb[j-2] && ra[i-2] == rb[j-1] {
+				v = min(v, prev2[j-2]+1)
+			}
+			cur[j] = v
+			best = min(best, v)
+		}
+		if best > limit {
+			return limit + 1
+		}
+		prev2, prev, cur = prev, cur, prev2
+	}
+	return prev[len(rb)]
+}

internal/kjv/state.go

@@ -0,0 +1,521 @@
+package kjv
+
+import (
+	"crypto/rand"
+	"crypto/subtle"
+	"database/sql"
+	"fmt"
+	"html"
+	"math"
+	"math/big"
+	"net"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+)
+
+// Reading sequences (/seq), reading plans (/plan) and shortcodes (/mksc) keep
+// their rows in the companion database.
+
+const (
+	idChars         = "0123456789ACEFHJKLMNPRTUVWXY"
+	planMaxPerDay   = 500
+	seqPurgeAfter   = 90 * 24 * time.Hour
+	stateTimeFormat = "2006-01-02 15:04:05.000000"
+)
+
+func generateShortID() string { // 5 characters
+	n, err := rand.Int(rand.Reader, big.NewInt(17_210_367-614_656))
+	if err != nil {
+		panic(err)
+	}
+	remain := 614_656 + int(n.Int64())
+	out := ""
+	for remain > 0 {
+		out = string(idChars[remain%len(idChars)]) + out
+		remain /= len(idChars)
+	}
+	return out
+}
+
+func stateNow() string { return time.Now().Format(stateTimeFormat) }
+
+var errNoState = fmt.Errorf("the database is read-only, so reading plans and shortcodes are unavailable")
+
+func (a *App) stateDB() (*sql.DB, error) {
+	if !a.store.stateOK {
+		return nil, errNoState
+	}
+	return a.store.db, nil
+}
+
+// insertID stores a row under a fresh random id.
+func insertID(db *sql.DB, query string, args ...any) (string, error) {
+	var err error
+	for i := 0; i < 5; i++ { // a clash on a random 5 character id is very unlikely
+		id := generateShortID()
+		if _, err = db.Exec(query, append([]any{id}, args...)...); err == nil {
+			return id, nil
+		}
+	}
+	return "", err
+}
+
+// ---------------------------------------------------------------------------
+// shortcodes
+
+func (a *App) handleMakeShortcode(query string) Resp {
+	query = cleanInput(query, searchChars+":-,")
+	if query == "" {
+		return htmlR("Empty query.")
+	}
+	db, err := a.stateDB()
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	var code string
+	if err := db.QueryRow("SELECT shortcode FROM shortcodes WHERE query = ?", query).Scan(&code); err == nil {
+		return htmlR(code) // the same query always gets the same code
+	}
+	code, err = insertID(db, "INSERT INTO shortcodes (shortcode, query, creation_date) VALUES (?, ?, ?)", query, stateNow())
+	if err != nil {
+		lg.Error("shortcode insert: %v", err)
+		return errJSON(http.StatusInternalServerError, "Could not create a shortcode")
+	}
+	lg.Info("Generated shortcode %s for query '%s'", code, query)
+	return htmlR(code)
+}
+
+func (a *App) handleShortcode(code string) Resp {
+	clean := cleanInput(code, idChars)
+	if clean == "" {
+		return htmlR("Empty shortcode.")
+	}
+	db, err := a.stateDB()
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	var query string
+	if err := db.QueryRow("SELECT query FROM shortcodes WHERE shortcode = ?", clean).Scan(&query); err != nil {
+		return a.notFound()
+	}
+	lg.Info("Resolved shortcode %s for query '%s'", clean, query)
+	return Resp{Status: http.StatusFound, Type: "text/html; charset=utf-8",
+		Header: map[string]string{"Location": "/find?kw=" + url.QueryEscape(query) + "&view=rich"}}
+}
+
+func (a *App) notFound() Resp {
+	return jsonStatus(map[string]string{
+		"error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."},
+		http.StatusNotFound)
+}
+
+func (a *App) tableReport(title, id, query string, header bool) Resp {
+	db, err := a.stateDB()
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	rows, err := db.Query(query)
+	if err != nil {
+		lg.Error("report: %v", err)
+		return errJSON(http.StatusInternalServerError, "Database error")
+	}
+	defer rows.Close()
+	var sb strings.Builder
+	fmt.Fprintf(&sb, "<h2>%s</h2><div class=\"table-wrap\"><table id=\"%s\">", title, id)
+	if header {
+		sb.WriteString("<tr><td>shortcode</td><td>query</td><td>creation_date</td></tr>")
+	}
+	for rows.Next() {
+		var c1, c2, c3 sql.NullString
+		rows.Scan(&c1, &c2, &c3)
+		fmt.Fprintf(&sb, "<tr><td>%s</td><td>%s</td><td>%s</td></tr>", html.EscapeString(c1.String), html.EscapeString(c2.String), html.EscapeString(c3.String))
+	}
+	sb.WriteString("</table></div>")
+	return htmlR(a.staticHTMLDoc(title, sb.String()))
+}
+
+// ---------------------------------------------------------------------------
+// /seq : hand out scripture a few verses at a time
+
+func (a *App) purgeSequences(db *sql.DB) {
+	rows, err := db.Query("SELECT uid, last_used FROM reading_tracker LIMIT 10")
+	if err != nil {
+		return
+	}
+	var stale []string
+	for rows.Next() {
+		var uid, last string
+		rows.Scan(&uid, &last)
+		if t, err := time.ParseInLocation(stateTimeFormat, last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
+			stale = append(stale, uid)
+		} else if t, err := time.ParseInLocation("2006-01-02 15:04:05", last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
+			stale = append(stale, uid)
+		}
+	}
+	rows.Close()
+	for _, uid := range stale {
+		lg.Info("Purging UID: %s", uid)
+		db.Exec("DELETE FROM reading_tracker WHERE uid = ?", uid)
+	}
+}
+
+func (a *App) startIndex(ref string) (int, bool) {
+	ref = strings.TrimSpace(ref)
+	if ref == "" {
+		return 0, true
+	}
+	resolved := a.res.Resolve(ref, true)
+	if resolved == nil {
+		return 0, false
+	}
+	i := a.store.VerseIndex(resolved.Refs[0].Start)
+	return i, i >= 0
+}
+
+func (a *App) handleSeq(q url.Values) Resp {
+	uid, getuid := q.Get("uid"), q.Get("getuid") != ""
+	num := 1
+	if q.Has("num") {
+		n, err := strconv.Atoi(q.Get("num"))
+		if err != nil {
+			return htmlR("Invalid num argument")
+		}
+		num = n
+	}
+	if uid == "" && !getuid {
+		return a.staticPage("html/seq.html")
+	}
+	num-- // 0 will mean "next"
+	num = min(num, len(a.store.verseIDs))
+	if num < 0 {
+		return jsonR(map[string]string{"error": "You may only increment forward."})
+	}
+	db, err := a.stateDB()
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	if getuid {
+		start, ok := a.startIndex(q.Get("start"))
+		if !ok {
+			return jsonR(map[string]string{"error": "Could not find that starting verse."})
+		}
+		id, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
+		if err != nil {
+			lg.Error("UID database failed to insert: %v", err)
+			return jsonR(map[string]string{"error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"})
+		}
+		lg.Info("New SEQ UID: %s", id)
+		return htmlR("Your new UID is: " + id)
+	}
+	uid = cleanInput(strings.ReplaceAll(uid, " ", ""), searchChars[:62]+":-,")
+	var next int
+	if err := db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&next); err != nil {
+		lg.Info("Invalid UID %s", uid)
+		return jsonR(map[string]string{"error": "UID Invalid or purged (90 days of inactivity)"})
+	}
+	a.purgeSequences(db)
+	total := len(a.store.verseIDs)
+	if next >= total || next < 0 { // finished the Bible: start over
+		next = 0
+	}
+	last := min(next+num, total-1)
+	verses := a.store.Range(a.store.verseIDs[next], a.store.verseIDs[last])
+	if _, err := db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, last+1, stateNow()); err != nil {
+		lg.Error("seq update: %v", err)
+	}
+	lg.Info("Served %d verses to UID %s", last-next+1, uid)
+	return a.render(verses, viewOf(q, "json"), "").resp()
+}
+
+// ---------------------------------------------------------------------------
+// /plan : a /seq style tracker that can go backwards. The plan is its UID; the
+// page URL (/plan/<uid>?n=<verses per day>) is what gets bookmarked.
+
+func planPerDay(raw string, def int) int {
+	n, err := strconv.Atoi(raw)
+	if err != nil {
+		return def
+	}
+	return max(1, min(planMaxPerDay, n))
+}
+
+func (a *App) planError(msg string) Resp {
+	r := htmlR(a.staticHTMLDoc("Reading Plan",
+		"<h2>Reading Plan</h2><p>"+html.EscapeString(msg)+"</p><p><a href='/plan'>Back to plan setup</a></p>"))
+	r.Status = http.StatusBadRequest
+	return r
+}
+
+func redirect303(to string) Resp {
+	return Resp{Status: http.StatusSeeOther, Type: "text/html; charset=utf-8", Header: map[string]string{"Location": to}}
+}
+
+func validPlanUID(uid string) bool { return uid != "" && cleanInput(uid, idChars) == uid }
+
+func (a *App) handlePlanNew(f url.Values) Resp {
+	db, err := a.stateDB()
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	total := len(a.store.verseIDs)
+	start, ok := a.startIndex(f.Get("start"))
+	if !ok {
+		return a.planError("We couldn't find that starting point. Try something like 'Matthew 1:1'.")
+	}
+	var perDay int
+	if f.Get("pace") == "verses" {
+		perDay = planPerDay(f.Get("verses"), 25)
+	} else {
+		days, err := strconv.Atoi(f.Get("days"))
+		if err != nil {
+			days = 365
+		}
+		days = max(1, min(3650, days))
+		perDay = planPerDay(strconv.Itoa(int(math.Ceil(float64(total-start)/float64(days)))), 25)
+	}
+	uid, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
+	if err != nil {
+		return a.planError("Could not create a plan, please try again.")
+	}
+	lg.Info("New reading plan %s start=%d per_day=%d", uid, start, perDay)
+	return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
+}
+
+func (a *App) planPosition(uid string) (int, bool, error) {
+	db, err := a.stateDB()
+	if err != nil {
+		return 0, false, err
+	}
+	var pos int
+	err = db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&pos)
+	return pos, err == nil, nil
+}
+
+func (a *App) handlePlanMove(uid string, f url.Values) Resp {
+	perDay := planPerDay(f.Get("n"), 25)
+	if !validPlanUID(uid) {
+		return a.planError("That reading plan id is not valid.")
+	}
+	pos, found, err := a.planPosition(uid)
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	if !found {
+		return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
+	}
+	total := len(a.store.verseIDs)
+	switch f.Get("action") {
+	case "next":
+		pos = min(total, pos+perDay)
+	case "back":
+		pos = max(0, pos-perDay)
+	case "restart":
+		pos = 0
+	case "goto":
+		target, ok := a.startIndex(f.Get("kw"))
+		if !ok {
+			return a.planError("We couldn't find that reference. Try something like 'John 3:1'.")
+		}
+		pos = target
+	}
+	a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, pos, stateNow())
+	return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
+}
+
+func (a *App) handlePlanView(uid string, q url.Values) Resp {
+	if !validPlanUID(uid) {
+		return a.planError("That reading plan id is not valid.")
+	}
+	perDay := planPerDay(q.Get("n"), 25)
+	position, found, err := a.planPosition(uid)
+	if err != nil {
+		return errJSON(http.StatusServiceUnavailable, err.Error())
+	}
+	if !found {
+		return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
+	}
+	// keeps an active plan from being purged as unused
+	a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, position, stateNow())
+
+	ids := a.store.verseIDs
+	total := len(ids)
+	finished := position >= total
+	lastIndex := min(total, position+perDay) - 1
+	percent := math.Round(1000*float64(min(position, total))/float64(total)) / 10
+	esc := html.EscapeString
+
+	controls := fmt.Sprintf("<form method='post' action='/plan/%s/move' class='plan-form'>\n<input type='hidden' name='n' value='%d'>", uid, perDay)
+	backDisabled := ""
+	if position <= 0 {
+		backDisabled = "disabled"
+	}
+	next := ""
+	if !finished {
+		next = "<button type='submit' name='action' value='next' class='plan-primary'>Done &mdash; next reading &rarr;</button>"
+	}
+	controls += fmt.Sprintf(`%s
+<button type='submit' name='action' value='back' class='plan-secondary' %s>&larr; Previous reading</button>
+</form>
+<form method='post' action='/plan/%s/move' class='plan-form plan-goto'>
+<input type='hidden' name='n' value='%d'>
+<input type='hidden' name='action' value='goto'>
+<label for='goto-kw'>Jump to</label>
+<div class='plan-goto-row'><input type='text' id='goto-kw' name='kw' data-book-suggest placeholder='e.g. Psalms 23:1' autocapitalize='off' autocomplete='off'>
+<button type='submit' class='plan-secondary'>Go</button></div>
+</form>`, next, backDisabled, uid, perDay)
+
+	var summary, versesHTML string
+	if finished {
+		summary = fmt.Sprintf("<div class='plan-summary'><strong>You've finished the plan!</strong><div>You read all %s verses. Use Previous reading to look back, or start over.</div></div>", commas(total))
+		controls += fmt.Sprintf("<form method='post' action='/plan/%s/move' class='plan-form'>\n<input type='hidden' name='n' value='%d'>\n<button type='submit' name='action' value='restart' class='plan-secondary'>Start over from Genesis 1:1</button></form>", uid, perDay)
+	} else {
+		first, last := a.store.FormatRef(ids[position]), a.store.FormatRef(ids[lastIndex])
+		summary = fmt.Sprintf("<div class='plan-summary'><strong>Today's reading</strong><div class='plan-range'>%s &ndash; %s</div>"+
+			"<div class='plan-meta'>%d verses &middot; %.1f%% of the Bible complete</div>"+
+			"<div class='plan-progress' role='img' aria-label='%.1f%% complete'><span style='width:%.1f%%'></span></div></div>",
+			esc(first), esc(last), lastIndex-position+1, percent, percent, percent)
+		var sb strings.Builder
+		for _, v := range a.store.Range(ids[position], ids[lastIndex]) {
+			sb.WriteString(a.responseRich(v))
+		}
+		versesHTML = sb.String()
+	}
+	note := "<div class='plan-note'>Bookmark this page &mdash; it is your reading plan. Come back each day, then press <em>Done</em> to move on.</div>"
+	store := fmt.Sprintf("<script>try{localStorage.setItem('kjv.plan',JSON.stringify({uid:'%s',n:%d,t:Date.now()}))}catch(e){}</script>", uid, perDay)
+	return htmlR(a.renderNavbar("", nil, "") + "<div class='responses'>" + summary + versesHTML +
+		"<div class='plan-controls'>" + controls + "</div>" + note + store + "</div>")
+}
+
+func commas(n int) string {
+	s := strconv.Itoa(n)
+	for i := len(s) - 3; i > 0; i -= 3 {
+		s = s[:i] + "," + s[i:]
+	}
+	return s
+}
+
+// ---------------------------------------------------------------------------
+// abuse limits
+
+// creationLimiter bounds how fast each visitor can add rows (shortcodes,
+// plans, sequences), with a looser limit over everyone together as a backstop.
+type creationLimiter struct {
+	mu      sync.Mutex
+	clients map[string]*bucket
+	all     bucket
+}
+
+type bucket struct {
+	tokens float64
+	last   time.Time
+}
+
+const (
+	clientRate, clientBurst = 0.5, 10.0 // per visitor: a new row every 2 seconds, 10 at once
+	totalRate, totalBurst   = 20.0, 100.0
+	maxTrackedClients       = 10000
+)
+
+var creations = &creationLimiter{clients: map[string]*bucket{}, all: bucket{totalBurst, time.Now()}}
+
+func (b *bucket) take(now time.Time, rate, burst float64) bool {
+	b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*rate)
+	b.last = now
+	if b.tokens < 1 {
+		return false
+	}
+	b.tokens--
+	return true
+}
+
+func (c *creationLimiter) allow(client string) bool {
+	c.mu.Lock()
+	defer c.mu.Unlock()
+	now := time.Now()
+	if len(c.clients) > maxTrackedClients { // forget visitors who have long since refilled
+		for k, b := range c.clients {
+			if now.Sub(b.last) > time.Minute {
+				delete(c.clients, k)
+			}
+		}
+	}
+	if isLoopback(client) { // the proxy did not say who this is: only the shared limit applies
+		return c.all.take(now, totalRate, totalBurst)
+	}
+	b := c.clients[client]
+	if b == nil {
+		b = &bucket{clientBurst, now}
+		c.clients[client] = b
+	}
+	if !b.take(now, clientRate, clientBurst) {
+		return false
+	}
+	return c.all.take(now, totalRate, totalBurst)
+}
+
+// clientAddr is the visitor as the reverse proxy saw them. Only the proxy can
+// reach this program (it listens on localhost), so the header it sets is
+// trusted; the last X-Forwarded-For entry is the one the proxy itself added.
+func clientAddr(r *http.Request) string {
+	if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); ip != "" {
+		return ip
+	}
+	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
+		parts := strings.Split(xff, ",")
+		return strings.TrimSpace(parts[len(parts)-1])
+	}
+	host, _, _ := net.SplitHostPort(r.RemoteAddr)
+	noClientHeader.Do(func() {
+		lg.Warn("requests arrive without X-Real-IP or X-Forwarded-For: the reverse proxy should pass the visitor's address on, " +
+			"otherwise creating plans and shortcodes is limited for all visitors together, not for each")
+	})
+	return host
+}
+
+var noClientHeader sync.Once
+
+func isLoopback(addr string) bool {
+	ip := net.ParseIP(addr)
+	return addr == "" || (ip != nil && ip.IsLoopback())
+}
+
+func tooMany() Resp {
+	r := errJSON(http.StatusTooManyRequests, "Too many requests, try again in a moment")
+	r.Header = map[string]string{"Retry-After": "5"}
+	return r
+}
+
+// maintain deletes reading plans nobody has used for 90 days (the pages promise this).
+func (a *App) maintain() {
+	for {
+		if a.store.stateOK {
+			cutoff := time.Now().Add(-seqPurgeAfter).Format(stateTimeFormat)
+			if res, err := a.store.db.Exec("DELETE FROM reading_tracker WHERE last_used < ?", cutoff); err == nil {
+				if n, _ := res.RowsAffected(); n > 0 {
+					lg.Info("Purged %d unused reading plans", n)
+				}
+			}
+		}
+		time.Sleep(6 * time.Hour)
+	}
+}
+
+// adminOnly protects the listings of every plan id and shortcode (a plan id is
+// all that is needed to move that plan). They need the KJV_SECRET_API_KEY as a
+// bearer token, and do not exist when no key is set.
+func (a *App) adminOnly(r *http.Request) (Resp, bool) {
+	if a.secretKey == "" {
+		return a.notFound(), false
+	}
+	token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
+	if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(a.secretKey)) != 1 {
+		resp := errJSON(http.StatusUnauthorized, "Invalid Authorization")
+		resp.Header = map[string]string{"WWW-Authenticate": "Bearer"}
+		return resp, false
+	}
+	return Resp{}, true
+}

internal/kjv/store.go

@@ -0,0 +1,227 @@
+package kjv
+
+import (
+	"database/sql"
+	"fmt"
+	"sort"
+
+	_ "modernc.org/sqlite"
+)
+
+// Verse is one row of the fts_kjv table. ID is book*1_000_000 + chapter*1000 + verse.
+type Verse struct {
+	ID      int64
+	Book    int
+	Chapter int
+	Verse   int
+	Text    string
+}
+
+type BookInfo struct {
+	Title    string
+	Category string
+	OTNT     string
+	Chapters int
+}
+
+// Store holds the companion database plus everything read from it at startup.
+// The whole text is only ~4MB, so verse lookups never touch SQLite; the FTS
+// index is still queried for keyword search.
+type Store struct {
+	db *sql.DB
+
+	verses      []Verse // sorted by ID
+	verseIDs    []int64
+	chapterList []int64 // chapter keys (ID/1000), sorted and unique
+	bookNames   []string
+	bookName    map[int]string
+	bookByName  map[string]int
+	chapterLen  map[int]int
+	bookInfo    []BookInfo
+	abbrevs     []abbrev
+
+	stateOK bool // reading plans and shortcodes can be written
+}
+
+type abbrev struct {
+	Name string
+	Book int
+}
+
+func OpenStore(path string) (*Store, error) {
+	dsn := "file:" + path + "?_pragma=busy_timeout(5000)"
+	db, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		return nil, err
+	}
+	s := &Store{db: db, bookName: map[int]string{}, bookByName: map[string]int{}, chapterLen: map[int]int{}}
+	if err := s.load(); err != nil {
+		db.Close()
+		return nil, fmt.Errorf("%s: %w", path, err)
+	}
+	s.stateOK = s.initState() == nil
+	return s, nil
+}
+
+func (s *Store) Close() error { return s.db.Close() }
+
+func (s *Store) load() error {
+	rows, err := s.db.Query("SELECT b, n FROM key_english ORDER BY b")
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var id int
+		var name string
+		if err := rows.Scan(&id, &name); err != nil {
+			rows.Close()
+			return err
+		}
+		s.bookNames = append(s.bookNames, name)
+		s.bookName[id] = name
+		s.bookByName[name] = id
+	}
+	rows.Close()
+	if len(s.bookNames) == 0 {
+		return fmt.Errorf("key_english is empty")
+	}
+
+	rows, err = s.db.Query("SELECT id, b, c, v, t FROM fts_kjv ORDER BY id")
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var v Verse
+		if err := rows.Scan(&v.ID, &v.Book, &v.Chapter, &v.Verse, &v.Text); err != nil {
+			rows.Close()
+			return err
+		}
+		s.verses = append(s.verses, v)
+	}
+	rows.Close()
+	if len(s.verses) == 0 {
+		return fmt.Errorf("fts_kjv is empty")
+	}
+	for _, v := range s.verses {
+		s.verseIDs = append(s.verseIDs, v.ID)
+		if key := v.ID / 1000; len(s.chapterList) == 0 || s.chapterList[len(s.chapterList)-1] != key {
+			s.chapterList = append(s.chapterList, key)
+		}
+		if v.Chapter > s.chapterLen[v.Book] {
+			s.chapterLen[v.Book] = v.Chapter
+		}
+	}
+
+	rows, err = s.db.Query(`SELECT title_short, category, otnt, chapters FROM book_info ORDER BY "order"`)
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var b BookInfo
+		if err := rows.Scan(&b.Title, &b.Category, &b.OTNT, &b.Chapters); err != nil {
+			rows.Close()
+			return err
+		}
+		s.bookInfo = append(s.bookInfo, b)
+	}
+	rows.Close()
+
+	rows, err = s.db.Query("SELECT a, b FROM key_abbreviations_english ORDER BY id")
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var a abbrev
+		if err := rows.Scan(&a.Name, &a.Book); err != nil {
+			rows.Close()
+			return err
+		}
+		s.abbrevs = append(s.abbrevs, a)
+	}
+	rows.Close()
+	return nil
+}
+
+// initState creates the tables that hold user-generated data (reading plans and
+// shortcodes) inside the companion database.
+func (s *Store) initState() error {
+	for _, stmt := range []string{
+		"CREATE TABLE IF NOT EXISTS reading_tracker (uid TEXT PRIMARY KEY, verse INTEGER, last_used TEXT)",
+		"CREATE TABLE IF NOT EXISTS shortcodes (shortcode TEXT PRIMARY KEY, query TEXT UNIQUE, creation_date TEXT)",
+	} {
+		if _, err := s.db.Exec(stmt); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *Store) BookName(id int) string { return s.bookName[id] }
+
+func (s *Store) FormatRef(verseID int64) string {
+	return fmt.Sprintf("%s %d:%d", s.BookName(int(verseID/1_000_000)), verseID/1000%1000, verseID%1000)
+}
+
+// Range returns the verses with start <= id <= end (end <= 0 means just start).
+func (s *Store) Range(start, end int64) []Verse {
+	i := sort.Search(len(s.verses), func(i int) bool { return s.verses[i].ID >= start })
+	if end <= 0 {
+		if i < len(s.verses) && s.verses[i].ID == start {
+			return s.verses[i : i+1]
+		}
+		return nil
+	}
+	j := sort.Search(len(s.verses), func(j int) bool { return s.verses[j].ID > end })
+	if i >= j {
+		return nil
+	}
+	return s.verses[i:j]
+}
+
+// VerseIndex is the position of a verse id in the whole Bible, or -1.
+func (s *Store) VerseIndex(id int64) int {
+	i := sort.Search(len(s.verseIDs), func(i int) bool { return s.verseIDs[i] >= id })
+	if i < len(s.verseIDs) && s.verseIDs[i] == id {
+		return i
+	}
+	return -1
+}
+
+// FTS runs a full text query and returns the matching verses in Bible order.
+func (s *Store) FTS(query string) ([]Verse, error) {
+	rows, err := s.db.Query("SELECT id FROM fts_kjv(?)", query)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []Verse
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		if v := s.Range(id, 0); len(v) == 1 {
+			out = append(out, v[0])
+		}
+	}
+	return out, rows.Err()
+}
+
+type crossRef struct{ Start, End int64 }
+
+func (s *Store) CrossReferences(verseID int64) ([]crossRef, error) {
+	rows, err := s.db.Query("SELECT sv, ev FROM cross_reference WHERE vid = ? ORDER BY r DESC LIMIT 15", verseID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []crossRef
+	for rows.Next() {
+		var c crossRef
+		if err := rows.Scan(&c.Start, &c.End); err != nil {
+			return nil, err
+		}
+		out = append(out, c)
+	}
+	return out, rows.Err()
+}