kjv_api/internal/kjv/state.go

package kjv

import (
	"crypto/rand"
	"crypto/subtle"
	"database/sql"
	"fmt"
	"html"
	"math"
	"math/big"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Reading sequences (/seq), reading plans (/plan) and shortcodes (/mksc) keep
// their rows in the companion database.

const (
	idChars         = "0123456789ACEFHJKLMNPRTUVWXY"
	planMaxPerDay   = 500
	seqPurgeAfter   = 90 * 24 * time.Hour
	stateTimeFormat = "2006-01-02 15:04:05.000000"
)

func generateShortID() string { // 5 characters
	n, err := rand.Int(rand.Reader, big.NewInt(17_210_367-614_656))
	if err != nil {
		panic(err)
	}
	remain := 614_656 + int(n.Int64())
	out := ""
	for remain > 0 {
		out = string(idChars[remain%len(idChars)]) + out
		remain /= len(idChars)
	}
	return out
}

func stateNow() string { return time.Now().Format(stateTimeFormat) }

var errNoState = fmt.Errorf("the database is read-only, so reading plans and shortcodes are unavailable")

func (a *App) stateDB() (*sql.DB, error) {
	if !a.store.stateOK {
		return nil, errNoState
	}
	return a.store.db, nil
}

// insertID stores a row under a fresh random id.
func insertID(db *sql.DB, query string, args ...any) (string, error) {
	var err error
	for i := 0; i < 5; i++ { // a clash on a random 5 character id is very unlikely
		id := generateShortID()
		if _, err = db.Exec(query, append([]any{id}, args...)...); err == nil {
			return id, nil
		}
	}
	return "", err
}

// ---------------------------------------------------------------------------
// shortcodes

func (a *App) handleMakeShortcode(query string) Resp {
	query = cleanInput(query, searchChars+":-,")
	if query == "" {
		return htmlR("Empty query.")
	}
	db, err := a.stateDB()
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	var code string
	if err := db.QueryRow("SELECT shortcode FROM shortcodes WHERE query = ?", query).Scan(&code); err == nil {
		return htmlR(code) // the same query always gets the same code
	}
	code, err = insertID(db, "INSERT INTO shortcodes (shortcode, query, creation_date) VALUES (?, ?, ?)", query, stateNow())
	if err != nil {
		lg.Error("shortcode insert: %v", err)
		return errJSON(http.StatusInternalServerError, "Could not create a shortcode")
	}
	lg.Info("Generated shortcode %s for query '%s'", code, query)
	return htmlR(code)
}

func (a *App) handleShortcode(code string) Resp {
	clean := cleanInput(code, idChars)
	if clean == "" {
		return htmlR("Empty shortcode.")
	}
	db, err := a.stateDB()
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	var query string
	if err := db.QueryRow("SELECT query FROM shortcodes WHERE shortcode = ?", clean).Scan(&query); err != nil {
		return a.notFound()
	}
	lg.Info("Resolved shortcode %s for query '%s'", clean, query)
	return Resp{Status: http.StatusFound, Type: "text/html; charset=utf-8",
		Header: map[string]string{"Location": "/find?kw=" + url.QueryEscape(query) + "&view=rich"}}
}

func (a *App) notFound() Resp {
	return jsonStatus(map[string]string{
		"error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."},
		http.StatusNotFound)
}

func (a *App) tableReport(title, id, query string, header bool) Resp {
	db, err := a.stateDB()
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	rows, err := db.Query(query)
	if err != nil {
		lg.Error("report: %v", err)
		return errJSON(http.StatusInternalServerError, "Database error")
	}
	defer rows.Close()
	var sb strings.Builder
	fmt.Fprintf(&sb, "<h2>%s</h2><div class=\"table-wrap\"><table id=\"%s\">", title, id)
	if header {
		sb.WriteString("<tr><td>shortcode</td><td>query</td><td>creation_date</td></tr>")
	}
	for rows.Next() {
		var c1, c2, c3 sql.NullString
		rows.Scan(&c1, &c2, &c3)
		fmt.Fprintf(&sb, "<tr><td>%s</td><td>%s</td><td>%s</td></tr>", html.EscapeString(c1.String), html.EscapeString(c2.String), html.EscapeString(c3.String))
	}
	sb.WriteString("</table></div>")
	return htmlR(a.staticHTMLDoc(title, sb.String()))
}

// ---------------------------------------------------------------------------
// /seq : hand out scripture a few verses at a time

func (a *App) purgeSequences(db *sql.DB) {
	rows, err := db.Query("SELECT uid, last_used FROM reading_tracker LIMIT 10")
	if err != nil {
		return
	}
	var stale []string
	for rows.Next() {
		var uid, last string
		rows.Scan(&uid, &last)
		if t, err := time.ParseInLocation(stateTimeFormat, last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
			stale = append(stale, uid)
		} else if t, err := time.ParseInLocation("2006-01-02 15:04:05", last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
			stale = append(stale, uid)
		}
	}
	rows.Close()
	for _, uid := range stale {
		lg.Info("Purging UID: %s", uid)
		db.Exec("DELETE FROM reading_tracker WHERE uid = ?", uid)
	}
}

func (a *App) startIndex(ref string) (int, bool) {
	ref = strings.TrimSpace(ref)
	if ref == "" {
		return 0, true
	}
	resolved := a.res.Resolve(ref, true)
	if resolved == nil {
		return 0, false
	}
	i := a.store.VerseIndex(resolved.Refs[0].Start)
	return i, i >= 0
}

func (a *App) handleSeq(q url.Values) Resp {
	uid, getuid := q.Get("uid"), q.Get("getuid") != ""
	num := 1
	if q.Has("num") {
		n, err := strconv.Atoi(q.Get("num"))
		if err != nil {
			return htmlR("Invalid num argument")
		}
		num = n
	}
	if uid == "" && !getuid {
		return a.staticPage("html/seq.html")
	}
	num-- // 0 will mean "next"
	num = min(num, len(a.store.verseIDs))
	if num < 0 {
		return jsonR(map[string]string{"error": "You may only increment forward."})
	}
	db, err := a.stateDB()
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	if getuid {
		start, ok := a.startIndex(q.Get("start"))
		if !ok {
			return jsonR(map[string]string{"error": "Could not find that starting verse."})
		}
		id, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
		if err != nil {
			lg.Error("UID database failed to insert: %v", err)
			return jsonR(map[string]string{"error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"})
		}
		lg.Info("New SEQ UID: %s", id)
		return htmlR("Your new UID is: " + id)
	}
	uid = cleanInput(strings.ReplaceAll(uid, " ", ""), searchChars[:62]+":-,")
	var next int
	if err := db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&next); err != nil {
		lg.Info("Invalid UID %s", uid)
		return jsonR(map[string]string{"error": "UID Invalid or purged (90 days of inactivity)"})
	}
	a.purgeSequences(db)
	total := len(a.store.verseIDs)
	if next >= total || next < 0 { // finished the Bible: start over
		next = 0
	}
	last := min(next+num, total-1)
	verses := a.store.Range(a.store.verseIDs[next], a.store.verseIDs[last])
	if _, err := db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, last+1, stateNow()); err != nil {
		lg.Error("seq update: %v", err)
	}
	lg.Info("Served %d verses to UID %s", last-next+1, uid)
	return a.render(verses, viewOf(q, "json"), "").resp()
}

// ---------------------------------------------------------------------------
// /plan : a /seq style tracker that can go backwards. The plan is its UID; the
// page URL (/plan/<uid>?n=<verses per day>) is what gets bookmarked.

func planPerDay(raw string, def int) int {
	n, err := strconv.Atoi(raw)
	if err != nil {
		return def
	}
	return max(1, min(planMaxPerDay, n))
}

func (a *App) planError(msg string) Resp {
	r := htmlR(a.staticHTMLDoc("Reading Plan",
		"<h2>Reading Plan</h2><p>"+html.EscapeString(msg)+"</p><p><a href='/plan'>Back to plan setup</a></p>"))
	r.Status = http.StatusBadRequest
	return r
}

func redirect303(to string) Resp {
	return Resp{Status: http.StatusSeeOther, Type: "text/html; charset=utf-8", Header: map[string]string{"Location": to}}
}

func validPlanUID(uid string) bool { return uid != "" && cleanInput(uid, idChars) == uid }

func (a *App) handlePlanNew(f url.Values) Resp {
	db, err := a.stateDB()
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	total := len(a.store.verseIDs)
	start, ok := a.startIndex(f.Get("start"))
	if !ok {
		return a.planError("We couldn't find that starting point. Try something like 'Matthew 1:1'.")
	}
	var perDay int
	if f.Get("pace") == "verses" {
		perDay = planPerDay(f.Get("verses"), 25)
	} else {
		days, err := strconv.Atoi(f.Get("days"))
		if err != nil {
			days = 365
		}
		days = max(1, min(3650, days))
		perDay = planPerDay(strconv.Itoa(int(math.Ceil(float64(total-start)/float64(days)))), 25)
	}
	uid, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
	if err != nil {
		return a.planError("Could not create a plan, please try again.")
	}
	lg.Info("New reading plan %s start=%d per_day=%d", uid, start, perDay)
	return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
}

func (a *App) planPosition(uid string) (int, bool, error) {
	db, err := a.stateDB()
	if err != nil {
		return 0, false, err
	}
	var pos int
	err = db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&pos)
	return pos, err == nil, nil
}

func (a *App) handlePlanMove(uid string, f url.Values) Resp {
	perDay := planPerDay(f.Get("n"), 25)
	if !validPlanUID(uid) {
		return a.planError("That reading plan id is not valid.")
	}
	pos, found, err := a.planPosition(uid)
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	if !found {
		return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
	}
	total := len(a.store.verseIDs)
	switch f.Get("action") {
	case "next":
		pos = min(total, pos+perDay)
	case "back":
		pos = max(0, pos-perDay)
	case "restart":
		pos = 0
	case "goto":
		target, ok := a.startIndex(f.Get("kw"))
		if !ok {
			return a.planError("We couldn't find that reference. Try something like 'John 3:1'.")
		}
		pos = target
	}
	a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, pos, stateNow())
	return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
}

func (a *App) handlePlanView(uid string, q url.Values) Resp {
	if !validPlanUID(uid) {
		return a.planError("That reading plan id is not valid.")
	}
	perDay := planPerDay(q.Get("n"), 25)
	position, found, err := a.planPosition(uid)
	if err != nil {
		return errJSON(http.StatusServiceUnavailable, err.Error())
	}
	if !found {
		return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
	}
	// keeps an active plan from being purged as unused
	a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, position, stateNow())

	ids := a.store.verseIDs
	total := len(ids)
	finished := position >= total
	lastIndex := min(total, position+perDay) - 1
	percent := math.Round(1000*float64(min(position, total))/float64(total)) / 10
	esc := html.EscapeString

	controls := fmt.Sprintf("<form method='post' action='/plan/%s/move' class='plan-form'>\n<input type='hidden' name='n' value='%d'>", uid, perDay)
	backDisabled := ""
	if position <= 0 {
		backDisabled = "disabled"
	}
	next := ""
	if !finished {
		next = "<button type='submit' name='action' value='next' class='plan-primary'>Done &mdash; next reading &rarr;</button>"
	}
	controls += fmt.Sprintf(`%s
<button type='submit' name='action' value='back' class='plan-secondary' %s>&larr; Previous reading</button>
</form>
<form method='post' action='/plan/%s/move' class='plan-form plan-goto'>
<input type='hidden' name='n' value='%d'>
<input type='hidden' name='action' value='goto'>
<label for='goto-kw'>Jump to</label>
<div class='plan-goto-row'><input type='text' id='goto-kw' name='kw' data-book-suggest placeholder='e.g. Psalms 23:1' autocapitalize='off' autocomplete='off'>
<button type='submit' class='plan-secondary'>Go</button></div>
</form>`, next, backDisabled, uid, perDay)

	var summary, versesHTML string
	if finished {
		summary = fmt.Sprintf("<div class='plan-summary'><strong>You've finished the plan!</strong><div>You read all %s verses. Use Previous reading to look back, or start over.</div></div>", commas(total))
		controls += fmt.Sprintf("<form method='post' action='/plan/%s/move' class='plan-form'>\n<input type='hidden' name='n' value='%d'>\n<button type='submit' name='action' value='restart' class='plan-secondary'>Start over from Genesis 1:1</button></form>", uid, perDay)
	} else {
		first, last := a.store.FormatRef(ids[position]), a.store.FormatRef(ids[lastIndex])
		summary = fmt.Sprintf("<div class='plan-summary'><strong>Today's reading</strong><div class='plan-range'>%s &ndash; %s</div>"+
			"<div class='plan-meta'>%d verses &middot; %.1f%% of the Bible complete</div>"+
			"<div class='plan-progress' role='img' aria-label='%.1f%% complete'><span style='width:%.1f%%'></span></div></div>",
			esc(first), esc(last), lastIndex-position+1, percent, percent, percent)
		var sb strings.Builder
		for _, v := range a.store.Range(ids[position], ids[lastIndex]) {
			sb.WriteString(a.responseRich(v))
		}
		versesHTML = sb.String()
	}
	note := "<div class='plan-note'>Bookmark this page &mdash; it is your reading plan. Come back each day, then press <em>Done</em> to move on.</div>"
	store := fmt.Sprintf("<script>try{localStorage.setItem('kjv.plan',JSON.stringify({uid:'%s',n:%d,t:Date.now()}))}catch(e){}</script>", uid, perDay)
	return htmlR(a.renderNavbar("", nil, "") + "<div class='responses'>" + summary + versesHTML +
		"<div class='plan-controls'>" + controls + "</div>" + note + store + "</div>")
}

func commas(n int) string {
	s := strconv.Itoa(n)
	for i := len(s) - 3; i > 0; i -= 3 {
		s = s[:i] + "," + s[i:]
	}
	return s
}

// ---------------------------------------------------------------------------
// abuse limits

// creationLimiter bounds how fast each visitor can add rows (shortcodes,
// plans, sequences), with a looser limit over everyone together as a backstop.
type creationLimiter struct {
	mu      sync.Mutex
	clients map[string]*bucket
	all     bucket
}

type bucket struct {
	tokens float64
	last   time.Time
}

const (
	clientRate, clientBurst = 0.5, 10.0 // per visitor: a new row every 2 seconds, 10 at once
	totalRate, totalBurst   = 20.0, 100.0
	maxTrackedClients       = 10000
)

var creations = &creationLimiter{clients: map[string]*bucket{}, all: bucket{totalBurst, time.Now()}}

func (b *bucket) take(now time.Time, rate, burst float64) bool {
	b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*rate)
	b.last = now
	if b.tokens < 1 {
		return false
	}
	b.tokens--
	return true
}

func (c *creationLimiter) allow(client string) bool {
	c.mu.Lock()
	defer c.mu.Unlock()
	now := time.Now()
	if len(c.clients) > maxTrackedClients { // forget visitors who have long since refilled
		for k, b := range c.clients {
			if now.Sub(b.last) > time.Minute {
				delete(c.clients, k)
			}
		}
	}
	if isLoopback(client) { // the proxy did not say who this is: only the shared limit applies
		return c.all.take(now, totalRate, totalBurst)
	}
	b := c.clients[client]
	if b == nil {
		b = &bucket{clientBurst, now}
		c.clients[client] = b
	}
	if !b.take(now, clientRate, clientBurst) {
		return false
	}
	return c.all.take(now, totalRate, totalBurst)
}

// clientAddr is the visitor as the reverse proxy saw them. Only the proxy can
// reach this program (it listens on localhost), so the header it sets is
// trusted; the last X-Forwarded-For entry is the one the proxy itself added.
func clientAddr(r *http.Request) string {
	if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); ip != "" {
		return ip
	}
	if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
		parts := strings.Split(xff, ",")
		return strings.TrimSpace(parts[len(parts)-1])
	}
	host, _, _ := net.SplitHostPort(r.RemoteAddr)
	noClientHeader.Do(func() {
		lg.Warn("requests arrive without X-Real-IP or X-Forwarded-For: the reverse proxy should pass the visitor's address on, " +
			"otherwise creating plans and shortcodes is limited for all visitors together, not for each")
	})
	return host
}

var noClientHeader sync.Once

func isLoopback(addr string) bool {
	ip := net.ParseIP(addr)
	return addr == "" || (ip != nil && ip.IsLoopback())
}

func tooMany() Resp {
	r := errJSON(http.StatusTooManyRequests, "Too many requests, try again in a moment")
	r.Header = map[string]string{"Retry-After": "5"}
	return r
}

// maintain deletes reading plans nobody has used for 90 days (the pages promise this).
func (a *App) maintain() {
	for {
		if a.store.stateOK {
			cutoff := time.Now().Add(-seqPurgeAfter).Format(stateTimeFormat)
			if res, err := a.store.db.Exec("DELETE FROM reading_tracker WHERE last_used < ?", cutoff); err == nil {
				if n, _ := res.RowsAffected(); n > 0 {
					lg.Info("Purged %d unused reading plans", n)
				}
			}
		}
		time.Sleep(6 * time.Hour)
	}
}

// adminOnly protects the listings of every plan id and shortcode (a plan id is
// all that is needed to move that plan). They need the KJV_SECRET_API_KEY as a
// bearer token, and do not exist when no key is set.
func (a *App) adminOnly(r *http.Request) (Resp, bool) {
	if a.secretKey == "" {
		return a.notFound(), false
	}
	token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
	if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(a.secretKey)) != 1 {
		resp := errJSON(http.StatusUnauthorized, "Invalid Authorization")
		resp.Header = map[string]string{"WWW-Authenticate": "Bearer"}
		return resp, false
	}
	return Resp{}, true
}