package kjv
import (
"crypto/rand"
"crypto/subtle"
"database/sql"
"fmt"
"html"
"math"
"math/big"
"net"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"time"
)
// Reading sequences (/seq), reading plans (/plan) and shortcodes (/mksc) keep
// their rows in the companion database.
const (
idChars = "0123456789ACEFHJKLMNPRTUVWXY"
planMaxPerDay = 500
seqPurgeAfter = 90 * 24 * time.Hour
stateTimeFormat = "2006-01-02 15:04:05.000000"
)
func generateShortID() string { // 5 characters
n, err := rand.Int(rand.Reader, big.NewInt(17_210_367-614_656))
if err != nil {
panic(err)
}
remain := 614_656 + int(n.Int64())
out := ""
for remain > 0 {
out = string(idChars[remain%len(idChars)]) + out
remain /= len(idChars)
}
return out
}
func stateNow() string { return time.Now().Format(stateTimeFormat) }
var errNoState = fmt.Errorf("the database is read-only, so reading plans and shortcodes are unavailable")
func (a *App) stateDB() (*sql.DB, error) {
if !a.store.stateOK {
return nil, errNoState
}
return a.store.db, nil
}
// insertID stores a row under a fresh random id.
func insertID(db *sql.DB, query string, args ...any) (string, error) {
var err error
for i := 0; i < 5; i++ { // a clash on a random 5 character id is very unlikely
id := generateShortID()
if _, err = db.Exec(query, append([]any{id}, args...)...); err == nil {
return id, nil
}
}
return "", err
}
// ---------------------------------------------------------------------------
// shortcodes
func (a *App) handleMakeShortcode(query string) Resp {
query = cleanInput(query, searchChars+":-,")
if query == "" {
return htmlR("Empty query.")
}
db, err := a.stateDB()
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
var code string
if err := db.QueryRow("SELECT shortcode FROM shortcodes WHERE query = ?", query).Scan(&code); err == nil {
return htmlR(code) // the same query always gets the same code
}
code, err = insertID(db, "INSERT INTO shortcodes (shortcode, query, creation_date) VALUES (?, ?, ?)", query, stateNow())
if err != nil {
lg.Error("shortcode insert: %v", err)
return errJSON(http.StatusInternalServerError, "Could not create a shortcode")
}
lg.Info("Generated shortcode %s for query '%s'", code, query)
return htmlR(code)
}
func (a *App) handleShortcode(code string) Resp {
clean := cleanInput(code, idChars)
if clean == "" {
return htmlR("Empty shortcode.")
}
db, err := a.stateDB()
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
var query string
if err := db.QueryRow("SELECT query FROM shortcodes WHERE shortcode = ?", clean).Scan(&query); err != nil {
return a.notFound()
}
lg.Info("Resolved shortcode %s for query '%s'", clean, query)
return Resp{Status: http.StatusFound, Type: "text/html; charset=utf-8",
Header: map[string]string{"Location": "/find?kw=" + url.QueryEscape(query) + "&view=rich"}}
}
func (a *App) notFound() Resp {
return jsonStatus(map[string]string{
"error": "You're lost! Isaiah 41:10 - Fear thou not; for I am with thee: be not dismayed; for I am thy God: I will strengthen thee; yea, I will help thee; yea, I will uphold thee with the right hand of my righteousness."},
http.StatusNotFound)
}
func (a *App) tableReport(title, id, query string, header bool) Resp {
db, err := a.stateDB()
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
rows, err := db.Query(query)
if err != nil {
lg.Error("report: %v", err)
return errJSON(http.StatusInternalServerError, "Database error")
}
defer rows.Close()
var sb strings.Builder
fmt.Fprintf(&sb, "
%s
", title, id)
if header {
sb.WriteString("| shortcode | query | creation_date |
")
}
for rows.Next() {
var c1, c2, c3 sql.NullString
rows.Scan(&c1, &c2, &c3)
fmt.Fprintf(&sb, "| %s | %s | %s |
", html.EscapeString(c1.String), html.EscapeString(c2.String), html.EscapeString(c3.String))
}
sb.WriteString("
")
return htmlR(a.staticHTMLDoc(title, sb.String()))
}
// ---------------------------------------------------------------------------
// /seq : hand out scripture a few verses at a time
func (a *App) purgeSequences(db *sql.DB) {
rows, err := db.Query("SELECT uid, last_used FROM reading_tracker LIMIT 10")
if err != nil {
return
}
var stale []string
for rows.Next() {
var uid, last string
rows.Scan(&uid, &last)
if t, err := time.ParseInLocation(stateTimeFormat, last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
stale = append(stale, uid)
} else if t, err := time.ParseInLocation("2006-01-02 15:04:05", last, time.Local); err == nil && time.Since(t) > seqPurgeAfter {
stale = append(stale, uid)
}
}
rows.Close()
for _, uid := range stale {
lg.Info("Purging UID: %s", uid)
db.Exec("DELETE FROM reading_tracker WHERE uid = ?", uid)
}
}
func (a *App) startIndex(ref string) (int, bool) {
ref = strings.TrimSpace(ref)
if ref == "" {
return 0, true
}
resolved := a.res.Resolve(ref, true)
if resolved == nil {
return 0, false
}
i := a.store.VerseIndex(resolved.Refs[0].Start)
return i, i >= 0
}
func (a *App) handleSeq(q url.Values) Resp {
uid, getuid := q.Get("uid"), q.Get("getuid") != ""
num := 1
if q.Has("num") {
n, err := strconv.Atoi(q.Get("num"))
if err != nil {
return htmlR("Invalid num argument")
}
num = n
}
if uid == "" && !getuid {
return a.staticPage("html/seq.html")
}
num-- // 0 will mean "next"
num = min(num, len(a.store.verseIDs))
if num < 0 {
return jsonR(map[string]string{"error": "You may only increment forward."})
}
db, err := a.stateDB()
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
if getuid {
start, ok := a.startIndex(q.Get("start"))
if !ok {
return jsonR(map[string]string{"error": "Could not find that starting verse."})
}
id, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
if err != nil {
lg.Error("UID database failed to insert: %v", err)
return jsonR(map[string]string{"error": "Something went wrong with the UID database. Please notify tyler@dinsmoor.us"})
}
lg.Info("New SEQ UID: %s", id)
return htmlR("Your new UID is: " + id)
}
uid = cleanInput(strings.ReplaceAll(uid, " ", ""), searchChars[:62]+":-,")
var next int
if err := db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&next); err != nil {
lg.Info("Invalid UID %s", uid)
return jsonR(map[string]string{"error": "UID Invalid or purged (90 days of inactivity)"})
}
a.purgeSequences(db)
total := len(a.store.verseIDs)
if next >= total || next < 0 { // finished the Bible: start over
next = 0
}
last := min(next+num, total-1)
verses := a.store.Range(a.store.verseIDs[next], a.store.verseIDs[last])
if _, err := db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, last+1, stateNow()); err != nil {
lg.Error("seq update: %v", err)
}
lg.Info("Served %d verses to UID %s", last-next+1, uid)
return a.render(verses, viewOf(q, "json"), "").resp()
}
// ---------------------------------------------------------------------------
// /plan : a /seq style tracker that can go backwards. The plan is its UID; the
// page URL (/plan/?n=) is what gets bookmarked.
func planPerDay(raw string, def int) int {
n, err := strconv.Atoi(raw)
if err != nil {
return def
}
return max(1, min(planMaxPerDay, n))
}
func (a *App) planError(msg string) Resp {
r := htmlR(a.staticHTMLDoc("Reading Plan",
"Reading Plan
"+html.EscapeString(msg)+"
Back to plan setup
"))
r.Status = http.StatusBadRequest
return r
}
func redirect303(to string) Resp {
return Resp{Status: http.StatusSeeOther, Type: "text/html; charset=utf-8", Header: map[string]string{"Location": to}}
}
func validPlanUID(uid string) bool { return uid != "" && cleanInput(uid, idChars) == uid }
func (a *App) handlePlanNew(f url.Values) Resp {
db, err := a.stateDB()
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
total := len(a.store.verseIDs)
start, ok := a.startIndex(f.Get("start"))
if !ok {
return a.planError("We couldn't find that starting point. Try something like 'Matthew 1:1'.")
}
var perDay int
if f.Get("pace") == "verses" {
perDay = planPerDay(f.Get("verses"), 25)
} else {
days, err := strconv.Atoi(f.Get("days"))
if err != nil {
days = 365
}
days = max(1, min(3650, days))
perDay = planPerDay(strconv.Itoa(int(math.Ceil(float64(total-start)/float64(days)))), 25)
}
uid, err := insertID(db, "INSERT INTO reading_tracker VALUES(?, ?, ?)", start, stateNow())
if err != nil {
return a.planError("Could not create a plan, please try again.")
}
lg.Info("New reading plan %s start=%d per_day=%d", uid, start, perDay)
return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
}
func (a *App) planPosition(uid string) (int, bool, error) {
db, err := a.stateDB()
if err != nil {
return 0, false, err
}
var pos int
err = db.QueryRow("SELECT verse FROM reading_tracker WHERE uid = ?", uid).Scan(&pos)
return pos, err == nil, nil
}
func (a *App) handlePlanMove(uid string, f url.Values) Resp {
perDay := planPerDay(f.Get("n"), 25)
if !validPlanUID(uid) {
return a.planError("That reading plan id is not valid.")
}
pos, found, err := a.planPosition(uid)
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
if !found {
return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
}
total := len(a.store.verseIDs)
switch f.Get("action") {
case "next":
pos = min(total, pos+perDay)
case "back":
pos = max(0, pos-perDay)
case "restart":
pos = 0
case "goto":
target, ok := a.startIndex(f.Get("kw"))
if !ok {
return a.planError("We couldn't find that reference. Try something like 'John 3:1'.")
}
pos = target
}
a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, pos, stateNow())
return redirect303(fmt.Sprintf("/plan/%s?n=%d", uid, perDay))
}
func (a *App) handlePlanView(uid string, q url.Values) Resp {
if !validPlanUID(uid) {
return a.planError("That reading plan id is not valid.")
}
perDay := planPerDay(q.Get("n"), 25)
position, found, err := a.planPosition(uid)
if err != nil {
return errJSON(http.StatusServiceUnavailable, err.Error())
}
if !found {
return a.planError("That reading plan was not found (plans unused for 90 days are removed).")
}
// keeps an active plan from being purged as unused
a.store.db.Exec("REPLACE INTO reading_tracker (uid, verse, last_used) VALUES(?, ?, ?)", uid, position, stateNow())
ids := a.store.verseIDs
total := len(ids)
finished := position >= total
lastIndex := min(total, position+perDay) - 1
percent := math.Round(1000*float64(min(position, total))/float64(total)) / 10
esc := html.EscapeString
controls := fmt.Sprintf("
`, next, backDisabled, uid, perDay)
var summary, versesHTML string
if finished {
summary = fmt.Sprintf("You've finished the plan!You read all %s verses. Use Previous reading to look back, or start over.
", commas(total))
controls += fmt.Sprintf("", uid, perDay)
} else {
first, last := a.store.FormatRef(ids[position]), a.store.FormatRef(ids[lastIndex])
summary = fmt.Sprintf("Today's reading%s – %s
"+
"
%d verses · %.1f%% of the Bible complete
"+
"
",
esc(first), esc(last), lastIndex-position+1, percent, percent, percent)
var sb strings.Builder
for _, v := range a.store.Range(ids[position], ids[lastIndex]) {
sb.WriteString(a.responseRich(v))
}
versesHTML = sb.String()
}
note := "Bookmark this page — it is your reading plan. Come back each day, then press Done to move on.
"
store := fmt.Sprintf("", uid, perDay)
return htmlR(a.renderNavbar("", nil, "") + "" + summary + versesHTML +
"
" + controls + "
" + note + store + "
")
}
func commas(n int) string {
s := strconv.Itoa(n)
for i := len(s) - 3; i > 0; i -= 3 {
s = s[:i] + "," + s[i:]
}
return s
}
// ---------------------------------------------------------------------------
// abuse limits
// creationLimiter bounds how fast each visitor can add rows (shortcodes,
// plans, sequences), with a looser limit over everyone together as a backstop.
type creationLimiter struct {
mu sync.Mutex
clients map[string]*bucket
all bucket
}
type bucket struct {
tokens float64
last time.Time
}
const (
clientRate, clientBurst = 0.5, 10.0 // per visitor: a new row every 2 seconds, 10 at once
totalRate, totalBurst = 20.0, 100.0
maxTrackedClients = 10000
)
var creations = &creationLimiter{clients: map[string]*bucket{}, all: bucket{totalBurst, time.Now()}}
func (b *bucket) take(now time.Time, rate, burst float64) bool {
b.tokens = min(burst, b.tokens+now.Sub(b.last).Seconds()*rate)
b.last = now
if b.tokens < 1 {
return false
}
b.tokens--
return true
}
func (c *creationLimiter) allow(client string) bool {
c.mu.Lock()
defer c.mu.Unlock()
now := time.Now()
if len(c.clients) > maxTrackedClients { // forget visitors who have long since refilled
for k, b := range c.clients {
if now.Sub(b.last) > time.Minute {
delete(c.clients, k)
}
}
}
if isLoopback(client) { // the proxy did not say who this is: only the shared limit applies
return c.all.take(now, totalRate, totalBurst)
}
b := c.clients[client]
if b == nil {
b = &bucket{clientBurst, now}
c.clients[client] = b
}
if !b.take(now, clientRate, clientBurst) {
return false
}
return c.all.take(now, totalRate, totalBurst)
}
// clientAddr is the visitor as the reverse proxy saw them. Only the proxy can
// reach this program (it listens on localhost), so the header it sets is
// trusted; the last X-Forwarded-For entry is the one the proxy itself added.
func clientAddr(r *http.Request) string {
if ip := strings.TrimSpace(r.Header.Get("X-Real-IP")); ip != "" {
return ip
}
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
parts := strings.Split(xff, ",")
return strings.TrimSpace(parts[len(parts)-1])
}
host, _, _ := net.SplitHostPort(r.RemoteAddr)
noClientHeader.Do(func() {
lg.Warn("requests arrive without X-Real-IP or X-Forwarded-For: the reverse proxy should pass the visitor's address on, " +
"otherwise creating plans and shortcodes is limited for all visitors together, not for each")
})
return host
}
var noClientHeader sync.Once
func isLoopback(addr string) bool {
ip := net.ParseIP(addr)
return addr == "" || (ip != nil && ip.IsLoopback())
}
func tooMany() Resp {
r := errJSON(http.StatusTooManyRequests, "Too many requests, try again in a moment")
r.Header = map[string]string{"Retry-After": "5"}
return r
}
// maintain deletes reading plans nobody has used for 90 days (the pages promise this).
func (a *App) maintain() {
for {
if a.store.stateOK {
cutoff := time.Now().Add(-seqPurgeAfter).Format(stateTimeFormat)
if res, err := a.store.db.Exec("DELETE FROM reading_tracker WHERE last_used < ?", cutoff); err == nil {
if n, _ := res.RowsAffected(); n > 0 {
lg.Info("Purged %d unused reading plans", n)
}
}
}
time.Sleep(6 * time.Hour)
}
}
// adminOnly protects the listings of every plan id and shortcode (a plan id is
// all that is needed to move that plan). They need the KJV_SECRET_API_KEY as a
// bearer token, and do not exist when no key is set.
func (a *App) adminOnly(r *http.Request) (Resp, bool) {
if a.secretKey == "" {
return a.notFound(), false
}
token, ok := strings.CutPrefix(r.Header.Get("Authorization"), "Bearer ")
if !ok || subtle.ConstantTimeCompare([]byte(token), []byte(a.secretKey)) != 1 {
resp := errJSON(http.StatusUnauthorized, "Invalid Authorization")
resp.Header = map[string]string{"WWW-Authenticate": "Bearer"}
return resp, false
}
return Resp{}, true
}