alt language suggestion, with reasonable concept search suggestion. Need to grow inter-reference data

app.py

@@ -97,6 +97,22 @@ if not kjv_cur:
 if not sermon_cur:
     print("WARN: Error connecting to Sermon Database")
 
+# Build grammatical variants lookup from word groups
+variant_groups = json.load(open("data/grammatical_variants.json", "r"))
+grammatical_variants_map = {}
+for group in variant_groups:
+    for word in group:
+        # Map each word to all other words in its group
+        grammatical_variants_map[word] = [w for w in group if w != word]
+
+# Build theological synonyms lookup from word groups
+theological_synonym_groups = json.load(open("data/theological_synonyms.json", "r"))
+theological_synonyms_map = {}
+for group in theological_synonym_groups:
+    for word in group:
+        # Map each word to other words in its group
+        theological_synonyms_map[word] = [w for w in group if w != word]
+
 cached: dict = {
     "verse_ids": [i[0] for i in kjv_cur.execute("SELECT id FROM fts_kjv;").fetchall()],
     "book_names": [
@@ -105,7 +121,8 @@ cached: dict = {
     "chapter_list": [],
     "books_chapter_lengths": {},
     "json_dictionary": json.load(open("data/1828_Webster_KJV.json", "r")),
-    "grammatical_variants": json.load(open("data/grammatical_variants.json", "r")),
+    "grammatical_variants": grammatical_variants_map,
+    "theological_synonyms": theological_synonyms_map,
     "html_response": open("html/response.html", "r").read(),
     "html_navbar": open("html/navbar.html", "r").read(),
     "css_navbar": open("styles/header.css", "r").read(),
@@ -777,11 +794,43 @@ def search():
         # PRIORITY 1: Try grammatical variant expansion first
         words = cleaned_keywords.split()
 
+        def sort_variants_by_similarity(original, variants):
+            """Sort variants by morphological similarity: forwards (longer/more specific) then backwards (shorter/simpler)"""
+            def get_common_prefix_length(w1, w2):
+                """Find how many characters match from the start"""
+                prefix_len = 0
+                for i in range(min(len(w1), len(w2))):
+                    if w1[i].lower() == w2[i].lower():
+                        prefix_len += 1
+                    else:
+                        break
+                return prefix_len
+
+            scored_variants = []
+            for variant in variants:
+                prefix_len = get_common_prefix_length(original, variant)
+
+                # Reject variants that don't share the stem (beginning doesn't match)
+                if prefix_len < min(2, len(original) // 2):
+                    continue
+
+                # Score: prioritize common prefix length (stem preservation) heavily,
+                # then prefer longer variants (forward/more specific) over shorter (backward/simpler)
+                # Prefix is weighted 1000x to ensure stem-preservation is primary
+                score = (prefix_len * 1000) + len(variant)
+                scored_variants.append((score, variant))
+
+            # Sort by score descending: highest score = most similar with longest form first
+            return [v for _, v in sorted(scored_variants, key=lambda x: x[0], reverse=True)]
+
         for word in words:
             variants = cached["grammatical_variants"].get(word.lower(), [])
             if variants:
-                # Try each variant individually
-                for variant in variants:
+                # Sort variants: forwards in likeness (longer/specific) then backwards (shorter/simple)
+                sorted_variants = sort_variants_by_similarity(word.lower(), variants)
+
+                # Try each variant in order of similarity
+                for variant in sorted_variants:
                     variant_query = cleaned_keywords.replace(word, variant, 1)
                     results = lookup_fts(variant_query)
                     if results:
@@ -832,17 +881,66 @@ def search():
                     response = jsonify({"corrected_from": cleaned_keywords, "corrected_to": corrected_keyword, "results": response_list})
                 return response
 
-        # PRIORITY 3: If we still have no results, show spelling suggestions
+        # PRIORITY 3: Try theological synonym expansion
+        for word in words:
+            synonyms = cached["theological_synonyms"].get(word.lower(), None)
+            if synonyms:
+                for synonym in synonyms:
+                    synonym_query = cleaned_keywords.replace(word, synonym, 1)
+                    results = lookup_fts(synonym_query)
+                    if results:
+                        response_list = scripture_response(results)
+                        arg_view = request.args.get("view", None)
+                        explanation = f"In the KJV, '{word}' and '{synonym}' often refer to similar theological concepts. Consider also searching for related verses."
+                        if arg_view == "plain":
+                            response = f"Showing results for '{synonym_query}' (theological equivalent of '{cleaned_keywords}')\n{explanation}\n<br>" + "<br>".join(response_list)
+                        elif arg_view == "rich":
+                            theological_notice = f"<div class='theological-notice'>Showing results for '<strong>{synonym_query}</strong>' (theological equivalent of '{cleaned_keywords}')<br><em>{explanation}</em></div>"
+                            response_str = "".join(response_list)
+                            response = response_str.replace("<div class='responses'>", f"<div class='responses'>{theological_notice}", 1)
+                        else:
+                            response = jsonify({"original": cleaned_keywords, "theological_equivalent": synonym_query, "explanation": explanation, "results": response_list})
+                        return response
+
+        # PRIORITY 4: If we still have no results, show spelling suggestions
         suggestions = h.suggest(cleaned_keywords)
         return "{} not found, try one of the following: {}".format(
             cleaned_keywords, ", ".join(suggestions) if suggestions else "No suggestions available"
         )
     response_list = scripture_response(results)
+
+    # Check if any words in the search have theological synonyms to suggest
+    theological_suggestions = []
+    words = cleaned_keywords.split()
+    for word in words:
+        synonyms = cached["theological_synonyms"].get(word.lower(), None)
+        if synonyms:
+            for synonym in synonyms:
+                theological_suggestions.append(synonym)
+
     arg_view = request.args.get("view", None)
     if arg_view == "plain":
         response = "<br>".join(response_list)
+        if theological_suggestions:
+            response += "<br><br>Also consider searching: "
+            for suggestion in theological_suggestions:
+                response += f"{suggestion}<br>"
     elif arg_view == "rich":
         response = "".join(response_list)
+        if theological_suggestions:
+            # Add suggestions before the LAST closing </div> which closes the responses container
+            suggestions_html = "<div class='theological-suggestions'><h3>Related Theological Concepts</h3>"
+            suggestions_html += "<p>In the KJV, these terms often refer to similar theological concepts. Consider also searching for related verses.</p>"
+            for suggestion in theological_suggestions:
+                search_link = f"/search?kw={suggestion}&view=rich"
+                suggestions_html += f"<div class='suggestion-item'>"
+                suggestions_html += f"<a href='{search_link}'><strong>{suggestion.capitalize()}</strong></a>"
+                suggestions_html += "</div>"
+            suggestions_html += "</div>"
+            # Find the last </div> and insert before it
+            last_div_pos = response.rfind("</div>")
+            if last_div_pos != -1:
+                response = response[:last_div_pos] + suggestions_html + response[last_div_pos:]
     else:
         response = jsonify(response_list)
     #logging.debug(f"Search Response: {response}")