From 733a41a35c6fb2ec251d2ce719de28d16f16bfa7 Mon Sep 17 00:00:00 2001 From: matty Date: Mon, 4 Aug 2025 00:21:10 +0000 Subject: [PATCH] revert to more "chat" based api calls --- src/main.ts | 68 +++++++++++++++++++++++++++++++++------------------ src/prisma.ts | 6 ++--- 2 files changed, 47 insertions(+), 27 deletions(-) diff --git a/src/main.ts b/src/main.ts index e150dee..10ed306 100644 --- a/src/main.ts +++ b/src/main.ts @@ -2,10 +2,8 @@ import { NewStatusBody, Notification, OllamaConfigOptions, - // OllamaChatRequest, - // OllamaChatResponse, - OllamaRequest, - OllamaResponse, + OllamaChatRequest, + OllamaChatResponse, PostAncestorsForModel, } from "../types.js"; // import striptags from "striptags"; @@ -53,7 +51,7 @@ const ollamaConfig: OllamaConfigOptions = { temperature: 0.6, top_p: 0.85, top_k: 40, - num_ctx: 2048, + num_ctx: 8192, repeat_penalty: 1.1, }; @@ -62,7 +60,7 @@ const ollamaConfig: OllamaConfigOptions = { const generateOllamaRequest = async ( notification: Notification -): Promise => { +): Promise => { const { whitelistOnly, ollamaModel, @@ -97,22 +95,42 @@ const generateOllamaRequest = async ( }); // console.log(conversationHistory); } - const oneOffPrompt = `${notification.status.account.fqn} says: ${notification.status.pleroma.content["text/plain"]}\n[/INST]`; - const contextPrompt = `<>[INST]\n${ollamaSystemPrompt}\nHere is the previous conversation context in JSON format:\n${JSON.stringify( - conversationHistory - )}\nAssume the {account_fqn} key is the user who posted the {plaintext_content} to the users in {mentions}\nReply as if you are a party to the conversation. If you see '@nice-ai' or 'nice-ai' in the {mentions}, you are an addressee of the conversation.\nAppend the '@' sign to each username at the beginning when addressing users.<>`; - const ollamaRequestBody: OllamaRequest = { + + // Simplified user message (remove [/INST] as it's not needed for Llama 3) + const userMessage = `${notification.status.account.fqn} says: ${notification.status.pleroma.content["text/plain"]}`; + + let systemContent = ollamaSystemPrompt; + if (replyWithContext) { + // Simplified context instructions (avoid heavy JSON; summarize for clarity) + systemContent = `${ollamaSystemPrompt}\n\nPrevious conversation context:\n${conversationHistory + .map( + (post) => + `${post.account_fqn} (to ${post.mentions.join(", ")}): ${ + post.plaintext_content + }` + ) + .join( + "\n" + )}\nReply as if you are a party to the conversation. If '@nice-ai' is mentioned, respond directly. Prefix usernames with '@' when addressing them.`; + } + + // Switch to chat request format (messages array auto-handles Llama 3 template) + const ollamaRequestBody: OllamaChatRequest = { model: ollamaModel, - prompt: oneOffPrompt, - system: replyWithContext ? contextPrompt : ollamaSystemPrompt, + messages: [ + { role: "system", content: systemContent as string }, + { role: "user", content: userMessage }, + ], stream: false, options: ollamaConfig, }; - const response = await fetch(`${ollamaUrl}/api/generate`, { + + // Change endpoint to /api/chat + const response = await fetch(`${ollamaUrl}/api/chat`, { method: "POST", body: JSON.stringify(ollamaRequestBody), }); - const ollamaResponse: OllamaResponse = await response.json(); + const ollamaResponse: OllamaChatResponse = await response.json(); await storePromptData(notification, ollamaResponse); return ollamaResponse; @@ -124,7 +142,7 @@ const generateOllamaRequest = async ( const postReplyToStatus = async ( notification: Notification, - ollamaResponseBody: OllamaResponse + ollamaResponseBody: OllamaChatResponse ) => { const { pleromaInstanceUrl, bearerToken } = envConfig; const emojiList = await getInstanceEmojis(); @@ -136,7 +154,7 @@ const postReplyToStatus = async ( let mentions: string[]; const statusBody: NewStatusBody = { content_type: "text/markdown", - status: `${ollamaResponseBody.response} :${randomEmoji}:`, + status: `${ollamaResponseBody.message.content} :${randomEmoji}:`, in_reply_to_id: notification.status.id, }; if ( @@ -176,26 +194,28 @@ const createTimelinePost = async () => { ollamaUrl, pleromaInstanceUrl, } = envConfig; - const ollamaRequestBody: OllamaRequest = { + const ollamaRequestBody: OllamaChatRequest = { model: ollamaModel, - prompt: "Say something random.", - system: ollamaSystemPrompt, + messages: [ + { role: "system", content: ollamaSystemPrompt as string }, + { role: "user", content: "Say something random." }, + ], stream: false, - // options: ollamaConfig, + options: ollamaConfig, }; try { - const response = await fetch(`${ollamaUrl}/api/generate`, { + const response = await fetch(`${ollamaUrl}/api/chat`, { method: "POST", body: JSON.stringify(ollamaRequestBody), }); if (!response.ok) throw new Error("Error generating ad-hoc Ollama response"); - const ollamaResponse: OllamaResponse = await response.json(); + const ollamaResponse: OllamaChatResponse = await response.json(); const newStatusBody: NewStatusBody = { content_type: "text/markdown", - status: ollamaResponse.response, + status: ollamaResponse.message.content, }; const pleromaResponse = await fetch( diff --git a/src/prisma.ts b/src/prisma.ts index 3a97ea5..f5d3e78 100644 --- a/src/prisma.ts +++ b/src/prisma.ts @@ -1,16 +1,16 @@ -import { Notification, OllamaResponse } from "../types.js"; +import { Notification, OllamaChatResponse } from "../types.js"; import { trimInputData } from "./util.js"; import { prisma } from "./main.js"; const storePromptData = async ( notification: Notification, - ollamaResponseBody: OllamaResponse + ollamaResponseBody: OllamaChatResponse ) => { try { await prisma.response.updateMany({ where: { pleromaNotificationId: notification.id }, data: { - response: ollamaResponseBody.response, + response: ollamaResponseBody.message.content, request: trimInputData(notification.status.content), to: notification.account.fqn, isProcessing: false,