From d311832a92f2d8f00d4e25ad5607aff497a44961 Mon Sep 17 00:00:00 2001 From: SpudGunMan Date: Wed, 20 Aug 2025 11:55:44 -0700 Subject: [PATCH] truncation --- modules/llm.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/modules/llm.py b/modules/llm.py index ff214af..cc9f034 100644 --- a/modules/llm.py +++ b/modules/llm.py @@ -13,7 +13,8 @@ from googlesearch import search # pip install googlesearch-python # LLM System Variables ollamaAPI = ollamaHostName + "/api/generate" rawQuery = True # if True, the input is sent raw to the LLM, if False, it is processed by the meshBotAI template -tokens = 450 # max tokens for the LLM response, this is the max length of the response +tokens = 450 # max charcters for the LLM response, this is the max length of the response also in prompts +requestTruncation = True # if True, the LLM "will" truncate the response openaiAPI = "https://api.openai.com/v1/completions" # not used, if you do push a enhancement! @@ -27,11 +28,12 @@ llmChat_history = {} trap_list_llm = ("ask:", "askai") meshbotAIinit = """ - You must keep your responses under 450 tokens - You can not ask for clarification, you must respond to the prompt as if you are a chatbot assistant. - You must respond in plain text standard ASCII characters, or emojis. + keep responses as short as possible. chatbot assistant no followuyp questions, no asking for clarification. + You must respond in plain text standard ASCII characters or emojis. """ +truncatePrompt = f"truncate this as short as possible:\n" + meshBotAI = """ FROM {llmModel} SYSTEM @@ -162,6 +164,23 @@ def llm_query(input, nodeID=0, location_name=None): # cleanup for message output response = result.strip().replace('\n', ' ') + + if rawQuery and requestTruncation and len(response) > 450: + #retryy loop to truncate the response + logger.warning(f"System: LLM Query: Response exceeded {tokens} characters, requesting truncation") + truncateQuery = {"model": llmModel, "prompt": truncatePrompt + response, "stream": False, "max_tokens": tokens} + truncateResult = requests.post(ollamaAPI, data=json.dumps(truncateQuery)) + if truncateResult.status_code == 200: + truncate_json = truncateResult.json() + result = truncate_json.get("response", "") + + else: + #use the original result if truncation fails + logger.warning("System: LLM Query: Truncation failed, using original response") + + # cleanup for message output + response = result.strip().replace('\n', ' ') + # done with the query, remove the user from the anti flood list antiFloodLLM.remove(nodeID)