diff --git a/config.template b/config.template index 783f7be..c705252 100644 --- a/config.template +++ b/config.template @@ -80,14 +80,17 @@ ollama = False # ollamaModel = gemma2:2b # server instance to use (defaults to local machine install) ollamaHostName = http://localhost:11434 + # Produce LLM replies to messages that aren't commands? # If False, the LLM only replies to the "ask:" and "askai" commands. llmReplyToNonCommands = True -# if True, the input is sent raw to the LLM, if False uses legacy template query +# if True, the input is sent raw to the LLM, if False uses SYSTEM prompt rawLLMQuery = True + # Enable Wikipedia/Kiwix integration with LLM for RAG (Retrieval Augmented Generation) # When enabled, LLM will automatically search Wikipedia/Kiwix and include context in responses llmUseWikiContext = False + # Use OpenWebUI instead of direct Ollama API (enables advanced RAG features) useOpenWebUI = False # OpenWebUI server URL (e.g., http://localhost:3000) diff --git a/modules/llm.md b/modules/llm.md new file mode 100644 index 0000000..9298d42 --- /dev/null +++ b/modules/llm.md @@ -0,0 +1,45 @@ +# How do I use this thing? +This is not a full turnkey setup for Docker yet? + + +# Ollama local +```bash +# bash +curl -fsSL https://ollama.com/install.sh | sh +# docker +docker run -d -p 3000:8080 --add-host=host.docker.internal:host-gateway -e OLLAMA_API_BASE_URL=http://host.docker.internal:11434 open-webui/open-webui +``` + +```ini +#service file addition +# https://github.com/ollama/ollama/issues/703 +[Service] +Environment="OLLAMA_HOST=0.0.0.0:11434" +``` +## validation +http://IP::11434 +`Ollama is running` + +# OpenWebUI (docker) +```bash +## ollama in docker +docker run -d -p 3000:8080 --gpus all -v open-webui:/app/backend/data --name open-webui ghcr.io/open-webui/open-webui:cuda + +## external ollama +docker run -d -p 3000:8080 -e OLLAMA_BASE_URL=https://IP:11434 -v open-webui:/app/backend/data --name open-webui --restart always ghcr.io/open-webui/open-webui:main +``` +wait for engine to build, update the config.ini for the bot + +```ini +# Use OpenWebUI instead of direct Ollama API (enables advanced RAG features) +useOpenWebUI = True +# OpenWebUI server URL (e.g., http://localhost:3000) +openWebUIURL = http://IP:3000 +``` + +## validation +http://IP:3000 +make a new admin user. +validate you have models imported or that the system is working for query. +set api endpoint [OpenWebUI API](https://docs.openwebui.com/getting-started/api-endpoints) +to quickly get started, go to admin ->settings ->connections ->Manage OpenAI API Connections ->Auth Type None \ No newline at end of file diff --git a/modules/llm.py b/modules/llm.py index ef2a651..d4b3152 100644 --- a/modules/llm.py +++ b/modules/llm.py @@ -201,7 +201,7 @@ def send_openwebui_query(prompt, model=None, max_tokens=450, context=''): } try: - result = requests.post(openWebUIChatAPI, headers=headers, json=data, timeout=10) + result = requests.post(openWebUIChatAPI, headers=headers, json=data, timeout=urlTimeoutSeconds * 4) if result.status_code == 200: result_json = result.json() # OpenWebUI returns OpenAI-compatible format @@ -274,15 +274,14 @@ def llm_query(input, nodeID=0, location_name=None, init=False): # classic model for gemma2, deepseek-r1, etc logger.debug(f"System: Using SYSTEM model framework, ideally for gemma2, deepseek-r1, etc") - - # Remove command bang if present - if cmdBang: - input = input[1:].strip() - if not location_name: location_name = "no location provided " + + # Remove command bang if present + if cmdBang and input.startswith('!'): + input = input.strip('!').strip() - # remove askai: and ask: from the input + # Remove any trap words from the start of the input for trap in trap_list_llm: if input.lower().startswith(trap): input = input[len(trap):].strip()