Add LMStudio agent support (generic) support (#1246)

* add LMStudio agent support (generic) support
"work" with non-tool callable LLMs, highly dependent on system specs

* add comments

* enable few-shot prompting per function for OSS models

.vscode/settings.json

@@ -17,6 +17,7 @@
     "hljs",
     "inferencing",
     "Langchain",
+    "lmstudio",
     "mbox",
     "Milvus",
     "Mintplex",

frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx

@@ -2,10 +2,11 @@ import React, { useEffect, useRef, useState } from "react";
 import AnythingLLMIcon from "@/media/logo/anything-llm-icon.png";
 import AgentLLMItem from "./AgentLLMItem";
 import { AVAILABLE_LLM_PROVIDERS } from "@/pages/GeneralSettings/LLMPreference";
-import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
+import { CaretUpDown, Gauge, MagnifyingGlass, X } from "@phosphor-icons/react";
 import AgentModelSelection from "../AgentModelSelection";
 
-const ENABLED_PROVIDERS = ["openai", "anthropic"];
+const ENABLED_PROVIDERS = ["openai", "anthropic", "lmstudio"];
+const WARN_PERFORMANCE = ["lmstudio"];
 
 const LLM_DEFAULT = {
   name: "Please make a selection",
@@ -62,6 +63,19 @@ export default function AgentLLMSelection({
   const selectedLLMObject = LLMS.find((llm) => llm.value === selectedLLM);
   return (
     <div className="border-b border-white/40 pb-8">
+      {WARN_PERFORMANCE.includes(selectedLLM) && (
+        <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
+          <div className="gap-x-2 flex items-center">
+            <Gauge className="shrink-0" size={25} />
+            <p className="text-sm">
+              Performance of LLMs that do not explicitly support tool-calling is
+              highly dependent on the model's capabilities and accuracy. Some
+              abilities may be limited or non-functional.
+            </p>
+          </div>
+        </div>
+      )}
+
       <div className="flex flex-col">
         <label htmlFor="name" className="block input-label">
           Workspace Agent LLM Provider

server/endpoints/agentWebsocket.js

@@ -51,7 +51,7 @@ function agentWebsocket(app) {
       await agentHandler.createAIbitat({ socket });
       await agentHandler.startAgentCluster();
     } catch (e) {
-      console.error(e.message);
+      console.error(e.message, e);
       socket?.send(JSON.stringify({ type: "wssFailure", content: e.message }));
       socket?.close();
     }

server/package.json

@@ -51,6 +51,7 @@
     "joi": "^17.11.0",
     "joi-password-complexity": "^5.2.0",
     "js-tiktoken": "^1.0.7",
+    "jsonrepair": "^3.7.0",
     "jsonwebtoken": "^8.5.1",
     "langchain": "0.1.36",
     "mime": "^3.0.0",

server/utils/agents/aibitat/index.js

@@ -603,6 +603,18 @@ ${this.getHistory({ to: route.to })
 
       // Execute the function and return the result to the provider
       fn.caller = byAgent || "agent";
+
+      // For OSS LLMs we really need to keep tabs on what they are calling
+      // so we can log it here.
+      if (provider?.verbose) {
+        this?.introspect?.(
+          `[debug]: ${fn.caller} is attempting to call \`${name}\` tool`
+        );
+        this.handlerProps.log(
+          `[debug]: ${fn.caller} is attempting to call \`${name}\` tool`
+        );
+      }
+
       const result = await fn.handler(args);
       Telemetry.sendTelemetry("agent_tool_call", { tool: name }, null, true);
       return await this.handleExecution(
@@ -727,6 +739,8 @@ ${this.getHistory({ to: route.to })
         return new Providers.OpenAIProvider({ model: config.model });
       case "anthropic":
         return new Providers.AnthropicProvider({ model: config.model });
+      case "lmstudio":
+        return new Providers.LMStudioProvider({});
 
       default:
         throw new Error(

server/utils/agents/aibitat/plugins/memory.js

@@ -16,7 +16,37 @@ const memory = {
           tracker: new Deduplicator(),
           name: this.name,
           description:
-            "Search against local documents for context that is relevant to the query or store a snippet of text into memory for retrieval later. Storing information should only be done when the user specifically requests for information to be remembered or saved to long-term memory. You should use this tool before search the internet for information.",
+            "Search against local documents for context that is relevant to the query or store a snippet of text into memory for retrieval later. Storing information should only be done when the user specifically requests for information to be remembered or saved to long-term memory. You should use this tool before search the internet for information. Do not use this tool unless you are explicity told to 'remember' or 'store' information.",
+          examples: [
+            {
+              prompt: "What is AnythingLLM?",
+              call: JSON.stringify({
+                action: "search",
+                content: "What is AnythingLLM?",
+              }),
+            },
+            {
+              prompt: "What do you know about Plato's motives?",
+              call: JSON.stringify({
+                action: "search",
+                content: "What are the facts about Plato's motives?",
+              }),
+            },
+            {
+              prompt: "Remember that you are a robot",
+              call: JSON.stringify({
+                action: "store",
+                content: "I am a robot, the user told me that i am.",
+              }),
+            },
+            {
+              prompt: "Save that to memory please.",
+              call: JSON.stringify({
+                action: "store",
+                content: "<insert summary of conversation until now>",
+              }),
+            },
+          ],
           parameters: {
             $schema: "http://json-schema.org/draft-07/schema#",
             type: "object",

server/utils/agents/aibitat/plugins/save-file-browser.js

@@ -16,6 +16,32 @@ const saveFileInBrowser = {
           name: this.name,
           description:
             "Save content to a file when the user explicity asks for a download of the file.",
+          examples: [
+            {
+              prompt: "Save me that to a file named 'output'",
+              call: JSON.stringify({
+                file_content:
+                  "<content of the file we will write previous conversation>",
+                filename: "output.txt",
+              }),
+            },
+            {
+              prompt: "Save me that to my desktop",
+              call: JSON.stringify({
+                file_content:
+                  "<content of the file we will write previous conversation>",
+                filename: "<relevant filename>.txt",
+              }),
+            },
+            {
+              prompt: "Save me that to a file",
+              call: JSON.stringify({
+                file_content:
+                  "<content of the file we will write from previous conversation>",
+                filename: "<descriptive filename>.txt",
+              }),
+            },
+          ],
           parameters: {
             $schema: "http://json-schema.org/draft-07/schema#",
             type: "object",

server/utils/agents/aibitat/plugins/summarize.js

@@ -19,6 +19,26 @@ const docSummarizer = {
           controller: new AbortController(),
           description:
             "Can get the list of files available to search with descriptions and can select a single file to open and summarize.",
+          examples: [
+            {
+              prompt: "Summarize example.txt",
+              call: JSON.stringify({
+                action: "summarize",
+                document_filename: "example.txt",
+              }),
+            },
+            {
+              prompt: "What files can you see?",
+              call: JSON.stringify({ action: "list", document_filename: null }),
+            },
+            {
+              prompt: "Tell me about readme.md",
+              call: JSON.stringify({
+                action: "summarize",
+                document_filename: "readme.md",
+              }),
+            },
+          ],
           parameters: {
             $schema: "http://json-schema.org/draft-07/schema#",
             type: "object",

server/utils/agents/aibitat/plugins/web-browsing.js

@@ -13,7 +13,21 @@ const webBrowsing = {
           super: aibitat,
           name: this.name,
           description:
-            "Searches for a given query online using a search engine.",
+            "Searches for a given query using a search engine to get better results for the user query.",
+          examples: [
+            {
+              prompt: "Who won the world series today?",
+              call: JSON.stringify({ query: "Winner of today's world series" }),
+            },
+            {
+              prompt: "What is AnythingLLM?",
+              call: JSON.stringify({ query: "AnythingLLM" }),
+            },
+            {
+              prompt: "Current AAPL stock price",
+              call: JSON.stringify({ query: "AAPL stock price today" }),
+            },
+          ],
           parameters: {
             $schema: "http://json-schema.org/draft-07/schema#",
             type: "object",

server/utils/agents/aibitat/plugins/web-scraping.js

@@ -16,15 +16,26 @@ const webScraping = {
           name: this.name,
           controller: new AbortController(),
           description:
-            "Scrapes the content of a webpage or online resource from a URL.",
+            "Scrapes the content of a webpage or online resource from a provided URL.",
+          examples: [
+            {
+              prompt: "What is useanything.com about?",
+              call: JSON.stringify({ uri: "https://useanything.com" }),
+            },
+            {
+              prompt: "Scrape https://example.com",
+              call: JSON.stringify({ uri: "https://example.com" }),
+            },
+          ],
           parameters: {
             $schema: "http://json-schema.org/draft-07/schema#",
             type: "object",
             properties: {
               url: {
                 type: "string",
                 format: "uri",
-                description: "A web URL.",
+                description:
+                  "A complete web address URL including protocol. Assumes https if not provided.",
               },
             },
             additionalProperties: false,

server/utils/agents/aibitat/providers/ai-provider.js

@@ -4,16 +4,25 @@
 
 const { ChatOpenAI } = require("@langchain/openai");
 const { ChatAnthropic } = require("@langchain/anthropic");
+const DEFAULT_WORKSPACE_PROMPT =
+  "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions.";
 
 class Provider {
   _client;
   constructor(client) {
     if (this.constructor == Provider) {
-      throw new Error("Class is of abstract type and can't be instantiated");
+      return;
     }
     this._client = client;
   }
 
+  providerLog(text, ...args) {
+    console.log(
+      `\x1b[36m[AgentLLM${this?.model ? ` - ${this.model}` : ""}]\x1b[0m ${text}`,
+      ...args
+    );
+  }
+
   get client() {
     return this._client;
   }
@@ -48,6 +57,15 @@ class Provider {
         return 8_000;
     }
   }
+
+  static systemPrompt(provider = null) {
+    switch (provider) {
+      case "lmstudio":
+        return "You are a helpful ai assistant who can assist the user and use tools available to help answer the users prompts and questions. Tools will be handled by another assistant and you will simply receive their responses to help answer the user prompt - always try to answer the user's prompt the best you can with the context available to you and your general knowledge.";
+      default:
+        return DEFAULT_WORKSPACE_PROMPT;
+    }
+  }
 }
 
 module.exports = Provider;

server/utils/agents/aibitat/providers/helpers/classes.js

@@ -0,0 +1,16 @@
+function InheritMultiple(bases = []) {
+  class Bases {
+    constructor() {
+      bases.forEach((base) => Object.assign(this, new base()));
+    }
+  }
+
+  bases.forEach((base) => {
+    Object.getOwnPropertyNames(base.prototype)
+      .filter((prop) => prop != "constructor")
+      .forEach((prop) => (Bases.prototype[prop] = base.prototype[prop]));
+  });
+  return Bases;
+}
+
+module.exports = InheritMultiple;