#530 - Chatbot API update

2026-08-09 10:22:55 +02:00 · 2023-03-10 11:54:11 +01:00
parent 5a56577695
commit b3a1628b77
11 changed files with 234 additions and 249 deletions
@@ -12,8 +12,8 @@

 ### ✨ New features

- [#530](https://github.com/estruyf/vscode-front-matter/issues/530): Implementation of the Front Matter AI 🤖 powered by [mendable.ai](https://mendable.ai)
 - [#513](https://github.com/estruyf/vscode-front-matter/issues/513): Added support for external UI scripts to add custom HTML on the dashboard elements
+- [#530](https://github.com/estruyf/vscode-front-matter/issues/530): Implementation of the Front Matter AI 🤖 powered by [mendable.ai](https://mendable.ai)

 ### 🎨 Enhancements

@@ -15,7 +15,6 @@
        "@headlessui/react": "1.5.0",
        "@heroicons/react": "1.0.4",
        "@iarna/toml": "2.2.3",
-        "@microsoft/fetch-event-source": "^2.0.1",
        "@octokit/rest": "^18.12.0",
        "@popperjs/core": "^2.11.6",
        "@sentry/react": "^6.13.3",
@@ -501,12 +500,6 @@
        "exenv-es6": "^1.0.0"
      }
    },
-    "node_modules/@microsoft/fetch-event-source": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@microsoft/fetch-event-source/-/fetch-event-source-2.0.1.tgz",
-      "integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==",
-      "dev": true
-    },
    "node_modules/@nodelib/fs.scandir": {
      "version": "2.1.5",
      "dev": true,
@@ -13178,12 +13171,6 @@
        "exenv-es6": "^1.0.0"
      }
    },
-    "@microsoft/fetch-event-source": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/@microsoft/fetch-event-source/-/fetch-event-source-2.0.1.tgz",
-      "integrity": "sha512-W6CLUJ2eBMw3Rec70qrsEW0jOm/3twwJv21mrmj2yORiaVmVYGS4sSS5yUwvQc1ZlDLYGPnClVWmUUMagKNsfA==",
-      "dev": true
-    },
    "@nodelib/fs.scandir": {
      "version": "2.1.5",
      "dev": true,
@@ -10,7 +10,8 @@
    "color": "#0e131f",
    "theme": "dark"
  },
-  "badges": [{
+  "badges": [
+    {
      "description": "version",
      "url": "https://img.shields.io/github/package-json/v/estruyf/vscode-front-matter?color=green&label=vscode-front-matter&style=flat-square",
      "href": "https://github.com/estruyf/vscode-front-matter"
@@ -54,7 +55,8 @@
  ],
  "main": "./dist/extension.js",
  "contributes": {
-    "keybindings": [{
+    "keybindings": [
+      {
        "command": "frontMatter.dashboard",
        "key": "alt+d"
      },
@@ -72,20 +74,24 @@
      }
    ],
    "viewsContainers": {
-      "activitybar": [{
-        "id": "frontmatter-explorer",
-        "title": "Front Matter",
-        "icon": "assets/frontmatter-short-min.svg"
-      }]
+      "activitybar": [
+        {
+          "id": "frontmatter-explorer",
+          "title": "Front Matter",
+          "icon": "assets/frontmatter-short-min.svg"
+        }
+      ]
    },
    "views": {
-      "frontmatter-explorer": [{
-        "id": "frontMatter.explorer",
-        "name": "Front Matter",
-        "icon": "assets/frontmatter-short-min.svg",
-        "contextualTitle": "Front Matter",
-        "type": "webview"
-      }]
+      "frontmatter-explorer": [
+        {
+          "id": "frontMatter.explorer",
+          "name": "Front Matter",
+          "icon": "assets/frontmatter-short-min.svg",
+          "contextualTitle": "Front Matter",
+          "type": "webview"
+        }
+      ]
    },
    "configuration": {
      "title": "Front Matter: use frontmatter.json for shared team settings",
@@ -120,7 +126,8 @@
        "frontMatter.content.defaultFileType": {
          "type": "string",
          "default": "md",
-          "oneOf": [{
+          "oneOf": [
+            {
              "enum": [
                "md",
                "mdx"
@@ -136,7 +143,8 @@
        "frontMatter.content.defaultSorting": {
          "type": "string",
          "default": "",
-          "oneOf": [{
+          "oneOf": [
+            {
              "enum": [
                "LastModifiedAsc",
                "LastModifiedDesc",
@@ -463,7 +471,8 @@
              "command": {
                "$id": "#scriptCommand",
                "type": "string",
-                "oneOf": [{
+                "oneOf": [
+                  {
                    "enum": [
                      "node",
                      "bash",
@@ -637,7 +646,8 @@
              "title",
              "file"
            ],
-            "anyOf": [{
+            "anyOf": [
+              {
                "required": [
                  "schema"
                ]
@@ -691,7 +701,8 @@
              "id",
              "path"
            ],
-            "anyOf": [{
+            "anyOf": [
+              {
                "required": [
                  "schema"
                ]
@@ -1037,7 +1048,8 @@
                      "default": "",
                      "description": "The ID of your taxonomy field. It cannot contain the \"tags\" or \"categories\" value.",
                      "not": {
-                        "anyOf": [{
+                        "anyOf": [
+                          {
                            "const": ""
                          },
                          {
@@ -1197,7 +1209,8 @@
                    "type",
                    "name"
                  ],
-                  "allOf": [{
+                  "allOf": [
+                    {
                      "if": {
                        "properties": {
                          "type": {
@@ -1355,48 +1368,51 @@
              "fields"
            ]
          },
-          "default": [{
-            "name": "default",
-            "pageBundle": false,
-            "fields": [{
-                "title": "Title",
-                "name": "title",
-                "type": "string"
-              },
-              {
-                "title": "Description",
-                "name": "description",
-                "type": "string"
-              },
-              {
-                "title": "Publishing date",
-                "name": "date",
-                "type": "datetime",
-                "default": "{{now}}",
-                "isPublishDate": true
-              },
-              {
-                "title": "Content preview",
-                "name": "preview",
-                "type": "image"
-              },
-              {
-                "title": "Is in draft",
-                "name": "draft",
-                "type": "boolean"
-              },
-              {
-                "title": "Tags",
-                "name": "tags",
-                "type": "tags"
-              },
-              {
-                "title": "Categories",
-                "name": "categories",
-                "type": "categories"
-              }
-            ]
-          }],
+          "default": [
+            {
+              "name": "default",
+              "pageBundle": false,
+              "fields": [
+                {
+                  "title": "Title",
+                  "name": "title",
+                  "type": "string"
+                },
+                {
+                  "title": "Description",
+                  "name": "description",
+                  "type": "string"
+                },
+                {
+                  "title": "Publishing date",
+                  "name": "date",
+                  "type": "datetime",
+                  "default": "{{now}}",
+                  "isPublishDate": true
+                },
+                {
+                  "title": "Content preview",
+                  "name": "preview",
+                  "type": "image"
+                },
+                {
+                  "title": "Is in draft",
+                  "name": "draft",
+                  "type": "boolean"
+                },
+                {
+                  "title": "Tags",
+                  "name": "tags",
+                  "type": "tags"
+                },
+                {
+                  "title": "Categories",
+                  "name": "categories",
+                  "type": "categories"
+                }
+              ]
+            }
+          ],
          "scope": "Taxonomy"
        },
        "frontMatter.taxonomy.customTaxonomy": {
@@ -1409,7 +1425,8 @@
                "type": "string",
                "description": "ID for your taxonomy field. It cannot contain the \"tags\" or \"categories\" value.",
                "not": {
-                  "anyOf": [{
+                  "anyOf": [
+                    {
                      "const": ""
                    },
                    {
@@ -1591,7 +1608,8 @@
        }
      }
    },
-    "commands": [{
+    "commands": [
+      {
        "command": "frontMatter.config.reload",
        "title": "Reload config",
        "category": "Front Matter"
@@ -1914,12 +1932,15 @@
        "category": "Front Matter"
      }
    ],
-    "submenus": [{
-      "id": "frontmatter.submenu",
-      "label": "Front Matter"
-    }],
+    "submenus": [
+      {
+        "id": "frontmatter.submenu",
+        "label": "Front Matter"
+      }
+    ],
    "menus": {
-      "editor/title": [{
+      "editor/title": [
+        {
          "command": "frontMatter.markup.heading",
          "group": "navigation@-133",
          "when": "frontMatter:file:isValid == true && frontMatter:markdown:wysiwyg"
@@ -2000,11 +2021,14 @@
          "when": "resourceFilename == 'frontmatter.json'"
        }
      ],
-      "explorer/context": [{
-        "submenu": "frontmatter.submenu",
-        "group": "frontmatter@1"
-      }],
-      "frontmatter.submenu": [{
+      "explorer/context": [
+        {
+          "submenu": "frontmatter.submenu",
+          "group": "frontmatter@1"
+        }
+      ],
+      "frontmatter.submenu": [
+        {
          "command": "frontMatter.createFromTemplate",
          "when": "explorerResourceIsFolder",
          "group": "frontmatter@1"
@@ -2020,7 +2044,8 @@
          "group": "frontmatter@3"
        }
      ],
-      "commandPalette": [{
+      "commandPalette": [
+        {
          "command": "frontMatter.init",
          "when": "frontMatterCanInit"
        },
@@ -2161,7 +2186,8 @@
          "when": "frontMatter:file:isValid == true"
        }
      ],
-      "view/title": [{
+      "view/title": [
+        {
          "command": "frontMatter.chatbot",
          "group": "navigation@0",
          "when": "view == frontMatter.explorer"
@@ -2183,52 +2209,57 @@
        }
      ]
    },
-    "grammars": [{
-      "path": "./syntaxes/hugo.tmLanguage.json",
-      "scopeName": "frontmatter.markdown.hugo",
-      "injectTo": [
-        "text.html.markdown"
-      ]
-    }],
-    "walkthroughs": [{
-      "id": "frontmatter.welcome",
-      "title": "Get started with Front Matter",
-      "description": "Discover the features of Front Matter and learn how to use the CMS for your SSG or static site.",
-      "steps": [{
-          "id": "frontmatter.welcome.init",
-          "title": "Get started",
-          "description": "Initial steps to get started.\n[Open dashboard](command:frontMatter.dashboard)",
-          "media": {
-            "markdown": "assets/walkthrough/get-started.md"
+    "grammars": [
+      {
+        "path": "./syntaxes/hugo.tmLanguage.json",
+        "scopeName": "frontmatter.markdown.hugo",
+        "injectTo": [
+          "text.html.markdown"
+        ]
+      }
+    ],
+    "walkthroughs": [
+      {
+        "id": "frontmatter.welcome",
+        "title": "Get started with Front Matter",
+        "description": "Discover the features of Front Matter and learn how to use the CMS for your SSG or static site.",
+        "steps": [
+          {
+            "id": "frontmatter.welcome.init",
+            "title": "Get started",
+            "description": "Initial steps to get started.\n[Open dashboard](command:frontMatter.dashboard)",
+            "media": {
+              "markdown": "assets/walkthrough/get-started.md"
+            },
+            "completionEvents": [
+              "onContext:frontMatterInitialized"
+            ]
          },
-          "completionEvents": [
-            "onContext:frontMatterInitialized"
-          ]
-        },
-        {
-          "id": "frontmatter.welcome.documentation",
-          "title": "Documentation",
-          "description": "Check out the documentation for Front Matter.\n[View our documentation](https://frontmatter.codes/docs)",
-          "media": {
-            "markdown": "assets/walkthrough/documentation.md"
+          {
+            "id": "frontmatter.welcome.documentation",
+            "title": "Documentation",
+            "description": "Check out the documentation for Front Matter.\n[View our documentation](https://frontmatter.codes/docs)",
+            "media": {
+              "markdown": "assets/walkthrough/documentation.md"
+            },
+            "completionEvents": [
+              "onLink:https://frontmatter.codes/docs"
+            ]
          },
-          "completionEvents": [
-            "onLink:https://frontmatter.codes/docs"
-          ]
-        },
-        {
-          "id": "frontmatter.welcome.supporter",
-          "title": "Support the project",
-          "description": "Become a supporter.\n[Support the project](https://github.com/sponsors/estruyf)",
-          "media": {
-            "markdown": "assets/walkthrough/support-the-project.md"
-          },
-          "completionEvents": [
-            "onLink:https://github.com/sponsors/estruyf"
-          ]
-        }
-      ]
-    }]
+          {
+            "id": "frontmatter.welcome.supporter",
+            "title": "Support the project",
+            "description": "Become a supporter.\n[Support the project](https://github.com/sponsors/estruyf)",
+            "media": {
+              "markdown": "assets/walkthrough/support-the-project.md"
+            },
+            "completionEvents": [
+              "onLink:https://github.com/sponsors/estruyf"
+            ]
+          }
+        ]
+      }
+    ]
  },
  "scripts": {
    "dev:ext": "npm run clean && npm-run-all --parallel watch:*",
@@ -2258,7 +2289,6 @@
    "@headlessui/react": "1.5.0",
    "@heroicons/react": "1.0.4",
    "@iarna/toml": "2.2.3",
-    "@microsoft/fetch-event-source": "^2.0.1",
    "@octokit/rest": "^18.12.0",
    "@popperjs/core": "^2.11.6",
    "@sentry/react": "^6.13.3",
@@ -2358,4 +2388,4 @@
  "vsce": {
    "dependencies": false
  }
-}
+}
@@ -1,7 +1,4 @@
 import * as React from 'react';
-import { InitResponse } from './models/InitResponse';
-import { NewConversationResponse } from './models/NewConversationResponse';
-import { fetchEventSource } from '@microsoft/fetch-event-source';
 import { useCallback, useEffect } from 'react';
 import { Header } from './Header';
 import { Chatbox } from './Chatbox';
@@ -9,11 +6,12 @@ import { Loader } from './Loader';
 import { QuestionAnswer } from './QuestionAnswer';
 import { Placeholder } from './Placeholder';
 import { useSettingsContext } from '../../providers/SettingsProvider';
+import { AiInitResponse } from './models/AiInitResponse';

 export interface IChatbotProps { }

 export const Chatbot: React.FunctionComponent<IChatbotProps> = ({ }: React.PropsWithChildren<IChatbotProps>) => {
-  const { aiKey, aiUrl } = useSettingsContext();
+  const { aiUrl } = useSettingsContext();
  const [company, setCompany] = React.useState<string | undefined>(undefined);
  const [chatId, setChatId] = React.useState<number | undefined>(undefined);
  const [questions, setQuestions] = React.useState<string[]>([]);
@@ -24,46 +22,29 @@ export const Chatbot: React.FunctionComponent<IChatbotProps> = ({ }: React.Props

  const init = async () => {
    setLoading(true);
-    const initResponse = await fetch(`${aiUrl}/initializeMendable`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        anon_key: aiKey
-      })
-    });
+    const initResponse = await fetch(`${aiUrl}/api/ai-init`);

    if (!initResponse.ok) {
      return;
    }

-    const initJson: InitResponse = await initResponse.json();
+    const initJson: AiInitResponse = await initResponse.json();

-    const newChatResponse = await fetch(`${aiUrl}/newConversation`, {
-      method: 'POST',
-      headers: {
-        'Content-Type': 'application/json',
-      },
-      body: JSON.stringify({
-        anon_key: aiKey,
-        messages: []
-      })
-    })
-
-    if (!newChatResponse.ok) {
+    if (!initJson.company || !initJson.chatId) {
      return;
    }

-    const newChat: NewConversationResponse = await newChatResponse.json();
-
-    setCompany(initJson.company.name);
-    setChatId(newChat.conversation_id);
+    setCompany(initJson.company);
+    setChatId(initJson.chatId);
    setLoading(false);
  };

-  const callChatbot = useCallback(async (message) => {
-    const nrOfQuestions = questions.length + 1;
+  const onTrigger = useCallback(async (message: string) => {
+    callChatbot(message, questions.length);
+  }, [questions, company, chatId]);
+
+  const callChatbot = useCallback(async (message, questionLenght) => {
+    const nrOfQuestions = questionLenght + 1;
    setLoading(true);

    setQuestions(prev => [...prev, message])
@@ -75,62 +56,69 @@ export const Chatbot: React.FunctionComponent<IChatbotProps> = ({ }: React.Props
      return;
    }

-    await fetchEventSource(`${aiUrl}/qaChat`, {
+    const response = await fetch(`${aiUrl}/api/ai-chat`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
-        'accept': 'application/json',
+        'accept': '*',
      },
      body: JSON.stringify({
-        company: company,
-        conversation_id: chatId,
-        history: [{ prompt: "", response: "", sources: [] }],
+        company,
+        chatId,
        question: message,
      }),
-      onmessage: (event) => {
-        setLoading(false);
-        const data = JSON.parse(event.data);
-        const chunk = data.chunk;
-
-        if (chunk === "<|source|>") {
-          setSources(prev => {
-            const metadata = [...new Set(data.metadata.map((m: any) => m.link))] as string[]
-
-            const crntSources: string[][] = Object.assign([], prev)
-            if (crntSources.length === nrOfQuestions) {
-              crntSources[nrOfQuestions - 1] = metadata;
-            } else {
-              crntSources.push(metadata);
-            }
-
-            return crntSources;
-          });
-        } else if (chunk === "<|message_id|>" && data.metadata) {
-          setAnswerIds(prev => {
-            const crntAnswerIds: number[] = Object.assign([], prev)
-            if (crntAnswerIds.length === nrOfQuestions) {
-              crntAnswerIds[nrOfQuestions - 1] = data.metadata;
-            } else {
-              crntAnswerIds.push(data.metadata);
-            }
-
-            return crntAnswerIds;
-          });
-        } else {
-          setAnswers(prev => {
-            const crntAnswers: string[] = Object.assign([], prev)
-            if (crntAnswers.length === nrOfQuestions) {
-              crntAnswers[nrOfQuestions - 1] = crntAnswers[nrOfQuestions - 1] + chunk;
-            } else {
-              crntAnswers.push(chunk);
-            }
-
-            return crntAnswers;
-          });
-        }
-      }
    });
-  }, [company, chatId, questions]);
+
+    setLoading(false);
+
+    if (!response.ok) {
+      return;
+    }
+
+    const data: {
+      answer: string;
+      answerId: number;
+      sources: string[];
+    } = await response.json();
+
+    if (!data.answer || !data.answerId) {
+      return;
+    }
+
+    setSources(prev => {
+      const metadata = [...new Set(data.sources || [])] as string[]
+      const crntSources: string[][] = Object.assign([], prev)
+      if (crntSources.length === nrOfQuestions) {
+        crntSources[nrOfQuestions - 1] = metadata;
+      } else {
+        crntSources.push(metadata);
+      }
+
+      return crntSources;
+    });
+
+    setAnswerIds(prev => {
+      const crntAnswerIds: number[] = Object.assign([], prev)
+      if (crntAnswerIds.length === nrOfQuestions) {
+        crntAnswerIds[nrOfQuestions - 1] = data.answerId;
+      } else {
+        crntAnswerIds.push(data.answerId);
+      }
+
+      return crntAnswerIds;
+    });
+
+    setAnswers(prev => {
+      const crntAnswers: string[] = Object.assign([], prev)
+      if (crntAnswers.length === nrOfQuestions) {
+        crntAnswers[nrOfQuestions - 1] = data.answer;
+      } else {
+        crntAnswers.push(data.answer);
+      }
+
+      return crntAnswers;
+    });
+  }, [company, chatId]);

  useEffect(() => {
    init();
@@ -161,7 +149,7 @@ export const Chatbot: React.FunctionComponent<IChatbotProps> = ({ }: React.Props
        }
      </main>

-      <Chatbox isLoading={loading} onTrigger={callChatbot} />
+      <Chatbox isLoading={loading} onTrigger={onTrigger} />

      <img className='hidden' src="https://api.visitorbadge.io/api/visitors?path=https%3A%2F%2Ffrontmatter.codes%2Fmetrics%2Fdashboards&slug=chatbot" alt="Chatbot metrics" />
    </div>
@@ -28,13 +28,14 @@ export const Feedback: React.FunctionComponent<IFeedbackProps> = ({
  }, []);

  const callVote = useCallback(async (vote: boolean) => {
-    await fetch(`${aiUrl}/updateMessageRating/${answerId}`, {
+    await fetch(`${aiUrl}/api/ai-feedback`, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
      },
      body: JSON.stringify({
-        ratingValue: vote ? 1 : -1,
+        answerId,
+        vote: vote ? 1 : -1,
      })
    })
  }, [answerId])
@@ -10,7 +10,7 @@ export interface IQuestionAnswerProps {

 export const QuestionAnswer: React.FunctionComponent<IQuestionAnswerProps> = ({ question, answer, answerId, sources }: React.PropsWithChildren<IQuestionAnswerProps>) => {
  return (
-    <ul className={`space-y-4 px-4`}>
+    <ul className={`mt-4 space-y-4 px-4`}>
      <li className='question'>{question}</li>

      <Answer
@@ -0,0 +1,4 @@
+export interface AiInitResponse {
+  company: string;
+  chatId: number;
+}
@@ -1,16 +0,0 @@
-export interface InitResponse {
-  company: Company;
-  project: Project;
-}
-
-export interface Project {
-  id: number;
-  created_at: string;
-  name: string;
-  company_id: number;
-}
-
-export interface Company {
-  company_id: number;
-  name: string;
-}
@@ -1,6 +0,0 @@
-export interface NewConversationResponse {
-  conversation_id: number;
-  start_time: string;
-  end_time?: any;
-  project_id: number;
-}
@@ -133,8 +133,7 @@ if (elm) {
  } else if (type === 'chatbot') {
    render(
      <SettingsProvider
-        aiKey='466f5321-12d9-4d64-9e5b-ea5db41ed2ba'
-        aiUrl='https://aijsplayground-production.up.railway.app'
+        aiUrl='https://frontmatter.codes'
        experimental={experimental === 'true'}
        version={version || ""}>
        <Chatbot />
@@ -4,19 +4,17 @@ interface ISettingsProviderProps {
  version?: string;
  experimental?: boolean;
  aiUrl?: string;
-  aiKey?: string;
 }

 const SettingsContext = React.createContext<ISettingsProviderProps | undefined>(undefined);

-const SettingsProvider: React.FunctionComponent<ISettingsProviderProps> = ({ version, experimental, aiKey, aiUrl, children }: React.PropsWithChildren<ISettingsProviderProps>) => {
+const SettingsProvider: React.FunctionComponent<ISettingsProviderProps> = ({ version, experimental, aiUrl, children }: React.PropsWithChildren<ISettingsProviderProps>) => {
  return (
    <SettingsContext.Provider
      value={{
        version,
        experimental,
-        aiUrl,
-        aiKey
+        aiUrl
      }}
    >
      {children}