From 467893db6709c63fff7013118d83e72b65dd1d3a Mon Sep 17 00:00:00 2001
From: Heiko Joerg Schick <heiko.schick@huawei.com>
Date: Thu, 14 Dec 2023 20:39:19 +0100
Subject: [PATCH] Added GPT-4-vision

---
 components/Chat/Chat.tsx        |   7 +-
 components/Chat/ChatInput.tsx   | 251 ++++++++++++++++++++------------
 components/Chat/ChatMessage.tsx |  33 +++--
 pages/api/chat.ts               |   3 +-
 types/chat.ts                   |  12 +-
 types/openai.ts                 |   9 +-
 utils/server/index.ts           |   4 +-
 7 files changed, 205 insertions(+), 114 deletions(-)

diff --git a/components/Chat/Chat.tsx b/components/Chat/Chat.tsx
index b7d782e..af97ac7 100644
--- a/components/Chat/Chat.tsx
+++ b/components/Chat/Chat.tsx
@@ -150,8 +150,9 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
         if (!plugin) {
           if (updatedConversation.messages.length === 1) {
             const { content } = message;
+            var textContent = content.filter(c => c.type == "text").map(c => c.text).join();
             const customName =
-              content.length > 30 ? content.substring(0, 30) + '...' : content;
+              textContent.length > 30 ? textContent.substring(0, 30) + '...' : textContent;
             updatedConversation = {
               ...updatedConversation,
               name: customName,
@@ -177,7 +178,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
               isFirst = false;
               const updatedMessages: Message[] = [
                 ...updatedConversation.messages,
-                { role: 'assistant', content: chunkValue },
+                { role: 'assistant', content: [{type: "text", text: chunkValue}] },
               ];
               updatedConversation = {
                 ...updatedConversation,
@@ -193,7 +194,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
                   if (index === updatedConversation.messages.length - 1) {
                     return {
                       ...message,
-                      content: text,
+                      content: [{type:"text", text}],
                     };
                   }
                   return message;
diff --git a/components/Chat/ChatInput.tsx b/components/Chat/ChatInput.tsx
index 64f8df6..758fa0b 100644
--- a/components/Chat/ChatInput.tsx
+++ b/components/Chat/ChatInput.tsx
@@ -2,6 +2,7 @@ import {
   IconArrowDown,
   IconBolt,
   IconBrandGoogle,
+  IconPhoto,
   IconPlayerStop,
   IconRepeat,
   IconSend,
@@ -18,7 +19,7 @@ import {
 
 import { useTranslation } from 'next-i18next';
 
-import { Message } from '@/types/chat';
+import { Content, Message } from '@/types/chat';
 import { Plugin } from '@/types/plugin';
 import { Prompt } from '@/types/prompt';
 
@@ -62,6 +63,40 @@ export const ChatInput = ({
   const [isModalVisible, setIsModalVisible] = useState(false);
   const [showPluginSelect, setShowPluginSelect] = useState(false);
   const [plugin, setPlugin] = useState<Plugin | null>(null);
+  const [images, setImages] = useState<string[]>([]);
+  const imageInputRef = useRef<HTMLInputElement | null>(null);
+
+  const handleImageInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+
+    if (file) {
+      const reader = new FileReader();
+      reader.onloadend = () => {
+        const base64Image = reader.result;
+        setImages((prevImages) => [...prevImages, base64Image as string]);
+      };
+      reader.readAsDataURL(file);
+    }
+  };
+
+  const ImagePreview = ({ images }: { images: string[] }) => {
+    return (
+      <div className="flex flex-wrap gap-2 mb-2">
+        {images.map((image, index) => (
+          <img
+            key={index}
+            src={image}
+            alt={`uploaded-${index}`}
+            className="w-20 h-20 object-cover rounded-md border"
+          />
+        ))}
+      </div>
+    );
+  };
+
+  const handleImageInputClick = () => {
+    imageInputRef.current?.click();
+  };
 
   const promptListRef = useRef<HTMLUListElement | null>(null);
 
@@ -97,7 +132,14 @@ export const ChatInput = ({
       return;
     }
 
-    onSend({ role: 'user', content }, plugin);
+    var messageContent:Content[] = [{"type": "text", "text": content}];
+    if(images && images.length >0){
+      var imageMessages = images.map(image => { return {type: "image_url", image_url:{"url": image}}});
+      messageContent = [...messageContent, ...imageMessages]
+    }
+
+    onSend({ role: 'user', content:messageContent }, plugin);
+    setImages([]);
     setContent('');
     setPlugin(null);
 
@@ -233,9 +275,8 @@ export const ChatInput = ({
     if (textareaRef && textareaRef.current) {
       textareaRef.current.style.height = 'inherit';
       textareaRef.current.style.height = `${textareaRef.current?.scrollHeight}px`;
-      textareaRef.current.style.overflow = `${
-        textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
-      }`;
+      textareaRef.current.style.overflow = `${textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
+        }`;
     }
   }, [content]);
 
@@ -280,103 +321,119 @@ export const ChatInput = ({
           )}
 
         <div className="relative mx-2 flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] dark:border-gray-900/50 dark:bg-[#40414F] dark:text-white dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] sm:mx-4">
-          <button
-            className="absolute left-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
-            onClick={() => setShowPluginSelect(!showPluginSelect)}
-            onKeyDown={(e) => {}}
-          >
-            {plugin ? <IconBrandGoogle size={20} /> : <IconBolt size={20} />}
-          </button>
+          <ImagePreview images={images} />
 
-          {showPluginSelect && (
-            <div className="absolute left-0 bottom-14 rounded bg-white dark:bg-[#343541]">
-              <PluginSelect
-                plugin={plugin}
-                onKeyDown={(e: any) => {
-                  if (e.key === 'Escape') {
-                    e.preventDefault();
+          <div className="relative mx-0 flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] dark:border-gray-900/50 dark:bg-[#40414F] dark:text-white dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] sm:mx-0">
+            <button
+              className="absolute left-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={() => setShowPluginSelect(!showPluginSelect)}
+              onKeyDown={(e) => { }}
+            >
+              {plugin ? <IconBrandGoogle size={20} /> : <IconBolt size={20} />}
+            </button>
+
+            {showPluginSelect && (
+              <div className="absolute left-0 bottom-14 rounded bg-white dark:bg-[#343541]">
+                <PluginSelect
+                  plugin={plugin}
+                  onKeyDown={(e: any) => {
+                    if (e.key === 'Escape') {
+                      e.preventDefault();
+                      setShowPluginSelect(false);
+                      textareaRef.current?.focus();
+                    }
+                  }}
+                  onPluginChange={(plugin: Plugin) => {
+                    setPlugin(plugin);
                     setShowPluginSelect(false);
-                    textareaRef.current?.focus();
-                  }
-                }}
-                onPluginChange={(plugin: Plugin) => {
-                  setPlugin(plugin);
-                  setShowPluginSelect(false);
 
-                  if (textareaRef && textareaRef.current) {
-                    textareaRef.current.focus();
-                  }
-                }}
-              />
-            </div>
-          )}
-
-          <textarea
-            ref={textareaRef}
-            className="m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-10 text-black dark:bg-transparent dark:text-white md:py-3 md:pl-10"
-            style={{
-              resize: 'none',
-              bottom: `${textareaRef?.current?.scrollHeight}px`,
-              maxHeight: '400px',
-              overflow: `${
-                textareaRef.current && textareaRef.current.scrollHeight > 400
-                  ? 'auto'
-                  : 'hidden'
-              }`,
-            }}
-            placeholder={
-              t('Type a message or type "/" to select a prompt...') || ''
-            }
-            value={content}
-            rows={1}
-            onCompositionStart={() => setIsTyping(true)}
-            onCompositionEnd={() => setIsTyping(false)}
-            onChange={handleChange}
-            onKeyDown={handleKeyDown}
-          />
-
-          <button
-            className="absolute right-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
-            onClick={handleSend}
-          >
-            {messageIsStreaming ? (
-              <div className="h-4 w-4 animate-spin rounded-full border-t-2 border-neutral-800 opacity-60 dark:border-neutral-100"></div>
-            ) : (
-              <IconSend size={18} />
+                    if (textareaRef && textareaRef.current) {
+                      textareaRef.current.focus();
+                    }
+                  }}
+                />
+              </div>
             )}
-          </button>
-
-          {showScrollDownButton && (
-            <div className="absolute bottom-12 right-0 lg:bottom-0 lg:-right-10">
-              <button
-                className="flex h-7 w-7 items-center justify-center rounded-full bg-neutral-300 text-gray-800 shadow-md hover:shadow-lg focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-neutral-200"
-                onClick={onScrollDownClick}
-              >
-                <IconArrowDown size={18} />
-              </button>
-            </div>
-          )}
-
-          {showPromptList && filteredPrompts.length > 0 && (
-            <div className="absolute bottom-12 w-full">
-              <PromptList
-                activePromptIndex={activePromptIndex}
-                prompts={filteredPrompts}
-                onSelect={handleInitModal}
-                onMouseOver={setActivePromptIndex}
-                promptListRef={promptListRef}
-              />
-            </div>
-          )}
-
-          {isModalVisible && (
-            <VariableModal
-              prompt={filteredPrompts[activePromptIndex]}
-              variables={variables}
-              onSubmit={handleSubmit}
-              onClose={() => setIsModalVisible(false)}
+           <input
+              ref={imageInputRef}
+              type="file"
+              accept="image/*"
+              style={{ display: 'none' }}
+              onChange={handleImageInputChange}
             />
-          )}
+
+            <button
+              className="absolute left-10 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={handleImageInputClick}
+            >
+              <IconPhoto size={20} />
+            </button>
+
+            <textarea
+              ref={textareaRef}
+              className="m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-20 text-black dark:bg-transparent dark:text-white md:py-3 md:pl-20"
+              style={{
+                resize: 'none',
+                bottom: `${textareaRef?.current?.scrollHeight}px`,
+                maxHeight: '400px',
+                overflow: `${textareaRef.current && textareaRef.current.scrollHeight > 400
+                    ? 'auto'
+                    : 'hidden'
+                  }`,
+              }}
+              placeholder={
+                t('Type a message or type "/" to select a prompt...') || ''
+              }
+              value={content}
+              rows={1}
+              onCompositionStart={() => setIsTyping(true)}
+              onCompositionEnd={() => setIsTyping(false)}
+              onChange={handleChange}
+              onKeyDown={handleKeyDown}
+            />
+          <button
+              className="absolute right-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={handleSend}
+            >
+              {messageIsStreaming ? (
+                <div className="h-4 w-4 animate-spin rounded-full border-t-2 border-neutral-800 opacity-60 dark:border-neutral-100"></div>
+              ) : (
+                <IconSend size={18} />
+              )}
+            </button>
+
+            {showScrollDownButton && (
+              <div className="absolute bottom-12 right-0 lg:bottom-0 lg:-right-10">
+                <button
+                  className="flex h-7 w-7 items-center justify-center rounded-full bg-neutral-300 text-gray-800 shadow-md hover:shadow-lg focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-neutral-200"
+                  onClick={onScrollDownClick}
+                >
+                  <IconArrowDown size={18} />
+                </button>
+              </div>
+            )}
+
+            {showPromptList && filteredPrompts.length > 0 && (
+              <div className="absolute bottom-12 w-full">
+                <PromptList
+                  activePromptIndex={activePromptIndex}
+                  prompts={filteredPrompts}
+                  onSelect={handleInitModal}
+                  onMouseOver={setActivePromptIndex}
+                  promptListRef={promptListRef}
+                />
+              </div>
+            )}
+
+            {isModalVisible && (
+              <VariableModal
+                prompt={filteredPrompts[activePromptIndex]}
+                variables={variables}
+                onSubmit={handleSubmit}
+                onClose={() => setIsModalVisible(false)}
+              />
+            )}
+          </div>
         </div>
       </div>
       <div className="px-3 pt-2 pb-3 text-center text-[12px] text-black/50 dark:text-white/50 md:px-4 md:pt-3 md:pb-6">
diff --git a/components/Chat/ChatMessage.tsx b/components/Chat/ChatMessage.tsx
index 512551f..b3d955d 100644
--- a/components/Chat/ChatMessage.tsx
+++ b/components/Chat/ChatMessage.tsx
@@ -40,6 +40,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   const [isEditing, setIsEditing] = useState<boolean>(false);
   const [isTyping, setIsTyping] = useState<boolean>(false);
   const [messageContent, setMessageContent] = useState(message.content);
+  const [messageTextContent, setMessageTextContent] = useState(message.content[0].text ?? "");
   const [messagedCopied, setMessageCopied] = useState(false);
 
   const textareaRef = useRef<HTMLTextAreaElement>(null);
@@ -49,7 +50,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   };
 
   const handleInputChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
-    setMessageContent(event.target.value);
+    // messageContent[0].text = event.target.value;
+    // setMessageContent(messageContent);
+    setMessageTextContent(event.target.value);
     if (textareaRef.current) {
       textareaRef.current.style.height = 'inherit';
       textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
@@ -57,8 +60,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   };
 
   const handleEditMessage = () => {
-    if (message.content != messageContent) {
+    if (message.content[0].text != messageTextContent) {
       if (selectedConversation && onEdit) {
+        messageContent[0].text = messageTextContent;
         onEdit({ ...message, content: messageContent });
       }
     }
@@ -104,7 +108,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   const copyOnClick = () => {
     if (!navigator.clipboard) return;
 
-    navigator.clipboard.writeText(message.content).then(() => {
+    var content = message.content;
+    var textContent = content.filter(c => c.type == "text").map(c => c.text).join();
+    navigator.clipboard.writeText(textContent).then(() => {
       setMessageCopied(true);
       setTimeout(() => {
         setMessageCopied(false);
@@ -113,7 +119,8 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   };
 
   useEffect(() => {
-    setMessageContent(message.content);
+    // setMessageContent(message.content);
+    setMessageTextContent(message.content[0].text ?? "");
   }, [message.content]);
 
 
@@ -150,7 +157,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                   <textarea
                     ref={textareaRef}
                     className="w-full resize-none whitespace-pre-wrap border-none dark:bg-[#343541]"
-                    value={messageContent}
+                    value={messageTextContent}
                     onChange={handleInputChange}
                     onKeyDown={handlePressEnter}
                     onCompositionStart={() => setIsTyping(true)}
@@ -169,14 +176,15 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                     <button
                       className="h-[40px] rounded-md bg-blue-500 px-4 py-1 text-sm font-medium text-white enabled:hover:bg-blue-600 disabled:opacity-50"
                       onClick={handleEditMessage}
-                      disabled={messageContent.trim().length <= 0}
+                      disabled={messageTextContent.length <= 0}
                     >
                       {t('Save & Submit')}
                     </button>
                     <button
                       className="h-[40px] rounded-md border border-neutral-300 px-4 py-1 text-sm font-medium text-neutral-700 hover:bg-neutral-100 dark:border-neutral-700 dark:text-neutral-300 dark:hover:bg-neutral-800"
                       onClick={() => {
-                        setMessageContent(message.content);
+                        // setMessageContent(message.content);
+                        setMessageTextContent(message.content[0].text ?? "");
                         setIsEditing(false);
                       }}
                     >
@@ -186,7 +194,14 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                 </div>
               ) : (
                 <div className="prose whitespace-pre-wrap dark:prose-invert flex-1">
-                  {message.content}
+                    <div>
+                    {messageTextContent}
+                  </div>
+                  <div className="flex flex-wrap justify-center">  
+                    {messageContent.filter(c => c.type === "image_url").map((c, index) => (  
+                      <img key={index} src={c.image_url?.url} alt="Message Content" className="max-w-full h-auto my-2" style={{objectFit: "contain"}} />  
+                    ))}  
+                  </div>
                 </div>
               )}
 
@@ -261,7 +276,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                   },
                 }}
               >
-                {`${message.content}${
+                {`${messageTextContent}${
                   messageIsStreaming && messageIndex == (selectedConversation?.messages.length ?? 0) - 1 ? '`▍`' : ''
                 }`}
               </MemoizedReactMarkdown>
diff --git a/pages/api/chat.ts b/pages/api/chat.ts
index f9f9c32..24693d7 100644
--- a/pages/api/chat.ts
+++ b/pages/api/chat.ts
@@ -41,7 +41,7 @@ const handler = async (req: Request): Promise<Response> => {
 
     for (let i = messages.length - 1; i >= 0; i--) {
       const message = messages[i];
-      const tokens = encoding.encode(message.content);
+      const tokens = encoding.encode(message.content[0].text ?? "");
 
       if (tokenCount + tokens.length + 1000 > model.tokenLimit) {
         break;
@@ -55,6 +55,7 @@ const handler = async (req: Request): Promise<Response> => {
     const stream = await OpenAIStream(model, promptToSend, temperatureToUse, key, messagesToSend);
 
     var resp = new Response(stream);
+    return resp;
 
     // let proxy services like nginx or argo tunnel know about pass the chunk immediately
     // similar to nginx option `proxy_buffering off;`
diff --git a/types/chat.ts b/types/chat.ts
index 1233f2c..fc31d6a 100644
--- a/types/chat.ts
+++ b/types/chat.ts
@@ -2,7 +2,17 @@ import { OpenAIModel } from './openai';
 
 export interface Message {
   role: Role;
-  content: string;
+  content: Content[];
+}
+
+export interface Content{
+  type: string;
+  text?: string;
+  image_url?: ImageUrl;
+}
+
+export interface ImageUrl{
+  url: string;
 }
 
 export type Role = 'assistant' | 'user';
diff --git a/types/openai.ts b/types/openai.ts
index 0159b21..0bf10dc 100644
--- a/types/openai.ts
+++ b/types/openai.ts
@@ -13,7 +13,8 @@ export enum OpenAIModelID {
   GPT_3_5_16K = 'gpt-3.5-turbo-16k',
   GPT_4 = 'gpt-4',
   GPT_4_32K = 'gpt-4-32k',
-  GPT_4_TURBO = 'gpt-4-1106-preview'
+  GPT_4_TURBO = 'gpt-4-1106-preview',
+  GPT_4_TURBO_VISION = 'gpt-4-vision-preview'
 }
 
 // in case the `DEFAULT_MODEL` environment variable is not set or set to an unsupported model
@@ -56,4 +57,10 @@ export const OpenAIModels: Record<OpenAIModelID, OpenAIModel> = {
     maxLength: 384000,
     tokenLimit: 128000,
   },
+  [OpenAIModelID.GPT_4_TURBO_VISION]: {
+    id: OpenAIModelID.GPT_4_TURBO_VISION,
+    name: 'GPT-4-TURBO-VISION',
+    maxLength: 384000,
+    tokenLimit: 128000,
+  },
 };
diff --git a/utils/server/index.ts b/utils/server/index.ts
index 7ccfc9e..bffb6a7 100644
--- a/utils/server/index.ts
+++ b/utils/server/index.ts
@@ -53,11 +53,11 @@ export const OpenAIStream = async (
       messages: [
         {
           role: 'system',
-          content: systemPrompt,
+          content: [{type:"text", text: systemPrompt}],
         },
         ...messages,
       ],
-      max_tokens: 1000,
+      max_tokens: 4096,
       temperature: temperature,
       stream: true,
     }),