Added GPT-4-vision

components/Chat/Chat.tsx

@@ -150,8 +150,9 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
         if (!plugin) {
           if (updatedConversation.messages.length === 1) {
             const { content } = message;
+            var textContent = content.filter(c => c.type == "text").map(c => c.text).join();
             const customName =
-              content.length > 30 ? content.substring(0, 30) + '...' : content;
+              textContent.length > 30 ? textContent.substring(0, 30) + '...' : textContent;
             updatedConversation = {
               ...updatedConversation,
               name: customName,
@@ -177,7 +178,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
               isFirst = false;
               const updatedMessages: Message[] = [
                 ...updatedConversation.messages,
-                { role: 'assistant', content: chunkValue },
+                { role: 'assistant', content: [{type: "text", text: chunkValue}] },
               ];
               updatedConversation = {
                 ...updatedConversation,
@@ -193,7 +194,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
                   if (index === updatedConversation.messages.length - 1) {
                     return {
                       ...message,
-                      content: text,
+                      content: [{type:"text", text}],
                     };
                   }
                   return message;

components/Chat/ChatInput.tsx

@@ -2,6 +2,7 @@ import {
   IconArrowDown,
   IconBolt,
   IconBrandGoogle,
+  IconPhoto,
   IconPlayerStop,
   IconRepeat,
   IconSend,
@@ -18,7 +19,7 @@ import {
 
 import { useTranslation } from 'next-i18next';
 
-import { Message } from '@/types/chat';
+import { Content, Message } from '@/types/chat';
 import { Plugin } from '@/types/plugin';
 import { Prompt } from '@/types/prompt';
 
@@ -62,6 +63,40 @@ export const ChatInput = ({
   const [isModalVisible, setIsModalVisible] = useState(false);
   const [showPluginSelect, setShowPluginSelect] = useState(false);
   const [plugin, setPlugin] = useState<Plugin | null>(null);
+  const [images, setImages] = useState<string[]>([]);
+  const imageInputRef = useRef<HTMLInputElement | null>(null);
+
+  const handleImageInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+
+    if (file) {
+      const reader = new FileReader();
+      reader.onloadend = () => {
+        const base64Image = reader.result;
+        setImages((prevImages) => [...prevImages, base64Image as string]);
+      };
+      reader.readAsDataURL(file);
+    }
+  };
+
+  const ImagePreview = ({ images }: { images: string[] }) => {
+    return (
+      <div className="flex flex-wrap gap-2 mb-2">
+        {images.map((image, index) => (
+          <img
+            key={index}
+            src={image}
+            alt={`uploaded-${index}`}
+            className="w-20 h-20 object-cover rounded-md border"
+          />
+        ))}
+      </div>
+    );
+  };
+
+  const handleImageInputClick = () => {
+    imageInputRef.current?.click();
+  };
 
   const promptListRef = useRef<HTMLUListElement | null>(null);
 
@@ -97,7 +132,14 @@ export const ChatInput = ({
       return;
     }
 
-    onSend({ role: 'user', content }, plugin);
+    var messageContent:Content[] = [{"type": "text", "text": content}];
+    if(images && images.length >0){
+      var imageMessages = images.map(image => { return {type: "image_url", image_url:{"url": image}}});
+      messageContent = [...messageContent, ...imageMessages]
+    }
+
+    onSend({ role: 'user', content:messageContent }, plugin);
+    setImages([]);
     setContent('');
     setPlugin(null);
 
@@ -233,8 +275,7 @@ export const ChatInput = ({
     if (textareaRef && textareaRef.current) {
       textareaRef.current.style.height = 'inherit';
       textareaRef.current.style.height = `${textareaRef.current?.scrollHeight}px`;
-      textareaRef.current.style.overflow = `${
+      textareaRef.current.style.overflow = `${textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
-        textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
         }`;
     }
   }, [content]);
@@ -280,6 +321,9 @@ export const ChatInput = ({
           )}
 
         <div className="relative mx-2 flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] dark:border-gray-900/50 dark:bg-[#40414F] dark:text-white dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] sm:mx-4">
+          <ImagePreview images={images} />
+
+          <div className="relative mx-0 flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] dark:border-gray-900/50 dark:bg-[#40414F] dark:text-white dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] sm:mx-0">
             <button
               className="absolute left-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
               onClick={() => setShowPluginSelect(!showPluginSelect)}
@@ -310,16 +354,29 @@ export const ChatInput = ({
                 />
               </div>
             )}
+           <input
+              ref={imageInputRef}
+              type="file"
+              accept="image/*"
+              style={{ display: 'none' }}
+              onChange={handleImageInputChange}
+            />
+
+            <button
+              className="absolute left-10 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={handleImageInputClick}
+            >
+              <IconPhoto size={20} />
+            </button>
 
             <textarea
               ref={textareaRef}
-            className="m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-10 text-black dark:bg-transparent dark:text-white md:py-3 md:pl-10"
+              className="m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-20 text-black dark:bg-transparent dark:text-white md:py-3 md:pl-20"
               style={{
                 resize: 'none',
                 bottom: `${textareaRef?.current?.scrollHeight}px`,
                 maxHeight: '400px',
-              overflow: `${
+                overflow: `${textareaRef.current && textareaRef.current.scrollHeight > 400
-                textareaRef.current && textareaRef.current.scrollHeight > 400
                     ? 'auto'
                     : 'hidden'
                   }`,
@@ -334,7 +391,6 @@ export const ChatInput = ({
               onChange={handleChange}
               onKeyDown={handleKeyDown}
             />
-
           <button
               className="absolute right-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
               onClick={handleSend}
@@ -379,6 +435,7 @@ export const ChatInput = ({
             )}
           </div>
         </div>
+      </div>
       <div className="px-3 pt-2 pb-3 text-center text-[12px] text-black/50 dark:text-white/50 md:px-4 md:pt-3 md:pb-6">
         <a
           href="https://github.com/mckaywrigley/chatbot-ui"

components/Chat/ChatMessage.tsx

@@ -40,6 +40,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   const [isEditing, setIsEditing] = useState<boolean>(false);
   const [isTyping, setIsTyping] = useState<boolean>(false);
   const [messageContent, setMessageContent] = useState(message.content);
+  const [messageTextContent, setMessageTextContent] = useState(message.content[0].text ?? "");
   const [messagedCopied, setMessageCopied] = useState(false);
 
   const textareaRef = useRef<HTMLTextAreaElement>(null);
@@ -49,7 +50,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   };
 
   const handleInputChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
-    setMessageContent(event.target.value);
+    // messageContent[0].text = event.target.value;
+    // setMessageContent(messageContent);
+    setMessageTextContent(event.target.value);
     if (textareaRef.current) {
       textareaRef.current.style.height = 'inherit';
       textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
@@ -57,8 +60,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   };
 
   const handleEditMessage = () => {
-    if (message.content != messageContent) {
+    if (message.content[0].text != messageTextContent) {
       if (selectedConversation && onEdit) {
+        messageContent[0].text = messageTextContent;
         onEdit({ ...message, content: messageContent });
       }
     }
@@ -104,7 +108,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   const copyOnClick = () => {
     if (!navigator.clipboard) return;
 
-    navigator.clipboard.writeText(message.content).then(() => {
+    var content = message.content;
+    var textContent = content.filter(c => c.type == "text").map(c => c.text).join();
+    navigator.clipboard.writeText(textContent).then(() => {
       setMessageCopied(true);
       setTimeout(() => {
         setMessageCopied(false);
@@ -113,7 +119,8 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
   };
 
   useEffect(() => {
-    setMessageContent(message.content);
+    // setMessageContent(message.content);
+    setMessageTextContent(message.content[0].text ?? "");
   }, [message.content]);
 
 
@@ -150,7 +157,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                   <textarea
                     ref={textareaRef}
                     className="w-full resize-none whitespace-pre-wrap border-none dark:bg-[#343541]"
-                    value={messageContent}
+                    value={messageTextContent}
                     onChange={handleInputChange}
                     onKeyDown={handlePressEnter}
                     onCompositionStart={() => setIsTyping(true)}
@@ -169,14 +176,15 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                     <button
                       className="h-[40px] rounded-md bg-blue-500 px-4 py-1 text-sm font-medium text-white enabled:hover:bg-blue-600 disabled:opacity-50"
                       onClick={handleEditMessage}
-                      disabled={messageContent.trim().length <= 0}
+                      disabled={messageTextContent.length <= 0}
                     >
                       {t('Save & Submit')}
                     </button>
                     <button
                       className="h-[40px] rounded-md border border-neutral-300 px-4 py-1 text-sm font-medium text-neutral-700 hover:bg-neutral-100 dark:border-neutral-700 dark:text-neutral-300 dark:hover:bg-neutral-800"
                       onClick={() => {
-                        setMessageContent(message.content);
+                        // setMessageContent(message.content);
+                        setMessageTextContent(message.content[0].text ?? "");
                         setIsEditing(false);
                       }}
                     >
@@ -186,7 +194,14 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                 </div>
               ) : (
                 <div className="prose whitespace-pre-wrap dark:prose-invert flex-1">
-                  {message.content}
+                    <div>
+                    {messageTextContent}
+                  </div>
+                  <div className="flex flex-wrap justify-center">  
+                    {messageContent.filter(c => c.type === "image_url").map((c, index) => (  
+                      <img key={index} src={c.image_url?.url} alt="Message Content" className="max-w-full h-auto my-2" style={{objectFit: "contain"}} />  
+                    ))}  
+                  </div>
                 </div>
               )}
 
@@ -261,7 +276,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                   },
                 }}
               >
-                {`${message.content}${
+                {`${messageTextContent}${
                   messageIsStreaming && messageIndex == (selectedConversation?.messages.length ?? 0) - 1 ? '`▍`' : ''
                 }`}
               </MemoizedReactMarkdown>

pages/api/chat.ts

@@ -41,7 +41,7 @@ const handler = async (req: Request): Promise<Response> => {
 
     for (let i = messages.length - 1; i >= 0; i--) {
       const message = messages[i];
-      const tokens = encoding.encode(message.content);
+      const tokens = encoding.encode(message.content[0].text ?? "");
 
       if (tokenCount + tokens.length + 1000 > model.tokenLimit) {
         break;
@@ -55,6 +55,7 @@ const handler = async (req: Request): Promise<Response> => {
     const stream = await OpenAIStream(model, promptToSend, temperatureToUse, key, messagesToSend);
 
     var resp = new Response(stream);
+    return resp;
 
     // let proxy services like nginx or argo tunnel know about pass the chunk immediately
     // similar to nginx option `proxy_buffering off;`

types/chat.ts

@@ -2,7 +2,17 @@ import { OpenAIModel } from './openai';
 
 export interface Message {
   role: Role;
-  content: string;
+  content: Content[];
+}
+
+export interface Content{
+  type: string;
+  text?: string;
+  image_url?: ImageUrl;
+}
+
+export interface ImageUrl{
+  url: string;
 }
 
 export type Role = 'assistant' | 'user';

types/openai.ts

@@ -13,7 +13,8 @@ export enum OpenAIModelID {
   GPT_3_5_16K = 'gpt-3.5-turbo-16k',
   GPT_4 = 'gpt-4',
   GPT_4_32K = 'gpt-4-32k',
-  GPT_4_TURBO = 'gpt-4-1106-preview'
+  GPT_4_TURBO = 'gpt-4-1106-preview',
+  GPT_4_TURBO_VISION = 'gpt-4-vision-preview'
 }
 
 // in case the `DEFAULT_MODEL` environment variable is not set or set to an unsupported model
@@ -56,4 +57,10 @@ export const OpenAIModels: Record<OpenAIModelID, OpenAIModel> = {
     maxLength: 384000,
     tokenLimit: 128000,
   },
+  [OpenAIModelID.GPT_4_TURBO_VISION]: {
+    id: OpenAIModelID.GPT_4_TURBO_VISION,
+    name: 'GPT-4-TURBO-VISION',
+    maxLength: 384000,
+    tokenLimit: 128000,
+  },
 };

utils/server/index.ts

@@ -53,11 +53,11 @@ export const OpenAIStream = async (
       messages: [
         {
           role: 'system',
-          content: systemPrompt,
+          content: [{type:"text", text: systemPrompt}],
         },
         ...messages,
       ],
-      max_tokens: 1000,
+      max_tokens: 4096,
       temperature: temperature,
       stream: true,
     }),