Added GPT-4-vision

2023-12-14 20:39:19 +01:00
parent 25df881a39
commit 467893db67
7 changed files with 205 additions and 114 deletions
@@ -150,8 +150,9 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
        if (!plugin) {
          if (updatedConversation.messages.length === 1) {
            const { content } = message;
+            var textContent = content.filter(c => c.type == "text").map(c => c.text).join();
            const customName =
-              content.length > 30 ? content.substring(0, 30) + '...' : content;
+              textContent.length > 30 ? textContent.substring(0, 30) + '...' : textContent;
            updatedConversation = {
              ...updatedConversation,
              name: customName,
@@ -177,7 +178,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
              isFirst = false;
              const updatedMessages: Message[] = [
                ...updatedConversation.messages,
-                { role: 'assistant', content: chunkValue },
+                { role: 'assistant', content: [{type: "text", text: chunkValue}] },
              ];
              updatedConversation = {
                ...updatedConversation,
@@ -193,7 +194,7 @@ export const Chat = memo(({ stopConversationRef }: Props) => {
                  if (index === updatedConversation.messages.length - 1) {
                    return {
                      ...message,
-                      content: text,
+                      content: [{type:"text", text}],
                    };
                  }
                  return message;
@@ -2,6 +2,7 @@ import {
  IconArrowDown,
  IconBolt,
  IconBrandGoogle,
+  IconPhoto,
  IconPlayerStop,
  IconRepeat,
  IconSend,
@@ -18,7 +19,7 @@ import {

 import { useTranslation } from 'next-i18next';

-import { Message } from '@/types/chat';
+import { Content, Message } from '@/types/chat';
 import { Plugin } from '@/types/plugin';
 import { Prompt } from '@/types/prompt';

@@ -62,6 +63,40 @@ export const ChatInput = ({
  const [isModalVisible, setIsModalVisible] = useState(false);
  const [showPluginSelect, setShowPluginSelect] = useState(false);
  const [plugin, setPlugin] = useState<Plugin | null>(null);
+  const [images, setImages] = useState<string[]>([]);
+  const imageInputRef = useRef<HTMLInputElement | null>(null);
+
+  const handleImageInputChange = (e: React.ChangeEvent<HTMLInputElement>) => {
+    const file = e.target.files?.[0];
+
+    if (file) {
+      const reader = new FileReader();
+      reader.onloadend = () => {
+        const base64Image = reader.result;
+        setImages((prevImages) => [...prevImages, base64Image as string]);
+      };
+      reader.readAsDataURL(file);
+    }
+  };
+
+  const ImagePreview = ({ images }: { images: string[] }) => {
+    return (
+      <div className="flex flex-wrap gap-2 mb-2">
+        {images.map((image, index) => (
+          <img
+            key={index}
+            src={image}
+            alt={`uploaded-${index}`}
+            className="w-20 h-20 object-cover rounded-md border"
+          />
+        ))}
+      </div>
+    );
+  };
+
+  const handleImageInputClick = () => {
+    imageInputRef.current?.click();
+  };

  const promptListRef = useRef<HTMLUListElement | null>(null);

@@ -97,7 +132,14 @@ export const ChatInput = ({
      return;
    }

-    onSend({ role: 'user', content }, plugin);
+    var messageContent:Content[] = [{"type": "text", "text": content}];
+    if(images && images.length >0){
+      var imageMessages = images.map(image => { return {type: "image_url", image_url:{"url": image}}});
+      messageContent = [...messageContent, ...imageMessages]
+    }
+
+    onSend({ role: 'user', content:messageContent }, plugin);
+    setImages([]);
    setContent('');
    setPlugin(null);

@@ -233,9 +275,8 @@ export const ChatInput = ({
    if (textareaRef && textareaRef.current) {
      textareaRef.current.style.height = 'inherit';
      textareaRef.current.style.height = `${textareaRef.current?.scrollHeight}px`;
-      textareaRef.current.style.overflow = `${
-        textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
-      }`;
+      textareaRef.current.style.overflow = `${textareaRef?.current?.scrollHeight > 400 ? 'auto' : 'hidden'
+        }`;
    }
  }, [content]);

@@ -280,103 +321,119 @@ export const ChatInput = ({
          )}

        <div className="relative mx-2 flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] dark:border-gray-900/50 dark:bg-[#40414F] dark:text-white dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] sm:mx-4">
-          <button
-            className="absolute left-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
-            onClick={() => setShowPluginSelect(!showPluginSelect)}
-            onKeyDown={(e) => {}}
-          >
-            {plugin ? <IconBrandGoogle size={20} /> : <IconBolt size={20} />}
-          </button>
+          <ImagePreview images={images} />

-          {showPluginSelect && (
-            <div className="absolute left-0 bottom-14 rounded bg-white dark:bg-[#343541]">
-              <PluginSelect
-                plugin={plugin}
-                onKeyDown={(e: any) => {
-                  if (e.key === 'Escape') {
-                    e.preventDefault();
+          <div className="relative mx-0 flex w-full flex-grow flex-col rounded-md border border-black/10 bg-white shadow-[0_0_10px_rgba(0,0,0,0.10)] dark:border-gray-900/50 dark:bg-[#40414F] dark:text-white dark:shadow-[0_0_15px_rgba(0,0,0,0.10)] sm:mx-0">
+            <button
+              className="absolute left-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={() => setShowPluginSelect(!showPluginSelect)}
+              onKeyDown={(e) => { }}
+            >
+              {plugin ? <IconBrandGoogle size={20} /> : <IconBolt size={20} />}
+            </button>
+
+            {showPluginSelect && (
+              <div className="absolute left-0 bottom-14 rounded bg-white dark:bg-[#343541]">
+                <PluginSelect
+                  plugin={plugin}
+                  onKeyDown={(e: any) => {
+                    if (e.key === 'Escape') {
+                      e.preventDefault();
+                      setShowPluginSelect(false);
+                      textareaRef.current?.focus();
+                    }
+                  }}
+                  onPluginChange={(plugin: Plugin) => {
+                    setPlugin(plugin);
                    setShowPluginSelect(false);
-                    textareaRef.current?.focus();
-                  }
-                }}
-                onPluginChange={(plugin: Plugin) => {
-                  setPlugin(plugin);
-                  setShowPluginSelect(false);

-                  if (textareaRef && textareaRef.current) {
-                    textareaRef.current.focus();
-                  }
-                }}
-              />
-            </div>
-          )}
-
-          <textarea
-            ref={textareaRef}
-            className="m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-10 text-black dark:bg-transparent dark:text-white md:py-3 md:pl-10"
-            style={{
-              resize: 'none',
-              bottom: `${textareaRef?.current?.scrollHeight}px`,
-              maxHeight: '400px',
-              overflow: `${
-                textareaRef.current && textareaRef.current.scrollHeight > 400
-                  ? 'auto'
-                  : 'hidden'
-              }`,
-            }}
-            placeholder={
-              t('Type a message or type "/" to select a prompt...') || ''
-            }
-            value={content}
-            rows={1}
-            onCompositionStart={() => setIsTyping(true)}
-            onCompositionEnd={() => setIsTyping(false)}
-            onChange={handleChange}
-            onKeyDown={handleKeyDown}
-          />
-
-          <button
-            className="absolute right-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
-            onClick={handleSend}
-          >
-            {messageIsStreaming ? (
-              <div className="h-4 w-4 animate-spin rounded-full border-t-2 border-neutral-800 opacity-60 dark:border-neutral-100"></div>
-            ) : (
-              <IconSend size={18} />
+                    if (textareaRef && textareaRef.current) {
+                      textareaRef.current.focus();
+                    }
+                  }}
+                />
+              </div>
            )}
-          </button>
-
-          {showScrollDownButton && (
-            <div className="absolute bottom-12 right-0 lg:bottom-0 lg:-right-10">
-              <button
-                className="flex h-7 w-7 items-center justify-center rounded-full bg-neutral-300 text-gray-800 shadow-md hover:shadow-lg focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-neutral-200"
-                onClick={onScrollDownClick}
-              >
-                <IconArrowDown size={18} />
-              </button>
-            </div>
-          )}
-
-          {showPromptList && filteredPrompts.length > 0 && (
-            <div className="absolute bottom-12 w-full">
-              <PromptList
-                activePromptIndex={activePromptIndex}
-                prompts={filteredPrompts}
-                onSelect={handleInitModal}
-                onMouseOver={setActivePromptIndex}
-                promptListRef={promptListRef}
-              />
-            </div>
-          )}
-
-          {isModalVisible && (
-            <VariableModal
-              prompt={filteredPrompts[activePromptIndex]}
-              variables={variables}
-              onSubmit={handleSubmit}
-              onClose={() => setIsModalVisible(false)}
+           <input
+              ref={imageInputRef}
+              type="file"
+              accept="image/*"
+              style={{ display: 'none' }}
+              onChange={handleImageInputChange}
            />
-          )}
+
+            <button
+              className="absolute left-10 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={handleImageInputClick}
+            >
+              <IconPhoto size={20} />
+            </button>
+
+            <textarea
+              ref={textareaRef}
+              className="m-0 w-full resize-none border-0 bg-transparent p-0 py-2 pr-8 pl-20 text-black dark:bg-transparent dark:text-white md:py-3 md:pl-20"
+              style={{
+                resize: 'none',
+                bottom: `${textareaRef?.current?.scrollHeight}px`,
+                maxHeight: '400px',
+                overflow: `${textareaRef.current && textareaRef.current.scrollHeight > 400
+                    ? 'auto'
+                    : 'hidden'
+                  }`,
+              }}
+              placeholder={
+                t('Type a message or type "/" to select a prompt...') || ''
+              }
+              value={content}
+              rows={1}
+              onCompositionStart={() => setIsTyping(true)}
+              onCompositionEnd={() => setIsTyping(false)}
+              onChange={handleChange}
+              onKeyDown={handleKeyDown}
+            />
+          <button
+              className="absolute right-2 top-2 rounded-sm p-1 text-neutral-800 opacity-60 hover:bg-neutral-200 hover:text-neutral-900 dark:bg-opacity-50 dark:text-neutral-100 dark:hover:text-neutral-200"
+              onClick={handleSend}
+            >
+              {messageIsStreaming ? (
+                <div className="h-4 w-4 animate-spin rounded-full border-t-2 border-neutral-800 opacity-60 dark:border-neutral-100"></div>
+              ) : (
+                <IconSend size={18} />
+              )}
+            </button>
+
+            {showScrollDownButton && (
+              <div className="absolute bottom-12 right-0 lg:bottom-0 lg:-right-10">
+                <button
+                  className="flex h-7 w-7 items-center justify-center rounded-full bg-neutral-300 text-gray-800 shadow-md hover:shadow-lg focus:outline-none focus:ring-2 focus:ring-blue-500 dark:bg-gray-700 dark:text-neutral-200"
+                  onClick={onScrollDownClick}
+                >
+                  <IconArrowDown size={18} />
+                </button>
+              </div>
+            )}
+
+            {showPromptList && filteredPrompts.length > 0 && (
+              <div className="absolute bottom-12 w-full">
+                <PromptList
+                  activePromptIndex={activePromptIndex}
+                  prompts={filteredPrompts}
+                  onSelect={handleInitModal}
+                  onMouseOver={setActivePromptIndex}
+                  promptListRef={promptListRef}
+                />
+              </div>
+            )}
+
+            {isModalVisible && (
+              <VariableModal
+                prompt={filteredPrompts[activePromptIndex]}
+                variables={variables}
+                onSubmit={handleSubmit}
+                onClose={() => setIsModalVisible(false)}
+              />
+            )}
+          </div>
        </div>
      </div>
      <div className="px-3 pt-2 pb-3 text-center text-[12px] text-black/50 dark:text-white/50 md:px-4 md:pt-3 md:pb-6">
@@ -40,6 +40,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
  const [isEditing, setIsEditing] = useState<boolean>(false);
  const [isTyping, setIsTyping] = useState<boolean>(false);
  const [messageContent, setMessageContent] = useState(message.content);
+  const [messageTextContent, setMessageTextContent] = useState(message.content[0].text ?? "");
  const [messagedCopied, setMessageCopied] = useState(false);

  const textareaRef = useRef<HTMLTextAreaElement>(null);
@@ -49,7 +50,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
  };

  const handleInputChange = (event: React.ChangeEvent<HTMLTextAreaElement>) => {
-    setMessageContent(event.target.value);
+    // messageContent[0].text = event.target.value;
+    // setMessageContent(messageContent);
+    setMessageTextContent(event.target.value);
    if (textareaRef.current) {
      textareaRef.current.style.height = 'inherit';
      textareaRef.current.style.height = `${textareaRef.current.scrollHeight}px`;
@@ -57,8 +60,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
  };

  const handleEditMessage = () => {
-    if (message.content != messageContent) {
+    if (message.content[0].text != messageTextContent) {
      if (selectedConversation && onEdit) {
+        messageContent[0].text = messageTextContent;
        onEdit({ ...message, content: messageContent });
      }
    }
@@ -104,7 +108,9 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
  const copyOnClick = () => {
    if (!navigator.clipboard) return;

-    navigator.clipboard.writeText(message.content).then(() => {
+    var content = message.content;
+    var textContent = content.filter(c => c.type == "text").map(c => c.text).join();
+    navigator.clipboard.writeText(textContent).then(() => {
      setMessageCopied(true);
      setTimeout(() => {
        setMessageCopied(false);
@@ -113,7 +119,8 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
  };

  useEffect(() => {
-    setMessageContent(message.content);
+    // setMessageContent(message.content);
+    setMessageTextContent(message.content[0].text ?? "");
  }, [message.content]);


@@ -150,7 +157,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                  <textarea
                    ref={textareaRef}
                    className="w-full resize-none whitespace-pre-wrap border-none dark:bg-[#343541]"
-                    value={messageContent}
+                    value={messageTextContent}
                    onChange={handleInputChange}
                    onKeyDown={handlePressEnter}
                    onCompositionStart={() => setIsTyping(true)}
@@ -169,14 +176,15 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                    <button
                      className="h-[40px] rounded-md bg-blue-500 px-4 py-1 text-sm font-medium text-white enabled:hover:bg-blue-600 disabled:opacity-50"
                      onClick={handleEditMessage}
-                      disabled={messageContent.trim().length <= 0}
+                      disabled={messageTextContent.length <= 0}
                    >
                      {t('Save & Submit')}
                    </button>
                    <button
                      className="h-[40px] rounded-md border border-neutral-300 px-4 py-1 text-sm font-medium text-neutral-700 hover:bg-neutral-100 dark:border-neutral-700 dark:text-neutral-300 dark:hover:bg-neutral-800"
                      onClick={() => {
-                        setMessageContent(message.content);
+                        // setMessageContent(message.content);
+                        setMessageTextContent(message.content[0].text ?? "");
                        setIsEditing(false);
                      }}
                    >
@@ -186,7 +194,14 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                </div>
              ) : (
                <div className="prose whitespace-pre-wrap dark:prose-invert flex-1">
-                  {message.content}
+                    <div>
+                    {messageTextContent}
+                  </div>
+                  <div className="flex flex-wrap justify-center">  
+                    {messageContent.filter(c => c.type === "image_url").map((c, index) => (  
+                      <img key={index} src={c.image_url?.url} alt="Message Content" className="max-w-full h-auto my-2" style={{objectFit: "contain"}} />  
+                    ))}  
+                  </div>
                </div>
              )}

@@ -261,7 +276,7 @@ export const ChatMessage: FC<Props> = memo(({ message, messageIndex, onEdit }) =
                  },
                }}
              >
-                {`${message.content}${
+                {`${messageTextContent}${
                  messageIsStreaming && messageIndex == (selectedConversation?.messages.length ?? 0) - 1 ? '`▍`' : ''
                }`}
              </MemoizedReactMarkdown>
@@ -41,7 +41,7 @@ const handler = async (req: Request): Promise<Response> => {

    for (let i = messages.length - 1; i >= 0; i--) {
      const message = messages[i];
-      const tokens = encoding.encode(message.content);
+      const tokens = encoding.encode(message.content[0].text ?? "");

      if (tokenCount + tokens.length + 1000 > model.tokenLimit) {
        break;
@@ -55,6 +55,7 @@ const handler = async (req: Request): Promise<Response> => {
    const stream = await OpenAIStream(model, promptToSend, temperatureToUse, key, messagesToSend);

    var resp = new Response(stream);
+    return resp;

    // let proxy services like nginx or argo tunnel know about pass the chunk immediately
    // similar to nginx option `proxy_buffering off;`
@@ -2,7 +2,17 @@ import { OpenAIModel } from './openai';

 export interface Message {
  role: Role;
-  content: string;
+  content: Content[];
+}
+
+export interface Content{
+  type: string;
+  text?: string;
+  image_url?: ImageUrl;
+}
+
+export interface ImageUrl{
+  url: string;
 }

 export type Role = 'assistant' | 'user';
@@ -13,7 +13,8 @@ export enum OpenAIModelID {
  GPT_3_5_16K = 'gpt-3.5-turbo-16k',
  GPT_4 = 'gpt-4',
  GPT_4_32K = 'gpt-4-32k',
-  GPT_4_TURBO = 'gpt-4-1106-preview'
+  GPT_4_TURBO = 'gpt-4-1106-preview',
+  GPT_4_TURBO_VISION = 'gpt-4-vision-preview'
 }

 // in case the `DEFAULT_MODEL` environment variable is not set or set to an unsupported model
@@ -56,4 +57,10 @@ export const OpenAIModels: Record<OpenAIModelID, OpenAIModel> = {
    maxLength: 384000,
    tokenLimit: 128000,
  },
+  [OpenAIModelID.GPT_4_TURBO_VISION]: {
+    id: OpenAIModelID.GPT_4_TURBO_VISION,
+    name: 'GPT-4-TURBO-VISION',
+    maxLength: 384000,
+    tokenLimit: 128000,
+  },
 };
@@ -53,11 +53,11 @@ export const OpenAIStream = async (
      messages: [
        {
          role: 'system',
-          content: systemPrompt,
+          content: [{type:"text", text: systemPrompt}],
        },
        ...messages,
      ],
-      max_tokens: 1000,
+      max_tokens: 4096,
      temperature: temperature,
      stream: true,
    }),