Fix performance issues fro streaming chat

src/lib/Chat.svelte

@@ -9,7 +9,13 @@
     showSetChatSettings,
     submitExitingPromptsNow,
     continueMessage,
-    getMessage
+    getMessage,
+
+    currentChatId,
+
+    currentChatMessages
+
+
   } from './Storage.svelte'
   import {
     type Message,
@@ -82,9 +88,9 @@
         submitForm(false, true)
       }
       if ($continueMessage) {
-        const message = getMessage(chat, $continueMessage)
+        const message = getMessage(chatId, $continueMessage)
         $continueMessage = ''
-        if (message && chat.messages.indexOf(message) === (chat.messages.length - 1)) {
+        if (message && $currentChatMessages.indexOf(message) === ($currentChatMessages.length - 1)) {
           submitForm(lastSubmitRecorded, true, message)
         }
       }
@@ -106,6 +112,9 @@
   onMount(async () => {
     if (!chat) return
 
+    $currentChatId = chatId
+    $currentChatMessages = chat.messages
+
     chatRequest = new ChatRequest()
     chatRequest.setChat(chat)
     // Focus the input on mount
@@ -164,9 +173,9 @@
   const addNewMessage = () => {
     if (chatRequest.updating) return
     let inputMessage: Message
-    const lastMessage = chat.messages[chat.messages.length - 1]
+    const lastMessage = $currentChatMessages[$currentChatMessages.length - 1]
     const uuid = uuidv4()
-    if (chat.messages.length === 0) {
+    if ($currentChatMessages.length === 0) {
       inputMessage = { role: 'system', content: input.value, uuid }
     } else if (lastMessage && lastMessage.role === 'user') {
       inputMessage = { role: 'assistant', content: input.value, uuid }
@@ -220,8 +229,8 @@
         // Compose the input message
         const inputMessage: Message = { role: 'user', content: input.value, uuid: uuidv4() }
         addMessage(chatId, inputMessage)
-      } else if (!fillMessage && chat.messages.length && chat.messages[chat.messages.length - 1].finish_reason === 'length') {
+      } else if (!fillMessage && $currentChatMessages.length && $currentChatMessages[$currentChatMessages.length - 1].finish_reason === 'length') {
-        fillMessage = chat.messages[chat.messages.length - 1]
+        fillMessage = $currentChatMessages[$currentChatMessages.length - 1]
       }
   
       // Clear the input value
@@ -237,7 +246,7 @@
     chatRequest.updatingMessage = ''
 
     try {
-      const response = await chatRequest.sendRequest(chat.messages, {
+      const response = await chatRequest.sendRequest($currentChatMessages, {
         chat,
         autoAddMessages: true, // Auto-add and update messages in array
         streaming: chatSettings.stream,
@@ -268,7 +277,7 @@
       uuid: uuidv4()
     }
 
-    const suggestMessages = chat.messages.slice(0, 10) // limit to first 10 messages
+    const suggestMessages = $currentChatMessages.slice(0, 10) // limit to first 10 messages
     suggestMessages.push(suggestMessage)
 
     const response = await chatRequest.sendRequest(suggestMessages, {
@@ -346,7 +355,7 @@
   </div>
 </nav>
 
-<Messages messages={chat.messages} chatId={chatId} />
+<Messages messages={$currentChatMessages} chatId={chatId} />
 
 {#if chatRequest.updating === true}
   <article class="message is-success assistant-message">
@@ -357,7 +366,7 @@
   </article>
 {/if}
 
-{#if chat.messages.length === 0 || (chat.messages.length === 1 && chat.messages[0].role === 'system')}
+{#if $currentChatMessages.length === 0 || ($currentChatMessages.length === 1 && $currentChatMessages[0].role === 'system')}
   <Prompts bind:input />
 {/if}
 </div>

src/lib/ChatRequest.svelte

@@ -3,12 +3,13 @@
     import { mergeProfileFields, prepareSummaryPrompt } from './Profiles.svelte'
     import { countMessageTokens, countPromptTokens, getModelMaxTokens } from './Stats.svelte'
     import type { Chat, ChatCompletionOpts, ChatSettings, Message, Model, Request, RequestImageGeneration } from './Types.svelte'
-    import { deleteMessage, getChatSettingValueNullDefault, insertMessages, saveChatStore, getApiKey, addError } from './Storage.svelte'
+    import { deleteMessage, getChatSettingValueNullDefault, insertMessages, getApiKey, addError, currentChatMessages, getMessages, updateMessages } from './Storage.svelte'
     import { scrollToBottom, scrollToMessage } from './Util.svelte'
     import { getRequestSettingList, defaultModel } from './Settings.svelte'
     import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
     import { getApiBase, getEndpointCompletions, getEndpointGenerations } from './ApiUtil.svelte'
     import { v4 as uuidv4 } from 'uuid'
+    import { get } from 'svelte/store'
 
 export class ChatRequest {
       constructor () {
@@ -317,7 +318,7 @@ export class ChatRequest {
         const maxTokens = getModelMaxTokens(model) // max tokens for model
 
         const continueRequest = async () => {
-          return await _this.sendRequest(chat.messages, {
+          return await _this.sendRequest(getMessages(chatId), {
             ...opts,
             didSummary: true
           }, overrides)
@@ -358,7 +359,7 @@ export class ChatRequest {
             promptSize = countPromptTokens(top.concat(rw), model) + countPadding
           }
           // Run a new request, now with the rolled messages hidden
-          return await _this.sendRequest(chat.messages, {
+          return await _this.sendRequest(get(currentChatMessages), {
             ...opts,
             didSummary: true // our "summary" was simply dropping some messages
           }, overrides)
@@ -466,11 +467,11 @@ export class ChatRequest {
           summaryResponse.summary = summarizedIds
           // Disable the messages we summarized so they still show in history
           rw.forEach((m, i) => { m.summarized = summaryIds })
-          saveChatStore()
+          updateMessages(chatId)
           // Re-run request with summarized prompts
           _this.updatingMessage = 'Continuing...'
           scrollToBottom(true)
-          return await _this.sendRequest(chat.messages, {
+          return await _this.sendRequest(get(currentChatMessages), {
             ...opts,
             didSummary: true
           },

src/lib/EditMessage.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import Code from './Code.svelte'
   import { afterUpdate, createEventDispatcher, onMount } from 'svelte'
-  import { deleteMessage, chatsStorage, deleteSummaryMessage, truncateFromMessage, submitExitingPromptsNow, saveChatStore, continueMessage } from './Storage.svelte'
+  import { deleteMessage, chatsStorage, deleteSummaryMessage, truncateFromMessage, submitExitingPromptsNow, continueMessage, updateMessages } from './Storage.svelte'
   import { getPrice } from './Stats.svelte'
   import SvelteMarkdown from 'svelte-markdown'
   import type { Message, Model, Chat } from './Types.svelte'
@@ -178,7 +178,7 @@
       return
     }
     message.suppress = value
-    saveChatStore()
+    updateMessages(chatId)
   }
 
   const downloadImage = () => {

src/lib/Profiles.svelte

@@ -2,7 +2,7 @@
   import { getChatDefaults, getExcludeFromProfile } from './Settings.svelte'
   import { get, writable } from 'svelte/store'
   // Profile definitions
-  import { addMessage, clearMessages, deleteMessage, getChat, getChatSettings, getCustomProfiles, getGlobalSettings, newName, resetChatSettings, saveChatStore, setGlobalSettingValueByKey, updateProfile } from './Storage.svelte'
+  import { addMessage, clearMessages, deleteMessage, getChat, getChatSettings, getCustomProfiles, getGlobalSettings, getMessages, newName, resetChatSettings, saveChatStore, setGlobalSettingValueByKey, setMessages, updateProfile } from './Storage.svelte'
   import type { Message, SelectOption, ChatSettings } from './Types.svelte'
   import { v4 as uuidv4 } from 'uuid'
 
@@ -90,15 +90,15 @@ export const prepareSummaryPrompt = (chatId:number, maxTokens:number) => {
 }
 
 export const setSystemPrompt = (chatId: number) => {
-    const chat = getChat(chatId)
+    const messages = getMessages(chatId)
     const systemPromptMessage:Message = {
       role: 'system',
       content: prepareProfilePrompt(chatId),
       uuid: uuidv4()
     }
-    if (chat.messages[0]?.role === 'system') deleteMessage(chatId, chat.messages[0].uuid)
+    if (messages[0]?.role === 'system') deleteMessage(chatId, messages[0].uuid)
-    chat.messages.unshift(systemPromptMessage)
+    messages.unshift(systemPromptMessage)
-    saveChatStore()
+    setMessages(chatId, messages.filter(m => true))
 }
 
 // Restart currently loaded profile

src/lib/Storage.svelte

@@ -19,6 +19,8 @@
   export let submitExitingPromptsNow = writable(false) // for them to go now.  Will not submit anything in the input
   export let pinMainMenu = writable(false) // Show menu (for mobile use)
   export let continueMessage = writable('') //
+  export let currentChatMessages = writable([] as Message[])
+  export let currentChatId = writable(0)
 
   const chatDefaults = getChatDefaults()
 
@@ -215,70 +217,84 @@
     chatsStorage.set(chats)
   }
 
+  export const getMessages = (chatId: number): Message[] => {
+    if (get(currentChatId) === chatId) return get(currentChatMessages)
+    return getChat(chatId).messages
+  }
+
+  let setMessagesTimer: any
+  export const setMessages = (chatId: number, messages: Message[]) => {
+    if (get(currentChatId) === chatId) {
+      // update current message cache right away
+      currentChatMessages.set(messages)
+      clearTimeout(setMessagesTimer)
+      // delay expensive all chats update for a bit
+      setMessagesTimer = setTimeout(() => {
+        getChat(chatId).messages = messages
+        saveChatStore()
+      }, 100)
+    } else {
+      getChat(chatId).messages = messages
+      saveChatStore()
+    }
+  }
+
+  export const updateMessages = (chatId: number) => {
+    setMessages(chatId, getMessages(chatId))
+  }
+
   export const addError = (chatId: number, error: string) => {
     addMessage(chatId, { content: error } as Message)
   }
 
   export const addMessage = (chatId: number, message: Message) => {
-    const chats = get(chatsStorage)
+    const messages = getMessages(chatId)
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
     if (!message.uuid) message.uuid = uuidv4()
-    if (chat.messages.indexOf(message) < 0) {
+    if (messages.indexOf(message) < 0) {
       // Don't have message, add it
-      chat.messages.push(message)
+      messages[messages.length] = message
     }
-    chatsStorage.set(chats)
+    setMessages(chatId, messages)
   }
 
-  export const getMessages = (chatId: number):Message[] => {
+  export const getMessage = (chatId: number, uuid:string):Message|undefined => {
-    const chats = get(chatsStorage)
+    return getMessages(chatId).find((m) => m.uuid === uuid)
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
-    return chat.messages
-  }
-
-  export const getMessage = (chat: Chat, uuid:string):Message|undefined => {
-    return chat.messages.find((m) => m.uuid === uuid)
   }
 
   export const insertMessages = (chatId: number, insertAfter: Message, newMessages: Message[]) => {
-    const chats = get(chatsStorage)
+    const messages = getMessages(chatId)
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
+    const index = messages.findIndex((m) => m.uuid === insertAfter.uuid)
-    const index = chat.messages.findIndex((m) => m.uuid === insertAfter.uuid)
     if (index === undefined || index < 0) {
       console.error("Couldn't insert after message:", insertAfter)
       return
     }
     newMessages.forEach(m => { m.uuid = m.uuid || uuidv4() })
-    chat.messages.splice(index + 1, 0, ...newMessages)
+    messages.splice(index + 1, 0, ...newMessages)
-    chatsStorage.set(chats)
+    setMessages(chatId, messages.filter(m => true))
   }
 
   export const deleteSummaryMessage = (chatId: number, uuid: string) => {
-    const chats = get(chatsStorage)
+    const message = getMessage(chatId, uuid)
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
-    const message = getMessage(chat, uuid)
     if (message && message.summarized) throw new Error('Unable to delete summarized message')
     if (message && message.summary) { // messages we summarized
       message.summary.forEach(sid => {
-        const m = getMessage(chat, sid)
+        const m = getMessage(chatId, sid)
         if (m) {
           delete m.summarized // unbind to this summary
         }
       })
       delete message.summary
     }
-    chatsStorage.set(chats)
+    updateMessages(chatId)
     deleteMessage(chatId, uuid)
   }
 
   export const deleteMessage = (chatId: number, uuid: string) => {
-    const chats = get(chatsStorage)
+    const messages = getMessages(chatId)
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
+    const index = messages.findIndex((m) => m.uuid === uuid)
-    const index = chat.messages.findIndex((m) => m.uuid === uuid)
+    const message = getMessage(chatId, uuid)
-    const message = getMessage(chat, uuid)
     if (message?.summarized) throw new Error('Unable to delete summarized message')
     if (message?.summary) throw new Error('Unable to directly delete message summary')
-    // const found = chat.messages.filter((m) => m.uuid === uuid)
     if (index < 0) {
       console.error(`Unable to find and delete message with ID: ${uuid}`)
       return
@@ -287,8 +303,8 @@
       deleteImage(chatId, message.image.id)
     }
     // console.warn(`Deleting message with ID: ${uuid}`, found, index)
-    chat.messages.splice(index, 1) // remove item
+    messages.splice(index, 1) // remove item
-    chatsStorage.set(chats)
+    setMessages(chatId, messages.filter(m => true))
   }
 
   const clearImages = (chatId: number, messages: Message[]) => {
@@ -298,38 +314,33 @@
   }
 
   export const truncateFromMessage = (chatId: number, uuid: string) => {
-    const chats = get(chatsStorage)
+    const messages = getMessages(chatId)
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
+    const index = messages.findIndex((m) => m.uuid === uuid)
-    const index = chat.messages.findIndex((m) => m.uuid === uuid)
+    const message = getMessage(chatId, uuid)
-    const message = getMessage(chat, uuid)
     if (message && message.summarized) throw new Error('Unable to truncate from a summarized message')
-    // const found = chat.messages.filter((m) => m.uuid === uuid)
     if (index < 0) {
       throw new Error(`Unable to find message with ID: ${uuid}`)
     }
-    const truncated = chat.messages.splice(index + 1) // remove every item after
+    const truncated = messages.splice(index + 1) // remove every item after
     clearImages(chatId, truncated)
-    chatsStorage.set(chats)
+    setMessages(chatId, messages.filter(m => true))
   }
 
   export const clearMessages = (chatId: number) => {
-    const chats = get(chatsStorage)
+    clearImages(chatId, getMessages(chatId))
-    const chat = chats.find((chat) => chat.id === chatId) as Chat
+    setMessages(chatId, [])
-    clearImages(chatId, chat.messages)
-    chat.messages = []
-    chatsStorage.set(chats)
   }
 
   export const deleteChat = (chatId: number) => {
     const chats = get(chatsStorage)
-    const chat = getChat(chatId)
+    clearImages(chatId, getMessages(chatId) || [])
-    clearImages(chatId, chat?.messages || [])
     chatsStorage.set(chats.filter((chat) => chat.id !== chatId))
   }
 
   export const updateChatImages = async (chatId: number, chat: Chat) => {
-    for (let i = 0; i < chat.messages.length; i++) {
+    const messages = getMessages(chatId)
-      const m = chat.messages[i]
+    for (let i = 0; i < messages.length; i++) {
+      const m = messages[i]
       if (m.image) m.image = await setImage(chatId, m.image)
     }
   }