Fix summary size issue

src/lib/Chat.svelte

@@ -233,20 +233,31 @@
     }
     focusInput()
 
-    const response = await chatRequest.sendRequest(chat.messages, {
+    chatRequest.updating = true
-      chat,
+    chatRequest.updatingMessage = ''
-      autoAddMessages: true, // Auto-add and update messages in array
+
-      streaming: chatSettings.stream,
+    try {
-      fillMessage,
+      const response = await chatRequest.sendRequest(chat.messages, {
-      onMessageChange: (messages) => {
+        chat,
-        scrollToBottom(true)
+        autoAddMessages: true, // Auto-add and update messages in array
+        streaming: chatSettings.stream,
+        fillMessage,
+        onMessageChange: (messages) => {
+          scrollToBottom(true)
+        }
+      })
+      await response.promiseToFinish()
+      const message = response.getMessages()[0]
+      if (message) {
+        ttsStart(message.content, recorded)
       }
-    })
+    } catch (e) {
-    await response.promiseToFinish()
+      console.error(e)
-    const message = response.getMessages()[0]
-    if (message) {
-      ttsStart(message.content, recorded)
     }
+
+    chatRequest.updating = false
+    chatRequest.updatingMessage = ''
+
     focusInput()
   }
 

src/lib/ChatRequest.svelte

@@ -8,6 +8,7 @@
     import { getRequestSettingList, defaultModel } from './Settings.svelte'
     import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
     import { getApiBase, getEndpointCompletions } from './ApiUtil.svelte'
+    import { v4 as uuidv4 } from 'uuid'
 
 export class ChatRequest {
       constructor () {
@@ -302,16 +303,18 @@ export class ChatRequest {
           const getSS = ():number => (ss < 1 && ss > 0)
             ? Math.round(reductionPoolSize * ss) // If summarySize between 0 and 1, use percentage of reduced
             : Math.min(ss, reductionPoolSize * 0.5) // If > 1, use token count
-          let promptSummary = prepareSummaryPrompt(chatId, reductionPoolSize)
+          let maxSummaryTokens = getSS()
+          let promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
           const summaryRequest = { role: 'user', content: promptSummary } as Message
           let promptSummarySize = countMessageTokens(summaryRequest, model)
           // Make sure there is enough room to generate the summary, and try to make sure
           // the last prompt is a user prompt as that seems to work better for summaries
-          while ((reductionPoolSize + promptSummarySize + getSS()) >= maxTokens ||
+          while ((reductionPoolSize + promptSummarySize + maxSummaryTokens) >= maxTokens ||
               (reductionPoolSize >= 100 && rw[rw.length - 1]?.role !== 'user')) {
             bottom.unshift(rw.pop() as Message)
             reductionPoolSize = countPromptTokens(rw, model)
-            promptSummary = prepareSummaryPrompt(chatId, reductionPoolSize)
+            maxSummaryTokens = getSS()
+            promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
             summaryRequest.content = promptSummary
             promptSummarySize = countMessageTokens(summaryRequest, model)
           }
@@ -321,13 +324,15 @@ export class ChatRequest {
           }
 
           // Create a message the summary will be loaded into
-          const summaryResponse = {
+          const srid = uuidv4()
+          const summaryResponse:Message = {
             role: 'assistant',
             content: '',
+            uuid: srid,
             streaming: opts.streaming,
             summary: [] as string[],
             model
-          } as Message
+          }
 
           // Insert summary completion prompt after that last message we're summarizing
           insertMessages(chatId, rw[rw.length - 1], [summaryResponse])
@@ -335,44 +340,51 @@ export class ChatRequest {
 
           // Request and load the summarization prompt
           _this.updatingMessage = 'Summarizing...'
-          const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), {
+          try {
-            summaryRequest: true,
+            const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), {
-            streaming: opts.streaming,
+              summaryRequest: true,
-            maxTokens: chatSettings.summarySize,
+              streaming: opts.streaming,
-            fillMessage: summaryResponse,
+              maxTokens: maxSummaryTokens,
-            autoAddMessages: true,
+              fillMessage: summaryResponse,
-            onMessageChange: (m) => {
+              autoAddMessages: true,
-              if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true)
+              onMessageChange: (m) => {
-            }
+                if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true)
-          } as ChatCompletionOpts, {
+              }
-            temperature: 0, // make summary more deterministic
+            } as ChatCompletionOpts, {
-            top_p: 0.5,
+              temperature: 0, // make summary more deterministic
-            presence_penalty: 0,
+              top_p: 0.5,
-            frequency_penalty: 0,
+              presence_penalty: 0,
-            ...overrides
+              frequency_penalty: 0,
-          } as ChatSettings)
+              ...overrides
-          // Wait for the response to complete
+            } as ChatSettings)
-          if (!summary.hasFinished()) await summary.promiseToFinish()
+            // Wait for the response to complete
-          if (summary.hasError()) {
+            if (!summary.hasFinished()) await summary.promiseToFinish()
+            if (summary.hasError()) {
             // Failed to some API issue. let the original caller handle it.
-            deleteMessage(chatId, summaryResponse.uuid)
+              deleteMessage(chatId, summaryResponse.uuid)
-            return summary
+              return summary
-          } else {
+            } else {
             // Looks like we got our summarized messages.
             // Mark the new summaries as such
-            summaryResponse.summary = rw.map(m => m.uuid)
+              summaryResponse.summary = rw.map(m => m.uuid)
-            const summaryIds = [summaryResponse.uuid]
+              const summaryIds = [summaryResponse.uuid]
-            // Disable the messages we summarized so they still show in history
+              // Disable the messages we summarized so they still show in history
-            rw.forEach((m, i) => { m.summarized = summaryIds })
+              rw.forEach((m, i) => { m.summarized = summaryIds })
-            saveChatStore()
+              saveChatStore()
-            // Re-run request with summarized prompts
+              // Re-run request with summarized prompts
-            // return { error: { message: "End for now" } } as Response
+              // return { error: { message: "End for now" } } as Response
-            _this.updatingMessage = 'Continuing...'
+              _this.updatingMessage = 'Continuing...'
-            scrollToBottom(true)
+              scrollToBottom(true)
-            return await _this.sendRequest(chat.messages, {
+              return await _this.sendRequest(chat.messages, {
-              ...opts,
+                ...opts,
-              didSummary: true
+                didSummary: true
-            })
+              })
+            }
+          } catch (e) {
+            _this.updating = false
+            _this.updatingMessage = ''
+            deleteMessage(chatId, srid)
+            throw e
           }
         } else {
           /***************