Allow automatic extension of truncated summary

src/lib/ChatRequest.svelte

@@ -59,8 +59,6 @@ export class ChatRequest {
         const promptTokenCount = countPromptTokens(messagePayload, model)
         const maxAllowed = maxTokens - (promptTokenCount + 1)
     
-        // Build and make the request
-        try {
         // Build the API request body
         const request: Request = {
           model: chatSettings.model,
@@ -94,14 +92,14 @@ export class ChatRequest {
           stream: opts.streaming
         }
 
+        // Set-up and make the request
+        try {
           // Add out token count to the response handler
           // (streaming doesn't return counts, so we need to do it client side)
           chatResponse.setPromptTokenCount(promptTokenCount)
 
           const signal = _this.controller.signal
     
-          // console.log('apikey', $apiKeyStorage)
-
           const fetchOptions = {
             method: 'POST',
             headers: {
@@ -297,6 +295,7 @@ export class ChatRequest {
            */
     
           const bottom = rw.slice(0 - pinBottom)
+          let continueCounter = chatSettings.summaryExtend + 1
           rw = rw.slice(0, 0 - pinBottom)
           let reductionPoolSize = countPromptTokens(rw, model)
           const ss = chatSettings.summarySize
@@ -340,6 +339,9 @@ export class ChatRequest {
 
           // Request and load the summarization prompt
           _this.updatingMessage = 'Summarizing...'
+          const summarizedIds = rw.map(m => m.uuid)
+          const summaryIds = [summaryResponse.uuid]
+          while (continueCounter-- > 0) {
             try {
               const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), {
                 summaryRequest: true,
@@ -351,8 +353,8 @@ export class ChatRequest {
                   if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true)
                 }
               } as ChatCompletionOpts, {
-              temperature: 0, // make summary more deterministic
+                temperature: 0.1, // make summary more deterministic
-              top_p: 0.5,
+                top_p: 1,
                 presence_penalty: 0,
                 frequency_penalty: 0,
                 ...overrides
@@ -360,26 +362,24 @@ export class ChatRequest {
               // Wait for the response to complete
               if (!summary.hasFinished()) await summary.promiseToFinish()
               if (summary.hasError()) {
-            // Failed to some API issue. let the original caller handle it.
+                // Failed for some API issue. let the original caller handle it.
-              deleteMessage(chatId, summaryResponse.uuid)
+                _this.updating = false
+                _this.updatingMessage = ''
+                deleteMessage(chatId, srid)
                 return summary
-            } else {
+              }
               // Looks like we got our summarized messages.
               // Mark the new summaries as such
-              summaryResponse.summary = rw.map(m => m.uuid)
+              // Need more?
-              const summaryIds = [summaryResponse.uuid]
+              if (summaryResponse.finish_reason === 'length' && continueCounter > 0) {
-              // Disable the messages we summarized so they still show in history
+                // Our summary was truncated
-              rw.forEach((m, i) => { m.summarized = summaryIds })
+                // Try to get more of it
-              saveChatStore()
+                delete summaryResponse.finish_reason
-              // Re-run request with summarized prompts
+                _this.updatingMessage = 'Summarizing more...'
-              // return { error: { message: "End for now" } } as Response
+                continue
-              _this.updatingMessage = 'Continuing...'
+              } else {
-              scrollToBottom(true)
+                // We're done
-              return await _this.sendRequest(chat.messages, {
+                continueCounter = 0
-                ...opts,
-                didSummary: true
-              },
-              overrides)
               }
             } catch (e) {
               _this.updating = false
@@ -387,6 +387,19 @@ export class ChatRequest {
               deleteMessage(chatId, srid)
               throw e
             }
+          }
+          summaryResponse.summary = summarizedIds
+          // Disable the messages we summarized so they still show in history
+          rw.forEach((m, i) => { m.summarized = summaryIds })
+          saveChatStore()
+          // Re-run request with summarized prompts
+          _this.updatingMessage = 'Continuing...'
+          scrollToBottom(true)
+          return await _this.sendRequest(chat.messages, {
+            ...opts,
+            didSummary: true
+          },
+          overrides)
         } else {
           /***************
            * Unknown mode.

src/lib/ChatSettingField.svelte

@@ -177,7 +177,7 @@
             placeholder={String(setting.placeholder || chatDefaults[setting.key])}
             on:change={e => queueSettingValueChange(e, setting)}
           />
-        {:else if setting.type === 'select'}
+        {:else if setting.type === 'select' || setting.type === 'select-number'}
           <!-- <div class="select"> -->
             <div class="select" class:control={fieldControls.length}>
             <select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } >

src/lib/Settings.svelte

@@ -60,7 +60,7 @@ const gptDefaults = {
   n: 1,
   stream: true,
   stop: null,
-  max_tokens: 500,
+  max_tokens: 512,
   presence_penalty: 0,
   frequency_penalty: 0,
   logit_bias: null,
@@ -77,6 +77,7 @@ const defaults:ChatSettings = {
   continuousChat: 'fifo',
   summaryThreshold: 3000,
   summarySize: 1000,
+  summaryExtend: 0,
   pinTop: 0,
   pinBottom: 6,
   summaryPrompt: '',
@@ -222,11 +223,23 @@ const summarySettings: ChatSetting[] = [
         name: 'Max Summary Size',
         title: 'Maximum number of tokens allowed for summary response.',
         min: 128,
-        max: 512,
+        max: 1024,
         step: 1,
         type: 'number',
         hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary'
       },
+      {
+        key: 'summaryExtend',
+        name: 'Summary Extend',
+        title: 'Number of times a truncated summary can be extended.',
+        type: 'select-number',
+        options: [
+          { value: 0, text: '0 - Summary must fit in first call.' },
+          { value: 1, text: '1 - Allow one extra API call to extend.' },
+          { value: 2, text: '2 - Allow two extra API calls to extend.' }
+        ],
+        hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary'
+      },
       {
         key: 'pinTop',
         name: 'Keep First Prompts',

src/lib/Storage.svelte

@@ -333,6 +333,7 @@
   export const cleanSettingValue = (type:string, value: any) => {
     switch (type) {
       case 'number':
+      case 'select-number':
         value = parseFloat(value)
         if (isNaN(value)) { value = null }
         return value

src/lib/Types.svelte

@@ -60,6 +60,7 @@
     continuousChat: (''|'fifo'|'summary');
     summaryThreshold: number;
     summarySize: number;
+    summaryExtend: number;
     pinTop: number;
     pinBottom: number;
     summaryPrompt: string;
@@ -141,7 +142,7 @@
   };
 
   export type SelectOption = {
-    value: string;
+    value: string|number;
     text: string;
   };
 
@@ -154,6 +155,11 @@ type SettingBoolean = {
     options: SelectOption[];
   };
 
+  export type SettingSelectNumber = {
+    type: 'select-number';
+    options: SelectOption[];
+  };
+
   export type SettingText = {
     type: 'text';
   };
@@ -199,7 +205,7 @@ type SettingBoolean = {
     fieldControls?: FieldControl[];
     beforeChange?: (chatId:number, setting:ChatSetting, value:any) => boolean;
     afterChange?: (chatId:number, setting:ChatSetting, value:any) => boolean;
-  } & (SettingNumber | SettingSelect | SettingBoolean | SettingText | SettingTextArea | SettingOther | SubSetting);
+  } & (SettingNumber | SettingSelect | SettingSelectNumber | SettingBoolean | SettingText | SettingTextArea | SettingOther | SubSetting);
 
 
   export type GlobalSetting = {