diff --git a/src/app.scss b/src/app.scss index e989b74..5f45928 100644 --- a/src/app.scss +++ b/src/app.scss @@ -529,7 +529,11 @@ aside.menu.main-menu .menu-expanse { .chat-message.summarized { opacity: 0.6; } +.chat-message.suppress .message-body { + opacity: 0.4; +} .tool-drawer, .tool-drawer-mask { + z-index: 1; position: absolute; visibility: hidden; width: 0%; diff --git a/src/lib/Chat.svelte b/src/lib/Chat.svelte index 987334a..559460c 100644 --- a/src/lib/Chat.svelte +++ b/src/lib/Chat.svelte @@ -154,8 +154,10 @@ let response: Response + const messageFilter = (m) => !m.suppress && m.role !== 'error' && m.content && !m.summarized + // Submit only the role and content of the messages, provide the previous messages as well for context - const filtered = messages.filter((message) => message.role !== 'error' && message.content && !message.summarized) + let filtered = messages.filter(messageFilter) // Get an estimate of the total prompt size we're sending const promptTokenCount:number = countPromptTokens(filtered, model) @@ -176,7 +178,8 @@ const systemPad = (filtered[0] || {} as Message).role === 'system' ? 1 : 0 const mlen = filtered.length - systemPad // always keep system prompt let diff = mlen - (pinTop + pinBottom) - while (diff <= 3 && (pinTop > 0 || pinBottom > 1)) { + const useRollMode = !prepareSummaryPrompt(chatId, 0) + while (!useRollMode && diff <= 3 && (pinTop > 0 || pinBottom > 1)) { // Not enough prompts exposed to summarize // try to open up pinTop and pinBottom to see if we can get more to summarize if (pinTop === 1 && pinBottom > 1) { @@ -188,7 +191,7 @@ } diff = mlen - (pinTop + pinBottom) } - if (diff > 0) { + if (!useRollMode && diff > 0) { // We've found at least one prompt we can try to summarize // Reduce to prompts we'll send in for summary // (we may need to update this to not include the pin-top, but the context it provides seems to help in the accuracy of the summary) @@ -197,6 +200,7 @@ let sourceTokenCount = countPromptTokens(summarize, model) // build summary prompt message let summaryPrompt = prepareSummaryPrompt(chatId, sourceTokenCount) + const summaryMessage = { role: 'user', content: summaryPrompt @@ -279,8 +283,23 @@ } else if (sourceTokenCount <= 20) { addMessage(chatId, { role: 'error', content: 'Unable to summarize. Not enough words in past content to summarize.', uuid: uuidv4() }) } - } else { + } else if (!useRollMode && diff < 1) { addMessage(chatId, { role: 'error', content: 'Unable to summarize. Not enough messages in past content to summarize.', uuid: uuidv4() }) + } else { + // roll-off mode + const top = filtered.slice(0,pinTop+systemPad) + let rollaway = filtered.slice(pinTop+systemPad) + let promptTokenCount = countPromptTokens(top.concat(rollaway), model) + // suppress messages we're rolling off + while (rollaway.length > (((promptTokenCount + (chatSettings.max_tokens||1)) > maxTokens) ? pinBottom||1 : 1) + && promptTokenCount >= chatSettings.summaryThreshold) { + const rollOff = rollaway.shift() + if (rollOff) rollOff.suppress = true + promptTokenCount = countPromptTokens(top.concat(rollaway), model) + } + saveChatStore() + // get a new list now excluding them + filtered = messages.filter(messageFilter) } } diff --git a/src/lib/EditMessage.svelte b/src/lib/EditMessage.svelte index ad05626..68039a1 100644 --- a/src/lib/EditMessage.svelte +++ b/src/lib/EditMessage.svelte @@ -1,12 +1,12 @@ {#key message.uuid} @@ -168,6 +190,7 @@ class:user-message={message.role === 'user' || message.role === 'system'} class:assistant-message={message.role === 'error' || message.role === 'assistant'} class:summarized={message.summarized} + class:suppress={message.suppress} class:editing={editing} >
@@ -244,20 +267,38 @@ {/if} {#if !message.summarized} - { - checkTruncate() - }} - > - {#if waitingForTruncateConfirm} - - {:else} - + { + checkTruncate() + }} + > + {#if waitingForTruncateConfirm} + + {:else} + + {/if} + {/if} - + {#if !message.summarized && message.role !== 'system'} + { + setSuppress(!message.suppress) + }} + > + {#if waitingForSuppressConfirm} + + {:else if message.suppress} + + {:else} + + {/if} + {/if}
diff --git a/src/lib/Messages.svelte b/src/lib/Messages.svelte index 8be7c81..0ae9022 100644 --- a/src/lib/Messages.svelte +++ b/src/lib/Messages.svelte @@ -13,7 +13,7 @@ {#each messages as message, i} - {#if !(message.summarized && $globalStorage.hideSummarized) && !(i === 0 && message.role === 'system' && !chatSettings.useSystemPrompt)} + {#if !((message.summarized) && $globalStorage.hideSummarized) && !(i === 0 && message.role === 'system' && !chatSettings.useSystemPrompt)} {/if} {/each} diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte index 8dcec01..6da05eb 100644 --- a/src/lib/Settings.svelte +++ b/src/lib/Settings.svelte @@ -100,7 +100,10 @@ const profileSetting: ChatSetting & SettingSelect = { afterChange: (chatId, setting) => { applyProfile(chatId, '', !getChat(chatId).sessionStarted) return true // Signal we should refresh the setting modal - } + }, + setDefault: (chatId, setting, value) => { + + }, } // Settings that will not be part of the API request @@ -159,16 +162,16 @@ const nonRequestSettings: ChatSetting[] = [ }, { key: 'useSummarization', - name: 'Enable Auto Summarize', - header: 'Continuous Chat - Summarization', + name: 'Enable Continuous Chat', + header: 'Continuous Chat - (Summarize or FIFO)', headerClass: 'is-info', - title: 'When out of token space, summarize past tokens and keep going.', + title: 'When out of token space, summarize or remove past prompts and keep going.', type: 'boolean' }, { key: 'summaryThreshold', - name: 'Summary Threshold', - title: 'When prompt history breaks this threshold, past prompts will be summarized to create space. 0 to disable.', + name: 'Token Threshold', + title: 'When prompt history breaks this threshold, past prompts will be summarized or rolled off to create space.', min: 0, max: 32000, step: 1, @@ -178,7 +181,7 @@ const nonRequestSettings: ChatSetting[] = [ { key: 'summarySize', name: 'Max Summary Size', - title: 'Maximum number of tokens to use for summarization response.', + title: 'Maximum number of tokens allowed for summary response.', min: 128, max: 2048, step: 1, @@ -187,7 +190,7 @@ const nonRequestSettings: ChatSetting[] = [ }, { key: 'pinTop', - name: 'Keep First Prompts During Summary', + name: 'Keep First Prompts', title: 'When we run out of space and need to summarize prompts, the top number of prompts will not be removed after summarization.', min: 0, max: 4, @@ -198,7 +201,7 @@ const nonRequestSettings: ChatSetting[] = [ }, { key: 'pinBottom', - name: 'Exclude Bottom Prompts From Summary', + name: 'Exclude Bottom Prompts', title: 'When we run out of space and need to summarize prompts, do not summarize the the last number prompts you set here.', min: 0, max: 20, // Will be auto adjusted down if needs more @@ -209,7 +212,7 @@ const nonRequestSettings: ChatSetting[] = [ }, { key: 'summaryPrompt', - name: 'Summary Generation Prompt', + name: 'Summary Generation Prompt (Empty will use FIFO instead.)', title: 'A prompt used to summarize past prompts.', placeholder: 'Enter a prompt that will be used to summarize past prompts here.', type: 'textarea', diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte index 96d4694..c8b4f7d 100644 --- a/src/lib/Types.svelte +++ b/src/lib/Types.svelte @@ -26,6 +26,7 @@ removed?: boolean; summarized?: string[]; summary?: string[]; + suppress?: boolean; }; export type Request = { @@ -41,7 +42,6 @@ frequency_penalty?: number; logit_bias?: Record | null; user?: string; - }; export type ChatSettings = { @@ -154,6 +154,7 @@ type SettingBoolean = { placeholder?: string; hide?: (chatId:number) => boolean; apiTransform?: (chatId:number, setting:ChatSetting, value:any) => any; + setDefault?: (chatId:number, setting:ChatSetting, value:any) => any; beforeChange?: (chatId:number, setting:ChatSetting, value:any) => boolean; afterChange?: (chatId:number, setting:ChatSetting, value:any) => boolean; } & (SettingNumber | SettingSelect | SettingBoolean | SettingText | SettingTextArea | SettingOther);