From 111209c5824f25f8ed22f37a48e0b223b1d8d0ca Mon Sep 17 00:00:00 2001 From: Webifi Date: Mon, 12 Jun 2023 20:29:51 -0500 Subject: [PATCH] Fix token summary counting issue --- src/lib/ChatRequest.svelte | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte index 00f62cd..7f46329 100644 --- a/src/lib/ChatRequest.svelte +++ b/src/lib/ChatRequest.svelte @@ -302,13 +302,14 @@ export class ChatRequest { const getSS = ():number => (ss < 1 && ss > 0) ? Math.round(reductionPoolSize * ss) // If summarySize between 0 and 1, use percentage of reduced : Math.min(ss, reductionPoolSize * 0.5) // If > 1, use token count + const topSize = countPromptTokens(top, model) let maxSummaryTokens = getSS() let promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens) const summaryRequest = { role: 'user', content: promptSummary } as Message let promptSummarySize = countMessageTokens(summaryRequest, model) // Make sure there is enough room to generate the summary, and try to make sure // the last prompt is a user prompt as that seems to work better for summaries - while ((reductionPoolSize + promptSummarySize + maxSummaryTokens) >= maxTokens || + while ((topSize + reductionPoolSize + promptSummarySize + maxSummaryTokens) >= maxTokens || (reductionPoolSize >= 100 && rw[rw.length - 1]?.role !== 'user')) { bottom.unshift(rw.pop() as Message) reductionPoolSize = countPromptTokens(rw, model)