Allow automatic extension of truncated summary

This commit is contained in:
Webifi 2023-06-12 16:52:02 -05:00
parent 28d68b3f55
commit a9a49f490a
5 changed files with 116 additions and 83 deletions

View File

@ -59,8 +59,6 @@ export class ChatRequest {
const promptTokenCount = countPromptTokens(messagePayload, model) const promptTokenCount = countPromptTokens(messagePayload, model)
const maxAllowed = maxTokens - (promptTokenCount + 1) const maxAllowed = maxTokens - (promptTokenCount + 1)
// Build and make the request
try {
// Build the API request body // Build the API request body
const request: Request = { const request: Request = {
model: chatSettings.model, model: chatSettings.model,
@ -94,14 +92,14 @@ export class ChatRequest {
stream: opts.streaming stream: opts.streaming
} }
// Set-up and make the request
try {
// Add out token count to the response handler // Add out token count to the response handler
// (streaming doesn't return counts, so we need to do it client side) // (streaming doesn't return counts, so we need to do it client side)
chatResponse.setPromptTokenCount(promptTokenCount) chatResponse.setPromptTokenCount(promptTokenCount)
const signal = _this.controller.signal const signal = _this.controller.signal
// console.log('apikey', $apiKeyStorage)
const fetchOptions = { const fetchOptions = {
method: 'POST', method: 'POST',
headers: { headers: {
@ -297,6 +295,7 @@ export class ChatRequest {
*/ */
const bottom = rw.slice(0 - pinBottom) const bottom = rw.slice(0 - pinBottom)
let continueCounter = chatSettings.summaryExtend + 1
rw = rw.slice(0, 0 - pinBottom) rw = rw.slice(0, 0 - pinBottom)
let reductionPoolSize = countPromptTokens(rw, model) let reductionPoolSize = countPromptTokens(rw, model)
const ss = chatSettings.summarySize const ss = chatSettings.summarySize
@ -340,6 +339,9 @@ export class ChatRequest {
// Request and load the summarization prompt // Request and load the summarization prompt
_this.updatingMessage = 'Summarizing...' _this.updatingMessage = 'Summarizing...'
const summarizedIds = rw.map(m => m.uuid)
const summaryIds = [summaryResponse.uuid]
while (continueCounter-- > 0) {
try { try {
const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), { const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), {
summaryRequest: true, summaryRequest: true,
@ -351,8 +353,8 @@ export class ChatRequest {
if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true) if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true)
} }
} as ChatCompletionOpts, { } as ChatCompletionOpts, {
temperature: 0, // make summary more deterministic temperature: 0.1, // make summary more deterministic
top_p: 0.5, top_p: 1,
presence_penalty: 0, presence_penalty: 0,
frequency_penalty: 0, frequency_penalty: 0,
...overrides ...overrides
@ -360,26 +362,24 @@ export class ChatRequest {
// Wait for the response to complete // Wait for the response to complete
if (!summary.hasFinished()) await summary.promiseToFinish() if (!summary.hasFinished()) await summary.promiseToFinish()
if (summary.hasError()) { if (summary.hasError()) {
// Failed to some API issue. let the original caller handle it. // Failed for some API issue. let the original caller handle it.
deleteMessage(chatId, summaryResponse.uuid) _this.updating = false
_this.updatingMessage = ''
deleteMessage(chatId, srid)
return summary return summary
} else { }
// Looks like we got our summarized messages. // Looks like we got our summarized messages.
// Mark the new summaries as such // Mark the new summaries as such
summaryResponse.summary = rw.map(m => m.uuid) // Need more?
const summaryIds = [summaryResponse.uuid] if (summaryResponse.finish_reason === 'length' && continueCounter > 0) {
// Disable the messages we summarized so they still show in history // Our summary was truncated
rw.forEach((m, i) => { m.summarized = summaryIds }) // Try to get more of it
saveChatStore() delete summaryResponse.finish_reason
// Re-run request with summarized prompts _this.updatingMessage = 'Summarizing more...'
// return { error: { message: "End for now" } } as Response continue
_this.updatingMessage = 'Continuing...' } else {
scrollToBottom(true) // We're done
return await _this.sendRequest(chat.messages, { continueCounter = 0
...opts,
didSummary: true
},
overrides)
} }
} catch (e) { } catch (e) {
_this.updating = false _this.updating = false
@ -387,6 +387,19 @@ export class ChatRequest {
deleteMessage(chatId, srid) deleteMessage(chatId, srid)
throw e throw e
} }
}
summaryResponse.summary = summarizedIds
// Disable the messages we summarized so they still show in history
rw.forEach((m, i) => { m.summarized = summaryIds })
saveChatStore()
// Re-run request with summarized prompts
_this.updatingMessage = 'Continuing...'
scrollToBottom(true)
return await _this.sendRequest(chat.messages, {
...opts,
didSummary: true
},
overrides)
} else { } else {
/*************** /***************
* Unknown mode. * Unknown mode.

View File

@ -177,7 +177,7 @@
placeholder={String(setting.placeholder || chatDefaults[setting.key])} placeholder={String(setting.placeholder || chatDefaults[setting.key])}
on:change={e => queueSettingValueChange(e, setting)} on:change={e => queueSettingValueChange(e, setting)}
/> />
{:else if setting.type === 'select'} {:else if setting.type === 'select' || setting.type === 'select-number'}
<!-- <div class="select"> --> <!-- <div class="select"> -->
<div class="select" class:control={fieldControls.length}> <div class="select" class:control={fieldControls.length}>
<select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } > <select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } >

View File

@ -60,7 +60,7 @@ const gptDefaults = {
n: 1, n: 1,
stream: true, stream: true,
stop: null, stop: null,
max_tokens: 500, max_tokens: 512,
presence_penalty: 0, presence_penalty: 0,
frequency_penalty: 0, frequency_penalty: 0,
logit_bias: null, logit_bias: null,
@ -77,6 +77,7 @@ const defaults:ChatSettings = {
continuousChat: 'fifo', continuousChat: 'fifo',
summaryThreshold: 3000, summaryThreshold: 3000,
summarySize: 1000, summarySize: 1000,
summaryExtend: 0,
pinTop: 0, pinTop: 0,
pinBottom: 6, pinBottom: 6,
summaryPrompt: '', summaryPrompt: '',
@ -222,11 +223,23 @@ const summarySettings: ChatSetting[] = [
name: 'Max Summary Size', name: 'Max Summary Size',
title: 'Maximum number of tokens allowed for summary response.', title: 'Maximum number of tokens allowed for summary response.',
min: 128, min: 128,
max: 512, max: 1024,
step: 1, step: 1,
type: 'number', type: 'number',
hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary' hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary'
}, },
{
key: 'summaryExtend',
name: 'Summary Extend',
title: 'Number of times a truncated summary can be extended.',
type: 'select-number',
options: [
{ value: 0, text: '0 - Summary must fit in first call.' },
{ value: 1, text: '1 - Allow one extra API call to extend.' },
{ value: 2, text: '2 - Allow two extra API calls to extend.' }
],
hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary'
},
{ {
key: 'pinTop', key: 'pinTop',
name: 'Keep First Prompts', name: 'Keep First Prompts',

View File

@ -333,6 +333,7 @@
export const cleanSettingValue = (type:string, value: any) => { export const cleanSettingValue = (type:string, value: any) => {
switch (type) { switch (type) {
case 'number': case 'number':
case 'select-number':
value = parseFloat(value) value = parseFloat(value)
if (isNaN(value)) { value = null } if (isNaN(value)) { value = null }
return value return value

View File

@ -60,6 +60,7 @@
continuousChat: (''|'fifo'|'summary'); continuousChat: (''|'fifo'|'summary');
summaryThreshold: number; summaryThreshold: number;
summarySize: number; summarySize: number;
summaryExtend: number;
pinTop: number; pinTop: number;
pinBottom: number; pinBottom: number;
summaryPrompt: string; summaryPrompt: string;
@ -141,7 +142,7 @@
}; };
export type SelectOption = { export type SelectOption = {
value: string; value: string|number;
text: string; text: string;
}; };
@ -154,6 +155,11 @@ type SettingBoolean = {
options: SelectOption[]; options: SelectOption[];
}; };
export type SettingSelectNumber = {
type: 'select-number';
options: SelectOption[];
};
export type SettingText = { export type SettingText = {
type: 'text'; type: 'text';
}; };
@ -199,7 +205,7 @@ type SettingBoolean = {
fieldControls?: FieldControl[]; fieldControls?: FieldControl[];
beforeChange?: (chatId:number, setting:ChatSetting, value:any) => boolean; beforeChange?: (chatId:number, setting:ChatSetting, value:any) => boolean;
afterChange?: (chatId:number, setting:ChatSetting, value:any) => boolean; afterChange?: (chatId:number, setting:ChatSetting, value:any) => boolean;
} & (SettingNumber | SettingSelect | SettingBoolean | SettingText | SettingTextArea | SettingOther | SubSetting); } & (SettingNumber | SettingSelect | SettingSelectNumber | SettingBoolean | SettingText | SettingTextArea | SettingOther | SubSetting);
export type GlobalSetting = { export type GlobalSetting = {