morgan
/
chatgpt-web
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allow automatic extension of truncated summary

This commit is contained in:
Webifi 2023-06-12 16:52:02 -05:00
parent 28d68b3f55
commit a9a49f490a
5 changed files with 116 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

163
src/lib/ChatRequest.svelte
View File

@ -58,50 +58,48 @@ export class ChatRequest {
const chatResponse = new ChatCompletionResponse(opts)
const promptTokenCount = countPromptTokens(messagePayload, model)
const maxAllowed = maxTokens - (promptTokenCount + 1)
// Build and make the request
try {
// Build the API request body
const request: Request = {
model: chatSettings.model,
messages: messagePayload,
// Provide the settings by mapping the settingsMap to key/value pairs
...getRequestSettingList().reduce((acc, setting) => {
const key = setting.key
let value = getChatSettingValueNullDefault(chatId, setting)
if (key in overrides) value = overrides[key]
if (typeof setting.apiTransform === 'function') {
value = setting.apiTransform(chatId, setting, value)
}
if (key === 'max_tokens') {
if (opts.maxTokens) value = opts.maxTokens // only as large as requested
if (value > maxAllowed || value < 1) value = null // if over max model, do not define max
}
if (key === 'n') {
if (opts.streaming || opts.summaryRequest) {
// Build the API request body
const request: Request = {
model: chatSettings.model,
messages: messagePayload,
// Provide the settings by mapping the settingsMap to key/value pairs
...getRequestSettingList().reduce((acc, setting) => {
const key = setting.key
let value = getChatSettingValueNullDefault(chatId, setting)
if (key in overrides) value = overrides[key]
if (typeof setting.apiTransform === 'function') {
value = setting.apiTransform(chatId, setting, value)
}
if (key === 'max_tokens') {
if (opts.maxTokens) value = opts.maxTokens // only as large as requested
if (value > maxAllowed || value < 1) value = null // if over max model, do not define max
}
if (key === 'n') {
if (opts.streaming || opts.summaryRequest) {
/*
Streaming goes insane with more than one completion.
Doesn't seem like there's any way to separate the jumbled mess of deltas for the
different completions.
Summary should only have one completion
*/
value = 1
}
value = 1
}
if (value !== null) acc[key] = value
return acc
}, {}),
stream: opts.streaming
}
}
if (value !== null) acc[key] = value
return acc
}, {}),
stream: opts.streaming
}
// Set-up and make the request
try {
// Add out token count to the response handler
// (streaming doesn't return counts, so we need to do it client side)
chatResponse.setPromptTokenCount(promptTokenCount)
const signal = _this.controller.signal
// console.log('apikey', $apiKeyStorage)
const fetchOptions = {
method: 'POST',
headers: {
@ -297,6 +295,7 @@ export class ChatRequest {
*/
const bottom = rw.slice(0 - pinBottom)
let continueCounter = chatSettings.summaryExtend + 1
rw = rw.slice(0, 0 - pinBottom)
let reductionPoolSize = countPromptTokens(rw, model)
const ss = chatSettings.summarySize
@ -340,53 +339,67 @@ export class ChatRequest {
// Request and load the summarization prompt
_this.updatingMessage = 'Summarizing...'
try {
const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), {
summaryRequest: true,
streaming: opts.streaming,
maxTokens: maxSummaryTokens,
fillMessage: summaryResponse,
autoAddMessages: true,
onMessageChange: (m) => {
if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true)
const summarizedIds = rw.map(m => m.uuid)
const summaryIds = [summaryResponse.uuid]
while (continueCounter-- > 0) {
try {
const summary = await _this.sendRequest(top.concat(rw).concat([summaryRequest]), {
summaryRequest: true,
streaming: opts.streaming,
maxTokens: maxSummaryTokens,
fillMessage: summaryResponse,
autoAddMessages: true,
onMessageChange: (m) => {
if (opts.streaming) scrollToMessage(summaryResponse.uuid, 150, true, true)
}
} as ChatCompletionOpts, {
temperature: 0.1, // make summary more deterministic
top_p: 1,
presence_penalty: 0,
frequency_penalty: 0,
...overrides
} as ChatSettings)
// Wait for the response to complete
if (!summary.hasFinished()) await summary.promiseToFinish()
if (summary.hasError()) {
// Failed for some API issue. let the original caller handle it.
_this.updating = false
_this.updatingMessage = ''
deleteMessage(chatId, srid)
return summary
}
} as ChatCompletionOpts, {
temperature: 0, // make summary more deterministic
top_p: 0.5,
presence_penalty: 0,
frequency_penalty: 0,
...overrides
} as ChatSettings)
// Wait for the response to complete
if (!summary.hasFinished()) await summary.promiseToFinish()
if (summary.hasError()) {
// Failed to some API issue. let the original caller handle it.
deleteMessage(chatId, summaryResponse.uuid)
return summary
} else {
// Looks like we got our summarized messages.
// Mark the new summaries as such
summaryResponse.summary = rw.map(m => m.uuid)
const summaryIds = [summaryResponse.uuid]
// Disable the messages we summarized so they still show in history
rw.forEach((m, i) => { m.summarized = summaryIds })
saveChatStore()
// Re-run request with summarized prompts
// return { error: { message: "End for now" } } as Response
_this.updatingMessage = 'Continuing...'
scrollToBottom(true)
return await _this.sendRequest(chat.messages, {
...opts,
didSummary: true
},
overrides)
// Looks like we got our summarized messages.
// Mark the new summaries as such
// Need more?
if (summaryResponse.finish_reason === 'length' && continueCounter > 0) {
// Our summary was truncated
// Try to get more of it
delete summaryResponse.finish_reason
_this.updatingMessage = 'Summarizing more...'
continue
} else {
// We're done
continueCounter = 0
}
} catch (e) {
_this.updating = false
_this.updatingMessage = ''
deleteMessage(chatId, srid)
throw e
}
} catch (e) {
_this.updating = false
_this.updatingMessage = ''
deleteMessage(chatId, srid)
throw e
}
summaryResponse.summary = summarizedIds
// Disable the messages we summarized so they still show in history
rw.forEach((m, i) => { m.summarized = summaryIds })
saveChatStore()
// Re-run request with summarized prompts
_this.updatingMessage = 'Continuing...'
scrollToBottom(true)
return await _this.sendRequest(chat.messages, {
...opts,
didSummary: true
},
overrides)
} else {
/***************
* Unknown mode.

2
src/lib/ChatSettingField.svelte
View File

@ -177,7 +177,7 @@
placeholder={String(setting.placeholder || chatDefaults[setting.key])}
on:change={e => queueSettingValueChange(e, setting)}
/>
{:else if setting.type === 'select'}
{:else if setting.type === 'select' || setting.type === 'select-number'}
<!-- <div class="select"> -->
<div class="select" class:control={fieldControls.length}>
<select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } >

17
src/lib/Settings.svelte
View File

@ -60,7 +60,7 @@ const gptDefaults = {
n: 1,
stream: true,
stop: null,
max_tokens: 500,
max_tokens: 512,
presence_penalty: 0,
frequency_penalty: 0,
logit_bias: null,
@ -77,6 +77,7 @@ const defaults:ChatSettings = {
continuousChat: 'fifo',
summaryThreshold: 3000,
summarySize: 1000,
summaryExtend: 0,
pinTop: 0,
pinBottom: 6,
summaryPrompt: '',
@ -222,11 +223,23 @@ const summarySettings: ChatSetting[] = [
name: 'Max Summary Size',
title: 'Maximum number of tokens allowed for summary response.',
min: 128,
max: 512,
max: 1024,
step: 1,
type: 'number',
hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary'
},
{
key: 'summaryExtend',
name: 'Summary Extend',
title: 'Number of times a truncated summary can be extended.',
type: 'select-number',
options: [
{ value: 0, text: '0 - Summary must fit in first call.' },
{ value: 1, text: '1 - Allow one extra API call to extend.' },
{ value: 2, text: '2 - Allow two extra API calls to extend.' }
],
hide: (chatId) => getChatSettings(chatId).continuousChat !== 'summary'
},
{
key: 'pinTop',
name: 'Keep First Prompts',

1
src/lib/Storage.svelte
View File

@ -333,6 +333,7 @@
export const cleanSettingValue = (type:string, value: any) => {
switch (type) {
case 'number':
case 'select-number':
value = parseFloat(value)
if (isNaN(value)) { value = null }
return value

16
src/lib/Types.svelte
View File

@ -60,6 +60,7 @@
continuousChat: (''|'fifo'|'summary');
summaryThreshold: number;
summarySize: number;
summaryExtend: number;
pinTop: number;
pinBottom: number;
summaryPrompt: string;
@ -141,19 +142,24 @@
};
export type SelectOption = {
value: string;
value: string|number;
text: string;
};
type SettingBoolean = {
type: 'boolean';
};
type SettingBoolean = {
type: 'boolean';
};
export type SettingSelect = {
type: 'select';
options: SelectOption[];
};
export type SettingSelectNumber = {
type: 'select-number';
options: SelectOption[];
};
export type SettingText = {
type: 'text';
};
@ -199,7 +205,7 @@ type SettingBoolean = {
fieldControls?: FieldControl[];
beforeChange?: (chatId:number, setting:ChatSetting, value:any) => boolean;
afterChange?: (chatId:number, setting:ChatSetting, value:any) => boolean;
} & (SettingNumber | SettingSelect | SettingBoolean | SettingText | SettingTextArea | SettingOther | SubSetting);
} & (SettingNumber | SettingSelect | SettingSelectNumber | SettingBoolean | SettingText | SettingTextArea | SettingOther | SubSetting);
export type GlobalSetting = {