diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte index 40b374d..cc53183 100644 --- a/src/lib/ChatRequest.svelte +++ b/src/lib/ChatRequest.svelte @@ -145,9 +145,13 @@ export class ChatRequest { const model = this.getModel() const maxTokens = getModelMaxTokens(model) - const messagePayload = filtered.map((m, i) => { return { role: m.role, content: m.content } }) as Message[] // Inject hidden prompts if requested - if (!opts.summaryRequest) this.buildHiddenPromptPrefixMessages(messagePayload, true) + if (!opts.summaryRequest) this.buildHiddenPromptPrefixMessages(filtered, true) + const messagePayload = filtered + .filter(m => { if (m.skipOnce) { delete m.skipOnce; return false } return true }) + .map(m => { + const content = m.content + (m.appendOnce || []).join('\n'); delete m.appendOnce; return { role: m.role, content } + }) as Message[] const chatResponse = new ChatCompletionResponse(opts) const promptTokenCount = countPromptTokens(messagePayload, model) @@ -291,7 +295,9 @@ export class ChatRequest { private buildHiddenPromptPrefixMessages (messages: Message[], insert:boolean = false): Message[] { const chatSettings = this.chat.settings const hiddenPromptPrefix = mergeProfileFields(chatSettings, chatSettings.hiddenPromptPrefix).trim() - if (hiddenPromptPrefix && messages.length && messages[messages.length - 1].role === 'user') { + const lastMessage = messages[messages.length - 1] + const isContinue = lastMessage?.role === 'assistant' && lastMessage.finish_reason === 'length' + if (hiddenPromptPrefix && (lastMessage?.role === 'user' || isContinue)) { const results = hiddenPromptPrefix.split(/[\s\r\n]*::EOM::[\s\r\n]*/).reduce((a, m) => { m = m.trim() if (m.length) { @@ -300,7 +306,17 @@ export class ChatRequest { return a }, [] as Message[]) if (insert) { - results.forEach(m => { messages.splice(messages.length - 1, 0, m) }) + results.forEach(m => { messages.splice(messages.length - (isContinue ? 2 : 1), 0, m) }) + const userMessage = messages[messages.length - 2] + if (chatSettings.hppContinuePrompt && isContinue && userMessage && userMessage.role === 'user') { + // If we're using a hiddenPromptPrefix and we're also continuing a truncated completion, + // stuff the continue completion request into the last user message to help the + // continuation be more influenced by the hiddenPromptPrefix + // (this will distort our token count estimates somewhat) + userMessage.appendOnce = userMessage.appendOnce || [] + userMessage.appendOnce.push('\n' + chatSettings.hppContinuePrompt + '\n' + lastMessage.content) + lastMessage.skipOnce = true + } } return results } diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte index a21ab26..519df55 100644 --- a/src/lib/Settings.svelte +++ b/src/lib/Settings.svelte @@ -90,6 +90,7 @@ const defaults:ChatSettings = { autoStartSession: false, trainingPrompts: [], hiddenPromptPrefix: '', + hppContinuePrompt: '', imageGenerationSize: '', // useResponseAlteration: false, // responseAlterations: [], @@ -205,6 +206,14 @@ const systemPromptSettings: ChatSetting[] = [ type: 'textarea', hide: (chatId) => !getChatSettings(chatId).useSystemPrompt }, + { + key: 'hppContinuePrompt', + name: 'Continue Truncation Prompt', + title: 'If using Hidden Prompts Prefix, a prompt that can be used to help continue a truncated completion.', + placeholder: 'Enter something like [Continue your response below:]', + type: 'textarea', + hide: (chatId) => !getChatSettings(chatId).useSystemPrompt || !(getChatSettings(chatId).hiddenPromptPrefix || '').trim() + }, { key: 'trainingPrompts', name: 'Training Prompts', diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte index dbc96ae..41721b5 100644 --- a/src/lib/Types.svelte +++ b/src/lib/Types.svelte @@ -39,6 +39,8 @@ export type Message = { streaming?: boolean; image?: ChatImage; created?: number; + skipOnce?: boolean; + appendOnce?: string[]; }; export type ResponseAlteration = { @@ -96,6 +98,7 @@ export type ChatSettings = { systemPrompt: string; autoStartSession: boolean; hiddenPromptPrefix: string; + hppContinuePrompt: string; // hiddenPromptPrefix used, optional glue when trying to continue truncated completion imageGenerationSize: ImageGenerationSizes; trainingPrompts?: Message[]; useResponseAlteration?: boolean;