Hidden prompt prefix when continuing completion

src/lib/ChatRequest.svelte

@@ -145,9 +145,13 @@ export class ChatRequest {
         const model = this.getModel()
         const maxTokens = getModelMaxTokens(model)
 
-        const messagePayload = filtered.map((m, i) => { return { role: m.role, content: m.content } }) as Message[]
         // Inject hidden prompts if requested
-        if (!opts.summaryRequest) this.buildHiddenPromptPrefixMessages(messagePayload, true)
+        if (!opts.summaryRequest) this.buildHiddenPromptPrefixMessages(filtered, true)
+        const messagePayload = filtered
+          .filter(m => { if (m.skipOnce) { delete m.skipOnce; return false } return true })
+          .map(m => {
+            const content = m.content + (m.appendOnce || []).join('\n'); delete m.appendOnce; return { role: m.role, content }
+          }) as Message[]
     
         const chatResponse = new ChatCompletionResponse(opts)
         const promptTokenCount = countPromptTokens(messagePayload, model)
@@ -291,7 +295,9 @@ export class ChatRequest {
       private buildHiddenPromptPrefixMessages (messages: Message[], insert:boolean = false): Message[] {
         const chatSettings = this.chat.settings
         const hiddenPromptPrefix = mergeProfileFields(chatSettings, chatSettings.hiddenPromptPrefix).trim()
-        if (hiddenPromptPrefix && messages.length && messages[messages.length - 1].role === 'user') {
+        const lastMessage = messages[messages.length - 1]
+        const isContinue = lastMessage?.role === 'assistant' && lastMessage.finish_reason === 'length'
+        if (hiddenPromptPrefix && (lastMessage?.role === 'user' || isContinue)) {
           const results = hiddenPromptPrefix.split(/[\s\r\n]*::EOM::[\s\r\n]*/).reduce((a, m) => {
             m = m.trim()
             if (m.length) {
@@ -300,7 +306,17 @@ export class ChatRequest {
             return a
           }, [] as Message[])
           if (insert) {
-            results.forEach(m => { messages.splice(messages.length - 1, 0, m) })
+            results.forEach(m => { messages.splice(messages.length - (isContinue ? 2 : 1), 0, m) })
+            const userMessage = messages[messages.length - 2]
+            if (chatSettings.hppContinuePrompt && isContinue && userMessage && userMessage.role === 'user') {
+              // If we're using a hiddenPromptPrefix and we're also continuing a truncated completion,
+              // stuff the continue completion request into the last user message to help the
+              // continuation be more influenced by the hiddenPromptPrefix
+              // (this will distort our token count estimates somewhat)
+              userMessage.appendOnce = userMessage.appendOnce || []
+              userMessage.appendOnce.push('\n' + chatSettings.hppContinuePrompt + '\n' + lastMessage.content)
+              lastMessage.skipOnce = true
+            }
           }
           return results
         }

src/lib/Settings.svelte

@@ -90,6 +90,7 @@ const defaults:ChatSettings = {
   autoStartSession: false,
   trainingPrompts: [],
   hiddenPromptPrefix: '',
+  hppContinuePrompt: '',
   imageGenerationSize: '',
   // useResponseAlteration: false,
   // responseAlterations: [],
@@ -205,6 +206,14 @@ const systemPromptSettings: ChatSetting[] = [
         type: 'textarea',
         hide: (chatId) => !getChatSettings(chatId).useSystemPrompt
       },
+      {
+        key: 'hppContinuePrompt',
+        name: 'Continue Truncation Prompt',
+        title: 'If using Hidden Prompts Prefix, a prompt that can be used to help continue a truncated completion.',
+        placeholder: 'Enter something like [Continue your response below:]',
+        type: 'textarea',
+        hide: (chatId) => !getChatSettings(chatId).useSystemPrompt || !(getChatSettings(chatId).hiddenPromptPrefix || '').trim()
+      },
       {
         key: 'trainingPrompts',
         name: 'Training Prompts',

src/lib/Types.svelte

@@ -39,6 +39,8 @@ export type Message = {
     streaming?: boolean;
     image?: ChatImage;
     created?: number;
+    skipOnce?: boolean;
+    appendOnce?: string[];
   };
 
 export type ResponseAlteration = {
@@ -96,6 +98,7 @@ export type ChatSettings = {
     systemPrompt: string;
     autoStartSession: boolean;
     hiddenPromptPrefix: string;
+    hppContinuePrompt: string; // hiddenPromptPrefix used, optional glue when trying to continue truncated completion
     imageGenerationSize: ImageGenerationSizes;
     trainingPrompts?: Message[];
     useResponseAlteration?: boolean;