diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte index ba1ec35..6c91d42 100644 --- a/src/lib/ChatRequest.svelte +++ b/src/lib/ChatRequest.svelte @@ -3,7 +3,7 @@ import { cleanContent, mergeProfileFields, prepareSummaryPrompt } from './Profiles.svelte' import { countMessageTokens, countPromptTokens, getModelMaxTokens } from './Stats.svelte' import type { Chat, ChatCompletionOpts, ChatSettings, Message, Model, Request } from './Types.svelte' - import { deleteMessage, getChatSettingValueNullDefault, insertMessages, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage } from './Storage.svelte' + import { deleteMessage, getChatSettingValueNullDefault, insertMessages, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage, getChat } from './Storage.svelte' import { scrollToBottom, scrollToMessage } from './Util.svelte' import { getDefaultModel, getRequestSettingList } from './Settings.svelte' import { v4 as uuidv4 } from 'uuid' @@ -62,7 +62,8 @@ export class ChatRequest { async sendRequest (messages: Message[], opts: ChatCompletionOpts, overrides: ChatSettings = {} as ChatSettings): Promise { // TODO: Continue to break this method down to smaller chunks const _this = this - const chat = _this.chat + const chat = getChat(_this.chat.id) + this.setChat(chat) const chatSettings = _this.chat.settings const chatId = chat.id const imagePromptDetect = /^\s*(please|can\s+you|will\s+you)*\s*(give|generate|create|show|build|design)\s+(me)*\s*(an|a|set|a\s+set\s+of)*\s*([0-9]+|one|two|three|four)*\s+(image|photo|picture|pic)s*\s*(for\s+me)*\s*(of|[^a-z0-9]+|about|that\s+has|showing|with|having|depicting)\s+[^a-z0-9]*(.*)$/i diff --git a/src/lib/providers/petals/models.svelte b/src/lib/providers/petals/models.svelte index 941f777..44467c7 100644 --- a/src/lib/providers/petals/models.svelte +++ b/src/lib/providers/petals/models.svelte @@ -22,14 +22,14 @@ const chatModelBase = { See this overview to start, though not all settings translate to Petals. Note that some models may mot be functional. See https://health.petals.dev for current status.`, check: checkModel, - start: '', + start: '###', stop: ['###', ''], - delimiter: '\n###\n\n', - userStart: 'User:\n', + delimiter: '\n###\n###', + userStart: ' User: ', userEnd: '', - assistantStart: '[[CHARACTER_NAME]]:\n', + assistantStart: ' [[CHARACTER_NAME]]: ', assistantEnd: '', - leadPrompt: '[[CHARACTER_NAME]]:\n', + leadPrompt: ' [[CHARACTER_NAME]]: ', systemEnd: '', prompt: 0.000000, // $0.000 per 1000 tokens prompt completion: 0.000000, // $0.000 per 1000 tokens completion @@ -52,35 +52,35 @@ const chatModelBase = { } as ModelDetail export const chatModels : Record = { - // 'enoch/llama-65b-hf': { - // ...chatModelBase, - // label: 'Petals - Llama-65b' - // }, - // 'codellama/CodeLlama-34b-Instruct-hf ': { - // ...chatModelBase, - // label: 'Petals - CodeLlama-34b', - // max: 2048 - // }, + 'enoch/llama-65b-hf': { + ...chatModelBase, + label: 'Petals - Llama-65b', + max: 2048 + }, 'timdettmers/guanaco-65b': { ...chatModelBase, label: 'Petals - Guanaco-65b', max: 2048 }, - 'meta-llama/Llama-2-70b-hf': { - ...chatModelBase, - label: 'Petals - Llama-2-70b' - }, + // 'codellama/CodeLlama-34b-Instruct-hf ': { + // ...chatModelBase, + // label: 'Petals - CodeLlama-34b', + // max: 2048 + // }, + // 'meta-llama/Llama-2-70b-hf': { + // ...chatModelBase, + // label: 'Petals - Llama-2-70b' + // }, 'meta-llama/Llama-2-70b-chat-hf': { ...chatModelBase, label: 'Petals - Llama-2-70b-chat', start: '', stop: ['', '[INST]', '[/INST]', '<>', '<>'], - delimiter: ' ', - userStart: '[INST][[SYSTEM_PROMPT]]', + delimiter: '', + userStart: '[INST] User: ', userEnd: ' [/INST]', - assistantStart: '[[SYSTEM_PROMPT]][[USER_PROMPT]]', - systemStart: '<>\n', - systemEnd: '\n<>\n\n' + systemStart: '[INST] <>\n', + systemEnd: '\n<> [/INST]' // leadPrompt: '' }, 'stabilityai/StableBeluga2': { diff --git a/src/lib/providers/petals/request.svelte b/src/lib/providers/petals/request.svelte index 53a8984..629925b 100644 --- a/src/lib/providers/petals/request.svelte +++ b/src/lib/providers/petals/request.svelte @@ -42,6 +42,8 @@ export const chatRequest = async ( const signal = chatRequest.controller.signal const providerData = chatRequest.providerData.petals || {} chatRequest.providerData.petals = providerData + const modelChanged = model !== providerData.lastModel + providerData.lastModel = model let ws: WebSocket = providerData.ws const abortListener = (e:Event) => { chatRequest.updating = false @@ -161,7 +163,14 @@ export const chatRequest = async ( let maxLen = Math.min(opts.maxTokens || chatSettings.max_tokens || maxTokens, maxTokens) - let inputPrompt = startSequence + let midDel = '' + for (let i = 0, l = delimiter.length; i < l; i++) { + const chk = delimiter.slice(0, i) + if ((providerData.knownBuffer || '').slice(0 - (i + 1)) === chk) midDel = chk + } + midDel = midDel.length ? delimiter.slice(0, 0 - midDel.length) : delimiter + + let inputPrompt = midDel const getNewWs = ():Promise => new Promise((resolve, reject) => { // console.warn('requesting new ws') @@ -183,7 +192,7 @@ export const chatRequest = async ( throw err } // console.warn('got new ws') - inputPrompt = lastPrompt + inputPrompt = lastPrompt + delimiter providerData.knownBuffer = '' providerData.ws = nws resolve(nws) @@ -221,7 +230,8 @@ export const chatRequest = async ( const kb = providerData.knownBuffer.replace(rgxp, '') const lp = lastPrompt.replace(rgxp, '') const lm = kb === lp - if (!lm || countTokens(model, providerData.knownBuffer + inputPrompt) >= maxTokens) { + if (!chatSettings.holdSocket || modelChanged || !lm || + countTokens(model, providerData.knownBuffer + inputPrompt) >= maxTokens) { wsOpen && ws.close() ws = await getNewWs() } @@ -231,7 +241,7 @@ export const chatRequest = async ( ws = await getNewWs() } - inputPrompt += delimiter + nextPrompt + inputPrompt += nextPrompt providerData.knownBuffer += inputPrompt // console.log(