From 47f96c0bb99aea1496943145da3853f977b618b5 Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 17:42:16 -0500 Subject: [PATCH 1/8] update chat settings update on each request --- src/lib/ChatRequest.svelte | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte index ba1ec35..6c91d42 100644 --- a/src/lib/ChatRequest.svelte +++ b/src/lib/ChatRequest.svelte @@ -3,7 +3,7 @@ import { cleanContent, mergeProfileFields, prepareSummaryPrompt } from './Profiles.svelte' import { countMessageTokens, countPromptTokens, getModelMaxTokens } from './Stats.svelte' import type { Chat, ChatCompletionOpts, ChatSettings, Message, Model, Request } from './Types.svelte' - import { deleteMessage, getChatSettingValueNullDefault, insertMessages, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage } from './Storage.svelte' + import { deleteMessage, getChatSettingValueNullDefault, insertMessages, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage, getChat } from './Storage.svelte' import { scrollToBottom, scrollToMessage } from './Util.svelte' import { getDefaultModel, getRequestSettingList } from './Settings.svelte' import { v4 as uuidv4 } from 'uuid' @@ -62,7 +62,8 @@ export class ChatRequest { async sendRequest (messages: Message[], opts: ChatCompletionOpts, overrides: ChatSettings = {} as ChatSettings): Promise { // TODO: Continue to break this method down to smaller chunks const _this = this - const chat = _this.chat + const chat = getChat(_this.chat.id) + this.setChat(chat) const chatSettings = _this.chat.settings const chatId = chat.id const imagePromptDetect = /^\s*(please|can\s+you|will\s+you)*\s*(give|generate|create|show|build|design)\s+(me)*\s*(an|a|set|a\s+set\s+of)*\s*([0-9]+|one|two|three|four)*\s+(image|photo|picture|pic)s*\s*(for\s+me)*\s*(of|[^a-z0-9]+|about|that\s+has|showing|with|having|depicting)\s+[^a-z0-9]*(.*)$/i From 872e6c8566e2e3ecd336738f768cda1e5bbdd2b0 Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 17:42:35 -0500 Subject: [PATCH 2/8] open new websocket if model changes --- src/lib/providers/petals/request.svelte | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/lib/providers/petals/request.svelte b/src/lib/providers/petals/request.svelte index 53a8984..6f9187b 100644 --- a/src/lib/providers/petals/request.svelte +++ b/src/lib/providers/petals/request.svelte @@ -42,6 +42,8 @@ export const chatRequest = async ( const signal = chatRequest.controller.signal const providerData = chatRequest.providerData.petals || {} chatRequest.providerData.petals = providerData + const modelChanged = model !== providerData.lastModel + providerData.lastModel = model let ws: WebSocket = providerData.ws const abortListener = (e:Event) => { chatRequest.updating = false @@ -221,7 +223,8 @@ export const chatRequest = async ( const kb = providerData.knownBuffer.replace(rgxp, '') const lp = lastPrompt.replace(rgxp, '') const lm = kb === lp - if (!lm || countTokens(model, providerData.knownBuffer + inputPrompt) >= maxTokens) { + if (!chatSettings.holdSocket || modelChanged || !lm || + countTokens(model, providerData.knownBuffer + inputPrompt) >= maxTokens) { wsOpen && ws.close() ws = await getNewWs() } From 69b107e8f7b8aff4feb00647fba35942f5eb1dbe Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 18:04:23 -0500 Subject: [PATCH 3/8] remove Llama-2-70b-hf model --- src/lib/providers/petals/models.svelte | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/lib/providers/petals/models.svelte b/src/lib/providers/petals/models.svelte index 941f777..21d60ec 100644 --- a/src/lib/providers/petals/models.svelte +++ b/src/lib/providers/petals/models.svelte @@ -66,10 +66,10 @@ export const chatModels : Record = { label: 'Petals - Guanaco-65b', max: 2048 }, - 'meta-llama/Llama-2-70b-hf': { - ...chatModelBase, - label: 'Petals - Llama-2-70b' - }, + // 'meta-llama/Llama-2-70b-hf': { + // ...chatModelBase, + // label: 'Petals - Llama-2-70b' + // }, 'meta-llama/Llama-2-70b-chat-hf': { ...chatModelBase, label: 'Petals - Llama-2-70b-chat', From 4f18d400008541e9d2dca9efbc030211bf92e542 Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 18:06:04 -0500 Subject: [PATCH 4/8] Add base llama-65b model --- src/lib/providers/petals/models.svelte | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/lib/providers/petals/models.svelte b/src/lib/providers/petals/models.svelte index 21d60ec..e48c582 100644 --- a/src/lib/providers/petals/models.svelte +++ b/src/lib/providers/petals/models.svelte @@ -52,20 +52,21 @@ const chatModelBase = { } as ModelDetail export const chatModels : Record = { - // 'enoch/llama-65b-hf': { - // ...chatModelBase, - // label: 'Petals - Llama-65b' - // }, - // 'codellama/CodeLlama-34b-Instruct-hf ': { - // ...chatModelBase, - // label: 'Petals - CodeLlama-34b', - // max: 2048 - // }, + 'enoch/llama-65b-hf': { + ...chatModelBase, + label: 'Petals - Llama-65b', + max: 2048 + }, 'timdettmers/guanaco-65b': { ...chatModelBase, label: 'Petals - Guanaco-65b', max: 2048 }, + // 'codellama/CodeLlama-34b-Instruct-hf ': { + // ...chatModelBase, + // label: 'Petals - CodeLlama-34b', + // max: 2048 + // }, // 'meta-llama/Llama-2-70b-hf': { // ...chatModelBase, // label: 'Petals - Llama-2-70b' From 01f7a657db3e22af2bb2029449f5cb83ff127c05 Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 19:14:27 -0500 Subject: [PATCH 5/8] Change default request config for Llama instruct --- src/lib/providers/petals/models.svelte | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib/providers/petals/models.svelte b/src/lib/providers/petals/models.svelte index e48c582..5432115 100644 --- a/src/lib/providers/petals/models.svelte +++ b/src/lib/providers/petals/models.svelte @@ -22,14 +22,14 @@ const chatModelBase = { See this overview to start, though not all settings translate to Petals. Note that some models may mot be functional. See https://health.petals.dev for current status.`, check: checkModel, - start: '', + start: '###', stop: ['###', ''], - delimiter: '\n###\n\n', - userStart: 'User:\n', + delimiter: '###', + userStart: ' User: ', userEnd: '', - assistantStart: '[[CHARACTER_NAME]]:\n', + assistantStart: ' [[CHARACTER_NAME]]: ', assistantEnd: '', - leadPrompt: '[[CHARACTER_NAME]]:\n', + leadPrompt: ' [[CHARACTER_NAME]]: ', systemEnd: '', prompt: 0.000000, // $0.000 per 1000 tokens prompt completion: 0.000000, // $0.000 per 1000 tokens completion From 4d27497bc7d0c2df261c3f5c76a93ce183280e1b Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 19:22:52 -0500 Subject: [PATCH 6/8] Simplify Llama-chat format for now --- src/lib/providers/petals/models.svelte | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/providers/petals/models.svelte b/src/lib/providers/petals/models.svelte index 5432115..b018004 100644 --- a/src/lib/providers/petals/models.svelte +++ b/src/lib/providers/petals/models.svelte @@ -76,12 +76,11 @@ export const chatModels : Record = { label: 'Petals - Llama-2-70b-chat', start: '', stop: ['', '[INST]', '[/INST]', '<>', '<>'], - delimiter: ' ', - userStart: '[INST][[SYSTEM_PROMPT]]', + delimiter: '', + userStart: '[INST] User: ', userEnd: ' [/INST]', - assistantStart: '[[SYSTEM_PROMPT]][[USER_PROMPT]]', - systemStart: '<>\n', - systemEnd: '\n<>\n\n' + systemStart: '[INST] <>\n', + systemEnd: '\n<> [/INST]' // leadPrompt: '' }, 'stabilityai/StableBeluga2': { From ddd17798e7e5fccc95fd764aed1b6f380154f248 Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 19:24:15 -0500 Subject: [PATCH 7/8] Merge delimiter with last known buffer --- src/lib/providers/petals/request.svelte | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/lib/providers/petals/request.svelte b/src/lib/providers/petals/request.svelte index 6f9187b..629925b 100644 --- a/src/lib/providers/petals/request.svelte +++ b/src/lib/providers/petals/request.svelte @@ -163,7 +163,14 @@ export const chatRequest = async ( let maxLen = Math.min(opts.maxTokens || chatSettings.max_tokens || maxTokens, maxTokens) - let inputPrompt = startSequence + let midDel = '' + for (let i = 0, l = delimiter.length; i < l; i++) { + const chk = delimiter.slice(0, i) + if ((providerData.knownBuffer || '').slice(0 - (i + 1)) === chk) midDel = chk + } + midDel = midDel.length ? delimiter.slice(0, 0 - midDel.length) : delimiter + + let inputPrompt = midDel const getNewWs = ():Promise => new Promise((resolve, reject) => { // console.warn('requesting new ws') @@ -185,7 +192,7 @@ export const chatRequest = async ( throw err } // console.warn('got new ws') - inputPrompt = lastPrompt + inputPrompt = lastPrompt + delimiter providerData.knownBuffer = '' providerData.ws = nws resolve(nws) @@ -234,7 +241,7 @@ export const chatRequest = async ( ws = await getNewWs() } - inputPrompt += delimiter + nextPrompt + inputPrompt += nextPrompt providerData.knownBuffer += inputPrompt // console.log( From 40e38d8b548feae8a9c933b4fa256181bc818fa7 Mon Sep 17 00:00:00 2001 From: Webifi Date: Fri, 1 Sep 2023 20:49:20 -0500 Subject: [PATCH 8/8] change default delimiter --- src/lib/providers/petals/models.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/providers/petals/models.svelte b/src/lib/providers/petals/models.svelte index b018004..44467c7 100644 --- a/src/lib/providers/petals/models.svelte +++ b/src/lib/providers/petals/models.svelte @@ -24,7 +24,7 @@ const chatModelBase = { check: checkModel, start: '###', stop: ['###', ''], - delimiter: '###', + delimiter: '\n###\n###', userStart: ' User: ', userEnd: '', assistantStart: ' [[CHARACTER_NAME]]: ',