From 8e35b198da22216aa6921c9675fa434449754af6 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Thu, 20 Jul 2023 20:32:36 -0500
Subject: [PATCH 01/17] Add [[LAST-PROMPT]] system prompt expansion

---
 src/lib/ChatRequest.svelte | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index 60dd80c..d0bc259 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -159,6 +159,8 @@ export class ChatRequest {
           const spl = chatSettings.sendSystemPromptLast
           const sp = messagePayload[0]
           if (sp) {
+            const lastSp = sp.content.split('::END-PROMPT::')
+            sp.content = lastSp[0].trim()
             if (messagePayload.length > 1) {
               sp.content = sp.content.replace(/::STARTUP::[\s\S]*::EOM::/, '::EOM::')
               sp.content = sp.content.replace(/::STARTUP::[\s\S]*::START-PROMPT::/, '::START-PROMPT::')
@@ -170,7 +172,7 @@ export class ChatRequest {
             if (spl) {
               messagePayload.shift()
               if (messagePayload[messagePayload.length - 1]?.role === 'user') {
-                messagePayload.splice(-2, 0, sp)
+                messagePayload.splice(-1, 0, sp)
               } else {
                 messagePayload.push(sp)
               }
@@ -196,6 +198,10 @@ export class ChatRequest {
               }).filter(m => m.content.length)
               messagePayload.splice(spl ? 0 : 1, 0, ...ms.concat(splitSystem.map(s => ({ role: 'system', content: s.trim() } as Message)).filter(m => m.content.length)))
             }
+            const lastSpC = lastSp[1]?.trim() || ''
+            if (lastSpC.length) {
+              messagePayload.push({ role: 'system', content: lastSpC } as Message)
+            }
           }
         }
 
@@ -356,9 +362,9 @@ export class ChatRequest {
           const results = hiddenPromptPrefix.split(/[\s\r\n]*::EOM::[\s\r\n]*/).reduce((a, m) => {
             m = m.trim()
             if (m.length) {
-              if (m.match(/[[USER_PROMPT]]/)) {
+              if (m.match(/\[\[USER_PROMPT\]\]/)) {
                 injectedPrompt = true
-                m.replace(/[[USER_PROMPT]]/g, lastMessage.content)
+                m.replace(/\[\[USER_PROMPT\]\]/g, lastMessage.content)
               }
               a.push({ role: a.length % 2 === 0 ? 'user' : 'assistant', content: m } as Message)
             }

From 914055f1f990ade466aae68227d2847f3a9bfb80 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 01:42:21 -0500
Subject: [PATCH 02/17] Initial test of Petals as alternative to OpenAI

---
 package-lock.json                     |   7 +
 package.json                          |   1 +
 src/lib/ApiUtil.svelte                |   2 +
 src/lib/ChatCompletionResponse.svelte |  12 +-
 src/lib/ChatRequest.svelte            | 269 ++++++++++++++++++--------
 src/lib/ChatSettingsModal.svelte      |  29 +--
 src/lib/Home.svelte                   |  73 ++++++-
 src/lib/Models.svelte                 | 123 +++++++++++-
 src/lib/Settings.svelte               |  24 ++-
 src/lib/Stats.svelte                  |  48 +++--
 src/lib/Types.svelte                  |  22 ++-
 11 files changed, 469 insertions(+), 141 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index e645d79..15510ce 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -27,6 +27,7 @@
         "eslint-plugin-svelte3": "^4.0.0",
         "flourite": "^1.2.4",
         "gpt-tokenizer": "^2.0.0",
+        "llama-tokenizer-js": "^1.1.1",
         "postcss": "^8.4.26",
         "sass": "^1.63.6",
         "stacking-order": "^2.0.0",
@@ -3182,6 +3183,12 @@
         "node": ">= 0.8.0"
       }
     },
+    "node_modules/llama-tokenizer-js": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/llama-tokenizer-js/-/llama-tokenizer-js-1.1.1.tgz",
+      "integrity": "sha512-5H2oSJnSufWGhOw6hcCGAqJeB3POmeIBzRklH3cXs0L4MSAYdwoYTodni4j5YVo6jApdhaqaNVU66gNRgXeBRg==",
+      "dev": true
+    },
     "node_modules/locate-path": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
diff --git a/package.json b/package.json
index a4dfe5d..ec0de52 100644
--- a/package.json
+++ b/package.json
@@ -33,6 +33,7 @@
     "eslint-plugin-svelte3": "^4.0.0",
     "flourite": "^1.2.4",
     "gpt-tokenizer": "^2.0.0",
+    "llama-tokenizer-js": "^1.1.1",
     "postcss": "^8.4.26",
     "sass": "^1.63.6",
     "stacking-order": "^2.0.0",
diff --git a/src/lib/ApiUtil.svelte b/src/lib/ApiUtil.svelte
index 77edbc7..ceded8b 100644
--- a/src/lib/ApiUtil.svelte
+++ b/src/lib/ApiUtil.svelte
@@ -5,10 +5,12 @@
   const endpointGenerations = import.meta.env.VITE_ENDPOINT_GENERATIONS || '/v1/images/generations'
   const endpointModels = import.meta.env.VITE_ENDPOINT_MODELS || '/v1/models'
   const endpointEmbeddings = import.meta.env.VITE_ENDPOINT_EMBEDDINGS || '/v1/embeddings'
+  const endpointPetalsV2Websocket = import.meta.env.VITE_PEDALS_WEBSOCKET || 'wss://chat.petals.dev/api/v2/generate'
 
   export const getApiBase = ():string => apiBase
   export const getEndpointCompletions = ():string => endpointCompletions
   export const getEndpointGenerations = ():string => endpointGenerations
   export const getEndpointModels = ():string => endpointModels
   export const getEndpointEmbeddings = ():string => endpointEmbeddings
+  export const getPetalsV2Websocket = ():string => endpointPetalsV2Websocket
 </script>
\ No newline at end of file
diff --git a/src/lib/ChatCompletionResponse.svelte b/src/lib/ChatCompletionResponse.svelte
index 03c1c31..a6743f6 100644
--- a/src/lib/ChatCompletionResponse.svelte
+++ b/src/lib/ChatCompletionResponse.svelte
@@ -1,9 +1,9 @@
 <script context="module" lang="ts">
 import { setImage } from './ImageStore.svelte'
+import { countTokens } from './Models.svelte'
 // TODO: Integrate API calls
-import { addMessage, getLatestKnownModel, saveChatStore, setLatestKnownModel, subtractRunningTotal, updateRunningTotal } from './Storage.svelte'
+import { addMessage, getLatestKnownModel, setLatestKnownModel, subtractRunningTotal, updateMessages, updateRunningTotal } from './Storage.svelte'
 import type { Chat, ChatCompletionOpts, ChatImage, Message, Model, Response, ResponseImage, Usage } from './Types.svelte'
-import { encode } from 'gpt-tokenizer'
 import { v4 as uuidv4 } from 'uuid'
 
 export class ChatCompletionResponse {
@@ -138,10 +138,10 @@ export class ChatCompletionResponse {
         message.content = this.initialFillMerge(message.content, choice.delta?.content)
         message.content += choice.delta.content
       }
-      completionTokenCount += encode(message.content).length
+      completionTokenCount += countTokens(this.model, message.content)
       message.model = response.model
       message.finish_reason = choice.finish_reason
-      message.streaming = choice.finish_reason === null && !this.finished
+      message.streaming = !choice.finish_reason && !this.finished
       this.messages[i] = message
     })
     // total up the tokens
@@ -209,10 +209,10 @@ export class ChatCompletionResponse {
   }
 
   private finish = (): void => {
+    this.messages.forEach(m => { m.streaming = false }) // make sure all are marked stopped
+    updateMessages(this.chat.id)
     if (this.finished) return
     this.finished = true
-    this.messages.forEach(m => { m.streaming = false }) // make sure all are marked stopped
-    saveChatStore()
     const message = this.messages[0]
     const model = this.model || getLatestKnownModel(this.chat.settings.model)
     if (message) {
diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index d0bc259..20b5626 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -7,9 +7,9 @@
     import { scrollToBottom, scrollToMessage } from './Util.svelte'
     import { getRequestSettingList, defaultModel } from './Settings.svelte'
     import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
-    import { getApiBase, getEndpointCompletions, getEndpointGenerations } from './ApiUtil.svelte'
     import { v4 as uuidv4 } from 'uuid'
     import { get } from 'svelte/store'
+    import { getEndpoint, getModelDetail, getRoleTag } from './Models.svelte'
 
 export class ChatRequest {
       constructor () {
@@ -77,7 +77,7 @@ export class ChatRequest {
         const chatResponse = new ChatCompletionResponse(opts)
 
         try {
-          const response = await fetch(getApiBase() + getEndpointGenerations(), fetchOptions)
+          const response = await fetch(getEndpoint('dall-e-' + size), fetchOptions)
           if (!response.ok) {
             await _this.handleError(response)
           } else {
@@ -206,7 +206,7 @@ export class ChatRequest {
         }
 
         // Get token counts
-        const promptTokenCount = countPromptTokens(messagePayload, model)
+        const promptTokenCount = countPromptTokens(messagePayload, model, chatSettings)
         const maxAllowed = maxTokens - (promptTokenCount + 1)
 
         // Build the API request body
@@ -245,96 +245,205 @@ export class ChatRequest {
 
         // Set-up and make the request
         const chatResponse = new ChatCompletionResponse(opts)
+
+        const modelDetail = getModelDetail(model)
+
         try {
           // Add out token count to the response handler
           // (streaming doesn't return counts, so we need to do it client side)
           chatResponse.setPromptTokenCount(promptTokenCount)
-
+    
           // fetchEventSource doesn't seem to throw on abort,
           // so we deal with it ourselves
           _this.controller = new AbortController()
           const signal = _this.controller.signal
-          const abortListener = (e:Event) => {
-            _this.updating = false
-            _this.updatingMessage = ''
-            chatResponse.updateFromError('User aborted request.')
-            signal.removeEventListener('abort', abortListener)
-          }
-          signal.addEventListener('abort', abortListener)
-    
-          const fetchOptions = {
-            method: 'POST',
-            headers: {
-              Authorization: `Bearer ${getApiKey()}`,
-              'Content-Type': 'application/json'
-            },
-            body: JSON.stringify(request),
-            signal
-          }
 
-          if (opts.streaming) {
-            /**
-             * Streaming request/response
-             * We'll get the response a token at a time, as soon as they are ready
-            */
+          if (modelDetail.type === 'PetalsV2Websocket') {
+            // Petals
+            const ws = new WebSocket(getEndpoint(model))
+            const abortListener = (e:Event) => {
+              _this.updating = false
+              _this.updatingMessage = ''
+              chatResponse.updateFromError('User aborted request.')
+              signal.removeEventListener('abort', abortListener)
+              ws.close()
+            }
+            signal.addEventListener('abort', abortListener)
+            const stopSequences = modelDetail.stop || ['###']
+            const stopSequencesC = stopSequences.slice()
+            const stopSequence = stopSequencesC.shift()
             chatResponse.onFinish(() => {
               _this.updating = false
               _this.updatingMessage = ''
             })
-            fetchEventSource(getApiBase() + getEndpointCompletions(), {
-              ...fetchOptions,
-              openWhenHidden: true,
-              onmessage (ev) {
-              // Remove updating indicator
-                _this.updating = 1 // hide indicator, but still signal we're updating
-                _this.updatingMessage = ''
-                // console.log('ev.data', ev.data)
-                if (!chatResponse.hasFinished()) {
-                  if (ev.data === '[DONE]') {
-                  // ?? anything to do when "[DONE]"?
-                  } else {
-                    const data = JSON.parse(ev.data)
-                    // console.log('data', data)
-                    window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
-                  }
+            ws.onopen = () => {
+              ws.send(JSON.stringify({
+                type: 'open_inference_session',
+                model,
+                max_length: maxTokens || opts.maxTokens
+              }))
+              ws.onmessage = event => {
+                const response = JSON.parse(event.data)
+                if (!response.ok) {
+                  const err = new Error('Error opening socket: ' + response.traceback)
+                  console.error(err)
+                  throw err
                 }
-              },
-              onclose () {
+                const rMessages = request.messages || [] as Message[]
+                const inputArray = (rMessages).reduce((a, m) => {
+                  const c = getRoleTag(m.role, model, chatSettings) + m.content
+                  a.push(c)
+                  return a
+                }, [] as string[])
+                const lastMessage = rMessages[rMessages.length - 1]
+                if (lastMessage && lastMessage.role !== 'assistant') {
+                  inputArray.push(getRoleTag('assistant', model, chatSettings))
+                }
+                const petalsRequest = {
+                  type: 'generate',
+                  inputs: (request.messages || [] as Message[]).reduce((a, m) => {
+                    const c = getRoleTag(m.role, model, chatSettings) + m.content
+                    a.push(c)
+                    return a
+                  }, [] as string[]).join(stopSequence),
+                  max_new_tokens: 3, // wait for up to 3 tokens before displaying
+                  stop_sequence: stopSequence,
+                  doSample: 1,
+                  temperature: request.temperature || 0,
+                  top_p: request.top_p || 0,
+                  extra_stop_sequences: stopSequencesC
+                }
+                ws.send(JSON.stringify(petalsRequest))
+                ws.onmessage = event => {
+                  // Remove updating indicator
+                  _this.updating = 1 // hide indicator, but still signal we're updating
+                  _this.updatingMessage = ''
+                  const response = JSON.parse(event.data)
+                  if (!response.ok) {
+                    const err = new Error('Error in response: ' + response.traceback)
+                    console.error(err)
+                    throw err
+                  }
+                  window.setTimeout(() => {
+                    chatResponse.updateFromAsyncResponse(
+                      {
+                        model,
+                        choices: [{
+                          delta: {
+                            content: response.outputs,
+                            role: 'assistant'
+                          },
+                          finish_reason: (response.stop ? 'stop' : null)
+                        }]
+                      } as any
+                    )
+                    if (response.stop) {
+                      const message = chatResponse.getMessages()[0]
+                      if (message) {
+                        for (let i = 0, l = stopSequences.length; i < l; i++) {
+                          if (message.content.endsWith(stopSequences[i])) {
+                            message.content = message.content.slice(0, message.content.length - stopSequences[i].length)
+                            updateMessages(chatId)
+                          }
+                        }
+                      }
+                    }
+                  }, 1)
+                }
+              }
+              ws.onclose = () => {
                 _this.updating = false
                 _this.updatingMessage = ''
                 chatResponse.updateFromClose()
-              },
-              onerror (err) {
+              }
+              ws.onerror = err => {
                 console.error(err)
                 throw err
-              },
-              async onopen (response) {
-                if (response.ok && response.headers.get('content-type') === EventStreamContentType) {
-                // everything's good
-                } else {
-                // client-side errors are usually non-retriable:
-                  await _this.handleError(response)
-                }
               }
-            }).catch(err => {
+            }
+          } else {
+            // OpenAI
+            const abortListener = (e:Event) => {
               _this.updating = false
               _this.updatingMessage = ''
-              chatResponse.updateFromError(err.message)
-            })
-          } else {
+              chatResponse.updateFromError('User aborted request.')
+              signal.removeEventListener('abort', abortListener)
+            }
+            signal.addEventListener('abort', abortListener)
+            const fetchOptions = {
+              method: 'POST',
+              headers: {
+                Authorization: `Bearer ${getApiKey()}`,
+                'Content-Type': 'application/json'
+              },
+              body: JSON.stringify(request),
+              signal
+            }
+
+            if (opts.streaming) {
+            /**
+             * Streaming request/response
+             * We'll get the response a token at a time, as soon as they are ready
+            */
+              chatResponse.onFinish(() => {
+                _this.updating = false
+                _this.updatingMessage = ''
+              })
+              fetchEventSource(getEndpoint(model), {
+                ...fetchOptions,
+                openWhenHidden: true,
+                onmessage (ev) {
+                  // Remove updating indicator
+                  _this.updating = 1 // hide indicator, but still signal we're updating
+                  _this.updatingMessage = ''
+                  // console.log('ev.data', ev.data)
+                  if (!chatResponse.hasFinished()) {
+                    if (ev.data === '[DONE]') {
+                      // ?? anything to do when "[DONE]"?
+                    } else {
+                      const data = JSON.parse(ev.data)
+                      // console.log('data', data)
+                      window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
+                    }
+                  }
+                },
+                onclose () {
+                  _this.updating = false
+                  _this.updatingMessage = ''
+                  chatResponse.updateFromClose()
+                },
+                onerror (err) {
+                  console.error(err)
+                  throw err
+                },
+                async onopen (response) {
+                  if (response.ok && response.headers.get('content-type') === EventStreamContentType) {
+                    // everything's good
+                  } else {
+                    // client-side errors are usually non-retriable:
+                    await _this.handleError(response)
+                  }
+                }
+              }).catch(err => {
+                _this.updating = false
+                _this.updatingMessage = ''
+                chatResponse.updateFromError(err.message)
+              })
+            } else {
             /**
              * Non-streaming request/response
              * We'll get the response all at once, after a long delay
              */
-            const response = await fetch(getApiBase() + getEndpointCompletions(), fetchOptions)
-            if (!response.ok) {
-              await _this.handleError(response)
-            } else {
-              const json = await response.json()
-              // Remove updating indicator
-              _this.updating = false
-              _this.updatingMessage = ''
-              chatResponse.updateFromSyncResponse(json)
+              const response = await fetch(getEndpoint(model), fetchOptions)
+              if (!response.ok) {
+                await _this.handleError(response)
+              } else {
+                const json = await response.json()
+                // Remove updating indicator
+                _this.updating = false
+                _this.updatingMessage = ''
+                chatResponse.updateFromSyncResponse(json)
+              }
             }
           }
         } catch (e) {
@@ -393,11 +502,11 @@ export class ChatRequest {
        * Gets an estimate of how many extra tokens will be added that won't be part of the visible messages
        * @param filtered
        */
-      private getTokenCountPadding (filtered: Message[]): number {
+      private getTokenCountPadding (filtered: Message[], settings: ChatSettings): number {
         let result = 0
         // add cost of hiddenPromptPrefix
         result += this.buildHiddenPromptPrefixMessages(filtered)
-          .reduce((a, m) => a + countMessageTokens(m, this.getModel()), 0)
+          .reduce((a, m) => a + countMessageTokens(m, this.getModel(), settings), 0)
         // more here eventually?
         return result
       }
@@ -419,10 +528,10 @@ export class ChatRequest {
         }
 
         // Get extra counts for when the prompts are finally sent.
-        const countPadding = this.getTokenCountPadding(filtered)
+        const countPadding = this.getTokenCountPadding(filtered, chatSettings)
 
         // See if we have enough to apply any of the reduction modes
-        const fullPromptSize = countPromptTokens(filtered, model) + countPadding
+        const fullPromptSize = countPromptTokens(filtered, model, chatSettings) + countPadding
         if (fullPromptSize < chatSettings.summaryThreshold) return await continueRequest() // nothing to do yet
         const overMax = fullPromptSize > maxTokens * 0.95
 
@@ -445,12 +554,12 @@ export class ChatRequest {
            * *************************************************************
            */
     
-          let promptSize = countPromptTokens(top.concat(rw), model) + countPadding
+          let promptSize = countPromptTokens(top.concat(rw), model, chatSettings) + countPadding
           while (rw.length && rw.length > pinBottom && promptSize >= chatSettings.summaryThreshold) {
             const rolled = rw.shift()
             // Hide messages we're "rolling"
             if (rolled) rolled.suppress = true
-            promptSize = countPromptTokens(top.concat(rw), model) + countPadding
+            promptSize = countPromptTokens(top.concat(rw), model, chatSettings) + countPadding
           }
           // Run a new request, now with the rolled messages hidden
           return await _this.sendRequest(get(currentChatMessages), {
@@ -466,26 +575,26 @@ export class ChatRequest {
           const bottom = rw.slice(0 - pinBottom)
           let continueCounter = chatSettings.summaryExtend + 1
           rw = rw.slice(0, 0 - pinBottom)
-          let reductionPoolSize = countPromptTokens(rw, model)
+          let reductionPoolSize = countPromptTokens(rw, model, chatSettings)
           const ss = Math.abs(chatSettings.summarySize)
           const getSS = ():number => (ss < 1 && ss > 0)
             ? Math.round(reductionPoolSize * ss) // If summarySize between 0 and 1, use percentage of reduced
             : Math.min(ss, reductionPoolSize * 0.5) // If > 1, use token count
-          const topSize = countPromptTokens(top, model)
+          const topSize = countPromptTokens(top, model, chatSettings)
           let maxSummaryTokens = getSS()
           let promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
           const summaryRequest = { role: 'user', content: promptSummary } as Message
-          let promptSummarySize = countMessageTokens(summaryRequest, model)
+          let promptSummarySize = countMessageTokens(summaryRequest, model, chatSettings)
           // Make sure there is enough room to generate the summary, and try to make sure
           // the last prompt is a user prompt as that seems to work better for summaries
           while ((topSize + reductionPoolSize + promptSummarySize + maxSummaryTokens) >= maxTokens ||
               (reductionPoolSize >= 100 && rw[rw.length - 1]?.role !== 'user')) {
             bottom.unshift(rw.pop() as Message)
-            reductionPoolSize = countPromptTokens(rw, model)
+            reductionPoolSize = countPromptTokens(rw, model, chatSettings)
             maxSummaryTokens = getSS()
             promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
             summaryRequest.content = promptSummary
-            promptSummarySize = countMessageTokens(summaryRequest, model)
+            promptSummarySize = countMessageTokens(summaryRequest, model, chatSettings)
           }
           if (reductionPoolSize < 50) {
             if (overMax) addError(chatId, 'Check summary settings. Unable to summarize enough messages.')
@@ -571,10 +680,10 @@ export class ChatRequest {
               // Try to get more of it
               delete summaryResponse.finish_reason
               _this.updatingMessage = 'Summarizing more...'
-              let _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model)
+              let _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chatSettings)
               while (rw.length && (_recount + maxSummaryTokens >= maxTokens)) {
                 rw.shift()
-                _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model)
+                _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chatSettings)
               }
               loopCount++
               continue
diff --git a/src/lib/ChatSettingsModal.svelte b/src/lib/ChatSettingsModal.svelte
index e41bd67..1ca0b64 100644
--- a/src/lib/ChatSettingsModal.svelte
+++ b/src/lib/ChatSettingsModal.svelte
@@ -3,7 +3,6 @@
   import { getChatDefaults, getChatSettingList, getChatSettingObjectByKey, getExcludeFromProfile } from './Settings.svelte'
   import {
     saveChatStore,
-    apiKeyStorage,
     chatsStorage,
     globalStorage,
     saveCustomProfile,
@@ -13,7 +12,7 @@
     checkStateChange,
     addChat
   } from './Storage.svelte'
-  import type { Chat, ChatSetting, ResponseModels, SettingSelect, SelectOption, ChatSettings } from './Types.svelte'
+  import type { Chat, ChatSetting, SettingSelect, ChatSettings } from './Types.svelte'
   import { errorNotice, sizeTextElements } from './Util.svelte'
   import Fa from 'svelte-fa/src/fa.svelte'
   import {
@@ -35,8 +34,7 @@
   import { replace } from 'svelte-spa-router'
   import { openModal } from 'svelte-modals'
   import PromptConfirm from './PromptConfirm.svelte'
-  import { getApiBase, getEndpointModels } from './ApiUtil.svelte'
-  import { supportedModelKeys } from './Models.svelte'
+  import { getModelOptions } from './Models.svelte'
 
   export let chatId:number
   export const show = () => { showSettings() }
@@ -184,31 +182,10 @@
 
     // Refresh settings modal
     showSettingsModal++
-  
-    // Load available models from OpenAI
-    const allModels = (await (
-      await fetch(getApiBase() + getEndpointModels(), {
-        method: 'GET',
-        headers: {
-          Authorization: `Bearer ${$apiKeyStorage}`,
-          'Content-Type': 'application/json'
-        }
-      })
-    ).json()) as ResponseModels
-    const filteredModels = supportedModelKeys.filter((model) => allModels.data.find((m) => m.id === model))
-
-    const modelOptions:SelectOption[] = filteredModels.reduce((a, m) => {
-      const o:SelectOption = {
-        value: m,
-        text: m
-      }
-      a.push(o)
-      return a
-    }, [] as SelectOption[])
 
     // Update the models in the settings
     if (modelSetting) {
-      modelSetting.options = modelOptions
+      modelSetting.options = await getModelOptions()
     }
     // Refresh settings modal
     showSettingsModal++
diff --git a/src/lib/Home.svelte b/src/lib/Home.svelte
index 16a2fc3..c86a17a 100644
--- a/src/lib/Home.svelte
+++ b/src/lib/Home.svelte
@@ -1,11 +1,14 @@
 <script lang="ts">
-  import { apiKeyStorage, lastChatId, getChat, started } from './Storage.svelte'
+  import { apiKeyStorage, globalStorage, lastChatId, getChat, started, setGlobalSettingValueByKey } from './Storage.svelte'
   import Footer from './Footer.svelte'
   import { replace } from 'svelte-spa-router'
   import { onMount } from 'svelte'
+  import { getPetalsV2Websocket } from './ApiUtil.svelte'
 
 $: apiKey = $apiKeyStorage
 
+let showPetalsSettings = $globalStorage.enablePetals
+
 onMount(() => {
     if (!$started) {
       $started = true
@@ -19,6 +22,12 @@ onMount(() => {
     $lastChatId = 0
 })
 
+const setPetalsEnabled = (event: Event) => {
+    const el = (event.target as HTMLInputElement)
+    setGlobalSettingValueByKey('enablePetals', !!el.checked)
+    showPetalsSettings = $globalStorage.enablePetals
+}
+
 </script>
 
 <section class="section">
@@ -60,6 +69,8 @@ onMount(() => {
         <p class="control">
           <button class="button is-info" type="submit">Save</button>
         </p>
+
+
       </form>
 
       {#if !apiKey}
@@ -70,6 +81,66 @@ onMount(() => {
       {/if}
     </div>
   </article>
+
+  
+  <article class="message" class:is-info={true}>
+    <div class="message-body">
+      <label class="label" for="enablePetals">
+        <input 
+        type="checkbox"
+        class="checkbox" 
+        id="enablePetals"
+        checked={!!$globalStorage.enablePetals} 
+        on:click={setPetalsEnabled}
+      >
+        Use Petals API and Models
+      </label>
+      {#if showPetalsSettings}
+        <p>Set Petals API Endpoint:</p>
+        <form
+          class="field has-addons has-addons-right"
+          on:submit|preventDefault={(event) => {
+            if (event.target && event.target[0].value) {
+              setGlobalSettingValueByKey('pedalsEndpoint', (event.target[0].value).trim())
+            } else {
+              setGlobalSettingValueByKey('pedalsEndpoint', '')
+            }
+          }}
+        >
+          <p class="control is-expanded">
+            <input
+              aria-label="PetalsAPI Endpoint"
+              type="text"
+              class="input"
+              placeholder={getPetalsV2Websocket()}
+              value={$globalStorage.pedalsEndpoint || ''}
+            />
+          </p>
+          <p class="control">
+            <button class="button is-info" type="submit">Save</button>
+          </p>
+
+          
+        </form>
+        <p>
+          Only use <u>{getPetalsV2Websocket()}</u> for testing.  You must set up your own Petals server for actual use. 
+        </p>
+        <p>
+          <b>Do not send sensitive information when using Petals.</b>
+        </p>
+        <p>
+            For more information on Petals, see 
+            <a href="https://github.com/petals-infra/chat.petals.dev">https://github.com/petals-infra/chat.petals.dev</a>
+        </p>
+      {/if}
+      {#if !apiKey}
+        <p class="help is-danger">
+          Please enter your <a href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use ChatGPT-web.
+          It is required to use ChatGPT-web.
+        </p>
+      {/if}
+    </div>
+  </article>
   {#if apiKey}
     <article class="message is-info">
       <div class="message-body">
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index ed8861d..1289939 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -1,43 +1,63 @@
 <script context="module" lang="ts">
-    import type { ModelDetail, Model } from './Types.svelte'
+    import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetalsV2Websocket } from './ApiUtil.svelte'
+    import { apiKeyStorage, globalStorage } from './Storage.svelte'
+    import { get } from 'svelte/store'
+    import type { ModelDetail, Model, ResponseModels, SelectOption, ChatSettings } from './Types.svelte'
+import { encode } from 'gpt-tokenizer'
+import llamaTokenizer from 'llama-tokenizer-js'
 
 // Reference: https://openai.com/pricing#language-models
 // Eventually we'll add API hosts and endpoints to this
 const modelDetails : Record<string, ModelDetail> = {
       'gpt-4-32k': {
+        type: 'OpenAIChat',
         prompt: 0.00006, // $0.06 per 1000 tokens prompt
         completion: 0.00012, // $0.12 per 1000 tokens completion
         max: 32768 // 32k max token buffer
       },
       'gpt-4': {
+        type: 'OpenAIChat',
         prompt: 0.00003, // $0.03 per 1000 tokens prompt
         completion: 0.00006, // $0.06 per 1000 tokens completion
         max: 8192 // 8k max token buffer
       },
       'gpt-3.5': {
+        type: 'OpenAIChat',
         prompt: 0.0000015, // $0.0015 per 1000 tokens prompt
         completion: 0.000002, // $0.002 per 1000 tokens completion
         max: 4096 // 4k max token buffer
       },
       'gpt-3.5-turbo-16k': {
+        type: 'OpenAIChat',
         prompt: 0.000003, // $0.003 per 1000 tokens prompt
         completion: 0.000004, // $0.004 per 1000 tokens completion
         max: 16384 // 16k max token buffer
+      },
+      'meta-llama/Llama-2-70b-chat-hf': {
+        type: 'PetalsV2Websocket',
+        label: 'Petals - Llama-2-70b-chat',
+        stop: ['###', '</s>'],
+        prompt: 0.000000, // $0.000 per 1000 tokens prompt
+        completion: 0.000000, // $0.000 per 1000 tokens completion
+        max: 4096 // 4k max token buffer
       }
 }
 
-const imageModels : Record<string, ModelDetail> = {
+export const imageModels : Record<string, ModelDetail> = {
       'dall-e-1024x1024': {
+        type: 'OpenAIDall-e',
         prompt: 0.00,
         completion: 0.020, // $0.020 per image
         max: 1000 // 1000 char prompt, max
       },
       'dall-e-512x512': {
+        type: 'OpenAIDall-e',
         prompt: 0.00,
         completion: 0.018, // $0.018 per image
         max: 1000 // 1000 char prompt, max
       },
       'dall-e-256x256': {
+        type: 'OpenAIDall-e',
         prompt: 0.00,
         completion: 0.016, // $0.016 per image
         max: 1000 // 1000 char prompt, max
@@ -47,8 +67,9 @@ const imageModels : Record<string, ModelDetail> = {
 const unknownDetail = {
   prompt: 0,
   completion: 0,
-  max: 4096
-}
+  max: 4096,
+  type: 'OpenAIChat'
+} as ModelDetail
 
 // See: https://platform.openai.com/docs/models/model-endpoint-compatibility
 // Eventually we'll add UI for managing this
@@ -62,7 +83,8 @@ export const supportedModels : Record<string, ModelDetail> = {
       'gpt-3.5-turbo': modelDetails['gpt-3.5'],
       'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
       'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
-      'gpt-3.5-turbo-0613': modelDetails['gpt-3.5']
+      'gpt-3.5-turbo-0613': modelDetails['gpt-3.5'],
+      'meta-llama/Llama-2-70b-chat-hf': modelDetails['meta-llama/Llama-2-70b-chat-hf']
 }
 
 const lookupList = {
@@ -75,7 +97,7 @@ export const supportedModelKeys = Object.keys({ ...supportedModels, ...imageMode
 
 const tpCache : Record<string, ModelDetail> = {}
 
-export const getModelDetail = (model: Model) => {
+export const getModelDetail = (model: Model): ModelDetail => {
       // First try to get exact match, then from cache
       let r = supportedModels[model] || tpCache[model]
       if (r) return r
@@ -93,4 +115,93 @@ export const getModelDetail = (model: Model) => {
       return r
 }
 
+export const getEndpoint = (model: Model): string => {
+  const modelDetails = getModelDetail(model)
+  const gSettings = get(globalStorage)
+  switch (modelDetails.type) {
+        case 'PetalsV2Websocket':
+          return gSettings.pedalsEndpoint || getPetalsV2Websocket()
+        case 'OpenAIDall-e':
+          return getApiBase() + getEndpointGenerations()
+        case 'OpenAIChat':
+        default:
+          return gSettings.openAICompletionEndpoint || (getApiBase() + getEndpointCompletions())
+  }
+}
+
+export const getRoleTag = (role: string, model: Model, settings: ChatSettings): string => {
+  const modelDetails = getModelDetail(model)
+  switch (modelDetails.type) {
+        case 'PetalsV2Websocket':
+          if (role === 'assistant') {
+            return ('Assistant') +
+              ': '
+          }
+          if (role === 'user') return 'Human: '
+          return ''
+        case 'OpenAIDall-e':
+          return role
+        case 'OpenAIChat':
+        default:
+          return role
+  }
+}
+
+export const getTokens = (model: Model, value: string): number[] => {
+  const modelDetails = getModelDetail(model)
+  switch (modelDetails.type) {
+        case 'PetalsV2Websocket':
+          return llamaTokenizer.encode(value)
+        case 'OpenAIDall-e':
+          return [0]
+        case 'OpenAIChat':
+        default:
+          return encode(value)
+  }
+}
+
+export const countTokens = (model: Model, value: string): number => {
+  return getTokens(model, value).length
+}
+
+export async function getModelOptions (): Promise<SelectOption[]> {
+  const gSettings = get(globalStorage)
+  const openAiKey = get(apiKeyStorage)
+  // Load available models from OpenAI
+  let openAiModels
+  try {
+        openAiModels = (await (
+          await fetch(getApiBase() + getEndpointModels(), {
+            method: 'GET',
+            headers: {
+              Authorization: `Bearer ${openAiKey}`,
+              'Content-Type': 'application/json'
+            }
+          })
+        ).json()) as ResponseModels
+  } catch (e) {
+        openAiModels = { data: [] }
+  }
+  const filteredModels = supportedModelKeys.filter((model) => {
+        switch (getModelDetail(model).type) {
+          case 'PetalsV2Websocket':
+            return gSettings.enablePetals
+          case 'OpenAIChat':
+          default:
+            return openAiModels.data.find((m) => m.id === model)
+        }
+  })
+
+  const modelOptions:SelectOption[] = filteredModels.reduce((a, m) => {
+        const o:SelectOption = {
+          value: m,
+          text: m
+        }
+        a.push(o)
+        return a
+  }, [] as SelectOption[])
+
+  return modelOptions
+}
+
 </script>
\ No newline at end of file
diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte
index f4540c6..fb525af 100644
--- a/src/lib/Settings.svelte
+++ b/src/lib/Settings.svelte
@@ -1,7 +1,6 @@
 <script context="module" lang="ts">
     import { applyProfile } from './Profiles.svelte'
     import { getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
-    import { encode } from 'gpt-tokenizer'
     import { faArrowDown91, faArrowDownAZ, faCheck, faThumbTack } from '@fortawesome/free-solid-svg-icons/index'
 // Setting definitions
 
@@ -18,6 +17,7 @@ import {
       type ChatSortOption
 
 } from './Types.svelte'
+    import { getTokens } from './Models.svelte'
 
 export const defaultModel:Model = 'gpt-3.5-turbo'
 
@@ -104,7 +104,10 @@ export const globalDefaults: GlobalSettings = {
   lastProfile: 'default',
   defaultProfile: 'default',
   hideSummarized: false,
-  chatSort: 'created'
+  chatSort: 'created',
+  openAICompletionEndpoint: '',
+  enablePetals: false,
+  pedalsEndpoint: ''
 }
 
 const excludeFromProfile = {
@@ -497,7 +500,7 @@ const chatSettingsList: ChatSetting[] = [
           // console.log('logit_bias', val, getChatSettings(chatId).logit_bias)
           if (!val) return null
           const tokenized:Record<number, number> = Object.entries(val).reduce((a, [k, v]) => {
-            const tokens:number[] = encode(k)
+            const tokens:number[] = getTokens(getChatSettings(chatId).model, k)
             tokens.forEach(t => { a[t] = v })
             return a
           }, {} as Record<number, number>)
@@ -536,6 +539,21 @@ const globalSettingsList:GlobalSetting[] = [
         key: 'hideSummarized',
         name: 'Hide Summarized Messages',
         type: 'boolean'
+      },
+      {
+        key: 'openAICompletionEndpoint',
+        name: 'OpenAI Completions Endpoint',
+        type: 'text'
+      },
+      {
+        key: 'enablePetals',
+        name: 'Enable Petals APIs',
+        type: 'boolean'
+      },
+      {
+        key: 'pedalsEndpoint',
+        name: 'Petals API Endpoint',
+        type: 'text'
       }
 ]
 
diff --git a/src/lib/Stats.svelte b/src/lib/Stats.svelte
index 9bb2366..efee712 100644
--- a/src/lib/Stats.svelte
+++ b/src/lib/Stats.svelte
@@ -1,25 +1,49 @@
 <script context="module" lang="ts">
-  import { getModelDetail } from './Models.svelte'
-  import type { Message, Model, Usage } from './Types.svelte'
-  import { encode } from 'gpt-tokenizer'
+  import { countTokens, getModelDetail, getRoleTag } from './Models.svelte'
+  import type { ChatSettings, Message, Model, Usage } from './Types.svelte'
 
   export const getPrice = (tokens: Usage, model: Model): number => {
     const t = getModelDetail(model)
     return ((tokens.prompt_tokens * t.prompt) + (tokens.completion_tokens * t.completion))
   }
 
-  export const countPromptTokens = (prompts:Message[], model:Model):number => {
-    return prompts.reduce((a, m) => {
-      a += countMessageTokens(m, model)
+  export const countPromptTokens = (prompts:Message[], model:Model, settings: ChatSettings):number => {
+    const detail = getModelDetail(model)
+    const count = prompts.reduce((a, m) => {
+      switch (detail.type) {
+        case 'PetalsV2Websocket':
+          a += countMessageTokens(m, model, settings)
+          break
+        case 'OpenAIChat':
+        default:
+          a += countMessageTokens(m, model, settings)
+      }
       return a
-    }, 0) + 3 // Always seems to be message counts + 3
+    }, 0)
+    switch (detail.type) {
+      case 'PetalsV2Websocket':
+        return count + (Math.max(prompts.length - 1, 0) * countTokens(model, (detail.stop && detail.stop[0]) || '###')) // todo, make stop per model?
+      case 'OpenAIChat':
+      default:
+        // Not sure how OpenAI formats it, but this seems to get close to the right counts.
+        // Would be nice to know. This works for gpt-3.5.  gpt-4 could be different.
+        // Complete stab in the dark here -- update if you know where all the extra tokens really come from.
+        return count + 3 // Always seems to be message counts + 3
+    }
   }
 
-  export const countMessageTokens = (message:Message, model:Model):number => {
-    // Not sure how OpenAI formats it, but this seems to get close to the right counts.
-    // Would be nice to know. This works for gpt-3.5.  gpt-4 could be different.
-    // Complete stab in the dark here -- update if you know where all the extra tokens really come from.
-    return encode('## ' + message.role + ' ##:\r\n\r\n' + message.content + '\r\n\r\n\r\n').length
+  export const countMessageTokens = (message:Message, model:Model, settings: ChatSettings):number => {
+    const detail = getModelDetail(model)
+    switch (detail.type) {
+      case 'PetalsV2Websocket':
+        return countTokens(model, getRoleTag(message.role, model, settings) + ': ' + message.content)
+      case 'OpenAIChat':
+      default:
+        // Not sure how OpenAI formats it, but this seems to get close to the right counts.
+        // Would be nice to know. This works for gpt-3.5.  gpt-4 could be different.
+        // Complete stab in the dark here -- update if you know where all the extra tokens really come from.
+        return countTokens(model, '## ' + message.role + ' ##:\r\n\r\n' + message.content + '\r\n\r\n\r\n')
+    }
   }
 
   export const getModelMaxTokens = (model:Model):number => {
diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte
index 1c99c97..5297c43 100644
--- a/src/lib/Types.svelte
+++ b/src/lib/Types.svelte
@@ -7,7 +7,12 @@ export type Model = typeof supportedModelKeys[number];
 
 export type ImageGenerationSizes = typeof imageGenerationSizeTypes[number];
 
+export type RequestType = 'OpenAIChat' | 'OpenAIDall-e' | 'PetalsV2Websocket'
+
 export type ModelDetail = {
+    type: RequestType;
+    label?: string;
+    stop?: string[];
     prompt: number;
     completion: number;
     max: number;
@@ -122,16 +127,16 @@ export type Chat = {
   };
 
   type ResponseOK = {
-    id: string;
-    object: string;
-    created: number;
-    choices: {
-      index: number;
+    id?: string;
+    object?: string;
+    created?: number;
+    choices?: {
+      index?: number;
       message: Message;
-      finish_reason: string;
+      finish_reason?: string;
       delta: Message;
     }[];
-    usage: Usage;
+    usage?: Usage;
     model: Model;
   };
 
@@ -172,6 +177,9 @@ export type GlobalSettings = {
     defaultProfile: string;
     hideSummarized: boolean;
     chatSort: ChatSortOptions;
+    openAICompletionEndpoint: string;
+    enablePetals: boolean;
+    pedalsEndpoint: string;
   };
 
   type SettingNumber = {

From cdbd17fa13a85c40b6045fb89d7c8a4016ae66b6 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 08:56:12 -0500
Subject: [PATCH 03/17] Don't show generation setting that don't apply

---
 src/lib/ChatSettingField.svelte | 16 +++++++++++++---
 src/lib/Settings.svelte         | 19 ++++++++++++++-----
 2 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/src/lib/ChatSettingField.svelte b/src/lib/ChatSettingField.svelte
index 394108b..a86f616 100644
--- a/src/lib/ChatSettingField.svelte
+++ b/src/lib/ChatSettingField.svelte
@@ -22,6 +22,14 @@
 
   const chatId = chat.id
   let show = false
+
+  const valueOf = (value: any) => {
+    if (typeof value === 'function') return value(chatId, setting)
+    return value
+  }
+
+  let header = valueOf(setting.header)
+  let headerClass = valueOf(setting.headerClass)
   
   const buildFieldControls = () => {
     fieldControls = (setting.fieldControls || [] as FieldControl[]).map(fc => {
@@ -38,6 +46,8 @@
 
   afterUpdate(() => {
     show = (typeof setting.hide !== 'function') || !setting.hide(chatId)
+    header = valueOf(setting.header)
+    headerClass = valueOf(setting.headerClass)
     buildFieldControls()
   })
 
@@ -146,9 +156,9 @@
 </script>
 
 {#if show}
-  {#if setting.header}
-  <p class="notification {setting.headerClass}">
-    {@html setting.header}
+  {#if header}
+  <p class="notification {headerClass}">
+    {@html header}
   </p>
   {/if}
   <div class="field is-horizontal">
diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte
index fb525af..09a3011 100644
--- a/src/lib/Settings.svelte
+++ b/src/lib/Settings.svelte
@@ -17,7 +17,7 @@ import {
       type ChatSortOption
 
 } from './Types.svelte'
-    import { getTokens } from './Models.svelte'
+    import { getModelDetail, getTokens } from './Models.svelte'
 
 export const defaultModel:Model = 'gpt-3.5-turbo'
 
@@ -410,6 +410,10 @@ const modelSetting: ChatSetting & SettingSelect = {
       afterChange: (chatId, setting) => true // refresh settings
 }
 
+const isNotOpenAI = (chatId) => {
+  return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
+}
+
 const chatSettingsList: ChatSetting[] = [
       profileSetting,
       ...systemPromptSettings,
@@ -420,7 +424,8 @@ const chatSettingsList: ChatSetting[] = [
         key: 'stream',
         name: 'Stream Response',
         title: 'Stream responses as they are generated.',
-        type: 'boolean'
+        type: 'boolean',
+        hide: isNotOpenAI
       },
       {
         key: 'temperature',
@@ -451,7 +456,8 @@ const chatSettingsList: ChatSetting[] = [
         min: 1,
         max: 10,
         step: 1,
-        type: 'number'
+        type: 'number',
+        hide: isNotOpenAI
       },
       {
         key: 'max_tokens',
@@ -463,6 +469,7 @@ const chatSettingsList: ChatSetting[] = [
         max: 32768,
         step: 1,
         type: 'number',
+        hide: isNotOpenAI,
         forceApi: true // Since default here is different than gpt default, will make sure we always send it
       },
       {
@@ -472,7 +479,8 @@ const chatSettingsList: ChatSetting[] = [
         min: -2,
         max: 2,
         step: 0.2,
-        type: 'number'
+        type: 'number',
+        hide: isNotOpenAI
       },
       {
         key: 'frequency_penalty',
@@ -481,7 +489,8 @@ const chatSettingsList: ChatSetting[] = [
         min: -2,
         max: 2,
         step: 0.2,
-        type: 'number'
+        type: 'number',
+        hide: isNotOpenAI
       },
       {
         // logit bias editor not implemented yet

From df222e7028ee17db227cf8f7e0ed520fa7f0c65d Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 13:23:24 -0500
Subject: [PATCH 04/17] Try to import chat name suggestion

---
 src/lib/Chat.svelte | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/src/lib/Chat.svelte b/src/lib/Chat.svelte
index 6826d9c..ebe6874 100644
--- a/src/lib/Chat.svelte
+++ b/src/lib/Chat.svelte
@@ -273,13 +273,16 @@
   const suggestName = async (): Promise<void> => {
     const suggestMessage: Message = {
       role: 'user',
-      content: "Using appropriate language, please give a 5 word summary of this conversation's topic.",
+      content: "Using appropriate language, please tell me a short 6 word summary of this conversation's topic for use as a book title. Only respond with the summary.",
       uuid: uuidv4()
     }
 
     const suggestMessages = $currentChatMessages.slice(0, 10) // limit to first 10 messages
     suggestMessages.push(suggestMessage)
 
+    chatRequest.updating = true
+    chatRequest.updatingMessage = 'Getting suggestion for chat name...'
+
     const response = await chatRequest.sendRequest(suggestMessages, {
       chat,
       autoAddMessages: false,
@@ -297,7 +300,7 @@
       })
     } else {
       response.getMessages().forEach(m => {
-        const name = m.content.split(/\s+/).slice(0, 8).join(' ').trim()
+        const name = m.content.split(/\s+/).slice(0, 8).join(' ').replace(/^[^a-z0-9!?]+|[^a-z0-9!?]+$/gi, '').trim()
         if (name) chat.name = name
       })
       saveChatStore()

From 9a6004c55d36bac64b393fe45ee832cc60b75910 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 13:24:18 -0500
Subject: [PATCH 05/17] More changes for Petals integration

---
 src/lib/ApiUtil.svelte                |   4 +-
 src/lib/ChatCompletionResponse.svelte |   4 +
 src/lib/ChatRequest.svelte            | 204 ++------------------------
 src/lib/ChatRequestOpenAi.svelte      | 100 +++++++++++++
 src/lib/ChatRequestPetals.svelte      | 126 ++++++++++++++++
 src/lib/Home.svelte                   |   6 +-
 src/lib/Models.svelte                 |  23 +--
 src/lib/Stats.svelte                  |  19 +--
 src/lib/Types.svelte                  |   3 +-
 9 files changed, 271 insertions(+), 218 deletions(-)
 create mode 100644 src/lib/ChatRequestOpenAi.svelte
 create mode 100644 src/lib/ChatRequestPetals.svelte

diff --git a/src/lib/ApiUtil.svelte b/src/lib/ApiUtil.svelte
index ceded8b..afd2f7f 100644
--- a/src/lib/ApiUtil.svelte
+++ b/src/lib/ApiUtil.svelte
@@ -5,12 +5,12 @@
   const endpointGenerations = import.meta.env.VITE_ENDPOINT_GENERATIONS || '/v1/images/generations'
   const endpointModels = import.meta.env.VITE_ENDPOINT_MODELS || '/v1/models'
   const endpointEmbeddings = import.meta.env.VITE_ENDPOINT_EMBEDDINGS || '/v1/embeddings'
-  const endpointPetalsV2Websocket = import.meta.env.VITE_PEDALS_WEBSOCKET || 'wss://chat.petals.dev/api/v2/generate'
+  const endpointPetals = import.meta.env.VITE_PEDALS_WEBSOCKET || 'wss://chat.petals.dev/api/v2/generate'
 
   export const getApiBase = ():string => apiBase
   export const getEndpointCompletions = ():string => endpointCompletions
   export const getEndpointGenerations = ():string => endpointGenerations
   export const getEndpointModels = ():string => endpointModels
   export const getEndpointEmbeddings = ():string => endpointEmbeddings
-  export const getPetalsV2Websocket = ():string => endpointPetalsV2Websocket
+  export const getPetals = ():string => endpointPetals
 </script>
\ No newline at end of file
diff --git a/src/lib/ChatCompletionResponse.svelte b/src/lib/ChatCompletionResponse.svelte
index a6743f6..ab5fcff 100644
--- a/src/lib/ChatCompletionResponse.svelte
+++ b/src/lib/ChatCompletionResponse.svelte
@@ -65,6 +65,10 @@ export class ChatCompletionResponse {
     this.promptTokenCount = tokens
   }
 
+  getPromptTokenCount (): number {
+    return this.promptTokenCount
+  }
+
   async updateImageFromSyncResponse (response: ResponseImage, prompt: string, model: Model) {
     this.setModel(model)
     for (let i = 0; i < response.data.length; i++) {
diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index 20b5626..40c966e 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -6,10 +6,11 @@
     import { deleteMessage, getChatSettingValueNullDefault, insertMessages, getApiKey, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage } from './Storage.svelte'
     import { scrollToBottom, scrollToMessage } from './Util.svelte'
     import { getRequestSettingList, defaultModel } from './Settings.svelte'
-    import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
     import { v4 as uuidv4 } from 'uuid'
     import { get } from 'svelte/store'
-    import { getEndpoint, getModelDetail, getRoleTag } from './Models.svelte'
+    import { getEndpoint, getModelDetail } from './Models.svelte'
+    import { runOpenAiCompletionRequest } from './ChatRequestOpenAi.svelte'
+    import { runPetalsCompletionRequest } from './ChatRequestPetals.svelte'
 
 export class ChatRequest {
       constructor () {
@@ -27,6 +28,14 @@ export class ChatRequest {
         this.chat = chat
       }
 
+      getChat (): Chat {
+        return this.chat
+      }
+
+      getChatSettings (): ChatSettings {
+        return this.chat.settings
+      }
+
       // Common error handler
       async handleError (response) {
         let errorResponse
@@ -258,193 +267,10 @@ export class ChatRequest {
           _this.controller = new AbortController()
           const signal = _this.controller.signal
 
-          if (modelDetail.type === 'PetalsV2Websocket') {
-            // Petals
-            const ws = new WebSocket(getEndpoint(model))
-            const abortListener = (e:Event) => {
-              _this.updating = false
-              _this.updatingMessage = ''
-              chatResponse.updateFromError('User aborted request.')
-              signal.removeEventListener('abort', abortListener)
-              ws.close()
-            }
-            signal.addEventListener('abort', abortListener)
-            const stopSequences = modelDetail.stop || ['###']
-            const stopSequencesC = stopSequences.slice()
-            const stopSequence = stopSequencesC.shift()
-            chatResponse.onFinish(() => {
-              _this.updating = false
-              _this.updatingMessage = ''
-            })
-            ws.onopen = () => {
-              ws.send(JSON.stringify({
-                type: 'open_inference_session',
-                model,
-                max_length: maxTokens || opts.maxTokens
-              }))
-              ws.onmessage = event => {
-                const response = JSON.parse(event.data)
-                if (!response.ok) {
-                  const err = new Error('Error opening socket: ' + response.traceback)
-                  console.error(err)
-                  throw err
-                }
-                const rMessages = request.messages || [] as Message[]
-                const inputArray = (rMessages).reduce((a, m) => {
-                  const c = getRoleTag(m.role, model, chatSettings) + m.content
-                  a.push(c)
-                  return a
-                }, [] as string[])
-                const lastMessage = rMessages[rMessages.length - 1]
-                if (lastMessage && lastMessage.role !== 'assistant') {
-                  inputArray.push(getRoleTag('assistant', model, chatSettings))
-                }
-                const petalsRequest = {
-                  type: 'generate',
-                  inputs: (request.messages || [] as Message[]).reduce((a, m) => {
-                    const c = getRoleTag(m.role, model, chatSettings) + m.content
-                    a.push(c)
-                    return a
-                  }, [] as string[]).join(stopSequence),
-                  max_new_tokens: 3, // wait for up to 3 tokens before displaying
-                  stop_sequence: stopSequence,
-                  doSample: 1,
-                  temperature: request.temperature || 0,
-                  top_p: request.top_p || 0,
-                  extra_stop_sequences: stopSequencesC
-                }
-                ws.send(JSON.stringify(petalsRequest))
-                ws.onmessage = event => {
-                  // Remove updating indicator
-                  _this.updating = 1 // hide indicator, but still signal we're updating
-                  _this.updatingMessage = ''
-                  const response = JSON.parse(event.data)
-                  if (!response.ok) {
-                    const err = new Error('Error in response: ' + response.traceback)
-                    console.error(err)
-                    throw err
-                  }
-                  window.setTimeout(() => {
-                    chatResponse.updateFromAsyncResponse(
-                      {
-                        model,
-                        choices: [{
-                          delta: {
-                            content: response.outputs,
-                            role: 'assistant'
-                          },
-                          finish_reason: (response.stop ? 'stop' : null)
-                        }]
-                      } as any
-                    )
-                    if (response.stop) {
-                      const message = chatResponse.getMessages()[0]
-                      if (message) {
-                        for (let i = 0, l = stopSequences.length; i < l; i++) {
-                          if (message.content.endsWith(stopSequences[i])) {
-                            message.content = message.content.slice(0, message.content.length - stopSequences[i].length)
-                            updateMessages(chatId)
-                          }
-                        }
-                      }
-                    }
-                  }, 1)
-                }
-              }
-              ws.onclose = () => {
-                _this.updating = false
-                _this.updatingMessage = ''
-                chatResponse.updateFromClose()
-              }
-              ws.onerror = err => {
-                console.error(err)
-                throw err
-              }
-            }
+          if (modelDetail.type === 'Petals') {
+            await runPetalsCompletionRequest(request, _this as any, chatResponse as any, signal, opts)
           } else {
-            // OpenAI
-            const abortListener = (e:Event) => {
-              _this.updating = false
-              _this.updatingMessage = ''
-              chatResponse.updateFromError('User aborted request.')
-              signal.removeEventListener('abort', abortListener)
-            }
-            signal.addEventListener('abort', abortListener)
-            const fetchOptions = {
-              method: 'POST',
-              headers: {
-                Authorization: `Bearer ${getApiKey()}`,
-                'Content-Type': 'application/json'
-              },
-              body: JSON.stringify(request),
-              signal
-            }
-
-            if (opts.streaming) {
-            /**
-             * Streaming request/response
-             * We'll get the response a token at a time, as soon as they are ready
-            */
-              chatResponse.onFinish(() => {
-                _this.updating = false
-                _this.updatingMessage = ''
-              })
-              fetchEventSource(getEndpoint(model), {
-                ...fetchOptions,
-                openWhenHidden: true,
-                onmessage (ev) {
-                  // Remove updating indicator
-                  _this.updating = 1 // hide indicator, but still signal we're updating
-                  _this.updatingMessage = ''
-                  // console.log('ev.data', ev.data)
-                  if (!chatResponse.hasFinished()) {
-                    if (ev.data === '[DONE]') {
-                      // ?? anything to do when "[DONE]"?
-                    } else {
-                      const data = JSON.parse(ev.data)
-                      // console.log('data', data)
-                      window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
-                    }
-                  }
-                },
-                onclose () {
-                  _this.updating = false
-                  _this.updatingMessage = ''
-                  chatResponse.updateFromClose()
-                },
-                onerror (err) {
-                  console.error(err)
-                  throw err
-                },
-                async onopen (response) {
-                  if (response.ok && response.headers.get('content-type') === EventStreamContentType) {
-                    // everything's good
-                  } else {
-                    // client-side errors are usually non-retriable:
-                    await _this.handleError(response)
-                  }
-                }
-              }).catch(err => {
-                _this.updating = false
-                _this.updatingMessage = ''
-                chatResponse.updateFromError(err.message)
-              })
-            } else {
-            /**
-             * Non-streaming request/response
-             * We'll get the response all at once, after a long delay
-             */
-              const response = await fetch(getEndpoint(model), fetchOptions)
-              if (!response.ok) {
-                await _this.handleError(response)
-              } else {
-                const json = await response.json()
-                // Remove updating indicator
-                _this.updating = false
-                _this.updatingMessage = ''
-                chatResponse.updateFromSyncResponse(json)
-              }
-            }
+            await runOpenAiCompletionRequest(request, _this as any, chatResponse as any, signal, opts)
           }
         } catch (e) {
         // console.error(e)
@@ -456,7 +282,7 @@ export class ChatRequest {
         return chatResponse
       }
 
-      private getModel (): Model {
+      getModel (): Model {
         return this.chat.settings.model || defaultModel
       }
 
diff --git a/src/lib/ChatRequestOpenAi.svelte b/src/lib/ChatRequestOpenAi.svelte
new file mode 100644
index 0000000..37495ef
--- /dev/null
+++ b/src/lib/ChatRequestOpenAi.svelte
@@ -0,0 +1,100 @@
+<script context="module" lang="ts">
+    import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
+    import ChatCompletionResponse from './ChatCompletionResponse.svelte'
+    import ChatRequest from './ChatRequest.svelte'
+    import { getEndpoint } from './Models.svelte'
+    import { getApiKey } from './Storage.svelte'
+    import type { ChatCompletionOpts, Request } from './Types.svelte'
+
+export const runOpenAiCompletionRequest = async (
+  request: Request,
+  chatRequest: ChatRequest,
+  chatResponse: ChatCompletionResponse,
+  signal: AbortSignal,
+  opts: ChatCompletionOpts) => {
+    // OpenAI Request
+      const model = chatRequest.getModel()
+      const abortListener = (e:Event) => {
+        chatRequest.updating = false
+        chatRequest.updatingMessage = ''
+        chatResponse.updateFromError('User aborted request.')
+        chatRequest.removeEventListener('abort', abortListener)
+      }
+      signal.addEventListener('abort', abortListener)
+      const fetchOptions = {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${getApiKey()}`,
+          'Content-Type': 'application/json'
+        },
+        body: JSON.stringify(request),
+        signal
+      }
+
+      if (opts.streaming) {
+      /**
+             * Streaming request/response
+             * We'll get the response a token at a time, as soon as they are ready
+            */
+        chatResponse.onFinish(() => {
+          chatRequest.updating = false
+          chatRequest.updatingMessage = ''
+        })
+        fetchEventSource(getEndpoint(model), {
+          ...fetchOptions,
+          openWhenHidden: true,
+          onmessage (ev) {
+          // Remove updating indicator
+            chatRequest.updating = 1 // hide indicator, but still signal we're updating
+            chatRequest.updatingMessage = ''
+            // console.log('ev.data', ev.data)
+            if (!chatResponse.hasFinished()) {
+              if (ev.data === '[DONE]') {
+              // ?? anything to do when "[DONE]"?
+              } else {
+                const data = JSON.parse(ev.data)
+                // console.log('data', data)
+                window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
+              }
+            }
+          },
+          onclose () {
+            chatRequest.updating = false
+            chatRequest.updatingMessage = ''
+            chatResponse.updateFromClose()
+          },
+          onerror (err) {
+            console.error(err)
+            throw err
+          },
+          async onopen (response) {
+            if (response.ok && response.headers.get('content-type') === EventStreamContentType) {
+            // everything's good
+            } else {
+            // client-side errors are usually non-retriable:
+              await chatRequest.handleError(response)
+            }
+          }
+        }).catch(err => {
+          chatRequest.updating = false
+          chatRequest.updatingMessage = ''
+          chatResponse.updateFromError(err.message)
+        })
+      } else {
+      /**
+             * Non-streaming request/response
+             * We'll get the response all at once, after a long delay
+             */
+        const response = await fetch(getEndpoint(model), fetchOptions)
+        if (!response.ok) {
+          await chatRequest.handleError(response)
+        } else {
+          const json = await response.json()
+          // Remove updating indicator
+          chatRequest.updating = false
+          chatRequest.updatingMessage = ''
+          chatResponse.updateFromSyncResponse(json)
+        }
+      }
+}
+</script>
\ No newline at end of file
diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
new file mode 100644
index 0000000..b0c1bac
--- /dev/null
+++ b/src/lib/ChatRequestPetals.svelte
@@ -0,0 +1,126 @@
+<script context="module" lang="ts">
+    import ChatCompletionResponse from './ChatCompletionResponse.svelte'
+    import ChatRequest from './ChatRequest.svelte'
+    import { getEndpoint, getModelDetail, getRoleTag } from './Models.svelte'
+    import type { ChatCompletionOpts, Message, Request } from './Types.svelte'
+    import { getModelMaxTokens } from './Stats.svelte'
+    import { updateMessages } from './Storage.svelte'
+
+export const runPetalsCompletionRequest = async (
+  request: Request,
+  chatRequest: ChatRequest,
+  chatResponse: ChatCompletionResponse,
+  signal: AbortSignal,
+  opts: ChatCompletionOpts) => {
+      // Petals
+      const model = chatRequest.getModel()
+      const modelDetail = getModelDetail(model)
+      const ws = new WebSocket(getEndpoint(model))
+      const abortListener = (e:Event) => {
+        chatRequest.updating = false
+        chatRequest.updatingMessage = ''
+        chatResponse.updateFromError('User aborted request.')
+        signal.removeEventListener('abort', abortListener)
+        ws.close()
+      }
+      signal.addEventListener('abort', abortListener)
+      const startSequences = modelDetail.start || []
+      const startSequence = startSequences[0] || ''
+      const stopSequences = modelDetail.stop || ['###']
+      const stopSequencesC = stopSequences.slice()
+      const stopSequence = stopSequencesC.shift()
+      const maxTokens = getModelMaxTokens(model)
+      let maxLen = Math.min(opts.maxTokens || chatRequest.chat.max_tokens || maxTokens, maxTokens)
+      const promptTokenCount = chatResponse.getPromptTokenCount()
+      if (promptTokenCount > maxLen) {
+        maxLen = Math.min(maxLen + promptTokenCount, maxTokens)
+      }
+      chatResponse.onFinish(() => {
+        chatRequest.updating = false
+        chatRequest.updatingMessage = ''
+      })
+      ws.onopen = () => {
+        ws.send(JSON.stringify({
+          type: 'open_inference_session',
+          model,
+          max_length: maxLen
+        }))
+        ws.onmessage = event => {
+          const response = JSON.parse(event.data)
+          if (!response.ok) {
+            const err = new Error('Error opening socket: ' + response.traceback)
+            console.error(err)
+            throw err
+          }
+          const rMessages = request.messages || [] as Message[]
+          const inputArray = (rMessages).reduce((a, m) => {
+            const c = getRoleTag(m.role, model, chatRequest.chat) + m.content
+            a.push(c)
+            return a
+          }, [] as string[])
+          const lastMessage = rMessages[rMessages.length - 1]
+          if (lastMessage && lastMessage.role !== 'assistant') {
+            inputArray.push(getRoleTag('assistant', model, chatRequest.chat))
+          }
+          const petalsRequest = {
+            type: 'generate',
+            inputs: inputArray.join(stopSequence),
+            max_new_tokens: 3, // wait for up to 3 tokens before displaying
+            stop_sequence: stopSequence,
+            doSample: 1,
+            temperature: request.temperature || 0,
+            top_p: request.top_p || 0,
+            extra_stop_sequences: stopSequencesC
+          }
+          ws.send(JSON.stringify(petalsRequest))
+          ws.onmessage = event => {
+            // Remove updating indicator
+            chatRequest.updating = 1 // hide indicator, but still signal we're updating
+            chatRequest.updatingMessage = ''
+            const response = JSON.parse(event.data)
+            if (!response.ok) {
+              const err = new Error('Error in response: ' + response.traceback)
+              console.error(err)
+              throw err
+            }
+            window.setTimeout(() => {
+              chatResponse.updateFromAsyncResponse(
+                      {
+                        model,
+                        choices: [{
+                          delta: {
+                            content: response.outputs,
+                            role: 'assistant'
+                          },
+                          finish_reason: (response.stop ? 'stop' : null)
+                        }]
+                      } as any
+              )
+              if (response.stop) {
+                const message = chatResponse.getMessages()[0]
+                if (message) {
+                  for (let i = 0, l = stopSequences.length; i < l; i++) {
+                    if (message.content.endsWith(stopSequences[i])) {
+                      message.content = message.content.slice(0, message.content.length - stopSequences[i].length)
+                      const startS = startSequence[i] || ''
+                      if (message.content.startsWith(startS)) message.content = message.content.slice(startS.length)
+                      updateMessages(chatRequest.getChat().id)
+                    }
+                  }
+                }
+              }
+            }, 1)
+          }
+        }
+        ws.onclose = () => {
+          chatRequest.updating = false
+          chatRequest.updatingMessage = ''
+          chatResponse.updateFromClose()
+        }
+        ws.onerror = err => {
+          console.error(err)
+          throw err
+        }
+      }
+}
+</script>
\ No newline at end of file
diff --git a/src/lib/Home.svelte b/src/lib/Home.svelte
index c86a17a..a69b1c2 100644
--- a/src/lib/Home.svelte
+++ b/src/lib/Home.svelte
@@ -3,7 +3,7 @@
   import Footer from './Footer.svelte'
   import { replace } from 'svelte-spa-router'
   import { onMount } from 'svelte'
-  import { getPetalsV2Websocket } from './ApiUtil.svelte'
+  import { getPetals } from './ApiUtil.svelte'
 
 $: apiKey = $apiKeyStorage
 
@@ -112,7 +112,7 @@ const setPetalsEnabled = (event: Event) => {
               aria-label="PetalsAPI Endpoint"
               type="text"
               class="input"
-              placeholder={getPetalsV2Websocket()}
+              placeholder={getPetals()}
               value={$globalStorage.pedalsEndpoint || ''}
             />
           </p>
@@ -123,7 +123,7 @@ const setPetalsEnabled = (event: Event) => {
           
         </form>
         <p>
-          Only use <u>{getPetalsV2Websocket()}</u> for testing.  You must set up your own Petals server for actual use. 
+          Only use <u>{getPetals()}</u> for testing.  You must set up your own Petals server for actual use. 
         </p>
         <p>
           <b>Do not send sensitive information when using Petals.</b>
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index 1289939..8f03e24 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -1,5 +1,5 @@
 <script context="module" lang="ts">
-    import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetalsV2Websocket } from './ApiUtil.svelte'
+    import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetals } from './ApiUtil.svelte'
     import { apiKeyStorage, globalStorage } from './Storage.svelte'
     import { get } from 'svelte/store'
     import type { ModelDetail, Model, ResponseModels, SelectOption, ChatSettings } from './Types.svelte'
@@ -34,9 +34,10 @@ const modelDetails : Record<string, ModelDetail> = {
         max: 16384 // 16k max token buffer
       },
       'meta-llama/Llama-2-70b-chat-hf': {
-        type: 'PetalsV2Websocket',
+        type: 'Petals',
         label: 'Petals - Llama-2-70b-chat',
-        stop: ['###', '</s>'],
+        start: [''],
+        stop: ['</s>'],
         prompt: 0.000000, // $0.000 per 1000 tokens prompt
         completion: 0.000000, // $0.000 per 1000 tokens completion
         max: 4096 // 4k max token buffer
@@ -119,8 +120,8 @@ export const getEndpoint = (model: Model): string => {
   const modelDetails = getModelDetail(model)
   const gSettings = get(globalStorage)
   switch (modelDetails.type) {
-        case 'PetalsV2Websocket':
-          return gSettings.pedalsEndpoint || getPetalsV2Websocket()
+        case 'Petals':
+          return gSettings.pedalsEndpoint || getPetals()
         case 'OpenAIDall-e':
           return getApiBase() + getEndpointGenerations()
         case 'OpenAIChat':
@@ -132,12 +133,12 @@ export const getEndpoint = (model: Model): string => {
 export const getRoleTag = (role: string, model: Model, settings: ChatSettings): string => {
   const modelDetails = getModelDetail(model)
   switch (modelDetails.type) {
-        case 'PetalsV2Websocket':
+        case 'Petals':
           if (role === 'assistant') {
-            return ('Assistant') +
-              ': '
+            if (settings.useSystemPrompt && settings.characterName) return '[' + settings.characterName + '] '
+            return '[Assistant] '
           }
-          if (role === 'user') return 'Human: '
+          if (role === 'user') return '[user] '
           return ''
         case 'OpenAIDall-e':
           return role
@@ -150,7 +151,7 @@ export const getRoleTag = (role: string, model: Model, settings: ChatSettings):
 export const getTokens = (model: Model, value: string): number[] => {
   const modelDetails = getModelDetail(model)
   switch (modelDetails.type) {
-        case 'PetalsV2Websocket':
+        case 'Petals':
           return llamaTokenizer.encode(value)
         case 'OpenAIDall-e':
           return [0]
@@ -184,7 +185,7 @@ export async function getModelOptions (): Promise<SelectOption[]> {
   }
   const filteredModels = supportedModelKeys.filter((model) => {
         switch (getModelDetail(model).type) {
-          case 'PetalsV2Websocket':
+          case 'Petals':
             return gSettings.enablePetals
           case 'OpenAIChat':
           default:
diff --git a/src/lib/Stats.svelte b/src/lib/Stats.svelte
index efee712..87d3466 100644
--- a/src/lib/Stats.svelte
+++ b/src/lib/Stats.svelte
@@ -10,19 +10,12 @@
   export const countPromptTokens = (prompts:Message[], model:Model, settings: ChatSettings):number => {
     const detail = getModelDetail(model)
     const count = prompts.reduce((a, m) => {
-      switch (detail.type) {
-        case 'PetalsV2Websocket':
-          a += countMessageTokens(m, model, settings)
-          break
-        case 'OpenAIChat':
-        default:
-          a += countMessageTokens(m, model, settings)
-      }
+      a += countMessageTokens(m, model, settings)
       return a
     }, 0)
     switch (detail.type) {
-      case 'PetalsV2Websocket':
-        return count + (Math.max(prompts.length - 1, 0) * countTokens(model, (detail.stop && detail.stop[0]) || '###')) // todo, make stop per model?
+      case 'Petals':
+        return count
       case 'OpenAIChat':
       default:
         // Not sure how OpenAI formats it, but this seems to get close to the right counts.
@@ -34,9 +27,11 @@
 
   export const countMessageTokens = (message:Message, model:Model, settings: ChatSettings):number => {
     const detail = getModelDetail(model)
+    const start = detail.start && detail.start[0]
+    const stop = detail.stop && detail.stop[0]
     switch (detail.type) {
-      case 'PetalsV2Websocket':
-        return countTokens(model, getRoleTag(message.role, model, settings) + ': ' + message.content)
+      case 'Petals':
+        return countTokens(model, (start || '') + getRoleTag(message.role, model, settings) + ': ' + message.content + (stop || '###'))
       case 'OpenAIChat':
       default:
         // Not sure how OpenAI formats it, but this seems to get close to the right counts.
diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte
index 5297c43..45d597e 100644
--- a/src/lib/Types.svelte
+++ b/src/lib/Types.svelte
@@ -7,12 +7,13 @@ export type Model = typeof supportedModelKeys[number];
 
 export type ImageGenerationSizes = typeof imageGenerationSizeTypes[number];
 
-export type RequestType = 'OpenAIChat' | 'OpenAIDall-e' | 'PetalsV2Websocket'
+export type RequestType = 'OpenAIChat' | 'OpenAIDall-e' | 'Petals'
 
 export type ModelDetail = {
     type: RequestType;
     label?: string;
     stop?: string[];
+    start?: string[];
     prompt: number;
     completion: number;
     max: number;

From 6d35a46d50ef164eb4b1cccec93693366f484511 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 14:40:12 -0500
Subject: [PATCH 06/17] Allow user to adjust message chaining strings

---
 src/lib/ChatRequest.svelte       | 31 ++++++++++---------
 src/lib/ChatRequestPetals.svelte | 12 +++-----
 src/lib/ChatSettingField.svelte  | 22 ++++++--------
 src/lib/Models.svelte            | 45 +++++++++++++++++++++------
 src/lib/Profiles.svelte          |  2 +-
 src/lib/Settings.svelte          | 52 ++++++++++++++++++++++++++++++++
 src/lib/Stats.svelte             | 15 +++++----
 src/lib/Types.svelte             | 12 ++++++--
 src/lib/Util.svelte              |  5 +++
 9 files changed, 142 insertions(+), 54 deletions(-)

diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index 40c966e..a157ccb 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -215,7 +215,7 @@ export class ChatRequest {
         }
 
         // Get token counts
-        const promptTokenCount = countPromptTokens(messagePayload, model, chatSettings)
+        const promptTokenCount = countPromptTokens(messagePayload, model, chat)
         const maxAllowed = maxTokens - (promptTokenCount + 1)
 
         // Build the API request body
@@ -287,7 +287,8 @@ export class ChatRequest {
       }
 
       private buildHiddenPromptPrefixMessages (messages: Message[], insert:boolean = false): Message[] {
-        const chatSettings = this.chat.settings
+        const chat = this.chat
+        const chatSettings = chat.settings
         const hiddenPromptPrefix = mergeProfileFields(chatSettings, chatSettings.hiddenPromptPrefix).trim()
         const lastMessage = messages[messages.length - 1]
         const isContinue = lastMessage?.role === 'assistant' && lastMessage.finish_reason === 'length'
@@ -328,11 +329,11 @@ export class ChatRequest {
        * Gets an estimate of how many extra tokens will be added that won't be part of the visible messages
        * @param filtered
        */
-      private getTokenCountPadding (filtered: Message[], settings: ChatSettings): number {
+      private getTokenCountPadding (filtered: Message[], chat: Chat): number {
         let result = 0
         // add cost of hiddenPromptPrefix
         result += this.buildHiddenPromptPrefixMessages(filtered)
-          .reduce((a, m) => a + countMessageTokens(m, this.getModel(), settings), 0)
+          .reduce((a, m) => a + countMessageTokens(m, this.getModel(), chat), 0)
         // more here eventually?
         return result
       }
@@ -354,10 +355,10 @@ export class ChatRequest {
         }
 
         // Get extra counts for when the prompts are finally sent.
-        const countPadding = this.getTokenCountPadding(filtered, chatSettings)
+        const countPadding = this.getTokenCountPadding(filtered, chat)
 
         // See if we have enough to apply any of the reduction modes
-        const fullPromptSize = countPromptTokens(filtered, model, chatSettings) + countPadding
+        const fullPromptSize = countPromptTokens(filtered, model, chat) + countPadding
         if (fullPromptSize < chatSettings.summaryThreshold) return await continueRequest() // nothing to do yet
         const overMax = fullPromptSize > maxTokens * 0.95
 
@@ -380,12 +381,12 @@ export class ChatRequest {
            * *************************************************************
            */
     
-          let promptSize = countPromptTokens(top.concat(rw), model, chatSettings) + countPadding
+          let promptSize = countPromptTokens(top.concat(rw), model, chat) + countPadding
           while (rw.length && rw.length > pinBottom && promptSize >= chatSettings.summaryThreshold) {
             const rolled = rw.shift()
             // Hide messages we're "rolling"
             if (rolled) rolled.suppress = true
-            promptSize = countPromptTokens(top.concat(rw), model, chatSettings) + countPadding
+            promptSize = countPromptTokens(top.concat(rw), model, chat) + countPadding
           }
           // Run a new request, now with the rolled messages hidden
           return await _this.sendRequest(get(currentChatMessages), {
@@ -401,26 +402,26 @@ export class ChatRequest {
           const bottom = rw.slice(0 - pinBottom)
           let continueCounter = chatSettings.summaryExtend + 1
           rw = rw.slice(0, 0 - pinBottom)
-          let reductionPoolSize = countPromptTokens(rw, model, chatSettings)
+          let reductionPoolSize = countPromptTokens(rw, model, chat)
           const ss = Math.abs(chatSettings.summarySize)
           const getSS = ():number => (ss < 1 && ss > 0)
             ? Math.round(reductionPoolSize * ss) // If summarySize between 0 and 1, use percentage of reduced
             : Math.min(ss, reductionPoolSize * 0.5) // If > 1, use token count
-          const topSize = countPromptTokens(top, model, chatSettings)
+          const topSize = countPromptTokens(top, model, chat)
           let maxSummaryTokens = getSS()
           let promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
           const summaryRequest = { role: 'user', content: promptSummary } as Message
-          let promptSummarySize = countMessageTokens(summaryRequest, model, chatSettings)
+          let promptSummarySize = countMessageTokens(summaryRequest, model, chat)
           // Make sure there is enough room to generate the summary, and try to make sure
           // the last prompt is a user prompt as that seems to work better for summaries
           while ((topSize + reductionPoolSize + promptSummarySize + maxSummaryTokens) >= maxTokens ||
               (reductionPoolSize >= 100 && rw[rw.length - 1]?.role !== 'user')) {
             bottom.unshift(rw.pop() as Message)
-            reductionPoolSize = countPromptTokens(rw, model, chatSettings)
+            reductionPoolSize = countPromptTokens(rw, model, chat)
             maxSummaryTokens = getSS()
             promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
             summaryRequest.content = promptSummary
-            promptSummarySize = countMessageTokens(summaryRequest, model, chatSettings)
+            promptSummarySize = countMessageTokens(summaryRequest, model, chat)
           }
           if (reductionPoolSize < 50) {
             if (overMax) addError(chatId, 'Check summary settings. Unable to summarize enough messages.')
@@ -506,10 +507,10 @@ export class ChatRequest {
               // Try to get more of it
               delete summaryResponse.finish_reason
               _this.updatingMessage = 'Summarizing more...'
-              let _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chatSettings)
+              let _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chat)
               while (rw.length && (_recount + maxSummaryTokens >= maxTokens)) {
                 rw.shift()
-                _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chatSettings)
+                _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chat)
               }
               loopCount++
               continue
diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
index b0c1bac..15f0278 100644
--- a/src/lib/ChatRequestPetals.svelte
+++ b/src/lib/ChatRequestPetals.svelte
@@ -1,7 +1,7 @@
 <script context="module" lang="ts">
     import ChatCompletionResponse from './ChatCompletionResponse.svelte'
     import ChatRequest from './ChatRequest.svelte'
-    import { getEndpoint, getModelDetail, getRoleTag } from './Models.svelte'
+    import { getEndpoint, getModelDetail, getRoleTag, getStopSequence } from './Models.svelte'
     import type { ChatCompletionOpts, Message, Request } from './Types.svelte'
     import { getModelMaxTokens } from './Stats.svelte'
     import { updateMessages } from './Storage.svelte'
@@ -13,6 +13,7 @@ export const runPetalsCompletionRequest = async (
   signal: AbortSignal,
   opts: ChatCompletionOpts) => {
       // Petals
+      const chat = chatRequest.getChat()
       const model = chatRequest.getModel()
       const modelDetail = getModelDetail(model)
       const ws = new WebSocket(getEndpoint(model))
@@ -24,11 +25,10 @@ export const runPetalsCompletionRequest = async (
         ws.close()
       }
       signal.addEventListener('abort', abortListener)
-      const startSequences = modelDetail.start || []
-      const startSequence = startSequences[0] || ''
       const stopSequences = modelDetail.stop || ['###']
+      const stopSequence = getStopSequence(chat)
       const stopSequencesC = stopSequences.slice()
-      const stopSequence = stopSequencesC.shift()
+      if (stopSequence === stopSequencesC[0]) stopSequencesC.shift()
       const maxTokens = getModelMaxTokens(model)
       let maxLen = Math.min(opts.maxTokens || chatRequest.chat.max_tokens || maxTokens, maxTokens)
       const promptTokenCount = chatResponse.getPromptTokenCount()
@@ -102,9 +102,7 @@ export const runPetalsCompletionRequest = async (
                   for (let i = 0, l = stopSequences.length; i < l; i++) {
                     if (message.content.endsWith(stopSequences[i])) {
                       message.content = message.content.slice(0, message.content.length - stopSequences[i].length)
-                      const startS = startSequence[i] || ''
-                      if (message.content.startsWith(startS)) message.content = message.content.slice(startS.length)
-                      updateMessages(chatRequest.getChat().id)
+                      updateMessages(chat.id)
                     }
                   }
                 }
diff --git a/src/lib/ChatSettingField.svelte b/src/lib/ChatSettingField.svelte
index a86f616..4a2e7c2 100644
--- a/src/lib/ChatSettingField.svelte
+++ b/src/lib/ChatSettingField.svelte
@@ -3,7 +3,7 @@
   // import { getProfile } from './Profiles.svelte'
   import { cleanSettingValue, setChatSettingValue } from './Storage.svelte'
   import type { Chat, ChatSetting, ChatSettings, ControlAction, FieldControl, SettingPrompt } from './Types.svelte'
-  import { autoGrowInputOnEvent, errorNotice } from './Util.svelte'
+  import { autoGrowInputOnEvent, errorNotice, valueOf } from './Util.svelte'
   // import { replace } from 'svelte-spa-router'
   import Fa from 'svelte-fa/src/fa.svelte'
   import { openModal } from 'svelte-modals'
@@ -23,13 +23,9 @@
   const chatId = chat.id
   let show = false
 
-  const valueOf = (value: any) => {
-    if (typeof value === 'function') return value(chatId, setting)
-    return value
-  }
-
-  let header = valueOf(setting.header)
-  let headerClass = valueOf(setting.headerClass)
+  let header = valueOf(chatId, setting.header)
+  let headerClass = valueOf(chatId, setting.headerClass)
+  let placeholder = valueOf(chatId, setting.placeholder)
   
   const buildFieldControls = () => {
     fieldControls = (setting.fieldControls || [] as FieldControl[]).map(fc => {
@@ -46,8 +42,9 @@
 
   afterUpdate(() => {
     show = (typeof setting.hide !== 'function') || !setting.hide(chatId)
-    header = valueOf(setting.header)
-    headerClass = valueOf(setting.headerClass)
+    header = valueOf(chatId, setting.header)
+    headerClass = valueOf(chatId, setting.headerClass)
+    placeholder = valueOf(chatId, setting.placeholder)
     buildFieldControls()
   })
 
@@ -181,7 +178,7 @@
       <label class="label" for="settings-{setting.key}" title="{setting.title}">{setting.name}</label>
       <textarea
         class="input is-info is-focused chat-input auto-size"
-        placeholder={setting.placeholder || ''}
+        placeholder={placeholder || ''}
         rows="1"
         on:input={e => autoGrowInputOnEvent(e)}
         on:change={e => { queueSettingValueChange(e, setting); autoGrowInputOnEvent(e) }}
@@ -205,7 +202,7 @@
             min={setting.min}
             max={setting.max}
             step={setting.step}
-            placeholder={String(setting.placeholder || chatDefaults[setting.key])}
+            placeholder={String(placeholder || chatDefaults[setting.key])}
             on:change={e => queueSettingValueChange(e, setting)}
           />
         {:else if setting.type === 'select' || setting.type === 'select-number'}
@@ -243,6 +240,7 @@
               title="{setting.title}"
               class="input" 
               value={chatSettings[setting.key]} 
+              placeholder={String(placeholder || chatDefaults[setting.key])}
               on:change={e => { queueSettingValueChange(e, setting) }}
             >
           </div>
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index 8f03e24..3dfa087 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -2,9 +2,12 @@
     import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetals } from './ApiUtil.svelte'
     import { apiKeyStorage, globalStorage } from './Storage.svelte'
     import { get } from 'svelte/store'
-    import type { ModelDetail, Model, ResponseModels, SelectOption, ChatSettings } from './Types.svelte'
+    import type { ModelDetail, Model, ResponseModels, SelectOption, Chat } from './Types.svelte'
 import { encode } from 'gpt-tokenizer'
 import llamaTokenizer from 'llama-tokenizer-js'
+    import { mergeProfileFields } from './Profiles.svelte'
+    import { getChatSettingObjectByKey } from './Settings.svelte'
+    import { valueOf } from './Util.svelte'
 
 // Reference: https://openai.com/pricing#language-models
 // Eventually we'll add API hosts and endpoints to this
@@ -36,8 +39,10 @@ const modelDetails : Record<string, ModelDetail> = {
       'meta-llama/Llama-2-70b-chat-hf': {
         type: 'Petals',
         label: 'Petals - Llama-2-70b-chat',
-        start: [''],
         stop: ['</s>'],
+        userStart: '[user]',
+        assistantStart: '[[[CHARACTER_NAME]]]',
+        systemStart: '',
         prompt: 0.000000, // $0.000 per 1000 tokens prompt
         completion: 0.000000, // $0.000 per 1000 tokens completion
         max: 4096 // 4k max token buffer
@@ -130,16 +135,38 @@ export const getEndpoint = (model: Model): string => {
   }
 }
 
-export const getRoleTag = (role: string, model: Model, settings: ChatSettings): string => {
+export const getStopSequence = (chat: Chat): string => {
+  return valueOf(chat.id, getChatSettingObjectByKey('stopSequence').placeholder)
+}
+
+export const getUserStart = (chat: Chat): string => {
+  return mergeProfileFields(
+        chat.settings,
+        valueOf(chat.id, getChatSettingObjectByKey('userMessageStart').placeholder)
+      )
+}
+
+export const getAssistantStart = (chat: Chat): string => {
+  return mergeProfileFields(
+        chat.settings,
+        valueOf(chat.id, getChatSettingObjectByKey('assistantMessageStart').placeholder)
+      )
+}
+
+export const getSystemStart = (chat: Chat): string => {
+  return mergeProfileFields(
+        chat.settings,
+        valueOf(chat.id, getChatSettingObjectByKey('systemMessageStart').placeholder)
+      )
+}
+
+export const getRoleTag = (role: string, model: Model, chat: Chat): string => {
   const modelDetails = getModelDetail(model)
   switch (modelDetails.type) {
         case 'Petals':
-          if (role === 'assistant') {
-            if (settings.useSystemPrompt && settings.characterName) return '[' + settings.characterName + '] '
-            return '[Assistant] '
-          }
-          if (role === 'user') return '[user] '
-          return ''
+          if (role === 'assistant') return getAssistantStart(chat) + ' '
+          if (role === 'user') return getUserStart(chat) + ' '
+          return getSystemStart(chat) + ' '
         case 'OpenAIDall-e':
           return role
         case 'OpenAIChat':
diff --git a/src/lib/Profiles.svelte b/src/lib/Profiles.svelte
index 4ee33ea..7e6a9d0 100644
--- a/src/lib/Profiles.svelte
+++ b/src/lib/Profiles.svelte
@@ -72,7 +72,7 @@ export const getProfile = (key:string, forReset:boolean = false):ChatSettings =>
 
 export const mergeProfileFields = (settings: ChatSettings, content: string|undefined, maxWords: number|undefined = undefined): string => {
     if (!content?.toString) return ''
-    content = (content + '').replaceAll('[[CHARACTER_NAME]]', settings.characterName || 'ChatGPT')
+    content = (content + '').replaceAll('[[CHARACTER_NAME]]', settings.characterName || 'Assistant')
     if (maxWords) content = (content + '').replaceAll('[[MAX_WORDS]]', maxWords.toString())
     return content
 }
diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte
index 09a3011..29f5be2 100644
--- a/src/lib/Settings.svelte
+++ b/src/lib/Settings.svelte
@@ -94,6 +94,10 @@ const defaults:ChatSettings = {
   hppContinuePrompt: '',
   hppWithSummaryPrompt: false,
   imageGenerationSize: '',
+  stopSequence: '',
+  userMessageStart: '',
+  assistantMessageStart: '',
+  systemMessageStart: '',
   // useResponseAlteration: false,
   // responseAlterations: [],
   isDirty: false
@@ -414,6 +418,10 @@ const isNotOpenAI = (chatId) => {
   return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
 }
 
+const isNotPetals = (chatId) => {
+  return getModelDetail(getChatSettings(chatId).model).type !== 'Petals'
+}
+
 const chatSettingsList: ChatSetting[] = [
       profileSetting,
       ...systemPromptSettings,
@@ -492,6 +500,50 @@ const chatSettingsList: ChatSetting[] = [
         type: 'number',
         hide: isNotOpenAI
       },
+      {
+        key: 'stopSequence',
+        name: 'Stop Sequence',
+        title: 'Characters used to separate messages in the message chain.',
+        type: 'text',
+        placeholder: (chatId) => {
+          const val = getModelDetail(getChatSettings(chatId).model).stop
+          return (val && val[0]) || ''
+        },
+        hide: isNotPetals
+      },
+      {
+        key: 'userMessageStart',
+        name: 'User Message Start Sequence',
+        title: 'Sequence to denote user messages in the message chain.',
+        type: 'text',
+        placeholder: (chatId) => {
+          const val = getModelDetail(getChatSettings(chatId).model).userStart
+          return val || ''
+        },
+        hide: isNotPetals
+      },
+      {
+        key: 'assistantMessageStart',
+        name: 'Assistant Message Start Sequence',
+        title: 'Sequence to denote assistant messages in the message chain.',
+        type: 'text',
+        placeholder: (chatId) => {
+          const val = getModelDetail(getChatSettings(chatId).model).assistantStart
+          return val || ''
+        },
+        hide: isNotPetals
+      },
+      {
+        key: 'systemMessageStart',
+        name: 'System Message Start Sequence',
+        title: 'Sequence to denote system messages in the message chain.',
+        type: 'text',
+        placeholder: (chatId) => {
+          const val = getModelDetail(getChatSettings(chatId).model).systemStart
+          return val || ''
+        },
+        hide: isNotPetals
+      },
       {
         // logit bias editor not implemented yet
         key: 'logit_bias',
diff --git a/src/lib/Stats.svelte b/src/lib/Stats.svelte
index 87d3466..183d921 100644
--- a/src/lib/Stats.svelte
+++ b/src/lib/Stats.svelte
@@ -1,16 +1,16 @@
 <script context="module" lang="ts">
-  import { countTokens, getModelDetail, getRoleTag } from './Models.svelte'
-  import type { ChatSettings, Message, Model, Usage } from './Types.svelte'
+  import { countTokens, getModelDetail, getRoleTag, getStopSequence } from './Models.svelte'
+  import type { Chat, Message, Model, Usage } from './Types.svelte'
 
   export const getPrice = (tokens: Usage, model: Model): number => {
     const t = getModelDetail(model)
     return ((tokens.prompt_tokens * t.prompt) + (tokens.completion_tokens * t.completion))
   }
 
-  export const countPromptTokens = (prompts:Message[], model:Model, settings: ChatSettings):number => {
+  export const countPromptTokens = (prompts:Message[], model:Model, chat: Chat):number => {
     const detail = getModelDetail(model)
     const count = prompts.reduce((a, m) => {
-      a += countMessageTokens(m, model, settings)
+      a += countMessageTokens(m, model, chat)
       return a
     }, 0)
     switch (detail.type) {
@@ -25,13 +25,12 @@
     }
   }
 
-  export const countMessageTokens = (message:Message, model:Model, settings: ChatSettings):number => {
+  export const countMessageTokens = (message:Message, model:Model, chat: Chat):number => {
     const detail = getModelDetail(model)
-    const start = detail.start && detail.start[0]
-    const stop = detail.stop && detail.stop[0]
+    const stop = getStopSequence(chat)
     switch (detail.type) {
       case 'Petals':
-        return countTokens(model, (start || '') + getRoleTag(message.role, model, settings) + ': ' + message.content + (stop || '###'))
+        return countTokens(model, getRoleTag(message.role, model, chat) + ': ' + message.content + (stop || '###'))
       case 'OpenAIChat':
       default:
         // Not sure how OpenAI formats it, but this seems to get close to the right counts.
diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte
index 45d597e..30e0319 100644
--- a/src/lib/Types.svelte
+++ b/src/lib/Types.svelte
@@ -13,7 +13,9 @@ export type ModelDetail = {
     type: RequestType;
     label?: string;
     stop?: string[];
-    start?: string[];
+    userStart?: string,
+    assistantStart?: string,
+    systemStart?: string,
     prompt: number;
     completion: number;
     max: number;
@@ -111,6 +113,10 @@ export type ChatSettings = {
     trainingPrompts?: Message[];
     useResponseAlteration?: boolean;
     responseAlterations?: ResponseAlteration[];
+    stopSequence: string;
+    userMessageStart: string;
+    assistantMessageStart: string;
+    systemMessageStart: string;
     isDirty?: boolean;
   } & Request;
 
@@ -245,6 +251,8 @@ export type SubSetting = {
     settings: any[];
   };
 
+export type ValueFn = (chatId:number) => string
+
 export type ChatSetting = {
     key: keyof ChatSettings;
     name: string;
@@ -253,7 +261,7 @@ export type ChatSetting = {
     hidden?: boolean; // Hide from setting menus
     header?: string;
     headerClass?: string;
-    placeholder?: string;
+    placeholder?: string | ValueFn;
     hide?: (chatId:number) => boolean;
     apiTransform?: (chatId:number, setting:ChatSetting, value:any) => any;
     fieldControls?: FieldControl[];
diff --git a/src/lib/Util.svelte b/src/lib/Util.svelte
index 3342362..81e1a3f 100644
--- a/src/lib/Util.svelte
+++ b/src/lib/Util.svelte
@@ -147,4 +147,9 @@
     newChat()
   }
 
+  export const valueOf = (chatId: number, value: any) => {
+    if (typeof value === 'function') return value(chatId)
+    return value
+  }
+
 </script> 
\ No newline at end of file

From 15dcd27e8f825778af3a274be627635f1038a1a4 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 16:48:26 -0500
Subject: [PATCH 07/17] Get temp and top_p working for Petals

---
 src/lib/ChatRequestPetals.svelte | 16 ++++++++++++----
 src/lib/Settings.svelte          | 26 ++++++++++++++++----------
 src/lib/Types.svelte             |  4 ++--
 3 files changed, 30 insertions(+), 16 deletions(-)

diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
index 15f0278..eefd31d 100644
--- a/src/lib/ChatRequestPetals.svelte
+++ b/src/lib/ChatRequestPetals.svelte
@@ -53,6 +53,14 @@ export const runPetalsCompletionRequest = async (
             throw err
           }
           const rMessages = request.messages || [] as Message[]
+          // make sure top_p and temperature are set the way we need
+          let temperature = request.temperature || 0
+          if (isNaN(temperature as any) || temperature === 1) temperature = 1
+          if (temperature === 0) temperature = 0.0001
+          let topP = request.top_p
+          if (isNaN(topP as any) || topP === 1) topP = 1
+          if (topP === 0) topP = 0.0001
+          // build the message array
           const inputArray = (rMessages).reduce((a, m) => {
             const c = getRoleTag(m.role, model, chatRequest.chat) + m.content
             a.push(c)
@@ -65,11 +73,11 @@ export const runPetalsCompletionRequest = async (
           const petalsRequest = {
             type: 'generate',
             inputs: inputArray.join(stopSequence),
-            max_new_tokens: 3, // wait for up to 3 tokens before displaying
+            max_new_tokens: 1, // wait for up to 1 tokens before displaying
             stop_sequence: stopSequence,
-            doSample: 1,
-            temperature: request.temperature || 0,
-            top_p: request.top_p || 0,
+            do_sample: 1, // enable top p and the like
+            temperature,
+            top_p: topP,
             extra_stop_sequences: stopSequencesC
           }
           ws.send(JSON.stringify(petalsRequest))
diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte
index 29f5be2..5f01837 100644
--- a/src/lib/Settings.svelte
+++ b/src/lib/Settings.svelte
@@ -55,6 +55,14 @@ export const getExcludeFromProfile = () => {
   return excludeFromProfile
 }
 
+const isNotOpenAI = (chatId) => {
+  return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
+}
+
+const isNotPetals = (chatId) => {
+  return getModelDetail(getChatSettings(chatId).model).type !== 'Petals'
+}
+
 const gptDefaults = {
   model: defaultModel,
   messages: [],
@@ -406,7 +414,13 @@ const modelSetting: ChatSetting & SettingSelect = {
       key: 'model',
       name: 'Model',
       title: 'The model to use - GPT-3.5 is cheaper, but GPT-4 is more powerful.',
-      header: 'Below are the settings that OpenAI allows to be changed for the API calls. See the <a target="_blank" href="https://platform.openai.com/docs/api-reference/chat/create">OpenAI API docs</a> for more details.',
+      header: (chatId) => {
+        if (isNotOpenAI(chatId)) {
+          return 'Below are the settings that can be changed for the API calls. See <a target="_blank" href="https://platform.openai.com/docs/api-reference/chat/create">this overview</a> to start, though not all settings translate to Petals.'
+        } else {
+          return 'Below are the settings that OpenAI allows to be changed for the API calls. See the <a target="_blank" href="https://platform.openai.com/docs/api-reference/chat/create">OpenAI API docs</a> for more details.'
+        }
+      },
       headerClass: 'is-warning',
       options: [],
       type: 'select',
@@ -414,14 +428,6 @@ const modelSetting: ChatSetting & SettingSelect = {
       afterChange: (chatId, setting) => true // refresh settings
 }
 
-const isNotOpenAI = (chatId) => {
-  return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
-}
-
-const isNotPetals = (chatId) => {
-  return getModelDetail(getChatSettings(chatId).model).type !== 'Petals'
-}
-
 const chatSettingsList: ChatSetting[] = [
       profileSetting,
       ...systemPromptSettings,
@@ -448,7 +454,7 @@ const chatSettingsList: ChatSetting[] = [
       },
       {
         key: 'top_p',
-        name: 'Nucleus Sampling',
+        name: 'Nucleus Sampling (Top-p)',
         title: 'An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n' +
               '\n' +
               'We generally recommend altering this or temperature but not both',
diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte
index 30e0319..3757c9f 100644
--- a/src/lib/Types.svelte
+++ b/src/lib/Types.svelte
@@ -259,8 +259,8 @@ export type ChatSetting = {
     title: string;
     forceApi?: boolean; // force in api requests, even if set to default
     hidden?: boolean; // Hide from setting menus
-    header?: string;
-    headerClass?: string;
+    header?: string | ValueFn;
+    headerClass?: string | ValueFn;
     placeholder?: string | ValueFn;
     hide?: (chatId:number) => boolean;
     apiTransform?: (chatId:number, setting:ChatSetting, value:any) => any;

From 7aadca3c5c52ccfce2e6c8eb28bb6bf367c66027 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 17:08:40 -0500
Subject: [PATCH 08/17] Better error handling for Petals

---
 src/lib/ChatCompletionResponse.svelte | 8 ++++----
 src/lib/ChatRequestPetals.svelte      | 2 ++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/lib/ChatCompletionResponse.svelte b/src/lib/ChatCompletionResponse.svelte
index ab5fcff..72fd4e0 100644
--- a/src/lib/ChatCompletionResponse.svelte
+++ b/src/lib/ChatCompletionResponse.svelte
@@ -175,15 +175,15 @@ export class ChatCompletionResponse {
       } as Message)
     }
     this.notifyMessageChange()
-    setTimeout(() => this.finish(), 250) // give others a chance to signal the finish first
+    setTimeout(() => this.finish(), 200) // give others a chance to signal the finish first
   }
 
   updateFromClose (force: boolean = false): void {
     if (!this.finished && !this.error && !this.messages?.find(m => m.content)) {
-      if (!force) return setTimeout(() => this.updateFromClose(true), 250) as any
-      return this.updateFromError('Unexpected connection termination')
+      if (!force) return setTimeout(() => this.updateFromClose(true), 300) as any
+      if (!this.finished) return this.updateFromError('Unexpected connection termination')
     }
-    setTimeout(() => this.finish(), 250) // give others a chance to signal the finish first
+    setTimeout(() => this.finish(), 260) // give others a chance to signal the finish first
   }
 
   onMessageChange = (listener: (m: Message[]) => void): number =>
diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
index eefd31d..356ae12 100644
--- a/src/lib/ChatRequestPetals.svelte
+++ b/src/lib/ChatRequestPetals.svelte
@@ -49,6 +49,7 @@ export const runPetalsCompletionRequest = async (
           const response = JSON.parse(event.data)
           if (!response.ok) {
             const err = new Error('Error opening socket: ' + response.traceback)
+            chatResponse.updateFromError(err.message)
             console.error(err)
             throw err
           }
@@ -89,6 +90,7 @@ export const runPetalsCompletionRequest = async (
             if (!response.ok) {
               const err = new Error('Error in response: ' + response.traceback)
               console.error(err)
+              chatResponse.updateFromError(err.message)
               throw err
             }
             window.setTimeout(() => {

From ca19bab19dae52f7966a7993fcc9d2130b766ec4 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Sat, 22 Jul 2023 17:21:01 -0500
Subject: [PATCH 09/17] Don't allow too low of temp or top_p

---
 src/lib/ChatRequestPetals.svelte | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
index 356ae12..9d72ebe 100644
--- a/src/lib/ChatRequestPetals.svelte
+++ b/src/lib/ChatRequestPetals.svelte
@@ -56,11 +56,11 @@ export const runPetalsCompletionRequest = async (
           const rMessages = request.messages || [] as Message[]
           // make sure top_p and temperature are set the way we need
           let temperature = request.temperature || 0
-          if (isNaN(temperature as any) || temperature === 1) temperature = 1
-          if (temperature === 0) temperature = 0.0001
+          if (isNaN(temperature as any)) temperature = 1
+          if (!temperature || temperature <= 0) temperature = 0.01
           let topP = request.top_p
-          if (isNaN(topP as any) || topP === 1) topP = 1
-          if (topP === 0) topP = 0.0001
+          if (topP === undefined || isNaN(topP as any)) topP = 1
+          if (!topP || topP <= 0) topP = 0.01
           // build the message array
           const inputArray = (rMessages).reduce((a, m) => {
             const c = getRoleTag(m.role, model, chatRequest.chat) + m.content

From f6380e1cc2ffe56e59820013dc438f65d1ba687b Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 15:26:17 -0500
Subject: [PATCH 10/17] Allow Petals and/or OpenAI

---
 src/lib/ChatMenuItem.svelte     |  4 +-
 src/lib/ChatOptionMenu.svelte   |  8 +--
 src/lib/ChatRequest.svelte      |  5 +-
 src/lib/ChatSettingField.svelte |  2 +-
 src/lib/Home.svelte             | 61 +++++++++++++---------
 src/lib/Models.svelte           | 90 ++++++++++++++++++++++++---------
 src/lib/Profiles.svelte         |  4 +-
 src/lib/Settings.svelte         | 13 +++--
 src/lib/Sidebar.svelte          |  8 +--
 src/lib/Storage.svelte          |  7 ++-
 src/lib/Types.svelte            |  1 +
 11 files changed, 139 insertions(+), 64 deletions(-)

diff --git a/src/lib/ChatMenuItem.svelte b/src/lib/ChatMenuItem.svelte
index 4503042..628fe7e 100644
--- a/src/lib/ChatMenuItem.svelte
+++ b/src/lib/ChatMenuItem.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
   import { replace } from 'svelte-spa-router'
   import type { Chat } from './Types.svelte'
-  import { apiKeyStorage, deleteChat, pinMainMenu, saveChatStore } from './Storage.svelte'
+  import { deleteChat, hasActiveModels, pinMainMenu, saveChatStore } from './Storage.svelte'
   import Fa from 'svelte-fa/src/fa.svelte'
   import { faTrash, faCircleCheck, faPencil } from '@fortawesome/free-solid-svg-icons/index'
   import { faMessage } from '@fortawesome/free-regular-svg-icons/index'
@@ -86,7 +86,7 @@
   <a 
     href={`#/chat/${chat.id}`}
     class="chat-menu-item"
-    class:is-waiting={waitingForConfirm} class:is-disabled={!$apiKeyStorage} class:is-active={activeChatId === chat.id}
+    class:is-waiting={waitingForConfirm} class:is-disabled={!hasActiveModels()} class:is-active={activeChatId === chat.id}
     on:click={() => { $pinMainMenu = false }} >
     {#if waitingForConfirm}
     <a class="is-pulled-right is-hidden px-1 py-0 has-text-weight-bold delete-button" href={'$'} on:click|preventDefault={() => delChat()}><Fa icon={faCircleCheck} /></a>
diff --git a/src/lib/ChatOptionMenu.svelte b/src/lib/ChatOptionMenu.svelte
index 1d40e40..5abdb1c 100644
--- a/src/lib/ChatOptionMenu.svelte
+++ b/src/lib/ChatOptionMenu.svelte
@@ -18,7 +18,7 @@
     faEyeSlash
   } from '@fortawesome/free-solid-svg-icons/index'
   import { faSquareMinus, faSquarePlus as faSquarePlusOutline } from '@fortawesome/free-regular-svg-icons/index'
-  import { apiKeyStorage, addChatFromJSON, chatsStorage, checkStateChange, clearChats, clearMessages, copyChat, globalStorage, setGlobalSettingValueByKey, showSetChatSettings, pinMainMenu, getChat, deleteChat, saveChatStore, saveCustomProfile } from './Storage.svelte'
+  import { addChatFromJSON, chatsStorage, checkStateChange, clearChats, clearMessages, copyChat, globalStorage, setGlobalSettingValueByKey, showSetChatSettings, pinMainMenu, getChat, deleteChat, saveChatStore, saveCustomProfile, hasActiveModels } from './Storage.svelte'
   import { exportAsMarkdown, exportChatAsJSON } from './Export.svelte'
   import { newNameForProfile, restartProfile } from './Profiles.svelte'
   import { replace } from 'svelte-spa-router'
@@ -173,7 +173,7 @@
         <span class="menu-icon"><Fa icon={faGear}/></span> Chat Profile Settings
       </a>
       <hr class="dropdown-divider">
-      <a href={'#'} class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { $apiKeyStorage && close(); $apiKeyStorage && startNewChatWithWarning(chatId) }} class="dropdown-item">
+      <a href={'#'} class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { hasActiveModels() && close(); hasActiveModels() && startNewChatWithWarning(chatId) }} class="dropdown-item">
         <span class="menu-icon"><Fa icon={faSquarePlus}/></span> New Chat from Default
       </a>
       <a href={'#'} class:is-disabled={!chatId} on:click|preventDefault={() => { chatId && close(); chatId && startNewChatFromChatId(chatId) }} class="dropdown-item">
@@ -196,14 +196,14 @@
       <a href={'#'} class="dropdown-item" class:is-disabled={!chatId} on:click|preventDefault={() => { close(); exportChatAsJSON(chatId) }}>
         <span class="menu-icon"><Fa icon={faDownload}/></span> Backup Chat JSON
       </a>
-      <a href={'#'} class="dropdown-item" class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { if (chatId) close(); chatFileInput.click() }}>
+      <a href={'#'} class="dropdown-item" class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { if (chatId) close(); chatFileInput.click() }}>
         <span class="menu-icon"><Fa icon={faUpload}/></span> Restore Chat JSON
       </a>
       <a href={'#'} class="dropdown-item" class:is-disabled={!chatId} on:click|preventDefault={() => { if (chatId) close(); exportAsMarkdown(chatId) }}>
         <span class="menu-icon"><Fa icon={faFileExport}/></span> Export Chat Markdown
       </a>
       <hr class="dropdown-divider">
-      <a href={'#'} class="dropdown-item" class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { if (chatId) close(); profileFileInput.click() }}>
+      <a href={'#'} class="dropdown-item" class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { if (chatId) close(); profileFileInput.click() }}>
         <span class="menu-icon"><Fa icon={faUpload}/></span> Restore Profile JSON
       </a>
       <hr class="dropdown-divider">
diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index a157ccb..7a17d2a 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -5,7 +5,7 @@
     import type { Chat, ChatCompletionOpts, ChatSettings, Message, Model, Request, RequestImageGeneration } from './Types.svelte'
     import { deleteMessage, getChatSettingValueNullDefault, insertMessages, getApiKey, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage } from './Storage.svelte'
     import { scrollToBottom, scrollToMessage } from './Util.svelte'
-    import { getRequestSettingList, defaultModel } from './Settings.svelte'
+    import { getDefaultModel, getRequestSettingList } from './Settings.svelte'
     import { v4 as uuidv4 } from 'uuid'
     import { get } from 'svelte/store'
     import { getEndpoint, getModelDetail } from './Models.svelte'
@@ -26,6 +26,7 @@ export class ChatRequest {
 
       setChat (chat: Chat) {
         this.chat = chat
+        this.chat.settings.model = this.getModel()
       }
 
       getChat (): Chat {
@@ -283,7 +284,7 @@ export class ChatRequest {
       }
 
       getModel (): Model {
-        return this.chat.settings.model || defaultModel
+        return this.chat.settings.model || getDefaultModel()
       }
 
       private buildHiddenPromptPrefixMessages (messages: Message[], insert:boolean = false): Message[] {
diff --git a/src/lib/ChatSettingField.svelte b/src/lib/ChatSettingField.svelte
index 4a2e7c2..0baf968 100644
--- a/src/lib/ChatSettingField.svelte
+++ b/src/lib/ChatSettingField.svelte
@@ -211,7 +211,7 @@
             {#key rkey}
             <select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } >
               {#each setting.options as option}
-                <option class:is-default={option.value === chatDefaults[setting.key]} value={option.value} selected={option.value === chatSettings[setting.key]}>{option.text}</option>
+                <option class:is-default={option.value === chatDefaults[setting.key]} value={option.value} selected={option.value === chatSettings[setting.key]} disabled={option.disabled}>{option.text}</option>
               {/each}
             </select>
             {/key}
diff --git a/src/lib/Home.svelte b/src/lib/Home.svelte
index a69b1c2..897e52c 100644
--- a/src/lib/Home.svelte
+++ b/src/lib/Home.svelte
@@ -1,19 +1,22 @@
 <script lang="ts">
-  import { apiKeyStorage, globalStorage, lastChatId, getChat, started, setGlobalSettingValueByKey } from './Storage.svelte'
+  import { apiKeyStorage, globalStorage, lastChatId, getChat, started, setGlobalSettingValueByKey, hasActiveModels, checkStateChange } from './Storage.svelte'
   import Footer from './Footer.svelte'
   import { replace } from 'svelte-spa-router'
-  import { onMount } from 'svelte'
+  import { afterUpdate, onMount } from 'svelte'
   import { getPetals } from './ApiUtil.svelte'
+  import { clearModelOptionCache } from './Models.svelte'
 
 $: apiKey = $apiKeyStorage
 
 let showPetalsSettings = $globalStorage.enablePetals
+let pedalsEndpoint = $globalStorage.pedalsEndpoint
+let hasModels = hasActiveModels()
 
 onMount(() => {
     if (!$started) {
       $started = true
       // console.log('started', apiKey, $lastChatId, getChat($lastChatId))
-      if (apiKey && getChat($lastChatId)) {
+      if (hasActiveModels() && getChat($lastChatId)) {
         const chatId = $lastChatId
         $lastChatId = 0
         replace(`/chat/${chatId}`)
@@ -22,6 +25,13 @@ onMount(() => {
     $lastChatId = 0
 })
 
+afterUpdate(() => {
+    clearModelOptionCache()
+    hasModels = hasActiveModels()
+    pedalsEndpoint = $globalStorage.pedalsEndpoint
+    $checkStateChange++
+})
+
 const setPetalsEnabled = (event: Event) => {
     const el = (event.target as HTMLInputElement)
     setGlobalSettingValueByKey('enablePetals', !!el.checked)
@@ -33,16 +43,21 @@ const setPetalsEnabled = (event: Event) => {
 <section class="section">
   <article class="message">
     <div class="message-body">
-      <strong><a href="https://github.com/Niek/chatgpt-web">ChatGPT-web</a></strong>
+    <p class="mb-4">
+      <strong><a href="https://github.com/Niek/chatgpt-web" target="_blank">ChatGPT-web</a></strong>
       is a simple one-page web interface to the OpenAI ChatGPT API. To use it, you need to register for
       <a href="https://platform.openai.com/account/api-keys" target="_blank" rel="noreferrer">an OpenAI API key</a>
       first. OpenAI bills per token (usage-based), which means it is a lot cheaper than
       <a href="https://openai.com/blog/chatgpt-plus" target="_blank" rel="noreferrer">ChatGPT Plus</a>, unless you use
       more than 10 million tokens per month. All messages are stored in your browser's local storage, so everything is
       <strong>private</strong>. You can also close the browser tab and come back later to continue the conversation.
+    </p>
+    <p>
+      As an alternative to OpenAI, you can also use Petals swarm as a free API option for open chat models like Llama 2. 
+    </p>
     </div>
   </article>
-  <article class="message" class:is-danger={!apiKey} class:is-warning={apiKey}>
+  <article class="message" class:is-danger={!hasModels} class:is-warning={!apiKey} class:is-info={apiKey}>
     <div class="message-body">
       Set your OpenAI API key below:
 
@@ -62,7 +77,8 @@ const setPetalsEnabled = (event: Event) => {
             type="password"
             autocomplete="off"
             class="input"
-            class:is-danger={!apiKey}
+            class:is-danger={!hasModels}
+            class:is-warning={!apiKey} class:is-info={apiKey}
             value={apiKey}
           />
         </p>
@@ -74,16 +90,16 @@ const setPetalsEnabled = (event: Event) => {
       </form>
 
       {#if !apiKey}
-        <p class="help is-danger">
-          Please enter your <a href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use ChatGPT-web.
-          It is required to use ChatGPT-web.
+        <p class:is-danger={!hasModels} class:is-warning={!apiKey}>
+          Please enter your <a target="_blank" href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use Open AI's ChatGPT API.
+          At least one API must be enabled to use ChatGPT-web.
         </p>
       {/if}
     </div>
   </article>
 
   
-  <article class="message" class:is-info={true}>
+  <article class="message" class:is-danger={!hasModels} class:is-warning={!showPetalsSettings} class:is-info={showPetalsSettings}>
     <div class="message-body">
       <label class="label" for="enablePetals">
         <input 
@@ -122,21 +138,20 @@ const setPetalsEnabled = (event: Event) => {
 
           
         </form>
-        <p>
-          Only use <u>{getPetals()}</u> for testing.  You must set up your own Petals server for actual use. 
+        
+        {#if !pedalsEndpoint}
+          <p class="help is-warning">
+            Please only use the default public API for testing. It's best to <a target="_blank" href="https://github.com/petals-infra/chat.petals.dev">configure a private endpoint</a> and enter it above for connection to the Petals swarm.
+          </p>
+        {/if}
+        <p class="my-4">
+          <a target="_blank" href="https://petals.dev/">Petals</a> lets you run large language models at home by connecting to a public swarm, BitTorrent-style, without hefty GPU requirements.
         </p>
-        <p>
-          <b>Do not send sensitive information when using Petals.</b>
+        <p class="mb-4">
+          You are encouraged to <a target="_blank" href="https://github.com/bigscience-workshop/petals/wiki/FAQ:-Frequently-asked-questions#running-a-server">set up a Petals server to share your GPU resources</a> with the public swarm. Minimum requirements to contribute Llama 2 completions are a GTX&nbsp;1080&nbsp;8GB, but the larger/faster the better.
         </p>
-        <p>
-            For more information on Petals, see 
-            <a href="https://github.com/petals-infra/chat.petals.dev">https://github.com/petals-infra/chat.petals.dev</a>
-        </p>
-      {/if}
-      {#if !apiKey}
-        <p class="help is-danger">
-          Please enter your <a href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use ChatGPT-web.
-          It is required to use ChatGPT-web.
+        <p class="help is-warning">
+          Because Petals uses a public swarm, <b>do not send sensitive information</b> when using Petals.
         </p>
       {/if}
     </div>
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index 3dfa087..1324e22 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -1,7 +1,7 @@
 <script context="module" lang="ts">
     import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetals } from './ApiUtil.svelte'
     import { apiKeyStorage, globalStorage } from './Storage.svelte'
-    import { get } from 'svelte/store'
+import { get, writable } from 'svelte/store'
     import type { ModelDetail, Model, ResponseModels, SelectOption, Chat } from './Types.svelte'
 import { encode } from 'gpt-tokenizer'
 import llamaTokenizer from 'llama-tokenizer-js'
@@ -9,6 +9,12 @@ import llamaTokenizer from 'llama-tokenizer-js'
     import { getChatSettingObjectByKey } from './Settings.svelte'
     import { valueOf } from './Util.svelte'
 
+/**
+ * TODO: All of this + what's scattered about need to be refactored to interfaces and classes
+ *       to make it all more modular
+ */
+const modelOptionCache = writable([] as SelectOption[])
+
 // Reference: https://openai.com/pricing#language-models
 // Eventually we'll add API hosts and endpoints to this
 const modelDetails : Record<string, ModelDetail> = {
@@ -46,6 +52,17 @@ const modelDetails : Record<string, ModelDetail> = {
         prompt: 0.000000, // $0.000 per 1000 tokens prompt
         completion: 0.000000, // $0.000 per 1000 tokens completion
         max: 4096 // 4k max token buffer
+      },
+      'timdettmers/guanaco-65b': {
+        type: 'Petals',
+        label: 'Petals - guanaco-65b',
+        stop: ['</s>'],
+        userStart: '[user]',
+        assistantStart: '[[[CHARACTER_NAME]]]',
+        systemStart: '',
+        prompt: 0.000000, // $0.000 per 1000 tokens prompt
+        completion: 0.000000, // $0.000 per 1000 tokens completion
+        max: 2048 // 2k max token buffer
       }
 }
 
@@ -80,17 +97,18 @@ const unknownDetail = {
 // See: https://platform.openai.com/docs/models/model-endpoint-compatibility
 // Eventually we'll add UI for managing this
 export const supportedModels : Record<string, ModelDetail> = {
+      'gpt-3.5-turbo': modelDetails['gpt-3.5'],
+      'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
+      'gpt-3.5-turbo-0613': modelDetails['gpt-3.5'],
+      'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
       'gpt-4': modelDetails['gpt-4'],
       'gpt-4-0314': modelDetails['gpt-4'],
       'gpt-4-0613': modelDetails['gpt-4'],
       'gpt-4-32k': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0314': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0613': modelDetails['gpt-4-32k'],
-      'gpt-3.5-turbo': modelDetails['gpt-3.5'],
-      'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
-      'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
-      'gpt-3.5-turbo-0613': modelDetails['gpt-3.5'],
       'meta-llama/Llama-2-70b-chat-hf': modelDetails['meta-llama/Llama-2-70b-chat-hf']
+      // 'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b']
 }
 
 const lookupList = {
@@ -192,43 +210,67 @@ export const countTokens = (model: Model, value: string): number => {
   return getTokens(model, value).length
 }
 
+export const clearModelOptionCache = () => {
+  modelOptionCache.set([])
+}
+
 export async function getModelOptions (): Promise<SelectOption[]> {
   const gSettings = get(globalStorage)
   const openAiKey = get(apiKeyStorage)
+  const cachedOptions = get(modelOptionCache)
+  if (cachedOptions && cachedOptions.length) return cachedOptions
   // Load available models from OpenAI
   let openAiModels
-  try {
-        openAiModels = (await (
-          await fetch(getApiBase() + getEndpointModels(), {
-            method: 'GET',
-            headers: {
-              Authorization: `Bearer ${openAiKey}`,
-              'Content-Type': 'application/json'
-            }
-          })
-        ).json()) as ResponseModels
-  } catch (e) {
+  let allowCache = true
+  if (openAiKey) {
+        try {
+          openAiModels = (await (
+            await fetch(getApiBase() + getEndpointModels(), {
+              method: 'GET',
+              headers: {
+                Authorization: `Bearer ${openAiKey}`,
+                'Content-Type': 'application/json'
+              }
+            })
+          ).json()) as ResponseModels
+        } catch (e) {
+          allowCache = false
+          openAiModels = { data: [] }
+        }
+  } else {
         openAiModels = { data: [] }
   }
-  const filteredModels = supportedModelKeys.filter((model) => {
-        switch (getModelDetail(model).type) {
+  // const filteredModels = Object.keys(supportedModels).filter((model) => {
+  //       switch (getModelDetail(model).type) {
+  //         case 'Petals':
+  //           return gSettings.enablePetals
+  //         case 'OpenAIChat':
+  //         default:
+  //           return openAiModels.data && openAiModels.data.find((m) => m.id === model)
+  //       }
+  // })
+
+  const modelOptions:SelectOption[] = Object.keys(supportedModels).reduce((a, m) => {
+        let disabled
+        switch (getModelDetail(m).type) {
           case 'Petals':
-            return gSettings.enablePetals
+            disabled = !gSettings.enablePetals
+            break
           case 'OpenAIChat':
           default:
-            return openAiModels.data.find((m) => m.id === model)
+            disabled = !(openAiModels.data && openAiModels.data.find((m) => m.id === m))
         }
-  })
-
-  const modelOptions:SelectOption[] = filteredModels.reduce((a, m) => {
         const o:SelectOption = {
           value: m,
-          text: m
+          text: m,
+          disabled
         }
         a.push(o)
         return a
   }, [] as SelectOption[])
 
+  if (allowCache) modelOptionCache.set(modelOptions)
+
   return modelOptions
 }
 
diff --git a/src/lib/Profiles.svelte b/src/lib/Profiles.svelte
index 7e6a9d0..2dacf17 100644
--- a/src/lib/Profiles.svelte
+++ b/src/lib/Profiles.svelte
@@ -1,5 +1,5 @@
 <script context="module" lang="ts">
-  import { getChatDefaults, getExcludeFromProfile } from './Settings.svelte'
+  import { getChatDefaults, getDefaultModel, getExcludeFromProfile } from './Settings.svelte'
   import { get, writable } from 'svelte/store'
   // Profile definitions
   import { addMessage, clearMessages, deleteMessage, getChat, getChatSettings, getCustomProfiles, getGlobalSettings, getMessages, newName, resetChatSettings, saveChatStore, setGlobalSettingValueByKey, setMessages, updateProfile } from './Storage.svelte'
@@ -22,7 +22,9 @@ export const getProfiles = (forceUpdate:boolean = false):Record<string, ChatSett
     }
     const result = Object.entries(profiles
     ).reduce((a, [k, v]) => {
+      v = JSON.parse(JSON.stringify(v))
       a[k] = v
+      v.model = v.model || getDefaultModel()
       return a
     }, {} as Record<string, ChatSettings>)
     Object.entries(getCustomProfiles()).forEach(([k, v]) => {
diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte
index 5f01837..f33324a 100644
--- a/src/lib/Settings.svelte
+++ b/src/lib/Settings.svelte
@@ -1,6 +1,7 @@
 <script context="module" lang="ts">
     import { applyProfile } from './Profiles.svelte'
-    import { getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
+    import { get } from 'svelte/store'
+    import { apiKeyStorage, getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
     import { faArrowDown91, faArrowDownAZ, faCheck, faThumbTack } from '@fortawesome/free-solid-svg-icons/index'
 // Setting definitions
 
@@ -19,7 +20,13 @@ import {
 } from './Types.svelte'
     import { getModelDetail, getTokens } from './Models.svelte'
 
-export const defaultModel:Model = 'gpt-3.5-turbo'
+const defaultModel:Model = 'gpt-3.5-turbo'
+const defaultModelPetals:Model = 'meta-llama/Llama-2-70b-chat-hf'
+
+export const getDefaultModel = (): Model => {
+  if (!get(apiKeyStorage)) return defaultModelPetals
+  return defaultModel
+}
 
 export const getChatSettingList = (): ChatSetting[] => {
       return chatSettingsList
@@ -64,7 +71,7 @@ const isNotPetals = (chatId) => {
 }
 
 const gptDefaults = {
-  model: defaultModel,
+  model: '',
   messages: [],
   temperature: 1,
   top_p: 1,
diff --git a/src/lib/Sidebar.svelte b/src/lib/Sidebar.svelte
index 8268a88..29be54e 100644
--- a/src/lib/Sidebar.svelte
+++ b/src/lib/Sidebar.svelte
@@ -1,7 +1,7 @@
 <script lang="ts">
   import { params } from 'svelte-spa-router'
   import ChatMenuItem from './ChatMenuItem.svelte'
-  import { apiKeyStorage, chatsStorage, pinMainMenu, checkStateChange, getChatSortOption, setChatSortOption } from './Storage.svelte'
+  import { apiKeyStorage, chatsStorage, pinMainMenu, checkStateChange, getChatSortOption, setChatSortOption, hasActiveModels } from './Storage.svelte'
   import Fa from 'svelte-fa/src/fa.svelte'
   import { faSquarePlus, faKey } from '@fortawesome/free-solid-svg-icons/index'
   import ChatOptionMenu from './ChatOptionMenu.svelte'
@@ -14,10 +14,12 @@
   $: activeChatId = $params && $params.chatId ? parseInt($params.chatId) : undefined
 
   let sortOption = getChatSortOption()
+  let hasModels = hasActiveModels()
 
   const onStateChange = (...args:any) => {
     sortOption = getChatSortOption()
     sortedChats = $chatsStorage.sort(sortOption.sortFn)
+    hasModels = hasActiveModels()
   }
 
   $: onStateChange($checkStateChange)
@@ -72,14 +74,14 @@
         </div>
       </div>
       <div class="level-right">
-        {#if !$apiKeyStorage}
+        {#if !hasModels}
         <div class="level-item">
           <a href={'#/'} class="panel-block" class:is-disabled={!$apiKeyStorage}
             ><span class="greyscale mr-1"><Fa icon={faKey} /></span> API key</a
           ></div>
         {:else}
         <div class="level-item">
-          <button on:click={() => { $pinMainMenu = false; startNewChatWithWarning(activeChatId) }} class="panel-block button" title="Start new chat with default profile" class:is-disabled={!$apiKeyStorage}
+          <button on:click={() => { $pinMainMenu = false; startNewChatWithWarning(activeChatId) }} class="panel-block button" title="Start new chat with default profile" class:is-disabled={!hasModels}
             ><span class="greyscale mr-1"><Fa icon={faSquarePlus} /></span> New chat</button>
           </div>
         {/if}
diff --git a/src/lib/Storage.svelte b/src/lib/Storage.svelte
index a5d3960..75619fb 100644
--- a/src/lib/Storage.svelte
+++ b/src/lib/Storage.svelte
@@ -25,11 +25,16 @@
   export let lastChatId = persisted('lastChatId', 0)
 
   const chatDefaults = getChatDefaults()
-
+  
   export const getApiKey = (): string => {
     return get(apiKeyStorage)
   }
 
+  export const hasActiveModels = (): boolean => {
+    const globalSettings = get(globalStorage) || {}
+    return !!get(apiKeyStorage) || !!globalSettings.enablePetals
+  }
+
   export const newChatID = (): number => {
     const chats = get(chatsStorage)
     const chatId = chats.reduce((maxId, chat) => Math.max(maxId, chat.id), 0) + 1
diff --git a/src/lib/Types.svelte b/src/lib/Types.svelte
index 3757c9f..7064115 100644
--- a/src/lib/Types.svelte
+++ b/src/lib/Types.svelte
@@ -199,6 +199,7 @@ export type GlobalSettings = {
 export type SelectOption = {
     value: string|number;
     text: string;
+    disabled?: boolean;
   };
 
 export type ChatSortOption = SelectOption & {

From f56e29b82911fc3177dcc4a597592eba7fc6a4ab Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 15:53:04 -0500
Subject: [PATCH 11/17] Show shorter model name

---
 src/lib/Chat.svelte        | 3 ++-
 src/lib/EditMessage.svelte | 3 ++-
 src/lib/Models.svelte      | 5 +++--
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/src/lib/Chat.svelte b/src/lib/Chat.svelte
index ebe6874..027173c 100644
--- a/src/lib/Chat.svelte
+++ b/src/lib/Chat.svelte
@@ -40,6 +40,7 @@
   import { openModal } from 'svelte-modals'
   import PromptInput from './PromptInput.svelte'
   import { ChatRequest } from './ChatRequest.svelte'
+  import { getModelDetail } from './Models.svelte'
 
   export let params = { chatId: '' }
   const chatId: number = parseInt(params.chatId)
@@ -423,7 +424,7 @@
   <div class="content has-text-centered running-total-container">
     {#each Object.entries(chat.usage || {}) as [model, usage]}
     <p class="is-size-7 running-totals">
-      <em>{model}</em> total <span class="has-text-weight-bold">{usage.total_tokens}</span>
+      <em>{getModelDetail(model || '').label || model}</em> total <span class="has-text-weight-bold">{usage.total_tokens}</span>
       tokens ~= <span class="has-text-weight-bold">${getPrice(usage, model).toFixed(6)}</span>
     </p>
     {/each}
diff --git a/src/lib/EditMessage.svelte b/src/lib/EditMessage.svelte
index 39c527e..05565d6 100644
--- a/src/lib/EditMessage.svelte
+++ b/src/lib/EditMessage.svelte
@@ -11,6 +11,7 @@
   import { openModal } from 'svelte-modals'
   import PromptConfirm from './PromptConfirm.svelte'
   import { getImage } from './ImageStore.svelte'
+  import { getModelDetail } from './Models.svelte'
 
   export let message:Message
   export let chatId:number
@@ -245,7 +246,7 @@
       <p class="is-size-7 message-note">System Prompt</p>
     {:else if message.usage}
       <p class="is-size-7 message-note">
-        <em>{message.model || defaultModel}</em> using <span class="has-text-weight-bold">{message.usage.total_tokens}</span>
+        <em>{getModelDetail(message.model || '').label || message.model || defaultModel}</em> using <span class="has-text-weight-bold">{message.usage.total_tokens}</span>
         tokens ~= <span class="has-text-weight-bold">${getPrice(message.usage, message.model || defaultModel).toFixed(6)}</span>
       </p>
     {/if}
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index 1324e22..b879b9e 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -252,7 +252,8 @@ export async function getModelOptions (): Promise<SelectOption[]> {
 
   const modelOptions:SelectOption[] = Object.keys(supportedModels).reduce((a, m) => {
         let disabled
-        switch (getModelDetail(m).type) {
+        const modelDetail = getModelDetail(m)
+        switch (modelDetail.type) {
           case 'Petals':
             disabled = !gSettings.enablePetals
             break
@@ -262,7 +263,7 @@ export async function getModelOptions (): Promise<SelectOption[]> {
         }
         const o:SelectOption = {
           value: m,
-          text: m,
+          text: modelDetail.label || m,
           disabled
         }
         a.push(o)

From 38d38bf9484389ac092291cd61aec3293fbab5f8 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 19:48:28 -0500
Subject: [PATCH 12/17] Fix some issues with stop sequences and role sequences

---
 src/lib/ChatRequestPetals.svelte | 47 ++++++++++++++++-------------
 src/lib/Models.svelte            | 52 +++++++++++++++++++++++---------
 2 files changed, 64 insertions(+), 35 deletions(-)

diff --git a/src/lib/ChatRequestPetals.svelte b/src/lib/ChatRequestPetals.svelte
index 9d72ebe..f7e7833 100644
--- a/src/lib/ChatRequestPetals.svelte
+++ b/src/lib/ChatRequestPetals.svelte
@@ -25,10 +25,16 @@ export const runPetalsCompletionRequest = async (
         ws.close()
       }
       signal.addEventListener('abort', abortListener)
-      const stopSequences = modelDetail.stop || ['###']
+      const stopSequences = (modelDetail.stop || ['###', '</s>']).slice()
       const stopSequence = getStopSequence(chat)
-      const stopSequencesC = stopSequences.slice()
-      if (stopSequence === stopSequencesC[0]) stopSequencesC.shift()
+      let stopSequenceC = stopSequence
+      if (stopSequence !== '###') {
+        stopSequences.push(stopSequence)
+        stopSequenceC = '</s>'
+      }
+      const stopSequencesC = stopSequences.filter((ss) => {
+        return ss !== '###' && ss !== stopSequenceC
+      })
       const maxTokens = getModelMaxTokens(model)
       let maxLen = Math.min(opts.maxTokens || chatRequest.chat.max_tokens || maxTokens, maxTokens)
       const promptTokenCount = chatResponse.getPromptTokenCount()
@@ -36,6 +42,16 @@ export const runPetalsCompletionRequest = async (
         maxLen = Math.min(maxLen + promptTokenCount, maxTokens)
       }
       chatResponse.onFinish(() => {
+        const message = chatResponse.getMessages()[0]
+        if (message) {
+          for (let i = 0, l = stopSequences.length; i < l; i++) {
+            const ss = stopSequences[i].trim()
+            if (message.content.trim().endsWith(ss)) {
+              message.content = message.content.trim().slice(0, message.content.trim().length - ss.length)
+              updateMessages(chat.id)
+            }
+          }
+        }
         chatRequest.updating = false
         chatRequest.updatingMessage = ''
       })
@@ -55,8 +71,8 @@ export const runPetalsCompletionRequest = async (
           }
           const rMessages = request.messages || [] as Message[]
           // make sure top_p and temperature are set the way we need
-          let temperature = request.temperature || 0
-          if (isNaN(temperature as any)) temperature = 1
+          let temperature = request.temperature
+          if (temperature === undefined || isNaN(temperature as any)) temperature = 1
           if (!temperature || temperature <= 0) temperature = 0.01
           let topP = request.top_p
           if (topP === undefined || isNaN(topP as any)) topP = 1
@@ -64,7 +80,7 @@ export const runPetalsCompletionRequest = async (
           // build the message array
           const inputArray = (rMessages).reduce((a, m) => {
             const c = getRoleTag(m.role, model, chatRequest.chat) + m.content
-            a.push(c)
+            a.push(c.trim())
             return a
           }, [] as string[])
           const lastMessage = rMessages[rMessages.length - 1]
@@ -75,12 +91,12 @@ export const runPetalsCompletionRequest = async (
             type: 'generate',
             inputs: inputArray.join(stopSequence),
             max_new_tokens: 1, // wait for up to 1 tokens before displaying
-            stop_sequence: stopSequence,
+            stop_sequence: stopSequenceC,
             do_sample: 1, // enable top p and the like
             temperature,
-            top_p: topP,
-            extra_stop_sequences: stopSequencesC
-          }
+            top_p: topP
+          } as any
+          if (stopSequencesC.length) petalsRequest.extra_stop_sequences = stopSequencesC
           ws.send(JSON.stringify(petalsRequest))
           ws.onmessage = event => {
             // Remove updating indicator
@@ -106,17 +122,6 @@ export const runPetalsCompletionRequest = async (
                         }]
                       } as any
               )
-              if (response.stop) {
-                const message = chatResponse.getMessages()[0]
-                if (message) {
-                  for (let i = 0, l = stopSequences.length; i < l; i++) {
-                    if (message.content.endsWith(stopSequences[i])) {
-                      message.content = message.content.slice(0, message.content.length - stopSequences[i].length)
-                      updateMessages(chat.id)
-                    }
-                  }
-                }
-              }
             }, 1)
           }
         }
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index b879b9e..f4ca2c1 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -42,27 +42,49 @@ const modelDetails : Record<string, ModelDetail> = {
         completion: 0.000004, // $0.004 per 1000 tokens completion
         max: 16384 // 16k max token buffer
       },
+      'enoch/llama-65b-hf': {
+        type: 'Petals',
+        label: 'Petals - Llama-65b',
+        stop: ['###', '</s>'],
+        userStart: '<|user|>',
+        assistantStart: '<|[[CHARACTER_NAME]]|>',
+        systemStart: '',
+        prompt: 0.000000, // $0.000 per 1000 tokens prompt
+        completion: 0.000000, // $0.000 per 1000 tokens completion
+        max: 2048 // 2k max token buffer
+      },
+      'timdettmers/guanaco-65b': {
+        type: 'Petals',
+        label: 'Petals - Guanaco-65b',
+        stop: ['###', '</s>'],
+        userStart: '<|user|>',
+        assistantStart: '<|[[CHARACTER_NAME]]|>',
+        systemStart: '',
+        prompt: 0.000000, // $0.000 per 1000 tokens prompt
+        completion: 0.000000, // $0.000 per 1000 tokens completion
+        max: 2048 // 2k max token buffer
+      },
       'meta-llama/Llama-2-70b-chat-hf': {
         type: 'Petals',
         label: 'Petals - Llama-2-70b-chat',
-        stop: ['</s>'],
-        userStart: '[user]',
-        assistantStart: '[[[CHARACTER_NAME]]]',
+        stop: ['###', '</s>'],
+        userStart: '<|user|>',
+        assistantStart: '<|[[CHARACTER_NAME]]|>',
         systemStart: '',
         prompt: 0.000000, // $0.000 per 1000 tokens prompt
         completion: 0.000000, // $0.000 per 1000 tokens completion
         max: 4096 // 4k max token buffer
       },
-      'timdettmers/guanaco-65b': {
+      'meta-llama/Llama-2-70b-hf': {
         type: 'Petals',
-        label: 'Petals - guanaco-65b',
-        stop: ['</s>'],
-        userStart: '[user]',
-        assistantStart: '[[[CHARACTER_NAME]]]',
+        label: 'Petals - Llama-2-70b',
+        stop: ['###', '</s>'],
+        userStart: '<|user|>',
+        assistantStart: '<|[[CHARACTER_NAME]]|>',
         systemStart: '',
         prompt: 0.000000, // $0.000 per 1000 tokens prompt
         completion: 0.000000, // $0.000 per 1000 tokens completion
-        max: 2048 // 2k max token buffer
+        max: 4096 // 4k max token buffer
       }
 }
 
@@ -107,8 +129,10 @@ export const supportedModels : Record<string, ModelDetail> = {
       'gpt-4-32k': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0314': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0613': modelDetails['gpt-4-32k'],
+      'enoch/llama-65b-hf': modelDetails['enoch/llama-65b-hf'],
+      'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b'],
+      'meta-llama/Llama-2-70b-hf': modelDetails['meta-llama/Llama-2-70b-hf'],
       'meta-llama/Llama-2-70b-chat-hf': modelDetails['meta-llama/Llama-2-70b-chat-hf']
-      // 'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b']
 }
 
 const lookupList = {
@@ -154,27 +178,27 @@ export const getEndpoint = (model: Model): string => {
 }
 
 export const getStopSequence = (chat: Chat): string => {
-  return valueOf(chat.id, getChatSettingObjectByKey('stopSequence').placeholder)
+  return chat.settings.stopSequence || valueOf(chat.id, getChatSettingObjectByKey('stopSequence').placeholder)
 }
 
 export const getUserStart = (chat: Chat): string => {
   return mergeProfileFields(
         chat.settings,
-        valueOf(chat.id, getChatSettingObjectByKey('userMessageStart').placeholder)
+        chat.settings.userMessageStart || valueOf(chat.id, getChatSettingObjectByKey('userMessageStart').placeholder)
       )
 }
 
 export const getAssistantStart = (chat: Chat): string => {
   return mergeProfileFields(
         chat.settings,
-        valueOf(chat.id, getChatSettingObjectByKey('assistantMessageStart').placeholder)
+        chat.settings.assistantMessageStart || valueOf(chat.id, getChatSettingObjectByKey('assistantMessageStart').placeholder)
       )
 }
 
 export const getSystemStart = (chat: Chat): string => {
   return mergeProfileFields(
         chat.settings,
-        valueOf(chat.id, getChatSettingObjectByKey('systemMessageStart').placeholder)
+        chat.settings.systemMessageStart || valueOf(chat.id, getChatSettingObjectByKey('systemMessageStart').placeholder)
       )
 }
 

From 190bf16ce685adbfdb5b4edb0c65071e24c45579 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 19:56:05 -0500
Subject: [PATCH 13/17] Drop llama 65b and guanaco 65b - unstable in swarm

---
 src/lib/Models.svelte | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte
index f4ca2c1..8019259 100644
--- a/src/lib/Models.svelte
+++ b/src/lib/Models.svelte
@@ -129,8 +129,8 @@ export const supportedModels : Record<string, ModelDetail> = {
       'gpt-4-32k': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0314': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0613': modelDetails['gpt-4-32k'],
-      'enoch/llama-65b-hf': modelDetails['enoch/llama-65b-hf'],
-      'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b'],
+      // 'enoch/llama-65b-hf': modelDetails['enoch/llama-65b-hf'],
+      // 'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b'],
       'meta-llama/Llama-2-70b-hf': modelDetails['meta-llama/Llama-2-70b-hf'],
       'meta-llama/Llama-2-70b-chat-hf': modelDetails['meta-llama/Llama-2-70b-chat-hf']
 }

From af08f5c99e1111f0be5f9c842f31fa1b4c207169 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 20:00:20 -0500
Subject: [PATCH 14/17] Update text

---
 src/lib/Home.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/Home.svelte b/src/lib/Home.svelte
index 897e52c..9bf2d1d 100644
--- a/src/lib/Home.svelte
+++ b/src/lib/Home.svelte
@@ -109,7 +109,7 @@ const setPetalsEnabled = (event: Event) => {
         checked={!!$globalStorage.enablePetals} 
         on:click={setPetalsEnabled}
       >
-        Use Petals API and Models
+        Use Petals API and Models (Llama 2)
       </label>
       {#if showPetalsSettings}
         <p>Set Petals API Endpoint:</p>

From 833633991aae68450fd5660170dda46c5b26bcac Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 22:20:52 -0500
Subject: [PATCH 15/17] Fix user prompt prefix injection

---
 src/lib/ChatRequest.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index 7a17d2a..3639875 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -301,7 +301,7 @@ export class ChatRequest {
             if (m.length) {
               if (m.match(/\[\[USER_PROMPT\]\]/)) {
                 injectedPrompt = true
-                m.replace(/\[\[USER_PROMPT\]\]/g, lastMessage.content)
+                m = m.replace(/\[\[USER_PROMPT\]\]/g, lastMessage.content)
               }
               a.push({ role: a.length % 2 === 0 ? 'user' : 'assistant', content: m } as Message)
             }

From 0ffdd788633b96fa26afe1ad5b45d75ab0fd9a1c Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Mon, 24 Jul 2023 23:14:28 -0500
Subject: [PATCH 16/17] Another prompt prefix issue

---
 src/lib/ChatRequest.svelte | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/lib/ChatRequest.svelte b/src/lib/ChatRequest.svelte
index 3639875..7156a38 100644
--- a/src/lib/ChatRequest.svelte
+++ b/src/lib/ChatRequest.svelte
@@ -320,7 +320,7 @@ export class ChatRequest {
               lastMessage.skipOnce = true
             }
           }
-          if (injectedPrompt) results.pop()
+          if (injectedPrompt) messages.pop()
           return results
         }
         return []

From ff3799637bf41811a15a61a82f6344812e599200 Mon Sep 17 00:00:00 2001
From: Webifi <john@webifi.com>
Date: Tue, 25 Jul 2023 00:21:04 -0500
Subject: [PATCH 17/17] Allow scrolling while streaming re: #241

---
 src/lib/Chat.svelte | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/src/lib/Chat.svelte b/src/lib/Chat.svelte
index 027173c..db0d065 100644
--- a/src/lib/Chat.svelte
+++ b/src/lib/Chat.svelte
@@ -246,6 +246,19 @@
     chatRequest.updating = true
     chatRequest.updatingMessage = ''
 
+    let doScroll = true
+    let didScroll = false
+
+    const checkUserScroll = (e: Event) => {
+      const el = e.target as HTMLElement
+      if (el && e.isTrusted && didScroll) {
+        // from user
+        doScroll = (window.innerHeight + window.scrollY + 10) >= document.body.offsetHeight
+      }
+    }
+
+    window.addEventListener('scroll', checkUserScroll)
+
     try {
       const response = await chatRequest.sendRequest($currentChatMessages, {
         chat,
@@ -253,7 +266,8 @@
         streaming: chatSettings.stream,
         fillMessage,
         onMessageChange: (messages) => {
-          scrollToBottom(true)
+          if (doScroll) scrollToBottom(true)
+          didScroll = !!messages[0]?.content
         }
       })
       await response.promiseToFinish()
@@ -264,6 +278,8 @@
     } catch (e) {
       console.error(e)
     }
+  
+    window.removeEventListener('scroll', checkUserScroll)
 
     chatRequest.updating = false
     chatRequest.updatingMessage = ''