Allow Petals and/or OpenAI

src/lib/ChatMenuItem.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import { replace } from 'svelte-spa-router'
   import type { Chat } from './Types.svelte'
-  import { apiKeyStorage, deleteChat, pinMainMenu, saveChatStore } from './Storage.svelte'
+  import { deleteChat, hasActiveModels, pinMainMenu, saveChatStore } from './Storage.svelte'
   import Fa from 'svelte-fa/src/fa.svelte'
   import { faTrash, faCircleCheck, faPencil } from '@fortawesome/free-solid-svg-icons/index'
   import { faMessage } from '@fortawesome/free-regular-svg-icons/index'
@@ -86,7 +86,7 @@
   <a 
     href={`#/chat/${chat.id}`}
     class="chat-menu-item"
-    class:is-waiting={waitingForConfirm} class:is-disabled={!$apiKeyStorage} class:is-active={activeChatId === chat.id}
+    class:is-waiting={waitingForConfirm} class:is-disabled={!hasActiveModels()} class:is-active={activeChatId === chat.id}
     on:click={() => { $pinMainMenu = false }} >
     {#if waitingForConfirm}
     <a class="is-pulled-right is-hidden px-1 py-0 has-text-weight-bold delete-button" href={'$'} on:click|preventDefault={() => delChat()}><Fa icon={faCircleCheck} /></a>

src/lib/ChatOptionMenu.svelte

@@ -18,7 +18,7 @@
     faEyeSlash
   } from '@fortawesome/free-solid-svg-icons/index'
   import { faSquareMinus, faSquarePlus as faSquarePlusOutline } from '@fortawesome/free-regular-svg-icons/index'
-  import { apiKeyStorage, addChatFromJSON, chatsStorage, checkStateChange, clearChats, clearMessages, copyChat, globalStorage, setGlobalSettingValueByKey, showSetChatSettings, pinMainMenu, getChat, deleteChat, saveChatStore, saveCustomProfile } from './Storage.svelte'
+  import { addChatFromJSON, chatsStorage, checkStateChange, clearChats, clearMessages, copyChat, globalStorage, setGlobalSettingValueByKey, showSetChatSettings, pinMainMenu, getChat, deleteChat, saveChatStore, saveCustomProfile, hasActiveModels } from './Storage.svelte'
   import { exportAsMarkdown, exportChatAsJSON } from './Export.svelte'
   import { newNameForProfile, restartProfile } from './Profiles.svelte'
   import { replace } from 'svelte-spa-router'
@@ -173,7 +173,7 @@
         <span class="menu-icon"><Fa icon={faGear}/></span> Chat Profile Settings
       </a>
       <hr class="dropdown-divider">
-      <a href={'#'} class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { $apiKeyStorage && close(); $apiKeyStorage && startNewChatWithWarning(chatId) }} class="dropdown-item">
+      <a href={'#'} class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { hasActiveModels() && close(); hasActiveModels() && startNewChatWithWarning(chatId) }} class="dropdown-item">
         <span class="menu-icon"><Fa icon={faSquarePlus}/></span> New Chat from Default
       </a>
       <a href={'#'} class:is-disabled={!chatId} on:click|preventDefault={() => { chatId && close(); chatId && startNewChatFromChatId(chatId) }} class="dropdown-item">
@@ -196,14 +196,14 @@
       <a href={'#'} class="dropdown-item" class:is-disabled={!chatId} on:click|preventDefault={() => { close(); exportChatAsJSON(chatId) }}>
         <span class="menu-icon"><Fa icon={faDownload}/></span> Backup Chat JSON
       </a>
-      <a href={'#'} class="dropdown-item" class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { if (chatId) close(); chatFileInput.click() }}>
+      <a href={'#'} class="dropdown-item" class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { if (chatId) close(); chatFileInput.click() }}>
         <span class="menu-icon"><Fa icon={faUpload}/></span> Restore Chat JSON
       </a>
       <a href={'#'} class="dropdown-item" class:is-disabled={!chatId} on:click|preventDefault={() => { if (chatId) close(); exportAsMarkdown(chatId) }}>
         <span class="menu-icon"><Fa icon={faFileExport}/></span> Export Chat Markdown
       </a>
       <hr class="dropdown-divider">
-      <a href={'#'} class="dropdown-item" class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { if (chatId) close(); profileFileInput.click() }}>
+      <a href={'#'} class="dropdown-item" class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { if (chatId) close(); profileFileInput.click() }}>
         <span class="menu-icon"><Fa icon={faUpload}/></span> Restore Profile JSON
       </a>
       <hr class="dropdown-divider">

src/lib/ChatRequest.svelte

@@ -5,7 +5,7 @@
     import type { Chat, ChatCompletionOpts, ChatSettings, Message, Model, Request, RequestImageGeneration } from './Types.svelte'
     import { deleteMessage, getChatSettingValueNullDefault, insertMessages, getApiKey, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage } from './Storage.svelte'
     import { scrollToBottom, scrollToMessage } from './Util.svelte'
-    import { getRequestSettingList, defaultModel } from './Settings.svelte'
+    import { getDefaultModel, getRequestSettingList } from './Settings.svelte'
     import { v4 as uuidv4 } from 'uuid'
     import { get } from 'svelte/store'
     import { getEndpoint, getModelDetail } from './Models.svelte'
@@ -26,6 +26,7 @@ export class ChatRequest {
 
       setChat (chat: Chat) {
         this.chat = chat
+        this.chat.settings.model = this.getModel()
       }
 
       getChat (): Chat {
@@ -283,7 +284,7 @@ export class ChatRequest {
       }
 
       getModel (): Model {
-        return this.chat.settings.model || defaultModel
+        return this.chat.settings.model || getDefaultModel()
       }
 
       private buildHiddenPromptPrefixMessages (messages: Message[], insert:boolean = false): Message[] {

src/lib/ChatSettingField.svelte

@@ -211,7 +211,7 @@
             {#key rkey}
             <select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } >
               {#each setting.options as option}
-                <option class:is-default={option.value === chatDefaults[setting.key]} value={option.value} selected={option.value === chatSettings[setting.key]}>{option.text}</option>
+                <option class:is-default={option.value === chatDefaults[setting.key]} value={option.value} selected={option.value === chatSettings[setting.key]} disabled={option.disabled}>{option.text}</option>
               {/each}
             </select>
             {/key}

src/lib/Home.svelte

@@ -1,19 +1,22 @@
 <script lang="ts">
-  import { apiKeyStorage, globalStorage, lastChatId, getChat, started, setGlobalSettingValueByKey } from './Storage.svelte'
+  import { apiKeyStorage, globalStorage, lastChatId, getChat, started, setGlobalSettingValueByKey, hasActiveModels, checkStateChange } from './Storage.svelte'
   import Footer from './Footer.svelte'
   import { replace } from 'svelte-spa-router'
-  import { onMount } from 'svelte'
+  import { afterUpdate, onMount } from 'svelte'
   import { getPetals } from './ApiUtil.svelte'
+  import { clearModelOptionCache } from './Models.svelte'
 
 $: apiKey = $apiKeyStorage
 
 let showPetalsSettings = $globalStorage.enablePetals
+let pedalsEndpoint = $globalStorage.pedalsEndpoint
+let hasModels = hasActiveModels()
 
 onMount(() => {
     if (!$started) {
       $started = true
       // console.log('started', apiKey, $lastChatId, getChat($lastChatId))
-      if (apiKey && getChat($lastChatId)) {
+      if (hasActiveModels() && getChat($lastChatId)) {
         const chatId = $lastChatId
         $lastChatId = 0
         replace(`/chat/${chatId}`)
@@ -22,6 +25,13 @@ onMount(() => {
     $lastChatId = 0
 })
 
+afterUpdate(() => {
+    clearModelOptionCache()
+    hasModels = hasActiveModels()
+    pedalsEndpoint = $globalStorage.pedalsEndpoint
+    $checkStateChange++
+})
+
 const setPetalsEnabled = (event: Event) => {
     const el = (event.target as HTMLInputElement)
     setGlobalSettingValueByKey('enablePetals', !!el.checked)
@@ -33,16 +43,21 @@ const setPetalsEnabled = (event: Event) => {
 <section class="section">
   <article class="message">
     <div class="message-body">
-      <strong><a href="https://github.com/Niek/chatgpt-web">ChatGPT-web</a></strong>
+    <p class="mb-4">
+      <strong><a href="https://github.com/Niek/chatgpt-web" target="_blank">ChatGPT-web</a></strong>
       is a simple one-page web interface to the OpenAI ChatGPT API. To use it, you need to register for
       <a href="https://platform.openai.com/account/api-keys" target="_blank" rel="noreferrer">an OpenAI API key</a>
       first. OpenAI bills per token (usage-based), which means it is a lot cheaper than
       <a href="https://openai.com/blog/chatgpt-plus" target="_blank" rel="noreferrer">ChatGPT Plus</a>, unless you use
       more than 10 million tokens per month. All messages are stored in your browser's local storage, so everything is
       <strong>private</strong>. You can also close the browser tab and come back later to continue the conversation.
+    </p>
+    <p>
+      As an alternative to OpenAI, you can also use Petals swarm as a free API option for open chat models like Llama 2. 
+    </p>
     </div>
   </article>
-  <article class="message" class:is-danger={!apiKey} class:is-warning={apiKey}>
+  <article class="message" class:is-danger={!hasModels} class:is-warning={!apiKey} class:is-info={apiKey}>
     <div class="message-body">
       Set your OpenAI API key below:
 
@@ -62,7 +77,8 @@ const setPetalsEnabled = (event: Event) => {
             type="password"
             autocomplete="off"
             class="input"
-            class:is-danger={!apiKey}
+            class:is-danger={!hasModels}
+            class:is-warning={!apiKey} class:is-info={apiKey}
             value={apiKey}
           />
         </p>
@@ -74,16 +90,16 @@ const setPetalsEnabled = (event: Event) => {
       </form>
 
       {#if !apiKey}
-        <p class="help is-danger">
+        <p class:is-danger={!hasModels} class:is-warning={!apiKey}>
-          Please enter your <a href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use ChatGPT-web.
+          Please enter your <a target="_blank" href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use Open AI's ChatGPT API.
-          It is required to use ChatGPT-web.
+          At least one API must be enabled to use ChatGPT-web.
         </p>
       {/if}
     </div>
   </article>
 
   
-  <article class="message" class:is-info={true}>
+  <article class="message" class:is-danger={!hasModels} class:is-warning={!showPetalsSettings} class:is-info={showPetalsSettings}>
     <div class="message-body">
       <label class="label" for="enablePetals">
         <input 
@@ -122,21 +138,20 @@ const setPetalsEnabled = (event: Event) => {
 
           
         </form>
-        <p>
+        
-          Only use <u>{getPetals()}</u> for testing.  You must set up your own Petals server for actual use. 
+        {#if !pedalsEndpoint}
+          <p class="help is-warning">
+            Please only use the default public API for testing. It's best to <a target="_blank" href="https://github.com/petals-infra/chat.petals.dev">configure a private endpoint</a> and enter it above for connection to the Petals swarm.
+          </p>
+        {/if}
+        <p class="my-4">
+          <a target="_blank" href="https://petals.dev/">Petals</a> lets you run large language models at home by connecting to a public swarm, BitTorrent-style, without hefty GPU requirements.
         </p>
-        <p>
+        <p class="mb-4">
-          <b>Do not send sensitive information when using Petals.</b>
+          You are encouraged to <a target="_blank" href="https://github.com/bigscience-workshop/petals/wiki/FAQ:-Frequently-asked-questions#running-a-server">set up a Petals server to share your GPU resources</a> with the public swarm. Minimum requirements to contribute Llama 2 completions are a GTX&nbsp;1080&nbsp;8GB, but the larger/faster the better.
         </p>
-        <p>
+        <p class="help is-warning">
-            For more information on Petals, see 
+          Because Petals uses a public swarm, <b>do not send sensitive information</b> when using Petals.
-            <a href="https://github.com/petals-infra/chat.petals.dev">https://github.com/petals-infra/chat.petals.dev</a>
-        </p>
-      {/if}
-      {#if !apiKey}
-        <p class="help is-danger">
-          Please enter your <a href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use ChatGPT-web.
-          It is required to use ChatGPT-web.
         </p>
       {/if}
     </div>

src/lib/Models.svelte

@@ -1,7 +1,7 @@
 <script context="module" lang="ts">
     import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetals } from './ApiUtil.svelte'
     import { apiKeyStorage, globalStorage } from './Storage.svelte'
-    import { get } from 'svelte/store'
+import { get, writable } from 'svelte/store'
     import type { ModelDetail, Model, ResponseModels, SelectOption, Chat } from './Types.svelte'
 import { encode } from 'gpt-tokenizer'
 import llamaTokenizer from 'llama-tokenizer-js'
@@ -9,6 +9,12 @@ import llamaTokenizer from 'llama-tokenizer-js'
     import { getChatSettingObjectByKey } from './Settings.svelte'
     import { valueOf } from './Util.svelte'
 
+/**
+ * TODO: All of this + what's scattered about need to be refactored to interfaces and classes
+ *       to make it all more modular
+ */
+const modelOptionCache = writable([] as SelectOption[])
+
 // Reference: https://openai.com/pricing#language-models
 // Eventually we'll add API hosts and endpoints to this
 const modelDetails : Record<string, ModelDetail> = {
@@ -46,6 +52,17 @@ const modelDetails : Record<string, ModelDetail> = {
         prompt: 0.000000, // $0.000 per 1000 tokens prompt
         completion: 0.000000, // $0.000 per 1000 tokens completion
         max: 4096 // 4k max token buffer
+      },
+      'timdettmers/guanaco-65b': {
+        type: 'Petals',
+        label: 'Petals - guanaco-65b',
+        stop: ['</s>'],
+        userStart: '[user]',
+        assistantStart: '[[[CHARACTER_NAME]]]',
+        systemStart: '',
+        prompt: 0.000000, // $0.000 per 1000 tokens prompt
+        completion: 0.000000, // $0.000 per 1000 tokens completion
+        max: 2048 // 2k max token buffer
       }
 }
 
@@ -80,17 +97,18 @@ const unknownDetail = {
 // See: https://platform.openai.com/docs/models/model-endpoint-compatibility
 // Eventually we'll add UI for managing this
 export const supportedModels : Record<string, ModelDetail> = {
+      'gpt-3.5-turbo': modelDetails['gpt-3.5'],
+      'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
+      'gpt-3.5-turbo-0613': modelDetails['gpt-3.5'],
+      'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
       'gpt-4': modelDetails['gpt-4'],
       'gpt-4-0314': modelDetails['gpt-4'],
       'gpt-4-0613': modelDetails['gpt-4'],
       'gpt-4-32k': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0314': modelDetails['gpt-4-32k'],
       'gpt-4-32k-0613': modelDetails['gpt-4-32k'],
-      'gpt-3.5-turbo': modelDetails['gpt-3.5'],
-      'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
-      'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
-      'gpt-3.5-turbo-0613': modelDetails['gpt-3.5'],
       'meta-llama/Llama-2-70b-chat-hf': modelDetails['meta-llama/Llama-2-70b-chat-hf']
+      // 'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b']
 }
 
 const lookupList = {
@@ -192,43 +210,67 @@ export const countTokens = (model: Model, value: string): number => {
   return getTokens(model, value).length
 }
 
+export const clearModelOptionCache = () => {
+  modelOptionCache.set([])
+}
+
 export async function getModelOptions (): Promise<SelectOption[]> {
   const gSettings = get(globalStorage)
   const openAiKey = get(apiKeyStorage)
+  const cachedOptions = get(modelOptionCache)
+  if (cachedOptions && cachedOptions.length) return cachedOptions
   // Load available models from OpenAI
   let openAiModels
-  try {
+  let allowCache = true
-        openAiModels = (await (
+  if (openAiKey) {
-          await fetch(getApiBase() + getEndpointModels(), {
+        try {
-            method: 'GET',
+          openAiModels = (await (
-            headers: {
+            await fetch(getApiBase() + getEndpointModels(), {
-              Authorization: `Bearer ${openAiKey}`,
+              method: 'GET',
-              'Content-Type': 'application/json'
+              headers: {
-            }
+                Authorization: `Bearer ${openAiKey}`,
-          })
+                'Content-Type': 'application/json'
-        ).json()) as ResponseModels
+              }
-  } catch (e) {
+            })
+          ).json()) as ResponseModels
+        } catch (e) {
+          allowCache = false
+          openAiModels = { data: [] }
+        }
+  } else {
         openAiModels = { data: [] }
   }
-  const filteredModels = supportedModelKeys.filter((model) => {
+  // const filteredModels = Object.keys(supportedModels).filter((model) => {
-        switch (getModelDetail(model).type) {
+  //       switch (getModelDetail(model).type) {
+  //         case 'Petals':
+  //           return gSettings.enablePetals
+  //         case 'OpenAIChat':
+  //         default:
+  //           return openAiModels.data && openAiModels.data.find((m) => m.id === model)
+  //       }
+  // })
+
+  const modelOptions:SelectOption[] = Object.keys(supportedModels).reduce((a, m) => {
+        let disabled
+        switch (getModelDetail(m).type) {
           case 'Petals':
-            return gSettings.enablePetals
+            disabled = !gSettings.enablePetals
+            break
           case 'OpenAIChat':
           default:
-            return openAiModels.data.find((m) => m.id === model)
+            disabled = !(openAiModels.data && openAiModels.data.find((m) => m.id === m))
         }
-  })
-
-  const modelOptions:SelectOption[] = filteredModels.reduce((a, m) => {
         const o:SelectOption = {
           value: m,
-          text: m
+          text: m,
+          disabled
         }
         a.push(o)
         return a
   }, [] as SelectOption[])
 
+  if (allowCache) modelOptionCache.set(modelOptions)
+
   return modelOptions
 }
 

src/lib/Profiles.svelte

@@ -1,5 +1,5 @@
 <script context="module" lang="ts">
-  import { getChatDefaults, getExcludeFromProfile } from './Settings.svelte'
+  import { getChatDefaults, getDefaultModel, getExcludeFromProfile } from './Settings.svelte'
   import { get, writable } from 'svelte/store'
   // Profile definitions
   import { addMessage, clearMessages, deleteMessage, getChat, getChatSettings, getCustomProfiles, getGlobalSettings, getMessages, newName, resetChatSettings, saveChatStore, setGlobalSettingValueByKey, setMessages, updateProfile } from './Storage.svelte'
@@ -22,7 +22,9 @@ export const getProfiles = (forceUpdate:boolean = false):Record<string, ChatSett
     }
     const result = Object.entries(profiles
     ).reduce((a, [k, v]) => {
+      v = JSON.parse(JSON.stringify(v))
       a[k] = v
+      v.model = v.model || getDefaultModel()
       return a
     }, {} as Record<string, ChatSettings>)
     Object.entries(getCustomProfiles()).forEach(([k, v]) => {

src/lib/Settings.svelte

@@ -1,6 +1,7 @@
 <script context="module" lang="ts">
     import { applyProfile } from './Profiles.svelte'
-    import { getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
+    import { get } from 'svelte/store'
+    import { apiKeyStorage, getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
     import { faArrowDown91, faArrowDownAZ, faCheck, faThumbTack } from '@fortawesome/free-solid-svg-icons/index'
 // Setting definitions
 
@@ -19,7 +20,13 @@ import {
 } from './Types.svelte'
     import { getModelDetail, getTokens } from './Models.svelte'
 
-export const defaultModel:Model = 'gpt-3.5-turbo'
+const defaultModel:Model = 'gpt-3.5-turbo'
+const defaultModelPetals:Model = 'meta-llama/Llama-2-70b-chat-hf'
+
+export const getDefaultModel = (): Model => {
+  if (!get(apiKeyStorage)) return defaultModelPetals
+  return defaultModel
+}
 
 export const getChatSettingList = (): ChatSetting[] => {
       return chatSettingsList
@@ -64,7 +71,7 @@ const isNotPetals = (chatId) => {
 }
 
 const gptDefaults = {
-  model: defaultModel,
+  model: '',
   messages: [],
   temperature: 1,
   top_p: 1,

src/lib/Sidebar.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import { params } from 'svelte-spa-router'
   import ChatMenuItem from './ChatMenuItem.svelte'
-  import { apiKeyStorage, chatsStorage, pinMainMenu, checkStateChange, getChatSortOption, setChatSortOption } from './Storage.svelte'
+  import { apiKeyStorage, chatsStorage, pinMainMenu, checkStateChange, getChatSortOption, setChatSortOption, hasActiveModels } from './Storage.svelte'
   import Fa from 'svelte-fa/src/fa.svelte'
   import { faSquarePlus, faKey } from '@fortawesome/free-solid-svg-icons/index'
   import ChatOptionMenu from './ChatOptionMenu.svelte'
@@ -14,10 +14,12 @@
   $: activeChatId = $params && $params.chatId ? parseInt($params.chatId) : undefined
 
   let sortOption = getChatSortOption()
+  let hasModels = hasActiveModels()
 
   const onStateChange = (...args:any) => {
     sortOption = getChatSortOption()
     sortedChats = $chatsStorage.sort(sortOption.sortFn)
+    hasModels = hasActiveModels()
   }
 
   $: onStateChange($checkStateChange)
@@ -72,14 +74,14 @@
         </div>
       </div>
       <div class="level-right">
-        {#if !$apiKeyStorage}
+        {#if !hasModels}
         <div class="level-item">
           <a href={'#/'} class="panel-block" class:is-disabled={!$apiKeyStorage}
             ><span class="greyscale mr-1"><Fa icon={faKey} /></span> API key</a
           ></div>
         {:else}
         <div class="level-item">
-          <button on:click={() => { $pinMainMenu = false; startNewChatWithWarning(activeChatId) }} class="panel-block button" title="Start new chat with default profile" class:is-disabled={!$apiKeyStorage}
+          <button on:click={() => { $pinMainMenu = false; startNewChatWithWarning(activeChatId) }} class="panel-block button" title="Start new chat with default profile" class:is-disabled={!hasModels}
             ><span class="greyscale mr-1"><Fa icon={faSquarePlus} /></span> New chat</button>
           </div>
         {/if}

src/lib/Storage.svelte

@@ -30,6 +30,11 @@
     return get(apiKeyStorage)
   }
 
+  export const hasActiveModels = (): boolean => {
+    const globalSettings = get(globalStorage) || {}
+    return !!get(apiKeyStorage) || !!globalSettings.enablePetals
+  }
+
   export const newChatID = (): number => {
     const chats = get(chatsStorage)
     const chatId = chats.reduce((maxId, chat) => Math.max(maxId, chat.id), 0) + 1

src/lib/Types.svelte

@@ -199,6 +199,7 @@ export type GlobalSettings = {
 export type SelectOption = {
     value: string|number;
     text: string;
+    disabled?: boolean;
   };
 
 export type ChatSortOption = SelectOption & {