Merge pull request #235 from Webifi/main
Experimental support for Petals/Llama 2
This commit is contained in:
Niek van der Maas
GitHub
commit
f877ac09ab
|
|
@ -27,6 +27,7 @@
|
|||
"eslint-plugin-svelte3": "^4.0.0",
|
||||
"flourite": "^1.2.4",
|
||||
"gpt-tokenizer": "^2.0.0",
|
||||
"llama-tokenizer-js": "^1.1.1",
|
||||
"postcss": "^8.4.26",
|
||||
"sass": "^1.63.6",
|
||||
"stacking-order": "^2.0.0",
|
||||
|
|
@ -3182,6 +3183,12 @@
|
|||
"node": ">= 0.8.0"
|
||||
}
|
||||
},
|
||||
"node_modules/llama-tokenizer-js": {
|
||||
"version": "1.1.1",
|
||||
"resolved": "https://registry.npmjs.org/llama-tokenizer-js/-/llama-tokenizer-js-1.1.1.tgz",
|
||||
"integrity": "sha512-5H2oSJnSufWGhOw6hcCGAqJeB3POmeIBzRklH3cXs0L4MSAYdwoYTodni4j5YVo6jApdhaqaNVU66gNRgXeBRg==",
|
||||
"dev": true
|
||||
},
|
||||
"node_modules/locate-path": {
|
||||
"version": "6.0.0",
|
||||
"resolved": "https://registry.npmjs.org/locate-path/-/locate-path-6.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@
|
|||
"eslint-plugin-svelte3": "^4.0.0",
|
||||
"flourite": "^1.2.4",
|
||||
"gpt-tokenizer": "^2.0.0",
|
||||
"llama-tokenizer-js": "^1.1.1",
|
||||
"postcss": "^8.4.26",
|
||||
"sass": "^1.63.6",
|
||||
"stacking-order": "^2.0.0",
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@
|
|||
const endpointGenerations = import.meta.env.VITE_ENDPOINT_GENERATIONS || '/v1/images/generations'
|
||||
const endpointModels = import.meta.env.VITE_ENDPOINT_MODELS || '/v1/models'
|
||||
const endpointEmbeddings = import.meta.env.VITE_ENDPOINT_EMBEDDINGS || '/v1/embeddings'
|
||||
const endpointPetals = import.meta.env.VITE_PEDALS_WEBSOCKET || 'wss://chat.petals.dev/api/v2/generate'
|
||||
|
||||
export const getApiBase = ():string => apiBase
|
||||
export const getEndpointCompletions = ():string => endpointCompletions
|
||||
export const getEndpointGenerations = ():string => endpointGenerations
|
||||
export const getEndpointModels = ():string => endpointModels
|
||||
export const getEndpointEmbeddings = ():string => endpointEmbeddings
|
||||
export const getPetals = ():string => endpointPetals
|
||||
</script>
|
||||
|
|
@ -40,6 +40,7 @@
|
|||
import { openModal } from 'svelte-modals'
|
||||
import PromptInput from './PromptInput.svelte'
|
||||
import { ChatRequest } from './ChatRequest.svelte'
|
||||
import { getModelDetail } from './Models.svelte'
|
||||
|
||||
export let params = { chatId: '' }
|
||||
const chatId: number = parseInt(params.chatId)
|
||||
|
|
@ -245,6 +246,19 @@
|
|||
chatRequest.updating = true
|
||||
chatRequest.updatingMessage = ''
|
||||
|
||||
let doScroll = true
|
||||
let didScroll = false
|
||||
|
||||
const checkUserScroll = (e: Event) => {
|
||||
const el = e.target as HTMLElement
|
||||
if (el && e.isTrusted && didScroll) {
|
||||
// from user
|
||||
doScroll = (window.innerHeight + window.scrollY + 10) >= document.body.offsetHeight
|
||||
}
|
||||
}
|
||||
|
||||
window.addEventListener('scroll', checkUserScroll)
|
||||
|
||||
try {
|
||||
const response = await chatRequest.sendRequest($currentChatMessages, {
|
||||
chat,
|
||||
|
|
@ -252,7 +266,8 @@
|
|||
streaming: chatSettings.stream,
|
||||
fillMessage,
|
||||
onMessageChange: (messages) => {
|
||||
scrollToBottom(true)
|
||||
if (doScroll) scrollToBottom(true)
|
||||
didScroll = !!messages[0]?.content
|
||||
}
|
||||
})
|
||||
await response.promiseToFinish()
|
||||
|
|
@ -264,6 +279,8 @@
|
|||
console.error(e)
|
||||
}
|
||||
|
||||
window.removeEventListener('scroll', checkUserScroll)
|
||||
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
|
||||
|
|
@ -273,13 +290,16 @@
|
|||
const suggestName = async (): Promise<void> => {
|
||||
const suggestMessage: Message = {
|
||||
role: 'user',
|
||||
content: "Using appropriate language, please give a 5 word summary of this conversation's topic.",
|
||||
content: "Using appropriate language, please tell me a short 6 word summary of this conversation's topic for use as a book title. Only respond with the summary.",
|
||||
uuid: uuidv4()
|
||||
}
|
||||
|
||||
const suggestMessages = $currentChatMessages.slice(0, 10) // limit to first 10 messages
|
||||
suggestMessages.push(suggestMessage)
|
||||
|
||||
chatRequest.updating = true
|
||||
chatRequest.updatingMessage = 'Getting suggestion for chat name...'
|
||||
|
||||
const response = await chatRequest.sendRequest(suggestMessages, {
|
||||
chat,
|
||||
autoAddMessages: false,
|
||||
|
|
@ -297,7 +317,7 @@
|
|||
})
|
||||
} else {
|
||||
response.getMessages().forEach(m => {
|
||||
const name = m.content.split(/\s+/).slice(0, 8).join(' ').trim()
|
||||
const name = m.content.split(/\s+/).slice(0, 8).join(' ').replace(/^[^a-z0-9!?]+|[^a-z0-9!?]+$/gi, '').trim()
|
||||
if (name) chat.name = name
|
||||
})
|
||||
saveChatStore()
|
||||
|
|
@ -420,7 +440,7 @@
|
|||
<div class="content has-text-centered running-total-container">
|
||||
{#each Object.entries(chat.usage || {}) as [model, usage]}
|
||||
<p class="is-size-7 running-totals">
|
||||
<em>{model}</em> total <span class="has-text-weight-bold">{usage.total_tokens}</span>
|
||||
<em>{getModelDetail(model || '').label || model}</em> total <span class="has-text-weight-bold">{usage.total_tokens}</span>
|
||||
tokens ~= <span class="has-text-weight-bold">${getPrice(usage, model).toFixed(6)}</span>
|
||||
</p>
|
||||
{/each}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
<script context="module" lang="ts">
|
||||
import { setImage } from './ImageStore.svelte'
|
||||
import { countTokens } from './Models.svelte'
|
||||
// TODO: Integrate API calls
|
||||
import { addMessage, getLatestKnownModel, saveChatStore, setLatestKnownModel, subtractRunningTotal, updateRunningTotal } from './Storage.svelte'
|
||||
import { addMessage, getLatestKnownModel, setLatestKnownModel, subtractRunningTotal, updateMessages, updateRunningTotal } from './Storage.svelte'
|
||||
import type { Chat, ChatCompletionOpts, ChatImage, Message, Model, Response, ResponseImage, Usage } from './Types.svelte'
|
||||
import { encode } from 'gpt-tokenizer'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
|
||||
export class ChatCompletionResponse {
|
||||
|
|
@ -65,6 +65,10 @@ export class ChatCompletionResponse {
|
|||
this.promptTokenCount = tokens
|
||||
}
|
||||
|
||||
getPromptTokenCount (): number {
|
||||
return this.promptTokenCount
|
||||
}
|
||||
|
||||
async updateImageFromSyncResponse (response: ResponseImage, prompt: string, model: Model) {
|
||||
this.setModel(model)
|
||||
for (let i = 0; i < response.data.length; i++) {
|
||||
|
|
@ -138,10 +142,10 @@ export class ChatCompletionResponse {
|
|||
message.content = this.initialFillMerge(message.content, choice.delta?.content)
|
||||
message.content += choice.delta.content
|
||||
}
|
||||
completionTokenCount += encode(message.content).length
|
||||
completionTokenCount += countTokens(this.model, message.content)
|
||||
message.model = response.model
|
||||
message.finish_reason = choice.finish_reason
|
||||
message.streaming = choice.finish_reason === null && !this.finished
|
||||
message.streaming = !choice.finish_reason && !this.finished
|
||||
this.messages[i] = message
|
||||
})
|
||||
// total up the tokens
|
||||
|
|
@ -171,15 +175,15 @@ export class ChatCompletionResponse {
|
|||
} as Message)
|
||||
}
|
||||
this.notifyMessageChange()
|
||||
setTimeout(() => this.finish(), 250) // give others a chance to signal the finish first
|
||||
setTimeout(() => this.finish(), 200) // give others a chance to signal the finish first
|
||||
}
|
||||
|
||||
updateFromClose (force: boolean = false): void {
|
||||
if (!this.finished && !this.error && !this.messages?.find(m => m.content)) {
|
||||
if (!force) return setTimeout(() => this.updateFromClose(true), 250) as any
|
||||
return this.updateFromError('Unexpected connection termination')
|
||||
if (!force) return setTimeout(() => this.updateFromClose(true), 300) as any
|
||||
if (!this.finished) return this.updateFromError('Unexpected connection termination')
|
||||
}
|
||||
setTimeout(() => this.finish(), 250) // give others a chance to signal the finish first
|
||||
setTimeout(() => this.finish(), 260) // give others a chance to signal the finish first
|
||||
}
|
||||
|
||||
onMessageChange = (listener: (m: Message[]) => void): number =>
|
||||
|
|
@ -209,10 +213,10 @@ export class ChatCompletionResponse {
|
|||
}
|
||||
|
||||
private finish = (): void => {
|
||||
this.messages.forEach(m => { m.streaming = false }) // make sure all are marked stopped
|
||||
updateMessages(this.chat.id)
|
||||
if (this.finished) return
|
||||
this.finished = true
|
||||
this.messages.forEach(m => { m.streaming = false }) // make sure all are marked stopped
|
||||
saveChatStore()
|
||||
const message = this.messages[0]
|
||||
const model = this.model || getLatestKnownModel(this.chat.settings.model)
|
||||
if (message) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<script lang="ts">
|
||||
import { replace } from 'svelte-spa-router'
|
||||
import type { Chat } from './Types.svelte'
|
||||
import { apiKeyStorage, deleteChat, pinMainMenu, saveChatStore } from './Storage.svelte'
|
||||
import { deleteChat, hasActiveModels, pinMainMenu, saveChatStore } from './Storage.svelte'
|
||||
import Fa from 'svelte-fa/src/fa.svelte'
|
||||
import { faTrash, faCircleCheck, faPencil } from '@fortawesome/free-solid-svg-icons/index'
|
||||
import { faMessage } from '@fortawesome/free-regular-svg-icons/index'
|
||||
|
|
@ -86,7 +86,7 @@
|
|||
<a
|
||||
href={`#/chat/${chat.id}`}
|
||||
class="chat-menu-item"
|
||||
class:is-waiting={waitingForConfirm} class:is-disabled={!$apiKeyStorage} class:is-active={activeChatId === chat.id}
|
||||
class:is-waiting={waitingForConfirm} class:is-disabled={!hasActiveModels()} class:is-active={activeChatId === chat.id}
|
||||
on:click={() => { $pinMainMenu = false }} >
|
||||
{#if waitingForConfirm}
|
||||
<a class="is-pulled-right is-hidden px-1 py-0 has-text-weight-bold delete-button" href={'$'} on:click|preventDefault={() => delChat()}><Fa icon={faCircleCheck} /></a>
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@
|
|||
faEyeSlash
|
||||
} from '@fortawesome/free-solid-svg-icons/index'
|
||||
import { faSquareMinus, faSquarePlus as faSquarePlusOutline } from '@fortawesome/free-regular-svg-icons/index'
|
||||
import { apiKeyStorage, addChatFromJSON, chatsStorage, checkStateChange, clearChats, clearMessages, copyChat, globalStorage, setGlobalSettingValueByKey, showSetChatSettings, pinMainMenu, getChat, deleteChat, saveChatStore, saveCustomProfile } from './Storage.svelte'
|
||||
import { addChatFromJSON, chatsStorage, checkStateChange, clearChats, clearMessages, copyChat, globalStorage, setGlobalSettingValueByKey, showSetChatSettings, pinMainMenu, getChat, deleteChat, saveChatStore, saveCustomProfile, hasActiveModels } from './Storage.svelte'
|
||||
import { exportAsMarkdown, exportChatAsJSON } from './Export.svelte'
|
||||
import { newNameForProfile, restartProfile } from './Profiles.svelte'
|
||||
import { replace } from 'svelte-spa-router'
|
||||
|
|
@ -173,7 +173,7 @@
|
|||
<span class="menu-icon"><Fa icon={faGear}/></span> Chat Profile Settings
|
||||
</a>
|
||||
<hr class="dropdown-divider">
|
||||
<a href={'#'} class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { $apiKeyStorage && close(); $apiKeyStorage && startNewChatWithWarning(chatId) }} class="dropdown-item">
|
||||
<a href={'#'} class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { hasActiveModels() && close(); hasActiveModels() && startNewChatWithWarning(chatId) }} class="dropdown-item">
|
||||
<span class="menu-icon"><Fa icon={faSquarePlus}/></span> New Chat from Default
|
||||
</a>
|
||||
<a href={'#'} class:is-disabled={!chatId} on:click|preventDefault={() => { chatId && close(); chatId && startNewChatFromChatId(chatId) }} class="dropdown-item">
|
||||
|
|
@ -196,14 +196,14 @@
|
|||
<a href={'#'} class="dropdown-item" class:is-disabled={!chatId} on:click|preventDefault={() => { close(); exportChatAsJSON(chatId) }}>
|
||||
<span class="menu-icon"><Fa icon={faDownload}/></span> Backup Chat JSON
|
||||
</a>
|
||||
<a href={'#'} class="dropdown-item" class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { if (chatId) close(); chatFileInput.click() }}>
|
||||
<a href={'#'} class="dropdown-item" class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { if (chatId) close(); chatFileInput.click() }}>
|
||||
<span class="menu-icon"><Fa icon={faUpload}/></span> Restore Chat JSON
|
||||
</a>
|
||||
<a href={'#'} class="dropdown-item" class:is-disabled={!chatId} on:click|preventDefault={() => { if (chatId) close(); exportAsMarkdown(chatId) }}>
|
||||
<span class="menu-icon"><Fa icon={faFileExport}/></span> Export Chat Markdown
|
||||
</a>
|
||||
<hr class="dropdown-divider">
|
||||
<a href={'#'} class="dropdown-item" class:is-disabled={!$apiKeyStorage} on:click|preventDefault={() => { if (chatId) close(); profileFileInput.click() }}>
|
||||
<a href={'#'} class="dropdown-item" class:is-disabled={!hasActiveModels()} on:click|preventDefault={() => { if (chatId) close(); profileFileInput.click() }}>
|
||||
<span class="menu-icon"><Fa icon={faUpload}/></span> Restore Profile JSON
|
||||
</a>
|
||||
<hr class="dropdown-divider">
|
||||
|
|
|
|||
|
|
@ -5,11 +5,12 @@
|
|||
import type { Chat, ChatCompletionOpts, ChatSettings, Message, Model, Request, RequestImageGeneration } from './Types.svelte'
|
||||
import { deleteMessage, getChatSettingValueNullDefault, insertMessages, getApiKey, addError, currentChatMessages, getMessages, updateMessages, deleteSummaryMessage } from './Storage.svelte'
|
||||
import { scrollToBottom, scrollToMessage } from './Util.svelte'
|
||||
import { getRequestSettingList, defaultModel } from './Settings.svelte'
|
||||
import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
|
||||
import { getApiBase, getEndpointCompletions, getEndpointGenerations } from './ApiUtil.svelte'
|
||||
import { getDefaultModel, getRequestSettingList } from './Settings.svelte'
|
||||
import { v4 as uuidv4 } from 'uuid'
|
||||
import { get } from 'svelte/store'
|
||||
import { getEndpoint, getModelDetail } from './Models.svelte'
|
||||
import { runOpenAiCompletionRequest } from './ChatRequestOpenAi.svelte'
|
||||
import { runPetalsCompletionRequest } from './ChatRequestPetals.svelte'
|
||||
|
||||
export class ChatRequest {
|
||||
constructor () {
|
||||
|
|
@ -25,6 +26,15 @@ export class ChatRequest {
|
|||
|
||||
setChat (chat: Chat) {
|
||||
this.chat = chat
|
||||
this.chat.settings.model = this.getModel()
|
||||
}
|
||||
|
||||
getChat (): Chat {
|
||||
return this.chat
|
||||
}
|
||||
|
||||
getChatSettings (): ChatSettings {
|
||||
return this.chat.settings
|
||||
}
|
||||
|
||||
// Common error handler
|
||||
|
|
@ -77,7 +87,7 @@ export class ChatRequest {
|
|||
const chatResponse = new ChatCompletionResponse(opts)
|
||||
|
||||
try {
|
||||
const response = await fetch(getApiBase() + getEndpointGenerations(), fetchOptions)
|
||||
const response = await fetch(getEndpoint('dall-e-' + size), fetchOptions)
|
||||
if (!response.ok) {
|
||||
await _this.handleError(response)
|
||||
} else {
|
||||
|
|
@ -159,6 +169,8 @@ export class ChatRequest {
|
|||
const spl = chatSettings.sendSystemPromptLast
|
||||
const sp = messagePayload[0]
|
||||
if (sp) {
|
||||
const lastSp = sp.content.split('::END-PROMPT::')
|
||||
sp.content = lastSp[0].trim()
|
||||
if (messagePayload.length > 1) {
|
||||
sp.content = sp.content.replace(/::STARTUP::[\s\S]*::EOM::/, '::EOM::')
|
||||
sp.content = sp.content.replace(/::STARTUP::[\s\S]*::START-PROMPT::/, '::START-PROMPT::')
|
||||
|
|
@ -170,7 +182,7 @@ export class ChatRequest {
|
|||
if (spl) {
|
||||
messagePayload.shift()
|
||||
if (messagePayload[messagePayload.length - 1]?.role === 'user') {
|
||||
messagePayload.splice(-2, 0, sp)
|
||||
messagePayload.splice(-1, 0, sp)
|
||||
} else {
|
||||
messagePayload.push(sp)
|
||||
}
|
||||
|
|
@ -196,11 +208,15 @@ export class ChatRequest {
|
|||
}).filter(m => m.content.length)
|
||||
messagePayload.splice(spl ? 0 : 1, 0, ...ms.concat(splitSystem.map(s => ({ role: 'system', content: s.trim() } as Message)).filter(m => m.content.length)))
|
||||
}
|
||||
const lastSpC = lastSp[1]?.trim() || ''
|
||||
if (lastSpC.length) {
|
||||
messagePayload.push({ role: 'system', content: lastSpC } as Message)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Get token counts
|
||||
const promptTokenCount = countPromptTokens(messagePayload, model)
|
||||
const promptTokenCount = countPromptTokens(messagePayload, model, chat)
|
||||
const maxAllowed = maxTokens - (promptTokenCount + 1)
|
||||
|
||||
// Build the API request body
|
||||
|
|
@ -239,6 +255,9 @@ export class ChatRequest {
|
|||
|
||||
// Set-up and make the request
|
||||
const chatResponse = new ChatCompletionResponse(opts)
|
||||
|
||||
const modelDetail = getModelDetail(model)
|
||||
|
||||
try {
|
||||
// Add out token count to the response handler
|
||||
// (streaming doesn't return counts, so we need to do it client side)
|
||||
|
|
@ -248,88 +267,11 @@ export class ChatRequest {
|
|||
// so we deal with it ourselves
|
||||
_this.controller = new AbortController()
|
||||
const signal = _this.controller.signal
|
||||
const abortListener = (e:Event) => {
|
||||
_this.updating = false
|
||||
_this.updatingMessage = ''
|
||||
chatResponse.updateFromError('User aborted request.')
|
||||
signal.removeEventListener('abort', abortListener)
|
||||
}
|
||||
signal.addEventListener('abort', abortListener)
|
||||
|
||||
const fetchOptions = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${getApiKey()}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
signal
|
||||
}
|
||||
|
||||
if (opts.streaming) {
|
||||
/**
|
||||
* Streaming request/response
|
||||
* We'll get the response a token at a time, as soon as they are ready
|
||||
*/
|
||||
chatResponse.onFinish(() => {
|
||||
_this.updating = false
|
||||
_this.updatingMessage = ''
|
||||
})
|
||||
fetchEventSource(getApiBase() + getEndpointCompletions(), {
|
||||
...fetchOptions,
|
||||
openWhenHidden: true,
|
||||
onmessage (ev) {
|
||||
// Remove updating indicator
|
||||
_this.updating = 1 // hide indicator, but still signal we're updating
|
||||
_this.updatingMessage = ''
|
||||
// console.log('ev.data', ev.data)
|
||||
if (!chatResponse.hasFinished()) {
|
||||
if (ev.data === '[DONE]') {
|
||||
// ?? anything to do when "[DONE]"?
|
||||
if (modelDetail.type === 'Petals') {
|
||||
await runPetalsCompletionRequest(request, _this as any, chatResponse as any, signal, opts)
|
||||
} else {
|
||||
const data = JSON.parse(ev.data)
|
||||
// console.log('data', data)
|
||||
window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose () {
|
||||
_this.updating = false
|
||||
_this.updatingMessage = ''
|
||||
chatResponse.updateFromClose()
|
||||
},
|
||||
onerror (err) {
|
||||
console.error(err)
|
||||
throw err
|
||||
},
|
||||
async onopen (response) {
|
||||
if (response.ok && response.headers.get('content-type') === EventStreamContentType) {
|
||||
// everything's good
|
||||
} else {
|
||||
// client-side errors are usually non-retriable:
|
||||
await _this.handleError(response)
|
||||
}
|
||||
}
|
||||
}).catch(err => {
|
||||
_this.updating = false
|
||||
_this.updatingMessage = ''
|
||||
chatResponse.updateFromError(err.message)
|
||||
})
|
||||
} else {
|
||||
/**
|
||||
* Non-streaming request/response
|
||||
* We'll get the response all at once, after a long delay
|
||||
*/
|
||||
const response = await fetch(getApiBase() + getEndpointCompletions(), fetchOptions)
|
||||
if (!response.ok) {
|
||||
await _this.handleError(response)
|
||||
} else {
|
||||
const json = await response.json()
|
||||
// Remove updating indicator
|
||||
_this.updating = false
|
||||
_this.updatingMessage = ''
|
||||
chatResponse.updateFromSyncResponse(json)
|
||||
}
|
||||
await runOpenAiCompletionRequest(request, _this as any, chatResponse as any, signal, opts)
|
||||
}
|
||||
} catch (e) {
|
||||
// console.error(e)
|
||||
|
|
@ -341,12 +283,13 @@ export class ChatRequest {
|
|||
return chatResponse
|
||||
}
|
||||
|
||||
private getModel (): Model {
|
||||
return this.chat.settings.model || defaultModel
|
||||
getModel (): Model {
|
||||
return this.chat.settings.model || getDefaultModel()
|
||||
}
|
||||
|
||||
private buildHiddenPromptPrefixMessages (messages: Message[], insert:boolean = false): Message[] {
|
||||
const chatSettings = this.chat.settings
|
||||
const chat = this.chat
|
||||
const chatSettings = chat.settings
|
||||
const hiddenPromptPrefix = mergeProfileFields(chatSettings, chatSettings.hiddenPromptPrefix).trim()
|
||||
const lastMessage = messages[messages.length - 1]
|
||||
const isContinue = lastMessage?.role === 'assistant' && lastMessage.finish_reason === 'length'
|
||||
|
|
@ -356,9 +299,9 @@ export class ChatRequest {
|
|||
const results = hiddenPromptPrefix.split(/[\s\r\n]*::EOM::[\s\r\n]*/).reduce((a, m) => {
|
||||
m = m.trim()
|
||||
if (m.length) {
|
||||
if (m.match(/[[USER_PROMPT]]/)) {
|
||||
if (m.match(/\[\[USER_PROMPT\]\]/)) {
|
||||
injectedPrompt = true
|
||||
m.replace(/[[USER_PROMPT]]/g, lastMessage.content)
|
||||
m = m.replace(/\[\[USER_PROMPT\]\]/g, lastMessage.content)
|
||||
}
|
||||
a.push({ role: a.length % 2 === 0 ? 'user' : 'assistant', content: m } as Message)
|
||||
}
|
||||
|
|
@ -377,7 +320,7 @@ export class ChatRequest {
|
|||
lastMessage.skipOnce = true
|
||||
}
|
||||
}
|
||||
if (injectedPrompt) results.pop()
|
||||
if (injectedPrompt) messages.pop()
|
||||
return results
|
||||
}
|
||||
return []
|
||||
|
|
@ -387,11 +330,11 @@ export class ChatRequest {
|
|||
* Gets an estimate of how many extra tokens will be added that won't be part of the visible messages
|
||||
* @param filtered
|
||||
*/
|
||||
private getTokenCountPadding (filtered: Message[]): number {
|
||||
private getTokenCountPadding (filtered: Message[], chat: Chat): number {
|
||||
let result = 0
|
||||
// add cost of hiddenPromptPrefix
|
||||
result += this.buildHiddenPromptPrefixMessages(filtered)
|
||||
.reduce((a, m) => a + countMessageTokens(m, this.getModel()), 0)
|
||||
.reduce((a, m) => a + countMessageTokens(m, this.getModel(), chat), 0)
|
||||
// more here eventually?
|
||||
return result
|
||||
}
|
||||
|
|
@ -413,10 +356,10 @@ export class ChatRequest {
|
|||
}
|
||||
|
||||
// Get extra counts for when the prompts are finally sent.
|
||||
const countPadding = this.getTokenCountPadding(filtered)
|
||||
const countPadding = this.getTokenCountPadding(filtered, chat)
|
||||
|
||||
// See if we have enough to apply any of the reduction modes
|
||||
const fullPromptSize = countPromptTokens(filtered, model) + countPadding
|
||||
const fullPromptSize = countPromptTokens(filtered, model, chat) + countPadding
|
||||
if (fullPromptSize < chatSettings.summaryThreshold) return await continueRequest() // nothing to do yet
|
||||
const overMax = fullPromptSize > maxTokens * 0.95
|
||||
|
||||
|
|
@ -439,12 +382,12 @@ export class ChatRequest {
|
|||
* *************************************************************
|
||||
*/
|
||||
|
||||
let promptSize = countPromptTokens(top.concat(rw), model) + countPadding
|
||||
let promptSize = countPromptTokens(top.concat(rw), model, chat) + countPadding
|
||||
while (rw.length && rw.length > pinBottom && promptSize >= chatSettings.summaryThreshold) {
|
||||
const rolled = rw.shift()
|
||||
// Hide messages we're "rolling"
|
||||
if (rolled) rolled.suppress = true
|
||||
promptSize = countPromptTokens(top.concat(rw), model) + countPadding
|
||||
promptSize = countPromptTokens(top.concat(rw), model, chat) + countPadding
|
||||
}
|
||||
// Run a new request, now with the rolled messages hidden
|
||||
return await _this.sendRequest(get(currentChatMessages), {
|
||||
|
|
@ -460,26 +403,26 @@ export class ChatRequest {
|
|||
const bottom = rw.slice(0 - pinBottom)
|
||||
let continueCounter = chatSettings.summaryExtend + 1
|
||||
rw = rw.slice(0, 0 - pinBottom)
|
||||
let reductionPoolSize = countPromptTokens(rw, model)
|
||||
let reductionPoolSize = countPromptTokens(rw, model, chat)
|
||||
const ss = Math.abs(chatSettings.summarySize)
|
||||
const getSS = ():number => (ss < 1 && ss > 0)
|
||||
? Math.round(reductionPoolSize * ss) // If summarySize between 0 and 1, use percentage of reduced
|
||||
: Math.min(ss, reductionPoolSize * 0.5) // If > 1, use token count
|
||||
const topSize = countPromptTokens(top, model)
|
||||
const topSize = countPromptTokens(top, model, chat)
|
||||
let maxSummaryTokens = getSS()
|
||||
let promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
|
||||
const summaryRequest = { role: 'user', content: promptSummary } as Message
|
||||
let promptSummarySize = countMessageTokens(summaryRequest, model)
|
||||
let promptSummarySize = countMessageTokens(summaryRequest, model, chat)
|
||||
// Make sure there is enough room to generate the summary, and try to make sure
|
||||
// the last prompt is a user prompt as that seems to work better for summaries
|
||||
while ((topSize + reductionPoolSize + promptSummarySize + maxSummaryTokens) >= maxTokens ||
|
||||
(reductionPoolSize >= 100 && rw[rw.length - 1]?.role !== 'user')) {
|
||||
bottom.unshift(rw.pop() as Message)
|
||||
reductionPoolSize = countPromptTokens(rw, model)
|
||||
reductionPoolSize = countPromptTokens(rw, model, chat)
|
||||
maxSummaryTokens = getSS()
|
||||
promptSummary = prepareSummaryPrompt(chatId, maxSummaryTokens)
|
||||
summaryRequest.content = promptSummary
|
||||
promptSummarySize = countMessageTokens(summaryRequest, model)
|
||||
promptSummarySize = countMessageTokens(summaryRequest, model, chat)
|
||||
}
|
||||
if (reductionPoolSize < 50) {
|
||||
if (overMax) addError(chatId, 'Check summary settings. Unable to summarize enough messages.')
|
||||
|
|
@ -565,10 +508,10 @@ export class ChatRequest {
|
|||
// Try to get more of it
|
||||
delete summaryResponse.finish_reason
|
||||
_this.updatingMessage = 'Summarizing more...'
|
||||
let _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model)
|
||||
let _recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chat)
|
||||
while (rw.length && (_recount + maxSummaryTokens >= maxTokens)) {
|
||||
rw.shift()
|
||||
_recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model)
|
||||
_recount = countPromptTokens(top.concat(rw).concat([summaryRequest]).concat([summaryResponse]), model, chat)
|
||||
}
|
||||
loopCount++
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -0,0 +1,100 @@
|
|||
<script context="module" lang="ts">
|
||||
import { EventStreamContentType, fetchEventSource } from '@microsoft/fetch-event-source'
|
||||
import ChatCompletionResponse from './ChatCompletionResponse.svelte'
|
||||
import ChatRequest from './ChatRequest.svelte'
|
||||
import { getEndpoint } from './Models.svelte'
|
||||
import { getApiKey } from './Storage.svelte'
|
||||
import type { ChatCompletionOpts, Request } from './Types.svelte'
|
||||
|
||||
export const runOpenAiCompletionRequest = async (
|
||||
request: Request,
|
||||
chatRequest: ChatRequest,
|
||||
chatResponse: ChatCompletionResponse,
|
||||
signal: AbortSignal,
|
||||
opts: ChatCompletionOpts) => {
|
||||
// OpenAI Request
|
||||
const model = chatRequest.getModel()
|
||||
const abortListener = (e:Event) => {
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
chatResponse.updateFromError('User aborted request.')
|
||||
chatRequest.removeEventListener('abort', abortListener)
|
||||
}
|
||||
signal.addEventListener('abort', abortListener)
|
||||
const fetchOptions = {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${getApiKey()}`,
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify(request),
|
||||
signal
|
||||
}
|
||||
|
||||
if (opts.streaming) {
|
||||
/**
|
||||
* Streaming request/response
|
||||
* We'll get the response a token at a time, as soon as they are ready
|
||||
*/
|
||||
chatResponse.onFinish(() => {
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
})
|
||||
fetchEventSource(getEndpoint(model), {
|
||||
...fetchOptions,
|
||||
openWhenHidden: true,
|
||||
onmessage (ev) {
|
||||
// Remove updating indicator
|
||||
chatRequest.updating = 1 // hide indicator, but still signal we're updating
|
||||
chatRequest.updatingMessage = ''
|
||||
// console.log('ev.data', ev.data)
|
||||
if (!chatResponse.hasFinished()) {
|
||||
if (ev.data === '[DONE]') {
|
||||
// ?? anything to do when "[DONE]"?
|
||||
} else {
|
||||
const data = JSON.parse(ev.data)
|
||||
// console.log('data', data)
|
||||
window.setTimeout(() => { chatResponse.updateFromAsyncResponse(data) }, 1)
|
||||
}
|
||||
}
|
||||
},
|
||||
onclose () {
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
chatResponse.updateFromClose()
|
||||
},
|
||||
onerror (err) {
|
||||
console.error(err)
|
||||
throw err
|
||||
},
|
||||
async onopen (response) {
|
||||
if (response.ok && response.headers.get('content-type') === EventStreamContentType) {
|
||||
// everything's good
|
||||
} else {
|
||||
// client-side errors are usually non-retriable:
|
||||
await chatRequest.handleError(response)
|
||||
}
|
||||
}
|
||||
}).catch(err => {
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
chatResponse.updateFromError(err.message)
|
||||
})
|
||||
} else {
|
||||
/**
|
||||
* Non-streaming request/response
|
||||
* We'll get the response all at once, after a long delay
|
||||
*/
|
||||
const response = await fetch(getEndpoint(model), fetchOptions)
|
||||
if (!response.ok) {
|
||||
await chatRequest.handleError(response)
|
||||
} else {
|
||||
const json = await response.json()
|
||||
// Remove updating indicator
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
chatResponse.updateFromSyncResponse(json)
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
@ -0,0 +1,139 @@
|
|||
<script context="module" lang="ts">
|
||||
import ChatCompletionResponse from './ChatCompletionResponse.svelte'
|
||||
import ChatRequest from './ChatRequest.svelte'
|
||||
import { getEndpoint, getModelDetail, getRoleTag, getStopSequence } from './Models.svelte'
|
||||
import type { ChatCompletionOpts, Message, Request } from './Types.svelte'
|
||||
import { getModelMaxTokens } from './Stats.svelte'
|
||||
import { updateMessages } from './Storage.svelte'
|
||||
|
||||
export const runPetalsCompletionRequest = async (
|
||||
request: Request,
|
||||
chatRequest: ChatRequest,
|
||||
chatResponse: ChatCompletionResponse,
|
||||
signal: AbortSignal,
|
||||
opts: ChatCompletionOpts) => {
|
||||
// Petals
|
||||
const chat = chatRequest.getChat()
|
||||
const model = chatRequest.getModel()
|
||||
const modelDetail = getModelDetail(model)
|
||||
const ws = new WebSocket(getEndpoint(model))
|
||||
const abortListener = (e:Event) => {
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
chatResponse.updateFromError('User aborted request.')
|
||||
signal.removeEventListener('abort', abortListener)
|
||||
ws.close()
|
||||
}
|
||||
signal.addEventListener('abort', abortListener)
|
||||
const stopSequences = (modelDetail.stop || ['###', '</s>']).slice()
|
||||
const stopSequence = getStopSequence(chat)
|
||||
let stopSequenceC = stopSequence
|
||||
if (stopSequence !== '###') {
|
||||
stopSequences.push(stopSequence)
|
||||
stopSequenceC = '</s>'
|
||||
}
|
||||
const stopSequencesC = stopSequences.filter((ss) => {
|
||||
return ss !== '###' && ss !== stopSequenceC
|
||||
})
|
||||
const maxTokens = getModelMaxTokens(model)
|
||||
let maxLen = Math.min(opts.maxTokens || chatRequest.chat.max_tokens || maxTokens, maxTokens)
|
||||
const promptTokenCount = chatResponse.getPromptTokenCount()
|
||||
if (promptTokenCount > maxLen) {
|
||||
maxLen = Math.min(maxLen + promptTokenCount, maxTokens)
|
||||
}
|
||||
chatResponse.onFinish(() => {
|
||||
const message = chatResponse.getMessages()[0]
|
||||
if (message) {
|
||||
for (let i = 0, l = stopSequences.length; i < l; i++) {
|
||||
const ss = stopSequences[i].trim()
|
||||
if (message.content.trim().endsWith(ss)) {
|
||||
message.content = message.content.trim().slice(0, message.content.trim().length - ss.length)
|
||||
updateMessages(chat.id)
|
||||
}
|
||||
}
|
||||
}
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
})
|
||||
ws.onopen = () => {
|
||||
ws.send(JSON.stringify({
|
||||
type: 'open_inference_session',
|
||||
model,
|
||||
max_length: maxLen
|
||||
}))
|
||||
ws.onmessage = event => {
|
||||
const response = JSON.parse(event.data)
|
||||
if (!response.ok) {
|
||||
const err = new Error('Error opening socket: ' + response.traceback)
|
||||
chatResponse.updateFromError(err.message)
|
||||
console.error(err)
|
||||
throw err
|
||||
}
|
||||
const rMessages = request.messages || [] as Message[]
|
||||
// make sure top_p and temperature are set the way we need
|
||||
let temperature = request.temperature
|
||||
if (temperature === undefined || isNaN(temperature as any)) temperature = 1
|
||||
if (!temperature || temperature <= 0) temperature = 0.01
|
||||
let topP = request.top_p
|
||||
if (topP === undefined || isNaN(topP as any)) topP = 1
|
||||
if (!topP || topP <= 0) topP = 0.01
|
||||
// build the message array
|
||||
const inputArray = (rMessages).reduce((a, m) => {
|
||||
const c = getRoleTag(m.role, model, chatRequest.chat) + m.content
|
||||
a.push(c.trim())
|
||||
return a
|
||||
}, [] as string[])
|
||||
const lastMessage = rMessages[rMessages.length - 1]
|
||||
if (lastMessage && lastMessage.role !== 'assistant') {
|
||||
inputArray.push(getRoleTag('assistant', model, chatRequest.chat))
|
||||
}
|
||||
const petalsRequest = {
|
||||
type: 'generate',
|
||||
inputs: inputArray.join(stopSequence),
|
||||
max_new_tokens: 1, // wait for up to 1 tokens before displaying
|
||||
stop_sequence: stopSequenceC,
|
||||
do_sample: 1, // enable top p and the like
|
||||
temperature,
|
||||
top_p: topP
|
||||
} as any
|
||||
if (stopSequencesC.length) petalsRequest.extra_stop_sequences = stopSequencesC
|
||||
ws.send(JSON.stringify(petalsRequest))
|
||||
ws.onmessage = event => {
|
||||
// Remove updating indicator
|
||||
chatRequest.updating = 1 // hide indicator, but still signal we're updating
|
||||
chatRequest.updatingMessage = ''
|
||||
const response = JSON.parse(event.data)
|
||||
if (!response.ok) {
|
||||
const err = new Error('Error in response: ' + response.traceback)
|
||||
console.error(err)
|
||||
chatResponse.updateFromError(err.message)
|
||||
throw err
|
||||
}
|
||||
window.setTimeout(() => {
|
||||
chatResponse.updateFromAsyncResponse(
|
||||
{
|
||||
model,
|
||||
choices: [{
|
||||
delta: {
|
||||
content: response.outputs,
|
||||
role: 'assistant'
|
||||
},
|
||||
finish_reason: (response.stop ? 'stop' : null)
|
||||
}]
|
||||
} as any
|
||||
)
|
||||
}, 1)
|
||||
}
|
||||
}
|
||||
ws.onclose = () => {
|
||||
chatRequest.updating = false
|
||||
chatRequest.updatingMessage = ''
|
||||
chatResponse.updateFromClose()
|
||||
}
|
||||
ws.onerror = err => {
|
||||
console.error(err)
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
|
@ -3,7 +3,7 @@
|
|||
// import { getProfile } from './Profiles.svelte'
|
||||
import { cleanSettingValue, setChatSettingValue } from './Storage.svelte'
|
||||
import type { Chat, ChatSetting, ChatSettings, ControlAction, FieldControl, SettingPrompt } from './Types.svelte'
|
||||
import { autoGrowInputOnEvent, errorNotice } from './Util.svelte'
|
||||
import { autoGrowInputOnEvent, errorNotice, valueOf } from './Util.svelte'
|
||||
// import { replace } from 'svelte-spa-router'
|
||||
import Fa from 'svelte-fa/src/fa.svelte'
|
||||
import { openModal } from 'svelte-modals'
|
||||
|
|
@ -23,6 +23,10 @@
|
|||
const chatId = chat.id
|
||||
let show = false
|
||||
|
||||
let header = valueOf(chatId, setting.header)
|
||||
let headerClass = valueOf(chatId, setting.headerClass)
|
||||
let placeholder = valueOf(chatId, setting.placeholder)
|
||||
|
||||
const buildFieldControls = () => {
|
||||
fieldControls = (setting.fieldControls || [] as FieldControl[]).map(fc => {
|
||||
return fc.getAction(chatId, setting, chatSettings[setting.key])
|
||||
|
|
@ -38,6 +42,9 @@
|
|||
|
||||
afterUpdate(() => {
|
||||
show = (typeof setting.hide !== 'function') || !setting.hide(chatId)
|
||||
header = valueOf(chatId, setting.header)
|
||||
headerClass = valueOf(chatId, setting.headerClass)
|
||||
placeholder = valueOf(chatId, setting.placeholder)
|
||||
buildFieldControls()
|
||||
})
|
||||
|
||||
|
|
@ -146,9 +153,9 @@
|
|||
</script>
|
||||
|
||||
{#if show}
|
||||
{#if setting.header}
|
||||
<p class="notification {setting.headerClass}">
|
||||
{@html setting.header}
|
||||
{#if header}
|
||||
<p class="notification {headerClass}">
|
||||
{@html header}
|
||||
</p>
|
||||
{/if}
|
||||
<div class="field is-horizontal">
|
||||
|
|
@ -171,7 +178,7 @@
|
|||
<label class="label" for="settings-{setting.key}" title="{setting.title}">{setting.name}</label>
|
||||
<textarea
|
||||
class="input is-info is-focused chat-input auto-size"
|
||||
placeholder={setting.placeholder || ''}
|
||||
placeholder={placeholder || ''}
|
||||
rows="1"
|
||||
on:input={e => autoGrowInputOnEvent(e)}
|
||||
on:change={e => { queueSettingValueChange(e, setting); autoGrowInputOnEvent(e) }}
|
||||
|
|
@ -195,7 +202,7 @@
|
|||
min={setting.min}
|
||||
max={setting.max}
|
||||
step={setting.step}
|
||||
placeholder={String(setting.placeholder || chatDefaults[setting.key])}
|
||||
placeholder={String(placeholder || chatDefaults[setting.key])}
|
||||
on:change={e => queueSettingValueChange(e, setting)}
|
||||
/>
|
||||
{:else if setting.type === 'select' || setting.type === 'select-number'}
|
||||
|
|
@ -204,7 +211,7 @@
|
|||
{#key rkey}
|
||||
<select id="settings-{setting.key}" title="{setting.title}" on:change={e => queueSettingValueChange(e, setting) } >
|
||||
{#each setting.options as option}
|
||||
<option class:is-default={option.value === chatDefaults[setting.key]} value={option.value} selected={option.value === chatSettings[setting.key]}>{option.text}</option>
|
||||
<option class:is-default={option.value === chatDefaults[setting.key]} value={option.value} selected={option.value === chatSettings[setting.key]} disabled={option.disabled}>{option.text}</option>
|
||||
{/each}
|
||||
</select>
|
||||
{/key}
|
||||
|
|
@ -233,6 +240,7 @@
|
|||
title="{setting.title}"
|
||||
class="input"
|
||||
value={chatSettings[setting.key]}
|
||||
placeholder={String(placeholder || chatDefaults[setting.key])}
|
||||
on:change={e => { queueSettingValueChange(e, setting) }}
|
||||
>
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
import { getChatDefaults, getChatSettingList, getChatSettingObjectByKey, getExcludeFromProfile } from './Settings.svelte'
|
||||
import {
|
||||
saveChatStore,
|
||||
apiKeyStorage,
|
||||
chatsStorage,
|
||||
globalStorage,
|
||||
saveCustomProfile,
|
||||
|
|
@ -13,7 +12,7 @@
|
|||
checkStateChange,
|
||||
addChat
|
||||
} from './Storage.svelte'
|
||||
import type { Chat, ChatSetting, ResponseModels, SettingSelect, SelectOption, ChatSettings } from './Types.svelte'
|
||||
import type { Chat, ChatSetting, SettingSelect, ChatSettings } from './Types.svelte'
|
||||
import { errorNotice, sizeTextElements } from './Util.svelte'
|
||||
import Fa from 'svelte-fa/src/fa.svelte'
|
||||
import {
|
||||
|
|
@ -35,8 +34,7 @@
|
|||
import { replace } from 'svelte-spa-router'
|
||||
import { openModal } from 'svelte-modals'
|
||||
import PromptConfirm from './PromptConfirm.svelte'
|
||||
import { getApiBase, getEndpointModels } from './ApiUtil.svelte'
|
||||
import { supportedModelKeys } from './Models.svelte'
|
||||
import { getModelOptions } from './Models.svelte'
|
||||
|
||||
export let chatId:number
|
||||
export const show = () => { showSettings() }
|
||||
|
|
@ -185,30 +183,9 @@
|
|||
// Refresh settings modal
|
||||
showSettingsModal++
|
||||
|
||||
// Load available models from OpenAI
|
||||
const allModels = (await (
|
||||
await fetch(getApiBase() + getEndpointModels(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${$apiKeyStorage}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
})
|
||||
).json()) as ResponseModels
|
||||
const filteredModels = supportedModelKeys.filter((model) => allModels.data.find((m) => m.id === model))
|
||||
|
||||
const modelOptions:SelectOption[] = filteredModels.reduce((a, m) => {
|
||||
const o:SelectOption = {
|
||||
value: m,
|
||||
text: m
|
||||
}
|
||||
a.push(o)
|
||||
return a
|
||||
}, [] as SelectOption[])
|
||||
|
||||
// Update the models in the settings
|
||||
if (modelSetting) {
|
||||
modelSetting.options = modelOptions
|
||||
modelSetting.options = await getModelOptions()
|
||||
}
|
||||
// Refresh settings modal
|
||||
showSettingsModal++
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@
|
|||
import { openModal } from 'svelte-modals'
|
||||
import PromptConfirm from './PromptConfirm.svelte'
|
||||
import { getImage } from './ImageStore.svelte'
|
||||
import { getModelDetail } from './Models.svelte'
|
||||
|
||||
export let message:Message
|
||||
export let chatId:number
|
||||
|
|
@ -245,7 +246,7 @@
|
|||
<p class="is-size-7 message-note">System Prompt</p>
|
||||
{:else if message.usage}
|
||||
<p class="is-size-7 message-note">
|
||||
<em>{message.model || defaultModel}</em> using <span class="has-text-weight-bold">{message.usage.total_tokens}</span>
|
||||
<em>{getModelDetail(message.model || '').label || message.model || defaultModel}</em> using <span class="has-text-weight-bold">{message.usage.total_tokens}</span>
|
||||
tokens ~= <span class="has-text-weight-bold">${getPrice(message.usage, message.model || defaultModel).toFixed(6)}</span>
|
||||
</p>
|
||||
{/if}
|
||||
|
|
|
|||
|
|
@ -1,16 +1,22 @@
|
|||
<script lang="ts">
|
||||
import { apiKeyStorage, lastChatId, getChat, started } from './Storage.svelte'
|
||||
import { apiKeyStorage, globalStorage, lastChatId, getChat, started, setGlobalSettingValueByKey, hasActiveModels, checkStateChange } from './Storage.svelte'
|
||||
import Footer from './Footer.svelte'
|
||||
import { replace } from 'svelte-spa-router'
|
||||
import { onMount } from 'svelte'
|
||||
import { afterUpdate, onMount } from 'svelte'
|
||||
import { getPetals } from './ApiUtil.svelte'
|
||||
import { clearModelOptionCache } from './Models.svelte'
|
||||
|
||||
$: apiKey = $apiKeyStorage
|
||||
|
||||
let showPetalsSettings = $globalStorage.enablePetals
|
||||
let pedalsEndpoint = $globalStorage.pedalsEndpoint
|
||||
let hasModels = hasActiveModels()
|
||||
|
||||
onMount(() => {
|
||||
if (!$started) {
|
||||
$started = true
|
||||
// console.log('started', apiKey, $lastChatId, getChat($lastChatId))
|
||||
if (apiKey && getChat($lastChatId)) {
|
||||
if (hasActiveModels() && getChat($lastChatId)) {
|
||||
const chatId = $lastChatId
|
||||
$lastChatId = 0
|
||||
replace(`/chat/${chatId}`)
|
||||
|
|
@ -19,21 +25,39 @@ onMount(() => {
|
|||
$lastChatId = 0
|
||||
})
|
||||
|
||||
afterUpdate(() => {
|
||||
clearModelOptionCache()
|
||||
hasModels = hasActiveModels()
|
||||
pedalsEndpoint = $globalStorage.pedalsEndpoint
|
||||
$checkStateChange++
|
||||
})
|
||||
|
||||
const setPetalsEnabled = (event: Event) => {
|
||||
const el = (event.target as HTMLInputElement)
|
||||
setGlobalSettingValueByKey('enablePetals', !!el.checked)
|
||||
showPetalsSettings = $globalStorage.enablePetals
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
<section class="section">
|
||||
<article class="message">
|
||||
<div class="message-body">
|
||||
<strong><a href="https://github.com/Niek/chatgpt-web">ChatGPT-web</a></strong>
|
||||
<p class="mb-4">
|
||||
<strong><a href="https://github.com/Niek/chatgpt-web" target="_blank">ChatGPT-web</a></strong>
|
||||
is a simple one-page web interface to the OpenAI ChatGPT API. To use it, you need to register for
|
||||
<a href="https://platform.openai.com/account/api-keys" target="_blank" rel="noreferrer">an OpenAI API key</a>
|
||||
first. OpenAI bills per token (usage-based), which means it is a lot cheaper than
|
||||
<a href="https://openai.com/blog/chatgpt-plus" target="_blank" rel="noreferrer">ChatGPT Plus</a>, unless you use
|
||||
more than 10 million tokens per month. All messages are stored in your browser's local storage, so everything is
|
||||
<strong>private</strong>. You can also close the browser tab and come back later to continue the conversation.
|
||||
</p>
|
||||
<p>
|
||||
As an alternative to OpenAI, you can also use Petals swarm as a free API option for open chat models like Llama 2.
|
||||
</p>
|
||||
</div>
|
||||
</article>
|
||||
<article class="message" class:is-danger={!apiKey} class:is-warning={apiKey}>
|
||||
<article class="message" class:is-danger={!hasModels} class:is-warning={!apiKey} class:is-info={apiKey}>
|
||||
<div class="message-body">
|
||||
Set your OpenAI API key below:
|
||||
|
||||
|
|
@ -53,19 +77,81 @@ onMount(() => {
|
|||
type="password"
|
||||
autocomplete="off"
|
||||
class="input"
|
||||
class:is-danger={!apiKey}
|
||||
class:is-danger={!hasModels}
|
||||
class:is-warning={!apiKey} class:is-info={apiKey}
|
||||
value={apiKey}
|
||||
/>
|
||||
</p>
|
||||
<p class="control">
|
||||
<button class="button is-info" type="submit">Save</button>
|
||||
</p>
|
||||
|
||||
|
||||
</form>
|
||||
|
||||
{#if !apiKey}
|
||||
<p class="help is-danger">
|
||||
Please enter your <a href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use ChatGPT-web.
|
||||
It is required to use ChatGPT-web.
|
||||
<p class:is-danger={!hasModels} class:is-warning={!apiKey}>
|
||||
Please enter your <a target="_blank" href="https://platform.openai.com/account/api-keys">OpenAI API key</a> above to use Open AI's ChatGPT API.
|
||||
At least one API must be enabled to use ChatGPT-web.
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
</article>
|
||||
|
||||
|
||||
<article class="message" class:is-danger={!hasModels} class:is-warning={!showPetalsSettings} class:is-info={showPetalsSettings}>
|
||||
<div class="message-body">
|
||||
<label class="label" for="enablePetals">
|
||||
<input
|
||||
type="checkbox"
|
||||
class="checkbox"
|
||||
id="enablePetals"
|
||||
checked={!!$globalStorage.enablePetals}
|
||||
on:click={setPetalsEnabled}
|
||||
>
|
||||
Use Petals API and Models (Llama 2)
|
||||
</label>
|
||||
{#if showPetalsSettings}
|
||||
<p>Set Petals API Endpoint:</p>
|
||||
<form
|
||||
class="field has-addons has-addons-right"
|
||||
on:submit|preventDefault={(event) => {
|
||||
if (event.target && event.target[0].value) {
|
||||
setGlobalSettingValueByKey('pedalsEndpoint', (event.target[0].value).trim())
|
||||
} else {
|
||||
setGlobalSettingValueByKey('pedalsEndpoint', '')
|
||||
}
|
||||
}}
|
||||
>
|
||||
<p class="control is-expanded">
|
||||
<input
|
||||
aria-label="PetalsAPI Endpoint"
|
||||
type="text"
|
||||
class="input"
|
||||
placeholder={getPetals()}
|
||||
value={$globalStorage.pedalsEndpoint || ''}
|
||||
/>
|
||||
</p>
|
||||
<p class="control">
|
||||
<button class="button is-info" type="submit">Save</button>
|
||||
</p>
|
||||
|
||||
|
||||
</form>
|
||||
|
||||
{#if !pedalsEndpoint}
|
||||
<p class="help is-warning">
|
||||
Please only use the default public API for testing. It's best to <a target="_blank" href="https://github.com/petals-infra/chat.petals.dev">configure a private endpoint</a> and enter it above for connection to the Petals swarm.
|
||||
</p>
|
||||
{/if}
|
||||
<p class="my-4">
|
||||
<a target="_blank" href="https://petals.dev/">Petals</a> lets you run large language models at home by connecting to a public swarm, BitTorrent-style, without hefty GPU requirements.
|
||||
</p>
|
||||
<p class="mb-4">
|
||||
You are encouraged to <a target="_blank" href="https://github.com/bigscience-workshop/petals/wiki/FAQ:-Frequently-asked-questions#running-a-server">set up a Petals server to share your GPU resources</a> with the public swarm. Minimum requirements to contribute Llama 2 completions are a GTX 1080 8GB, but the larger/faster the better.
|
||||
</p>
|
||||
<p class="help is-warning">
|
||||
Because Petals uses a public swarm, <b>do not send sensitive information</b> when using Petals.
|
||||
</p>
|
||||
{/if}
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -1,43 +1,108 @@
|
|||
<script context="module" lang="ts">
|
||||
import type { ModelDetail, Model } from './Types.svelte'
|
||||
import { getApiBase, getEndpointCompletions, getEndpointGenerations, getEndpointModels, getPetals } from './ApiUtil.svelte'
|
||||
import { apiKeyStorage, globalStorage } from './Storage.svelte'
|
||||
import { get, writable } from 'svelte/store'
|
||||
import type { ModelDetail, Model, ResponseModels, SelectOption, Chat } from './Types.svelte'
|
||||
import { encode } from 'gpt-tokenizer'
|
||||
import llamaTokenizer from 'llama-tokenizer-js'
|
||||
import { mergeProfileFields } from './Profiles.svelte'
|
||||
import { getChatSettingObjectByKey } from './Settings.svelte'
|
||||
import { valueOf } from './Util.svelte'
|
||||
|
||||
/**
|
||||
* TODO: All of this + what's scattered about need to be refactored to interfaces and classes
|
||||
* to make it all more modular
|
||||
*/
|
||||
const modelOptionCache = writable([] as SelectOption[])
|
||||
|
||||
// Reference: https://openai.com/pricing#language-models
|
||||
// Eventually we'll add API hosts and endpoints to this
|
||||
const modelDetails : Record<string, ModelDetail> = {
|
||||
'gpt-4-32k': {
|
||||
type: 'OpenAIChat',
|
||||
prompt: 0.00006, // $0.06 per 1000 tokens prompt
|
||||
completion: 0.00012, // $0.12 per 1000 tokens completion
|
||||
max: 32768 // 32k max token buffer
|
||||
},
|
||||
'gpt-4': {
|
||||
type: 'OpenAIChat',
|
||||
prompt: 0.00003, // $0.03 per 1000 tokens prompt
|
||||
completion: 0.00006, // $0.06 per 1000 tokens completion
|
||||
max: 8192 // 8k max token buffer
|
||||
},
|
||||
'gpt-3.5': {
|
||||
type: 'OpenAIChat',
|
||||
prompt: 0.0000015, // $0.0015 per 1000 tokens prompt
|
||||
completion: 0.000002, // $0.002 per 1000 tokens completion
|
||||
max: 4096 // 4k max token buffer
|
||||
},
|
||||
'gpt-3.5-turbo-16k': {
|
||||
type: 'OpenAIChat',
|
||||
prompt: 0.000003, // $0.003 per 1000 tokens prompt
|
||||
completion: 0.000004, // $0.004 per 1000 tokens completion
|
||||
max: 16384 // 16k max token buffer
|
||||
},
|
||||
'enoch/llama-65b-hf': {
|
||||
type: 'Petals',
|
||||
label: 'Petals - Llama-65b',
|
||||
stop: ['###', '</s>'],
|
||||
userStart: '<|user|>',
|
||||
assistantStart: '<|[[CHARACTER_NAME]]|>',
|
||||
systemStart: '',
|
||||
prompt: 0.000000, // $0.000 per 1000 tokens prompt
|
||||
completion: 0.000000, // $0.000 per 1000 tokens completion
|
||||
max: 2048 // 2k max token buffer
|
||||
},
|
||||
'timdettmers/guanaco-65b': {
|
||||
type: 'Petals',
|
||||
label: 'Petals - Guanaco-65b',
|
||||
stop: ['###', '</s>'],
|
||||
userStart: '<|user|>',
|
||||
assistantStart: '<|[[CHARACTER_NAME]]|>',
|
||||
systemStart: '',
|
||||
prompt: 0.000000, // $0.000 per 1000 tokens prompt
|
||||
completion: 0.000000, // $0.000 per 1000 tokens completion
|
||||
max: 2048 // 2k max token buffer
|
||||
},
|
||||
'meta-llama/Llama-2-70b-chat-hf': {
|
||||
type: 'Petals',
|
||||
label: 'Petals - Llama-2-70b-chat',
|
||||
stop: ['###', '</s>'],
|
||||
userStart: '<|user|>',
|
||||
assistantStart: '<|[[CHARACTER_NAME]]|>',
|
||||
systemStart: '',
|
||||
prompt: 0.000000, // $0.000 per 1000 tokens prompt
|
||||
completion: 0.000000, // $0.000 per 1000 tokens completion
|
||||
max: 4096 // 4k max token buffer
|
||||
},
|
||||
'meta-llama/Llama-2-70b-hf': {
|
||||
type: 'Petals',
|
||||
label: 'Petals - Llama-2-70b',
|
||||
stop: ['###', '</s>'],
|
||||
userStart: '<|user|>',
|
||||
assistantStart: '<|[[CHARACTER_NAME]]|>',
|
||||
systemStart: '',
|
||||
prompt: 0.000000, // $0.000 per 1000 tokens prompt
|
||||
completion: 0.000000, // $0.000 per 1000 tokens completion
|
||||
max: 4096 // 4k max token buffer
|
||||
}
|
||||
}
|
||||
|
||||
const imageModels : Record<string, ModelDetail> = {
|
||||
export const imageModels : Record<string, ModelDetail> = {
|
||||
'dall-e-1024x1024': {
|
||||
type: 'OpenAIDall-e',
|
||||
prompt: 0.00,
|
||||
completion: 0.020, // $0.020 per image
|
||||
max: 1000 // 1000 char prompt, max
|
||||
},
|
||||
'dall-e-512x512': {
|
||||
type: 'OpenAIDall-e',
|
||||
prompt: 0.00,
|
||||
completion: 0.018, // $0.018 per image
|
||||
max: 1000 // 1000 char prompt, max
|
||||
},
|
||||
'dall-e-256x256': {
|
||||
type: 'OpenAIDall-e',
|
||||
prompt: 0.00,
|
||||
completion: 0.016, // $0.016 per image
|
||||
max: 1000 // 1000 char prompt, max
|
||||
|
|
@ -47,22 +112,27 @@ const imageModels : Record<string, ModelDetail> = {
|
|||
const unknownDetail = {
|
||||
prompt: 0,
|
||||
completion: 0,
|
||||
max: 4096
|
||||
}
|
||||
max: 4096,
|
||||
type: 'OpenAIChat'
|
||||
} as ModelDetail
|
||||
|
||||
// See: https://platform.openai.com/docs/models/model-endpoint-compatibility
|
||||
// Eventually we'll add UI for managing this
|
||||
export const supportedModels : Record<string, ModelDetail> = {
|
||||
'gpt-3.5-turbo': modelDetails['gpt-3.5'],
|
||||
'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
|
||||
'gpt-3.5-turbo-0613': modelDetails['gpt-3.5'],
|
||||
'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
|
||||
'gpt-4': modelDetails['gpt-4'],
|
||||
'gpt-4-0314': modelDetails['gpt-4'],
|
||||
'gpt-4-0613': modelDetails['gpt-4'],
|
||||
'gpt-4-32k': modelDetails['gpt-4-32k'],
|
||||
'gpt-4-32k-0314': modelDetails['gpt-4-32k'],
|
||||
'gpt-4-32k-0613': modelDetails['gpt-4-32k'],
|
||||
'gpt-3.5-turbo': modelDetails['gpt-3.5'],
|
||||
'gpt-3.5-turbo-16k': modelDetails['gpt-3.5-turbo-16k'],
|
||||
'gpt-3.5-turbo-0301': modelDetails['gpt-3.5'],
|
||||
'gpt-3.5-turbo-0613': modelDetails['gpt-3.5']
|
||||
// 'enoch/llama-65b-hf': modelDetails['enoch/llama-65b-hf'],
|
||||
// 'timdettmers/guanaco-65b': modelDetails['timdettmers/guanaco-65b'],
|
||||
'meta-llama/Llama-2-70b-hf': modelDetails['meta-llama/Llama-2-70b-hf'],
|
||||
'meta-llama/Llama-2-70b-chat-hf': modelDetails['meta-llama/Llama-2-70b-chat-hf']
|
||||
}
|
||||
|
||||
const lookupList = {
|
||||
|
|
@ -75,7 +145,7 @@ export const supportedModelKeys = Object.keys({ ...supportedModels, ...imageMode
|
|||
|
||||
const tpCache : Record<string, ModelDetail> = {}
|
||||
|
||||
export const getModelDetail = (model: Model) => {
|
||||
export const getModelDetail = (model: Model): ModelDetail => {
|
||||
// First try to get exact match, then from cache
|
||||
let r = supportedModels[model] || tpCache[model]
|
||||
if (r) return r
|
||||
|
|
@ -93,4 +163,140 @@ export const getModelDetail = (model: Model) => {
|
|||
return r
|
||||
}
|
||||
|
||||
export const getEndpoint = (model: Model): string => {
|
||||
const modelDetails = getModelDetail(model)
|
||||
const gSettings = get(globalStorage)
|
||||
switch (modelDetails.type) {
|
||||
case 'Petals':
|
||||
return gSettings.pedalsEndpoint || getPetals()
|
||||
case 'OpenAIDall-e':
|
||||
return getApiBase() + getEndpointGenerations()
|
||||
case 'OpenAIChat':
|
||||
default:
|
||||
return gSettings.openAICompletionEndpoint || (getApiBase() + getEndpointCompletions())
|
||||
}
|
||||
}
|
||||
|
||||
export const getStopSequence = (chat: Chat): string => {
|
||||
return chat.settings.stopSequence || valueOf(chat.id, getChatSettingObjectByKey('stopSequence').placeholder)
|
||||
}
|
||||
|
||||
export const getUserStart = (chat: Chat): string => {
|
||||
return mergeProfileFields(
|
||||
chat.settings,
|
||||
chat.settings.userMessageStart || valueOf(chat.id, getChatSettingObjectByKey('userMessageStart').placeholder)
|
||||
)
|
||||
}
|
||||
|
||||
export const getAssistantStart = (chat: Chat): string => {
|
||||
return mergeProfileFields(
|
||||
chat.settings,
|
||||
chat.settings.assistantMessageStart || valueOf(chat.id, getChatSettingObjectByKey('assistantMessageStart').placeholder)
|
||||
)
|
||||
}
|
||||
|
||||
export const getSystemStart = (chat: Chat): string => {
|
||||
return mergeProfileFields(
|
||||
chat.settings,
|
||||
chat.settings.systemMessageStart || valueOf(chat.id, getChatSettingObjectByKey('systemMessageStart').placeholder)
|
||||
)
|
||||
}
|
||||
|
||||
export const getRoleTag = (role: string, model: Model, chat: Chat): string => {
|
||||
const modelDetails = getModelDetail(model)
|
||||
switch (modelDetails.type) {
|
||||
case 'Petals':
|
||||
if (role === 'assistant') return getAssistantStart(chat) + ' '
|
||||
if (role === 'user') return getUserStart(chat) + ' '
|
||||
return getSystemStart(chat) + ' '
|
||||
case 'OpenAIDall-e':
|
||||
return role
|
||||
case 'OpenAIChat':
|
||||
default:
|
||||
return role
|
||||
}
|
||||
}
|
||||
|
||||
export const getTokens = (model: Model, value: string): number[] => {
|
||||
const modelDetails = getModelDetail(model)
|
||||
switch (modelDetails.type) {
|
||||
case 'Petals':
|
||||
return llamaTokenizer.encode(value)
|
||||
case 'OpenAIDall-e':
|
||||
return [0]
|
||||
case 'OpenAIChat':
|
||||
default:
|
||||
return encode(value)
|
||||
}
|
||||
}
|
||||
|
||||
export const countTokens = (model: Model, value: string): number => {
|
||||
return getTokens(model, value).length
|
||||
}
|
||||
|
||||
export const clearModelOptionCache = () => {
|
||||
modelOptionCache.set([])
|
||||
}
|
||||
|
||||
export async function getModelOptions (): Promise<SelectOption[]> {
|
||||
const gSettings = get(globalStorage)
|
||||
const openAiKey = get(apiKeyStorage)
|
||||
const cachedOptions = get(modelOptionCache)
|
||||
if (cachedOptions && cachedOptions.length) return cachedOptions
|
||||
// Load available models from OpenAI
|
||||
let openAiModels
|
||||
let allowCache = true
|
||||
if (openAiKey) {
|
||||
try {
|
||||
openAiModels = (await (
|
||||
await fetch(getApiBase() + getEndpointModels(), {
|
||||
method: 'GET',
|
||||
headers: {
|
||||
Authorization: `Bearer ${openAiKey}`,
|
||||
'Content-Type': 'application/json'
|
||||
}
|
||||
})
|
||||
).json()) as ResponseModels
|
||||
} catch (e) {
|
||||
allowCache = false
|
||||
openAiModels = { data: [] }
|
||||
}
|
||||
} else {
|
||||
openAiModels = { data: [] }
|
||||
}
|
||||
// const filteredModels = Object.keys(supportedModels).filter((model) => {
|
||||
// switch (getModelDetail(model).type) {
|
||||
// case 'Petals':
|
||||
// return gSettings.enablePetals
|
||||
// case 'OpenAIChat':
|
||||
// default:
|
||||
// return openAiModels.data && openAiModels.data.find((m) => m.id === model)
|
||||
// }
|
||||
// })
|
||||
|
||||
const modelOptions:SelectOption[] = Object.keys(supportedModels).reduce((a, m) => {
|
||||
let disabled
|
||||
const modelDetail = getModelDetail(m)
|
||||
switch (modelDetail.type) {
|
||||
case 'Petals':
|
||||
disabled = !gSettings.enablePetals
|
||||
break
|
||||
case 'OpenAIChat':
|
||||
default:
|
||||
disabled = !(openAiModels.data && openAiModels.data.find((m) => m.id === m))
|
||||
}
|
||||
const o:SelectOption = {
|
||||
value: m,
|
||||
text: modelDetail.label || m,
|
||||
disabled
|
||||
}
|
||||
a.push(o)
|
||||
return a
|
||||
}, [] as SelectOption[])
|
||||
|
||||
if (allowCache) modelOptionCache.set(modelOptions)
|
||||
|
||||
return modelOptions
|
||||
}
|
||||
|
||||
</script>
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
<script context="module" lang="ts">
|
||||
import { getChatDefaults, getExcludeFromProfile } from './Settings.svelte'
|
||||
import { getChatDefaults, getDefaultModel, getExcludeFromProfile } from './Settings.svelte'
|
||||
import { get, writable } from 'svelte/store'
|
||||
// Profile definitions
|
||||
import { addMessage, clearMessages, deleteMessage, getChat, getChatSettings, getCustomProfiles, getGlobalSettings, getMessages, newName, resetChatSettings, saveChatStore, setGlobalSettingValueByKey, setMessages, updateProfile } from './Storage.svelte'
|
||||
|
|
@ -22,7 +22,9 @@ export const getProfiles = (forceUpdate:boolean = false):Record<string, ChatSett
|
|||
}
|
||||
const result = Object.entries(profiles
|
||||
).reduce((a, [k, v]) => {
|
||||
v = JSON.parse(JSON.stringify(v))
|
||||
a[k] = v
|
||||
v.model = v.model || getDefaultModel()
|
||||
return a
|
||||
}, {} as Record<string, ChatSettings>)
|
||||
Object.entries(getCustomProfiles()).forEach(([k, v]) => {
|
||||
|
|
@ -72,7 +74,7 @@ export const getProfile = (key:string, forReset:boolean = false):ChatSettings =>
|
|||
|
||||
export const mergeProfileFields = (settings: ChatSettings, content: string|undefined, maxWords: number|undefined = undefined): string => {
|
||||
if (!content?.toString) return ''
|
||||
content = (content + '').replaceAll('[[CHARACTER_NAME]]', settings.characterName || 'ChatGPT')
|
||||
content = (content + '').replaceAll('[[CHARACTER_NAME]]', settings.characterName || 'Assistant')
|
||||
if (maxWords) content = (content + '').replaceAll('[[MAX_WORDS]]', maxWords.toString())
|
||||
return content
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<script context="module" lang="ts">
|
||||
import { applyProfile } from './Profiles.svelte'
|
||||
import { getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
|
||||
import { encode } from 'gpt-tokenizer'
|
||||
import { get } from 'svelte/store'
|
||||
import { apiKeyStorage, getChatSettings, getGlobalSettings, setGlobalSettingValueByKey } from './Storage.svelte'
|
||||
import { faArrowDown91, faArrowDownAZ, faCheck, faThumbTack } from '@fortawesome/free-solid-svg-icons/index'
|
||||
// Setting definitions
|
||||
|
||||
|
|
@ -18,8 +18,15 @@ import {
|
|||
type ChatSortOption
|
||||
|
||||
} from './Types.svelte'
|
||||
import { getModelDetail, getTokens } from './Models.svelte'
|
||||
|
||||
export const defaultModel:Model = 'gpt-3.5-turbo'
|
||||
const defaultModel:Model = 'gpt-3.5-turbo'
|
||||
const defaultModelPetals:Model = 'meta-llama/Llama-2-70b-chat-hf'
|
||||
|
||||
export const getDefaultModel = (): Model => {
|
||||
if (!get(apiKeyStorage)) return defaultModelPetals
|
||||
return defaultModel
|
||||
}
|
||||
|
||||
export const getChatSettingList = (): ChatSetting[] => {
|
||||
return chatSettingsList
|
||||
|
|
@ -55,8 +62,16 @@ export const getExcludeFromProfile = () => {
|
|||
return excludeFromProfile
|
||||
}
|
||||
|
||||
const isNotOpenAI = (chatId) => {
|
||||
return getModelDetail(getChatSettings(chatId).model).type !== 'OpenAIChat'
|
||||
}
|
||||
|
||||
const isNotPetals = (chatId) => {
|
||||
return getModelDetail(getChatSettings(chatId).model).type !== 'Petals'
|
||||
}
|
||||
|
||||
const gptDefaults = {
|
||||
model: defaultModel,
|
||||
model: '',
|
||||
messages: [],
|
||||
temperature: 1,
|
||||
top_p: 1,
|
||||
|
|
@ -94,6 +109,10 @@ const defaults:ChatSettings = {
|
|||
hppContinuePrompt: '',
|
||||
hppWithSummaryPrompt: false,
|
||||
imageGenerationSize: '',
|
||||
stopSequence: '',
|
||||
userMessageStart: '',
|
||||
assistantMessageStart: '',
|
||||
systemMessageStart: '',
|
||||
// useResponseAlteration: false,
|
||||
// responseAlterations: [],
|
||||
isDirty: false
|
||||
|
|
@ -104,7 +123,10 @@ export const globalDefaults: GlobalSettings = {
|
|||
lastProfile: 'default',
|
||||
defaultProfile: 'default',
|
||||
hideSummarized: false,
|
||||
chatSort: 'created'
|
||||
chatSort: 'created',
|
||||
openAICompletionEndpoint: '',
|
||||
enablePetals: false,
|
||||
pedalsEndpoint: ''
|
||||
}
|
||||
|
||||
const excludeFromProfile = {
|
||||
|
|
@ -399,7 +421,13 @@ const modelSetting: ChatSetting & SettingSelect = {
|
|||
key: 'model',
|
||||
name: 'Model',
|
||||
title: 'The model to use - GPT-3.5 is cheaper, but GPT-4 is more powerful.',
|
||||
header: 'Below are the settings that OpenAI allows to be changed for the API calls. See the <a target="_blank" href="https://platform.openai.com/docs/api-reference/chat/create">OpenAI API docs</a> for more details.',
|
||||
header: (chatId) => {
|
||||
if (isNotOpenAI(chatId)) {
|
||||
return 'Below are the settings that can be changed for the API calls. See <a target="_blank" href="https://platform.openai.com/docs/api-reference/chat/create">this overview</a> to start, though not all settings translate to Petals.'
|
||||
} else {
|
||||
return 'Below are the settings that OpenAI allows to be changed for the API calls. See the <a target="_blank" href="https://platform.openai.com/docs/api-reference/chat/create">OpenAI API docs</a> for more details.'
|
||||
}
|
||||
},
|
||||
headerClass: 'is-warning',
|
||||
options: [],
|
||||
type: 'select',
|
||||
|
|
@ -417,7 +445,8 @@ const chatSettingsList: ChatSetting[] = [
|
|||
key: 'stream',
|
||||
name: 'Stream Response',
|
||||
title: 'Stream responses as they are generated.',
|
||||
type: 'boolean'
|
||||
type: 'boolean',
|
||||
hide: isNotOpenAI
|
||||
},
|
||||
{
|
||||
key: 'temperature',
|
||||
|
|
@ -432,7 +461,7 @@ const chatSettingsList: ChatSetting[] = [
|
|||
},
|
||||
{
|
||||
key: 'top_p',
|
||||
name: 'Nucleus Sampling',
|
||||
name: 'Nucleus Sampling (Top-p)',
|
||||
title: 'An alternative to sampling with temperature, called nucleus sampling, where the model considers the results of the tokens with top_p probability mass. So 0.1 means only the tokens comprising the top 10% probability mass are considered.\n' +
|
||||
'\n' +
|
||||
'We generally recommend altering this or temperature but not both',
|
||||
|
|
@ -448,7 +477,8 @@ const chatSettingsList: ChatSetting[] = [
|
|||
min: 1,
|
||||
max: 10,
|
||||
step: 1,
|
||||
type: 'number'
|
||||
type: 'number',
|
||||
hide: isNotOpenAI
|
||||
},
|
||||
{
|
||||
key: 'max_tokens',
|
||||
|
|
@ -460,6 +490,7 @@ const chatSettingsList: ChatSetting[] = [
|
|||
max: 32768,
|
||||
step: 1,
|
||||
type: 'number',
|
||||
hide: isNotOpenAI,
|
||||
forceApi: true // Since default here is different than gpt default, will make sure we always send it
|
||||
},
|
||||
{
|
||||
|
|
@ -469,7 +500,8 @@ const chatSettingsList: ChatSetting[] = [
|
|||
min: -2,
|
||||
max: 2,
|
||||
step: 0.2,
|
||||
type: 'number'
|
||||
type: 'number',
|
||||
hide: isNotOpenAI
|
||||
},
|
||||
{
|
||||
key: 'frequency_penalty',
|
||||
|
|
@ -478,7 +510,52 @@ const chatSettingsList: ChatSetting[] = [
|
|||
min: -2,
|
||||
max: 2,
|
||||
step: 0.2,
|
||||
type: 'number'
|
||||
type: 'number',
|
||||
hide: isNotOpenAI
|
||||
},
|
||||
{
|
||||
key: 'stopSequence',
|
||||
name: 'Stop Sequence',
|
||||
title: 'Characters used to separate messages in the message chain.',
|
||||
type: 'text',
|
||||
placeholder: (chatId) => {
|
||||
const val = getModelDetail(getChatSettings(chatId).model).stop
|
||||
return (val && val[0]) || ''
|
||||
},
|
||||
hide: isNotPetals
|
||||
},
|
||||
{
|
||||
key: 'userMessageStart',
|
||||
name: 'User Message Start Sequence',
|
||||
title: 'Sequence to denote user messages in the message chain.',
|
||||
type: 'text',
|
||||
placeholder: (chatId) => {
|
||||
const val = getModelDetail(getChatSettings(chatId).model).userStart
|
||||
return val || ''
|
||||
},
|
||||
hide: isNotPetals
|
||||
},
|
||||
{
|
||||
key: 'assistantMessageStart',
|
||||
name: 'Assistant Message Start Sequence',
|
||||
title: 'Sequence to denote assistant messages in the message chain.',
|
||||
type: 'text',
|
||||
placeholder: (chatId) => {
|
||||
const val = getModelDetail(getChatSettings(chatId).model).assistantStart
|
||||
return val || ''
|
||||
},
|
||||
hide: isNotPetals
|
||||
},
|
||||
{
|
||||
key: 'systemMessageStart',
|
||||
name: 'System Message Start Sequence',
|
||||
title: 'Sequence to denote system messages in the message chain.',
|
||||
type: 'text',
|
||||
placeholder: (chatId) => {
|
||||
const val = getModelDetail(getChatSettings(chatId).model).systemStart
|
||||
return val || ''
|
||||
},
|
||||
hide: isNotPetals
|
||||
},
|
||||
{
|
||||
// logit bias editor not implemented yet
|
||||
|
|
@ -497,7 +574,7 @@ const chatSettingsList: ChatSetting[] = [
|
|||
// console.log('logit_bias', val, getChatSettings(chatId).logit_bias)
|
||||
if (!val) return null
|
||||
const tokenized:Record<number, number> = Object.entries(val).reduce((a, [k, v]) => {
|
||||
const tokens:number[] = encode(k)
|
||||
const tokens:number[] = getTokens(getChatSettings(chatId).model, k)
|
||||
tokens.forEach(t => { a[t] = v })
|
||||
return a
|
||||
}, {} as Record<number, number>)
|
||||
|
|
@ -536,6 +613,21 @@ const globalSettingsList:GlobalSetting[] = [
|
|||
key: 'hideSummarized',
|
||||
name: 'Hide Summarized Messages',
|
||||
type: 'boolean'
|
||||
},
|
||||
{
|
||||
key: 'openAICompletionEndpoint',
|
||||
name: 'OpenAI Completions Endpoint',
|
||||
type: 'text'
|
||||
},
|
||||
{
|
||||
key: 'enablePetals',
|
||||
name: 'Enable Petals APIs',
|
||||
type: 'boolean'
|
||||
},
|
||||
{
|
||||
key: 'pedalsEndpoint',
|
||||
name: 'Petals API Endpoint',
|
||||
type: 'text'
|
||||
}
|
||||
]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
<script lang="ts">
|
||||
import { params } from 'svelte-spa-router'
|
||||
import ChatMenuItem from './ChatMenuItem.svelte'
|
||||
import { apiKeyStorage, chatsStorage, pinMainMenu, checkStateChange, getChatSortOption, setChatSortOption } from './Storage.svelte'
|
||||
import { apiKeyStorage, chatsStorage, pinMainMenu, checkStateChange, getChatSortOption, setChatSortOption, hasActiveModels } from './Storage.svelte'
|
||||
import Fa from 'svelte-fa/src/fa.svelte'
|
||||
import { faSquarePlus, faKey } from '@fortawesome/free-solid-svg-icons/index'
|
||||
import ChatOptionMenu from './ChatOptionMenu.svelte'
|
||||
|
|
@ -14,10 +14,12 @@
|
|||
$: activeChatId = $params && $params.chatId ? parseInt($params.chatId) : undefined
|
||||
|
||||
let sortOption = getChatSortOption()
|
||||
let hasModels = hasActiveModels()
|
||||
|
||||
const onStateChange = (...args:any) => {
|
||||
sortOption = getChatSortOption()
|
||||
sortedChats = $chatsStorage.sort(sortOption.sortFn)
|
||||
hasModels = hasActiveModels()
|
||||
}
|
||||
|
||||
$: onStateChange($checkStateChange)
|
||||
|
|
@ -72,14 +74,14 @@
|
|||
</div>
|
||||
</div>
|
||||
<div class="level-right">
|
||||
{#if !$apiKeyStorage}
|
||||
{#if !hasModels}
|
||||
<div class="level-item">
|
||||
<a href={'#/'} class="panel-block" class:is-disabled={!$apiKeyStorage}
|
||||
><span class="greyscale mr-1"><Fa icon={faKey} /></span> API key</a
|
||||
></div>
|
||||
{:else}
|
||||
<div class="level-item">
|
||||
<button on:click={() => { $pinMainMenu = false; startNewChatWithWarning(activeChatId) }} class="panel-block button" title="Start new chat with default profile" class:is-disabled={!$apiKeyStorage}
|
||||
<button on:click={() => { $pinMainMenu = false; startNewChatWithWarning(activeChatId) }} class="panel-block button" title="Start new chat with default profile" class:is-disabled={!hasModels}
|
||||
><span class="greyscale mr-1"><Fa icon={faSquarePlus} /></span> New chat</button>
|
||||
</div>
|
||||
{/if}
|
||||
|
|
|
|||
|
|
@ -1,25 +1,43 @@
|
|||
<script context="module" lang="ts">
|
||||
import { getModelDetail } from './Models.svelte'
|
||||
import type { Message, Model, Usage } from './Types.svelte'
|
||||
import { encode } from 'gpt-tokenizer'
|
||||
import { countTokens, getModelDetail, getRoleTag, getStopSequence } from './Models.svelte'
|
||||
import type { Chat, Message, Model, Usage } from './Types.svelte'
|
||||
|
||||
export const getPrice = (tokens: Usage, model: Model): number => {
|
||||
const t = getModelDetail(model)
|
||||
return ((tokens.prompt_tokens * t.prompt) + (tokens.completion_tokens * t.completion))
|
||||
}
|
||||
|
||||
export const countPromptTokens = (prompts:Message[], model:Model):number => {
|
||||
return prompts.reduce((a, m) => {
|
||||
a += countMessageTokens(m, model)
|
||||
export const countPromptTokens = (prompts:Message[], model:Model, chat: Chat):number => {
|
||||
const detail = getModelDetail(model)
|
||||
const count = prompts.reduce((a, m) => {
|
||||
a += countMessageTokens(m, model, chat)
|
||||
return a
|
||||
}, 0) + 3 // Always seems to be message counts + 3
|
||||
}
|
||||
|
||||
export const countMessageTokens = (message:Message, model:Model):number => {
|
||||
}, 0)
|
||||
switch (detail.type) {
|
||||
case 'Petals':
|
||||
return count
|
||||
case 'OpenAIChat':
|
||||
default:
|
||||
// Not sure how OpenAI formats it, but this seems to get close to the right counts.
|
||||
// Would be nice to know. This works for gpt-3.5. gpt-4 could be different.
|
||||
// Complete stab in the dark here -- update if you know where all the extra tokens really come from.
|
||||
return encode('## ' + message.role + ' ##:\r\n\r\n' + message.content + '\r\n\r\n\r\n').length
|
||||
return count + 3 // Always seems to be message counts + 3
|
||||
}
|
||||
}
|
||||
|
||||
export const countMessageTokens = (message:Message, model:Model, chat: Chat):number => {
|
||||
const detail = getModelDetail(model)
|
||||
const stop = getStopSequence(chat)
|
||||
switch (detail.type) {
|
||||
case 'Petals':
|
||||
return countTokens(model, getRoleTag(message.role, model, chat) + ': ' + message.content + (stop || '###'))
|
||||
case 'OpenAIChat':
|
||||
default:
|
||||
// Not sure how OpenAI formats it, but this seems to get close to the right counts.
|
||||
// Would be nice to know. This works for gpt-3.5. gpt-4 could be different.
|
||||
// Complete stab in the dark here -- update if you know where all the extra tokens really come from.
|
||||
return countTokens(model, '## ' + message.role + ' ##:\r\n\r\n' + message.content + '\r\n\r\n\r\n')
|
||||
}
|
||||
}
|
||||
|
||||
export const getModelMaxTokens = (model:Model):number => {
|
||||
|
|
|
|||
|
|
@ -30,6 +30,11 @@
|
|||
return get(apiKeyStorage)
|
||||
}
|
||||
|
||||
export const hasActiveModels = (): boolean => {
|
||||
const globalSettings = get(globalStorage) || {}
|
||||
return !!get(apiKeyStorage) || !!globalSettings.enablePetals
|
||||
}
|
||||
|
||||
export const newChatID = (): number => {
|
||||
const chats = get(chatsStorage)
|
||||
const chatId = chats.reduce((maxId, chat) => Math.max(maxId, chat.id), 0) + 1
|
||||
|
|
|
|||
|
|
@ -7,7 +7,15 @@ export type Model = typeof supportedModelKeys[number];
|
|||
|
||||
export type ImageGenerationSizes = typeof imageGenerationSizeTypes[number];
|
||||
|
||||
export type RequestType = 'OpenAIChat' | 'OpenAIDall-e' | 'Petals'
|
||||
|
||||
export type ModelDetail = {
|
||||
type: RequestType;
|
||||
label?: string;
|
||||
stop?: string[];
|
||||
userStart?: string,
|
||||
assistantStart?: string,
|
||||
systemStart?: string,
|
||||
prompt: number;
|
||||
completion: number;
|
||||
max: number;
|
||||
|
|
@ -105,6 +113,10 @@ export type ChatSettings = {
|
|||
trainingPrompts?: Message[];
|
||||
useResponseAlteration?: boolean;
|
||||
responseAlterations?: ResponseAlteration[];
|
||||
stopSequence: string;
|
||||
userMessageStart: string;
|
||||
assistantMessageStart: string;
|
||||
systemMessageStart: string;
|
||||
isDirty?: boolean;
|
||||
} & Request;
|
||||
|
||||
|
|
@ -122,16 +134,16 @@ export type Chat = {
|
|||
};
|
||||
|
||||
type ResponseOK = {
|
||||
id: string;
|
||||
object: string;
|
||||
created: number;
|
||||
choices: {
|
||||
index: number;
|
||||
id?: string;
|
||||
object?: string;
|
||||
created?: number;
|
||||
choices?: {
|
||||
index?: number;
|
||||
message: Message;
|
||||
finish_reason: string;
|
||||
finish_reason?: string;
|
||||
delta: Message;
|
||||
}[];
|
||||
usage: Usage;
|
||||
usage?: Usage;
|
||||
model: Model;
|
||||
};
|
||||
|
||||
|
|
@ -172,6 +184,9 @@ export type GlobalSettings = {
|
|||
defaultProfile: string;
|
||||
hideSummarized: boolean;
|
||||
chatSort: ChatSortOptions;
|
||||
openAICompletionEndpoint: string;
|
||||
enablePetals: boolean;
|
||||
pedalsEndpoint: string;
|
||||
};
|
||||
|
||||
type SettingNumber = {
|
||||
|
|
@ -184,6 +199,7 @@ export type GlobalSettings = {
|
|||
export type SelectOption = {
|
||||
value: string|number;
|
||||
text: string;
|
||||
disabled?: boolean;
|
||||
};
|
||||
|
||||
export type ChatSortOption = SelectOption & {
|
||||
|
|
@ -236,15 +252,17 @@ export type SubSetting = {
|
|||
settings: any[];
|
||||
};
|
||||
|
||||
export type ValueFn = (chatId:number) => string
|
||||
|
||||
export type ChatSetting = {
|
||||
key: keyof ChatSettings;
|
||||
name: string;
|
||||
title: string;
|
||||
forceApi?: boolean; // force in api requests, even if set to default
|
||||
hidden?: boolean; // Hide from setting menus
|
||||
header?: string;
|
||||
headerClass?: string;
|
||||
placeholder?: string;
|
||||
header?: string | ValueFn;
|
||||
headerClass?: string | ValueFn;
|
||||
placeholder?: string | ValueFn;
|
||||
hide?: (chatId:number) => boolean;
|
||||
apiTransform?: (chatId:number, setting:ChatSetting, value:any) => any;
|
||||
fieldControls?: FieldControl[];
|
||||
|
|
|
|||
|
|
@ -147,4 +147,9 @@
|
|||
newChat()
|
||||
}
|
||||
|
||||
export const valueOf = (chatId: number, value: any) => {
|
||||
if (typeof value === 'function') return value(chatId)
|
||||
return value
|
||||
}
|
||||
|
||||
</script>
|
||||
Loading…
Reference in New Issue