From a8764b9768195e42ddb195abac9635189bedd99b Mon Sep 17 00:00:00 2001 From: Jannis Fedoruk-Betschki Date: Fri, 21 Jun 2024 11:03:52 +0200 Subject: [PATCH 1/6] [AI-32] Add option for custom OpenAI-compatible endpoint --- src/lib/ApiUtil.svelte | 2 - src/lib/Home.svelte | 55 ++++++++++++++++++++------ src/lib/Models.svelte | 38 +++++++++++------- src/lib/Settings.svelte | 27 +++++++++---- src/lib/Storage.svelte | 12 ++++++ src/lib/Types.svelte | 1 + src/lib/providers/openai/models.svelte | 15 ++++++- src/lib/providers/openai/util.svelte | 4 +- 8 files changed, 114 insertions(+), 40 deletions(-) diff --git a/src/lib/ApiUtil.svelte b/src/lib/ApiUtil.svelte index 74b15e5f..e1173d86 100644 --- a/src/lib/ApiUtil.svelte +++ b/src/lib/ApiUtil.svelte @@ -1,6 +1,5 @@ @@ -54,7 +54,7 @@ const setPetalsEnabled = (event: Event) => { private. You can also close the browser tab and come back later to continue the conversation.

- As an alternative to OpenAI, you can also use Petals swarm as a free API option for open chat models like Llama 2. + As an alternative to OpenAI, you can enter your own OpenAI-compatabile API endpoint, or use Petals swarm as a free API option for open chat models like Llama 2.

@@ -100,6 +100,37 @@ const setPetalsEnabled = (event: Event) => { +
+
+ Set the API BASE URI for alternative OpenAI-compatible endpoints: +
{ + let val = '' + if (event.target && event.target[0].value) { + val = (event.target[0].value).trim() + setGlobalSettingValueByKey('openAiEndpoint', val) + } else { + setGlobalSettingValueByKey('openAiEndpoint', '') + } + }} + > +

+ +

+

+ +

+
+
+
+
diff --git a/src/lib/Models.svelte b/src/lib/Models.svelte index e19f620f..6c229a06 100644 --- a/src/lib/Models.svelte +++ b/src/lib/Models.svelte @@ -36,25 +36,35 @@ export const supportedChatModelKeys = Object.keys({ ...supportedChatModels }) const tpCache : Record = {} export const getModelDetail = (model: Model): ModelDetail => { - // First try to get exact match, then from cache - let r = lookupList[model] || tpCache[model] - if (r) return r - // If no exact match, find closest match - const k = Object.keys(lookupList) - .sort((a, b) => b.length - a.length) // Longest to shortest for best match - .find((k) => model.startsWith(k)) - if (k) { - r = lookupList[k] + // Ensure model is a string for typesafety + if (typeof model !== 'string') { + console.warn('Invalid type for model:', model) + return unknownDetail } - if (!r) { - console.warn('Unable to find model detail for:', model, lookupList) - r = unknownDetail + + // Attempt to fetch the model details directly from lookupList or cache + let result = lookupList[model] || tpCache[model] + if (result) { + return result + } + + // No direct match found, attempting to find the closest match + const sortedKeys = Object.keys(lookupList).sort((a, b) => b.length - a.length) // Longest to shortest for best match + const bestMatchKey = sortedKeys.find(key => model.startsWith(key)) + + if (bestMatchKey) { + result = lookupList[bestMatchKey] + } else { + console.warn('Unable to find model detail for:', model) + result = unknownDetail // Assign a default detail for undefined models } + // Cache it so we don't need to do that again - tpCache[model] = r - return r + tpCache[model] = result + return result } + export const getEndpoint = (model: Model): string => { return getModelDetail(model).getEndpoint(model) } diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte index 2cdf58c1..ae18c62f 100644 --- a/src/lib/Settings.svelte +++ b/src/lib/Settings.svelte @@ -1,7 +1,7 @@ + await restartProfile(chatId) + replace(`/chat/${chatId}`) + }) + \ No newline at end of file diff --git a/src/lib/Profiles.svelte b/src/lib/Profiles.svelte index 5ca44d68..91c07a8e 100644 --- a/src/lib/Profiles.svelte +++ b/src/lib/Profiles.svelte @@ -15,7 +15,9 @@ export const isStaticProfile = (key:string):boolean => { return !!profiles[key] } -export const getProfiles = (forceUpdate:boolean = false):Record => { +export const getProfiles = async (forceUpdate:boolean = false):Promise> => { + const defaultModel = await getDefaultModel() + const pc = get(profileCache) if (!forceUpdate && Object.keys(pc).length) { return pc @@ -24,7 +26,7 @@ export const getProfiles = (forceUpdate:boolean = false):Record { v = JSON.parse(JSON.stringify(v)) a[k] = v - v.model = v.model || getDefaultModel() + v.model = v.model || defaultModel return a }, {} as Record) Object.entries(getCustomProfiles()).forEach(([k, v]) => { @@ -42,22 +44,23 @@ export const getProfiles = (forceUpdate:boolean = false):Record { - return Object.entries(getProfiles()).reduce((a, [k, v]) => { +export const getProfileSelect = async ():Promise => { + const allProfiles = await getProfiles() + return Object.entries(allProfiles).reduce((a, [k, v]) => { a.push({ value: k, text: v.profileName } as SelectOption) return a }, [] as SelectOption[]) } -export const getDefaultProfileKey = ():string => { - const allProfiles = getProfiles() +export const getDefaultProfileKey = async ():Promise => { + const allProfiles = await getProfiles() return (allProfiles[getGlobalSettings().defaultProfile || ''] || profiles[defaultProfile] || profiles[Object.keys(profiles)[0]]).profile } -export const getProfile = (key:string, forReset:boolean = false):ChatSettings => { - const allProfiles = getProfiles() +export const getProfile = async (key:string, forReset:boolean = false):Promise => { + const allProfiles = await getProfiles() let profile = allProfiles[key] || allProfiles[getGlobalSettings().defaultProfile || ''] || profiles[defaultProfile] || @@ -108,9 +111,9 @@ export const setSystemPrompt = (chatId: number) => { } // Restart currently loaded profile -export const restartProfile = (chatId:number, noApply:boolean = false) => { +export const restartProfile = async (chatId:number, noApply:boolean = false) => { const settings = getChatSettings(chatId) - if (!settings.profile && !noApply) return applyProfile(chatId, '', true) + if (!settings.profile && !noApply) return await applyProfile(chatId, '', true) // Clear current messages clearMessages(chatId) // Add the system prompt @@ -129,16 +132,16 @@ export const restartProfile = (chatId:number, noApply:boolean = false) => { setGlobalSettingValueByKey('lastProfile', settings.profile) } -export const newNameForProfile = (name:string) => { - const profiles = getProfileSelect() +export const newNameForProfile = async (name:string) => { + const profiles = await getProfileSelect() return newName(name, profiles.reduce((a, p) => { a[p.text] = p; return a }, {})) } // Apply currently selected profile -export const applyProfile = (chatId:number, key:string = '', resetChat:boolean = false) => { - resetChatSettings(chatId, resetChat) // Fully reset +export const applyProfile = async (chatId:number, key:string = '', resetChat:boolean = false) => { + await resetChatSettings(chatId, resetChat) // Fully reset if (!resetChat) return - return restartProfile(chatId, true) + return await restartProfile(chatId, true) } const summaryPrompts = { diff --git a/src/lib/Settings.svelte b/src/lib/Settings.svelte index dba8fec7..f46992d7 100644 --- a/src/lib/Settings.svelte +++ b/src/lib/Settings.svelte @@ -18,15 +18,17 @@ import { type ChatSortOption } from './Types.svelte' -import { getModelDetail, getTokens } from './Models.svelte' +import { getChatModelOptions, getModelDetail, getTokens } from './Models.svelte' // We are adding default model names explicitly here to avoid // circular dependencies. Alternative would be a big refactor, // which we want to avoid for now. -export const getDefaultModel = (): Model => { +export const getDefaultModel = async (): Promise => { if (!get(apiKeyStorage)) return 'stabilityai/StableBeluga2' - return 'gpt-3.5-turbo' + const models = await getChatModelOptions() + + return models[0].text } export const getChatSettingList = (): ChatSetting[] => { diff --git a/src/lib/Sidebar.svelte b/src/lib/Sidebar.svelte index e090c9ad..0ee2b8a9 100644 --- a/src/lib/Sidebar.svelte +++ b/src/lib/Sidebar.svelte @@ -82,7 +82,7 @@ >
{:else}
-
{/if} diff --git a/src/lib/Storage.svelte b/src/lib/Storage.svelte index 30d64a98..60f078f2 100644 --- a/src/lib/Storage.svelte +++ b/src/lib/Storage.svelte @@ -49,13 +49,13 @@ return chatId } - export const addChat = (profile:ChatSettings|undefined = undefined): number => { + export const addChat = async (profile:ChatSettings|undefined = undefined): Promise => { const chats = get(chatsStorage) // Find the max chatId const chatId = newChatID() - profile = JSON.parse(JSON.stringify(profile || getProfile(''))) as ChatSettings + profile = JSON.parse(JSON.stringify(profile || await getProfile(''))) as ChatSettings const nameMap = chats.reduce((a, chat) => { a[chat.name] = chat; return a }, {}) // Add a new chat @@ -73,7 +73,7 @@ }) chatsStorage.set(chats) // Apply defaults and prepare it to start - restartProfile(chatId) + await restartProfile(chatId) return chatId } @@ -164,10 +164,10 @@ } // Reset all setting to current profile defaults - export const resetChatSettings = (chatId, resetAll:boolean = false) => { + export const resetChatSettings = async (chatId, resetAll:boolean = false) => { const chats = get(chatsStorage) const chat = chats.find((chat) => chat.id === chatId) as Chat - const profile = getProfile(chat.settings.profile) + const profile = await getProfile(chat.settings.profile) const exclude = getExcludeFromProfile() if (resetAll) { // Reset to base defaults first, then apply profile @@ -495,7 +495,7 @@ return store.profiles || {} } - export const deleteCustomProfile = (chatId:number, profileId:string) => { + export const deleteCustomProfile = async (chatId:number, profileId:string) => { if (isStaticProfile(profileId)) { throw new Error('Sorry, you can\'t delete a static profile.') } @@ -507,10 +507,10 @@ } delete store.profiles[profileId] globalStorage.set(store) - getProfiles(true) // force update profile cache + await getProfiles(true) // force update profile cache } - export const saveCustomProfile = (profile:ChatSettings) => { + export const saveCustomProfile = async (profile:ChatSettings) => { const store = get(globalStorage) let profiles = store.profiles if (!profiles) { @@ -533,7 +533,7 @@ if (isStaticProfile(profile.profile)) { // throw new Error('Sorry, you can\'t modify a static profile. You can clone it though!') // Save static profile as new custom - profile.profileName = newNameForProfile(profile.profileName) + profile.profileName = await newNameForProfile(profile.profileName) profile.profile = uuidv4() } const clone = JSON.parse(JSON.stringify(profile)) // Always store a copy @@ -549,7 +549,7 @@ globalStorage.set(store) profile.isDirty = false saveChatStore() - getProfiles(true) // force update profile cache + await getProfiles(true) // force update profile cache } export const getChatSortOption = (): ChatSortOption => { diff --git a/src/lib/Util.svelte b/src/lib/Util.svelte index dda585c3..7ddb6b62 100644 --- a/src/lib/Util.svelte +++ b/src/lib/Util.svelte @@ -122,15 +122,15 @@ }) } - export const startNewChatFromChatId = (chatId: number) => { - const newChatId = addChat(getChat(chatId).settings) + export const startNewChatFromChatId = async (chatId: number) => { + const newChatId = await addChat(getChat(chatId).settings) // go to new chat replace(`/chat/${newChatId}`) } - export const startNewChatWithWarning = (activeChatId: number|undefined, profile?: ChatSettings|undefined) => { - const newChat = () => { - const chatId = addChat(profile) + export const startNewChatWithWarning = async (activeChatId: number|undefined, profile?: ChatSettings|undefined) => { + const newChat = async () => { + const chatId = await addChat(profile) replace(`/chat/${chatId}`) } // if (activeChatId && getChat(activeChatId).settings.isDirty) { @@ -146,7 +146,7 @@ // } else { // newChat() // } - newChat() + await newChat() } export const valueOf = (chatId: number, value: any) => { diff --git a/src/lib/providers/openai/request.svelte b/src/lib/providers/openai/request.svelte index c35daa19..8e6e977e 100644 --- a/src/lib/providers/openai/request.svelte +++ b/src/lib/providers/openai/request.svelte @@ -12,7 +12,7 @@ export const chatRequest = async ( chatResponse: ChatCompletionResponse, opts: ChatCompletionOpts): Promise => { // OpenAI Request - const model = chatRequest.getModel() + const model = await chatRequest.getModel() const signal = chatRequest.controller.signal const abortListener = (e:Event) => { chatRequest.updating = false diff --git a/src/lib/providers/petals/request.svelte b/src/lib/providers/petals/request.svelte index 70c777d7..8d5b8e50 100644 --- a/src/lib/providers/petals/request.svelte +++ b/src/lib/providers/petals/request.svelte @@ -37,7 +37,7 @@ export const chatRequest = async ( // Petals const chat = chatRequest.getChat() const chatSettings = chat.settings - const model = chatRequest.getModel() + const model = await chatRequest.getModel() const modelDetail = getModelDetail(model) const signal = chatRequest.controller.signal const providerData = chatRequest.providerData.petals || {} From e52666875325f714ce14658d4880ffe5838dcf55 Mon Sep 17 00:00:00 2001 From: Jannis Fedoruk-Betschki Date: Fri, 4 Oct 2024 08:57:46 +0200 Subject: [PATCH 6/6] [AI-32] Check /models endpoint before saving API URL --- src/lib/Home.svelte | 56 +++++++++++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/src/lib/Home.svelte b/src/lib/Home.svelte index 4cf79dd2..66e5045c 100644 --- a/src/lib/Home.svelte +++ b/src/lib/Home.svelte @@ -6,17 +6,18 @@ import { getPetalsBase, getPetalsWebsocket } from './ApiUtil.svelte' import { set as setOpenAI } from './providers/openai/util.svelte' import { hasActiveModels } from './Models.svelte' + import { get } from 'svelte/store' $: apiKey = $apiKeyStorage const openAiEndpoint = $globalStorage.openAiEndpoint || '' let showPetalsSettings = $globalStorage.enablePetals let pedalsEndpoint = $globalStorage.pedalsEndpoint let hasModels = hasActiveModels() + let apiError: string = '' onMount(() => { if (!$started) { $started = true - // console.log('started', apiKey, $lastChatId, getChat($lastChatId)) if (hasActiveModels() && getChat($lastChatId)) { const chatId = $lastChatId $lastChatId = 0 @@ -39,13 +40,30 @@ hasModels = hasActiveModels() } + async function testApiEndpoint (baseUri: string): Promise { + try { + const response = await fetch(`${baseUri}/v1/models`, { + headers: { Authorization: `Bearer ${get(apiKeyStorage)}` } + }) + if (!response.ok) { + apiError = `There was an error connecting to this endpoint: ${response.statusText}` + return false + } + apiError = '' + return true + } catch (error) { + console.error('Failed to connect:', error) + apiError = `There was an error connecting to this endpoint: ${error.message}` + return false + } + }
-

- ChatGPT-web +

+ ChatGPT-web is a simple one-page web interface to the OpenAI ChatGPT API. To use it, you need to register for an OpenAI API key first. OpenAI bills per token (usage-based), which means it is a lot cheaper than @@ -64,7 +82,7 @@

{ + on:submit|preventDefault={async (event) => { let val = '' if (event.target && event.target[0].value) { val = (event.target[0].value).trim() @@ -80,7 +98,8 @@ autocomplete="off" class="input" class:is-danger={!hasModels} - class:is-warning={!apiKey} class:is-info={apiKey} + class:is-warning={!apiKey} + class:is-info={apiKey} value={apiKey} />

@@ -100,18 +119,18 @@
-
+
Set the API BASE URI for alternative OpenAI-compatible endpoints: { + on:submit|preventDefault={async (event) => { let val = '' if (event.target && event.target[0].value) { val = (event.target[0].value).trim() + } + if (await testApiEndpoint(val)) { setGlobalSettingValueByKey('openAiEndpoint', val) - } else { - setGlobalSettingValueByKey('openAiEndpoint', '') } }} > @@ -120,6 +139,7 @@ aria-label="API BASE URI" type="text" class="input" + class:is-danger={apiError} placeholder="https://api.openai.com" value={openAiEndpoint} /> @@ -128,20 +148,22 @@

+ {#if apiError} +

{apiError}

+ {/if}
- - +
{#if showPetalsSettings}