llmproxy/worker.js

const DEBUG = 0

const DEFAULT_PROVIDERS = [
    {
      NAME: "OpenAI",
      ENDPOINT: "https://api.openai.com/v1",
      KEY: "REDACTED"
    },
    {
      NAME: "Anthropic",   // It needs to be OpenAI compatible
      ENDPOINT: "https://claude.api.morgan.kr/v1",
      KEY: "REDACTED"
    },
    {
      NAME: "DeepSeek",
      ENDPOINT: "https://api.deepseek.com/v1",
      KEY: "REDACTED"
    },
    {
      NAME: "GroqCloud",
      ENDPOINT: "https://api.groq.com/openai/v1",
      KEY: "REDACTED"
    },
    {
      NAME: "Google",
      ENDPOINT: "https://generativelanguage.googleapis.com/v1beta/openai",
      KEY: "REDACTED"
    },
  ]

const AUTH_KEY = "REDACTED"

export default {
  async fetch(request, env, ctx) {
    if (request.method === 'OPTIONS') {
      return new Response(null, {
        status: 204,
        headers: corsHeaders()
      })
    }
    
    const requestClone = request.clone()

    // 1. Authorization
    const authorization = request.headers.get('authorization');
    if (authorization) {
      let auth = authorization.split(' ')[1];
      if (auth !== AUTH_KEY) {
        return new Response('Unauthorized', {
          status: 401,
          headers: corsHeaders()
        })
      }
    } else {
      return new Response('Unauthorized', {
        status: 401,
        headers: corsHeaders()
      })
    }

    // 2. Process request
    try {
      const contentType = requestClone.headers.get('content-type') || ''
      let body

      const path = new URL(request.url).pathname

      // Get Providers Config
      // IF. Model is cached on KV
      let config = await env.KV.get("config");
      let PROVIDERS;
      if (config) {
        PROVIDERS = JSON.parse(config)
      } else {
        PROVIDERS = DEFAULT_PROVIDERS
        await env.KV.put("config", JSON.stringify(DEFAULT_PROVIDERS));
      }

      // ELSE. Update model on KV
      if (path === '/v1/models.reload') {
        const allModelsArrays = await Promise.all(
          PROVIDERS.map(prov => getProviders(env, prov, true))
        );
        const allModels = allModelsArrays.flat();
        const resp = { object: 'list', data: allModels };
        return new Response(JSON.stringify(resp), {
          status: 200,
          headers: {
            ...corsHeaders(),
            'Content-Type': 'application/json'
          }
        });
      }

      // 2-1. Models. (from KV storage?)
      else if (path === '/v1/models') {
        let allModels = [];
        for (let prov of PROVIDERS) {
          let model = await getProviders(env, prov);
          allModels = allModels.concat(model);
        }
        let resp = {object: 'list', data: allModels}
        return new Response(JSON.stringify(resp), {
            status: 200,
            headers: {
              ...corsHeaders(),
              'Content-Type': 'application/json'
            }
        });
      }

      // 2-2. Completion?
      else if (path === '/v1/chat/completions') {
        return await handleChatCompletions(requestClone, env, PROVIDERS)
      }

      else {
        return new Response(`Error: Unknown Endpoint`, {
          status: 500,
          headers: corsHeaders()
        })
      }


    } catch (error) {
      return new Response(`Error: ${error.message}`, {
        status: 500,
        headers: corsHeaders()
      })
    }
  }
}


async function handleChatCompletions(request, env, PROVIDERS) {
  // Parse JSON body.
  const body = await request.json();
  const { model } = body;
  
  console.log(body);

  const getProvider = async (model) => {
    for (let provider of PROVIDERS) {
      const models = await getProviders(env, provider);
      if (models.some(m => m.id === model)) {
        return provider;
      }
    }
    return null;
  };

  const provider = await getProvider(model);
  if (!provider) {
    return new Response('Model not supported by any provider', {
      status: 400,
      headers: corsHeaders()
    });
  }

  const { ENDPOINT, KEY: apiKey } = provider;
  if (!apiKey) {
    return new Response('Unauthorized: Invalid Provider', {
      status: 401,
      headers: corsHeaders()
    });
  }

  if (DEBUG) { console.log({model: model, provider: provider, endpoint: ENDPOINT}) }

  const targetHeaders = new Headers(request.headers);
  targetHeaders.set('Authorization', `Bearer ${apiKey}`);
  targetHeaders.set('Content-Type', 'application/json');

  const url = new URL(request.url);
  const upstreamUrl = ENDPOINT + url.pathname.replace(/^\/v1\//, '/'); + url.search;
  const upstreamResponse = await fetch(upstreamUrl, {
    method: request.method,
    headers: targetHeaders,
    body: JSON.stringify(body)
  });

  const filteredUpstreamHeaders = Object.fromEntries(
    [...upstreamResponse.headers].filter(([key]) => 
      !key.toLowerCase().startsWith('access-control-')
    )
  );
  const responseHeaders = {
    ...filteredUpstreamHeaders,
    ...corsHeaders()
  };

  const contentType = upstreamResponse.headers.get('content-type') || '';
  if (contentType.includes('text/event-stream')) {
    delete responseHeaders['content-length'];
    responseHeaders['Cache-Control'] = 'no-cache';
  }

  // const { readable, writable } = new TransformStream();
  // upstreamResponse.body.pipeTo(writable).catch((err) => {
  //   console.error('Stream error:', err);
  // });

  return new Response(upstreamResponse.body, {
    status: upstreamResponse.status,
    statusText: upstreamResponse.statusText,
    headers: responseHeaders
  });
}


async function getProviders(env, provider, refresh=false) {
  let value = await env.KV.get(provider.NAME);
  if (value === null || refresh) {
    const headers = new Headers();
    headers.set('Authorization', `Bearer ${provider.KEY}`)
    headers.set('Content-Type', 'application/json')
    const response = await fetch(provider.ENDPOINT + "/models", {headers})
    let models = (await response.json()).data;
    await env.KV.put(provider.NAME, JSON.stringify(models));
    value = JSON.stringify(models);
  }
  return JSON.parse(value);
}


function corsHeaders() {
  return {
    'Access-Control-Allow-Origin': '*',
    'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
    'Access-Control-Allow-Headers': 'Content-Type, Authorization'
  }
}