Skip to content

Commit

Permalink
⚡️ perf: improve tool calling streaming (lobehub#2460)
Browse files Browse the repository at this point in the history
* 🐛 fix: fix token count

* ⚡️ perf: reduce plugin store fetcher

* ⚡️ perf: support tool calling smoothing

* ✅ test: fix test

* ♻️ refactor: refactor the dalle generation

* 🚸 style: improve dalle plugin error handle

* ✅ test: fix test

* ✅ test: fix test
  • Loading branch information
arvinxx authored May 12, 2024
1 parent 24c81f2 commit 2a252f9
Show file tree
Hide file tree
Showing 56 changed files with 1,101 additions and 433 deletions.
58 changes: 29 additions & 29 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -81,25 +81,25 @@
]
},
"dependencies": {
"@ant-design/icons": "^5.3.6",
"@ant-design/icons": "^5.3.7",
"@anthropic-ai/sdk": "^0.20.9",
"@auth/core": "0.28.0",
"@aws-sdk/client-bedrock-runtime": "^3.565.0",
"@azure/openai": "^1.0.0-beta.12",
"@aws-sdk/client-bedrock-runtime": "^3.574.0",
"@azure/openai": "1.0.0-beta.12",
"@cfworker/json-schema": "^1.12.8",
"@clerk/localizations": "2.0.0",
"@clerk/nextjs": "^5.0.6",
"@clerk/themes": "^2.0.0",
"@clerk/nextjs": "^5.0.8",
"@clerk/themes": "^2.1.3",
"@google/generative-ai": "^0.10.0",
"@icons-pack/react-simple-icons": "^9.4.1",
"@icons-pack/react-simple-icons": "^9.5.0",
"@lobehub/chat-plugin-sdk": "latest",
"@lobehub/chat-plugins-gateway": "latest",
"@lobehub/icons": "latest",
"@lobehub/tts": "latest",
"@lobehub/ui": "^1.138.17",
"@lobehub/ui": "^1.138.23",
"@microsoft/fetch-event-source": "^2.0.1",
"@next/third-parties": "^14.2.3",
"@sentry/nextjs": "^7.112.2",
"@sentry/nextjs": "^7.114.0",
"@t3-oss/env-nextjs": "^0.10.1",
"@trpc/client": "next",
"@trpc/next": "next",
Expand All @@ -118,15 +118,15 @@
"diff": "^5.2.0",
"fast-deep-equal": "^3.1.3",
"gpt-tokenizer": "^2.1.2",
"i18next": "^23.11.3",
"i18next": "^23.11.4",
"i18next-browser-languagedetector": "^7.2.1",
"i18next-resources-to-backend": "^1.2.1",
"idb-keyval": "^6.2.1",
"immer": "^10.1.1",
"ip": "^2.0.1",
"jose": "^5.2.4",
"langfuse": "^3.8.0",
"langfuse-core": "^3.8.0",
"jose": "^5.3.0",
"langfuse": "^3.10.0",
"langfuse-core": "^3.10.0",
"lodash-es": "^4.17.21",
"lucide-react": "latest",
"modern-screenshot": "^4.4.39",
Expand All @@ -135,12 +135,12 @@
"next-auth": "5.0.0-beta.15",
"next-sitemap": "^4.2.3",
"numeral": "^2.0.6",
"nuqs": "^1.17.1",
"ollama": "^0.5.0",
"openai": "^4.39.0",
"nuqs": "^1.17.2",
"ollama": "^0.5.1",
"openai": "^4.45.0",
"pino": "^9.0.0",
"polished": "^4.3.1",
"posthog-js": "^1.130.1",
"posthog-js": "^1.131.4",
"query-string": "^9.0.0",
"random-words": "^2.0.1",
"react": "^18.3.1",
Expand All @@ -155,7 +155,7 @@
"remark-gfm": "^3.0.1",
"remark-html": "^15.0.2",
"rtl-detect": "^1.1.2",
"semver": "^7.6.0",
"semver": "^7.6.2",
"sharp": "^0.33.3",
"superjson": "^2.2.1",
"swr": "^2.2.5",
Expand All @@ -170,13 +170,13 @@
"y-webrtc": "^10.3.0",
"yaml": "^2.4.2",
"yjs": "^13.6.15",
"zod": "^3.23.5",
"zod": "^3.23.8",
"zustand": "^4.5.2",
"zustand-utils": "^1.3.2"
},
"devDependencies": {
"@commitlint/cli": "^19.3.0",
"@ducanh2912/next-pwa": "^10.2.6",
"@ducanh2912/next-pwa": "^10.2.7",
"@edge-runtime/vm": "^3.2.0",
"@lobehub/i18n-cli": "^1.18.1",
"@lobehub/lint": "^1.23.4",
Expand All @@ -185,24 +185,24 @@
"@next/eslint-plugin-next": "^14.2.3",
"@peculiar/webcrypto": "^1.4.6",
"@testing-library/jest-dom": "^6.4.5",
"@testing-library/react": "^15.0.6",
"@testing-library/react": "^15.0.7",
"@types/chroma-js": "^2.4.4",
"@types/debug": "^4.1.12",
"@types/diff": "^5.2.0",
"@types/diff": "^5.2.1",
"@types/ip": "^1.1.3",
"@types/json-schema": "^7.0.15",
"@types/lodash": "^4.17.0",
"@types/lodash": "^4.17.1",
"@types/lodash-es": "^4.17.12",
"@types/node": "^20.12.7",
"@types/node": "^20.12.11",
"@types/numeral": "^2.0.5",
"@types/react": "^18.3.1",
"@types/react": "^18.3.2",
"@types/react-dom": "^18.3.0",
"@types/rtl-detect": "^1.0.3",
"@types/semver": "^7.5.8",
"@types/systemjs": "^6.13.5",
"@types/ua-parser-js": "^0.7.39",
"@types/uuid": "^9.0.8",
"@umijs/lint": "^4.1.10",
"@umijs/lint": "^4.2.2",
"@vitest/coverage-v8": "~1.2.2",
"ajv-keywords": "^5.1.0",
"commitlint": "^19.3.0",
Expand All @@ -211,9 +211,9 @@
"eslint": "^8.57.0",
"eslint-plugin-mdx": "^2.3.4",
"fake-indexeddb": "^5.0.2",
"glob": "^10.3.12",
"glob": "^10.3.15",
"gray-matter": "^4.0.3",
"happy-dom": "^14.7.1",
"happy-dom": "^14.10.1",
"husky": "^9.0.11",
"just-diff": "^6.0.2",
"lint-staged": "^15.2.2",
Expand All @@ -227,11 +227,11 @@
"remark-parse": "^10.0.2",
"semantic-release": "^21.1.2",
"stylelint": "^15.11.0",
"tsx": "^4.7.3",
"tsx": "^4.10.0",
"typescript": "^5.4.5",
"unified": "^11.0.4",
"unist-util-visit": "^5.0.0",
"vite": "^5.2.10",
"vite": "^5.2.11",
"vitest": "~1.2.2",
"vitest-canvas-mock": "^0.3.3"
},
Expand Down
4 changes: 2 additions & 2 deletions src/app/api/chat/[provider]/route.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,18 @@
import { getAuth } from '@clerk/nextjs/server';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';

import { checkAuthMethod, getJWTPayload } from '@/app/api/middleware/auth/utils';
import { LOBE_CHAT_AUTH_HEADER, OAUTH_AUTHORIZED } from '@/const/auth';
import { AgentRuntime, LobeRuntimeAI } from '@/libs/agent-runtime';
import { ChatErrorType } from '@/types/fetch';

import { checkAuthMethod, getJWTPayload } from '../auth/utils';
import { POST } from './route';

vi.mock('@clerk/nextjs/server', () => ({
getAuth: vi.fn(),
}));

vi.mock('../auth/utils', () => ({
vi.mock('../../middleware/auth/utils', () => ({
getJWTPayload: vi.fn(),
checkAuthMethod: vi.fn(),
}));
Expand Down
2 changes: 1 addition & 1 deletion src/app/api/chat/[provider]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { ChatErrorType } from '@/types/fetch';
import { ChatStreamPayload } from '@/types/openai/chat';
import { getTracePayload } from '@/utils/trace';

import { checkAuth } from '../../middleware/auth';
import { createTraceOptions, initAgentRuntimeWithUserPayload } from '../agentRuntime';
import { checkAuth } from '../auth';

export const runtime = 'edge';

Expand Down
2 changes: 1 addition & 1 deletion src/app/api/chat/models/[provider]/route.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ import { createErrorResponse } from '@/app/api/errorResponse';
import { ChatCompletionErrorPayload, ModelProvider } from '@/libs/agent-runtime';
import { ChatErrorType } from '@/types/fetch';

import { checkAuth } from '../../../middleware/auth';
import { initAgentRuntimeWithUserPayload } from '../../agentRuntime';
import { checkAuth } from '../../auth';

export const runtime = 'edge';

Expand Down
52 changes: 1 addition & 51 deletions src/app/api/config.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import { describe, expect, it, vi } from 'vitest';

import { checkAuth } from './auth';
import { getPreferredRegion } from './config';
import { checkAuth } from './openai/createBizOpenAI/auth';

// Stub the global process object to safely mock environment variables
vi.stubGlobal('process', {
Expand Down Expand Up @@ -41,53 +41,3 @@ describe('getPreferredRegion', () => {
expect(preferredRegion).toStrictEqual(['ida1', 'sfo1']);
});
});

describe('ACCESS_CODE', () => {
let auth = false;

beforeEach(() => {
auth = false;
process.env.ACCESS_CODE = undefined;
// Reset environment variables before each test case
vi.restoreAllMocks();
});

it('set multiple access codes', () => {
process.env.ACCESS_CODE = ',code1,code2,code3';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({ accessCode: 'code2' }));
expect(auth).toBe(true);
({ auth } = checkAuth({ accessCode: 'code1,code2' }));
expect(auth).toBe(false);
});

it('set individual access code', () => {
process.env.ACCESS_CODE = 'code1';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({ accessCode: 'code2' }));
expect(auth).toBe(false);
});

it('no access code', () => {
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({}));
expect(auth).toBe(true);
});

it('empty access code', () => {
process.env.ACCESS_CODE = '';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({}));
expect(auth).toBe(true);

process.env.ACCESS_CODE = ',,';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({}));
expect(auth).toBe(true);
});
});
File renamed without changes.
File renamed without changes.
File renamed without changes.
52 changes: 52 additions & 0 deletions src/app/api/openai/createBizOpenAI/auth.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
import { checkAuth } from './auth';

describe('ACCESS_CODE', () => {
let auth = false;

beforeEach(() => {
auth = false;
process.env.ACCESS_CODE = undefined;
// Reset environment variables before each test case
vi.restoreAllMocks();
});

it('set multiple access codes', () => {
process.env.ACCESS_CODE = ',code1,code2,code3';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({ accessCode: 'code2' }));
expect(auth).toBe(true);
({ auth } = checkAuth({ accessCode: 'code1,code2' }));
expect(auth).toBe(false);
});

it('set individual access code', () => {
process.env.ACCESS_CODE = 'code1';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({ accessCode: 'code2' }));
expect(auth).toBe(false);
});

it('no access code', () => {
delete process.env.ACCESS_CODE;
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({}));
expect(auth).toBe(true);
});

it('empty access code', () => {
process.env.ACCESS_CODE = '';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({}));
expect(auth).toBe(true);

process.env.ACCESS_CODE = ',,';
({ auth } = checkAuth({ accessCode: 'code1' }));
expect(auth).toBe(true);
({ auth } = checkAuth({}));
expect(auth).toBe(true);
});
});
File renamed without changes.
2 changes: 1 addition & 1 deletion src/app/api/openai/createBizOpenAI/index.ts
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
import OpenAI from 'openai';

import { checkAuth } from '@/app/api/auth';
import { getOpenAIAuthFromRequest } from '@/const/fetch';
import { ChatErrorType, ErrorType } from '@/types/fetch';

import { createErrorResponse } from '../../errorResponse';
import { checkAuth } from './auth';
import { createOpenai } from './createOpenai';

/**
Expand Down
26 changes: 0 additions & 26 deletions src/app/api/openai/images/createImageGeneration.ts

This file was deleted.

16 changes: 0 additions & 16 deletions src/app/api/openai/images/route.ts

This file was deleted.

2 changes: 1 addition & 1 deletion src/app/api/plugin/gateway/route.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
import { PluginRequestPayload } from '@lobehub/chat-plugin-sdk';
import { createGatewayOnEdgeRuntime } from '@lobehub/chat-plugins-gateway';

import { getJWTPayload } from '@/app/api/chat/auth/utils';
import { createErrorResponse } from '@/app/api/errorResponse';
import { getJWTPayload } from '@/app/api/middleware/auth/utils';
import { getServerConfig } from '@/config/server';
import { LOBE_CHAT_AUTH_HEADER, OAUTH_AUTHORIZED, enableNextAuth } from '@/const/auth';
import { LOBE_CHAT_TRACE_ID, TraceNameMap } from '@/const/trace';
Expand Down
Loading

0 comments on commit 2a252f9

Please sign in to comment.