diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 7ecf264add..f1ef4843a1 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -10,7 +10,7 @@ }, "dependencies": { "bulma": "^0.9.3", - "next": "14.1.1", + "next": "14.2.10", "node-fetch": "^3.2.10", "sass": "^1.52.3", "react": "17.0.2", diff --git a/examples/nextjs-api-sales/yarn.lock b/examples/nextjs-api-sales/yarn.lock index a44956ba21..9acbdfdeb6 100644 --- a/examples/nextjs-api-sales/yarn.lock +++ b/examples/nextjs-api-sales/yarn.lock @@ -46,10 +46,10 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== -"@next/env@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/env/-/env-14.1.1.tgz#80150a8440eb0022a73ba353c6088d419b908bac" - integrity sha512-7CnQyD5G8shHxQIIg3c7/pSeYFeMhsNbpU/bmvH7ZnDql7mNRgg8O2JZrhrc/soFnfBnKP4/xXNiiSIPn2w8gA== +"@next/env@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/env/-/env-14.2.10.tgz#1d3178340028ced2d679f84140877db4f420333c" + integrity sha512-dZIu93Bf5LUtluBXIv4woQw2cZVZ2DJTjax5/5DOs3lzEOeKLy7GxRSr4caK9/SCPdaW6bCgpye6+n4Dh9oJPw== "@next/eslint-plugin-next@12.1.0": version "12.1.0" @@ -58,50 +58,50 @@ dependencies: glob "7.1.7" -"@next/swc-darwin-arm64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.1.tgz#b74ba7c14af7d05fa2848bdeb8ee87716c939b64" - integrity sha512-yDjSFKQKTIjyT7cFv+DqQfW5jsD+tVxXTckSe1KIouKk75t1qZmj/mV3wzdmFb0XHVGtyRjDMulfVG8uCKemOQ== +"@next/swc-darwin-arm64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.10.tgz#49d10ca4086fbd59ee68e204f75d7136eda2aa80" + integrity sha512-V3z10NV+cvMAfxQUMhKgfQnPbjw+Ew3cnr64b0lr8MDiBJs3eLnM6RpGC46nhfMZsiXgQngCJKWGTC/yDcgrDQ== -"@next/swc-darwin-x64@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.1.tgz#82c3e67775e40094c66e76845d1a36cc29c9e78b" - integrity sha512-KCQmBL0CmFmN8D64FHIZVD9I4ugQsDBBEJKiblXGgwn7wBCSe8N4Dx47sdzl4JAg39IkSN5NNrr8AniXLMb3aw== +"@next/swc-darwin-x64@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.10.tgz#0ebeae3afb8eac433882b79543295ab83624a1a8" + integrity sha512-Y0TC+FXbFUQ2MQgimJ/7Ina2mXIKhE7F+GUe1SgnzRmwFY3hX2z8nyVCxE82I2RicspdkZnSWMn4oTjIKz4uzA== -"@next/swc-linux-arm64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.1.tgz#4f4134457b90adc5c3d167d07dfb713c632c0caa" - integrity sha512-YDQfbWyW0JMKhJf/T4eyFr4b3tceTorQ5w2n7I0mNVTFOvu6CGEzfwT3RSAQGTi/FFMTFcuspPec/7dFHuP7Eg== +"@next/swc-linux-arm64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.10.tgz#7e602916d2fb55a3c532f74bed926a0137c16f20" + integrity sha512-ZfQ7yOy5zyskSj9rFpa0Yd7gkrBnJTkYVSya95hX3zeBG9E55Z6OTNPn1j2BTFWvOVVj65C3T+qsjOyVI9DQpA== -"@next/swc-linux-arm64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.1.tgz#594bedafaeba4a56db23a48ffed2cef7cd09c31a" - integrity sha512-fiuN/OG6sNGRN/bRFxRvV5LyzLB8gaL8cbDH5o3mEiVwfcMzyE5T//ilMmaTrnA8HLMS6hoz4cHOu6Qcp9vxgQ== +"@next/swc-linux-arm64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.10.tgz#6b143f628ccee490b527562e934f8de578d4be47" + integrity sha512-n2i5o3y2jpBfXFRxDREr342BGIQCJbdAUi/K4q6Env3aSx8erM9VuKXHw5KNROK9ejFSPf0LhoSkU/ZiNdacpQ== -"@next/swc-linux-x64-gnu@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.1.tgz#cb4e75f1ff2b9bcadf2a50684605928ddfc58528" - integrity sha512-rv6AAdEXoezjbdfp3ouMuVqeLjE1Bin0AuE6qxE6V9g3Giz5/R3xpocHoAi7CufRR+lnkuUjRBn05SYJ83oKNQ== +"@next/swc-linux-x64-gnu@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.10.tgz#086f2f16a0678890a1eb46518c4dda381b046082" + integrity sha512-GXvajAWh2woTT0GKEDlkVhFNxhJS/XdDmrVHrPOA83pLzlGPQnixqxD8u3bBB9oATBKB//5e4vpACnx5Vaxdqg== -"@next/swc-linux-x64-musl@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.1.tgz#15f26800df941b94d06327f674819ab64b272e25" - integrity sha512-YAZLGsaNeChSrpz/G7MxO3TIBLaMN8QWMr3X8bt6rCvKovwU7GqQlDu99WdvF33kI8ZahvcdbFsy4jAFzFX7og== +"@next/swc-linux-x64-musl@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.10.tgz#1befef10ed8dbcc5047b5d637a25ae3c30a0bfc3" + integrity sha512-opFFN5B0SnO+HTz4Wq4HaylXGFV+iHrVxd3YvREUX9K+xfc4ePbRrxqOuPOFjtSuiVouwe6uLeDtabjEIbkmDA== -"@next/swc-win32-arm64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.1.tgz#060c134fa7fa843666e3e8574972b2b723773dd9" - integrity sha512-1L4mUYPBMvVDMZg1inUYyPvFSduot0g73hgfD9CODgbr4xiTYe0VOMTZzaRqYJYBA9mana0x4eaAaypmWo1r5A== +"@next/swc-win32-arm64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.10.tgz#731f52c3ae3c56a26cf21d474b11ae1529531209" + integrity sha512-9NUzZuR8WiXTvv+EiU/MXdcQ1XUvFixbLIMNQiVHuzs7ZIFrJDLJDaOF1KaqttoTujpcxljM/RNAOmw1GhPPQQ== -"@next/swc-win32-ia32-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.1.tgz#5c06889352b1f77e3807834a0d0afd7e2d2d1da2" - integrity sha512-jvIE9tsuj9vpbbXlR5YxrghRfMuG0Qm/nZ/1KDHc+y6FpnZ/apsgh+G6t15vefU0zp3WSpTMIdXRUsNl/7RSuw== +"@next/swc-win32-ia32-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.10.tgz#32723ef7f04e25be12af357cc72ddfdd42fd1041" + integrity sha512-fr3aEbSd1GeW3YUMBkWAu4hcdjZ6g4NBl1uku4gAn661tcxd1bHs1THWYzdsbTRLcCKLjrDZlNp6j2HTfrw+Bg== -"@next/swc-win32-x64-msvc@14.1.1": - version "14.1.1" - resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.1.tgz#d38c63a8f9b7f36c1470872797d3735b4a9c5c52" - integrity sha512-S6K6EHDU5+1KrBDLko7/c1MNy/Ya73pIAmvKeFwsF4RmBFJSO7/7YeD4FnZ4iBdzE69PpQ4sOMU9ORKeNuxe8A== +"@next/swc-win32-x64-msvc@14.2.10": + version "14.2.10" + resolved "https://registry.yarnpkg.com/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.10.tgz#ee1d036cb5ec871816f96baee7991035bb242455" + integrity sha512-UjeVoRGKNL2zfbcQ6fscmgjBAS/inHBh63mjIlfPg/NG8Yn2ztqylXt5qilYb6hoHIwaU2ogHknHWWmahJjgZQ== "@nodelib/fs.scandir@2.1.5": version "2.1.5" @@ -129,11 +129,17 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.1.0.tgz#7f698254aadf921e48dda8c0a6b304026b8a9323" integrity sha512-JLo+Y592QzIE+q7Dl2pMUtt4q8SKYI5jDrZxrozEQxnGVOyYE+GWK9eLkwTaeN9DDctlaRAQ3TBmzZ1qdLE30A== -"@swc/helpers@0.5.2": - version "0.5.2" - resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.2.tgz#85ea0c76450b61ad7d10a37050289eded783c27d" - integrity sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw== +"@swc/counter@^0.1.3": + version "0.1.3" + resolved "https://registry.yarnpkg.com/@swc/counter/-/counter-0.1.3.tgz#cc7463bd02949611c6329596fccd2b0ec782b0e9" + integrity sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ== + +"@swc/helpers@0.5.5": + version "0.5.5" + resolved "https://registry.yarnpkg.com/@swc/helpers/-/helpers-0.5.5.tgz#12689df71bfc9b21c4f4ca00ae55f2f16c8b77c0" + integrity sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A== dependencies: + "@swc/counter" "^0.1.3" tslib "^2.4.0" "@types/json5@^0.0.29": @@ -1245,28 +1251,28 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= -next@14.1.1: - version "14.1.1" - resolved "https://registry.yarnpkg.com/next/-/next-14.1.1.tgz#92bd603996c050422a738e90362dff758459a171" - integrity sha512-McrGJqlGSHeaz2yTRPkEucxQKe5Zq7uPwyeHNmJaZNY4wx9E9QdxmTp310agFRoMuIYgQrCrT3petg13fSVOww== +next@14.2.10: + version "14.2.10" + resolved "https://registry.yarnpkg.com/next/-/next-14.2.10.tgz#331981a4fecb1ae8af1817d4db98fc9687ee1cb6" + integrity sha512-sDDExXnh33cY3RkS9JuFEKaS4HmlWmDKP1VJioucCG6z5KuA008DPsDZOzi8UfqEk3Ii+2NCQSJrfbEWtZZfww== dependencies: - "@next/env" "14.1.1" - "@swc/helpers" "0.5.2" + "@next/env" "14.2.10" + "@swc/helpers" "0.5.5" busboy "1.6.0" caniuse-lite "^1.0.30001579" graceful-fs "^4.2.11" postcss "8.4.31" styled-jsx "5.1.1" optionalDependencies: - "@next/swc-darwin-arm64" "14.1.1" - "@next/swc-darwin-x64" "14.1.1" - "@next/swc-linux-arm64-gnu" "14.1.1" - "@next/swc-linux-arm64-musl" "14.1.1" - "@next/swc-linux-x64-gnu" "14.1.1" - "@next/swc-linux-x64-musl" "14.1.1" - "@next/swc-win32-arm64-msvc" "14.1.1" - "@next/swc-win32-ia32-msvc" "14.1.1" - "@next/swc-win32-x64-msvc" "14.1.1" + "@next/swc-darwin-arm64" "14.2.10" + "@next/swc-darwin-x64" "14.2.10" + "@next/swc-linux-arm64-gnu" "14.2.10" + "@next/swc-linux-arm64-musl" "14.2.10" + "@next/swc-linux-x64-gnu" "14.2.10" + "@next/swc-linux-x64-musl" "14.2.10" + "@next/swc-win32-arm64-msvc" "14.2.10" + "@next/swc-win32-ia32-msvc" "14.2.10" + "@next/swc-win32-x64-msvc" "14.2.10" node-domexception@^1.0.0: version "1.0.0" diff --git a/packages/backend-core/src/configs/configs.ts b/packages/backend-core/src/configs/configs.ts index 0d189e3f7d..e4f4a874a5 100644 --- a/packages/backend-core/src/configs/configs.ts +++ b/packages/backend-core/src/configs/configs.ts @@ -1,4 +1,5 @@ import { + AIConfig, Config, ConfigType, GoogleConfig, @@ -254,3 +255,9 @@ export async function getSCIMConfig(): Promise { const config = await getConfig(ConfigType.SCIM) return config?.config } + +// AI + +export async function getAIConfig(): Promise { + return getConfig(ConfigType.AI) +} diff --git a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte index 2ac1609e7c..b60ea24dbc 100644 --- a/packages/builder/src/pages/builder/portal/settings/ai/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/ai/index.svelte @@ -56,10 +56,13 @@ } else { // We don't store the default BB AI config in the DB delete fullAIConfig.config.budibase_ai + // unset the default value from other configs if default is set if (editingAIConfig.isDefault) { for (let key in fullAIConfig.config) { - fullAIConfig.config[key].isDefault = false + if (key !== id) { + fullAIConfig.config[key].isDefault = false + } } } // Add new or update existing custom AI Config diff --git a/packages/pro b/packages/pro index 922431260e..e2fe0f9cc8 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit 922431260e90d558a1ca55398475412e75088057 +Subproject commit e2fe0f9cc856b4ee1a97df96d623b2d87d4e8733 diff --git a/packages/server/package.json b/packages/server/package.json index 0b36d49c2e..76dd03b5a8 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -101,7 +101,7 @@ "mysql2": "3.9.8", "node-fetch": "2.6.7", "object-sizeof": "2.6.1", - "openai": "^4.52.1", + "openai": "4.59.0", "openapi-types": "9.3.1", "oracledb": "6.5.1", "pg": "8.10.0", diff --git a/packages/server/src/automations/steps/openai.ts b/packages/server/src/automations/steps/openai.ts index d02ba56b70..b1dfa3df5b 100644 --- a/packages/server/src/automations/steps/openai.ts +++ b/packages/server/src/automations/steps/openai.ts @@ -10,6 +10,7 @@ import { } from "@budibase/types" import { env } from "@budibase/backend-core" import * as automationUtils from "../automationUtils" +import * as pro from "@budibase/pro" enum Model { GPT_35_TURBO = "gpt-3.5-turbo", @@ -62,19 +63,33 @@ export const definition: AutomationStepDefinition = { }, } +/** + * Maintains backward compatibility with automation steps created before the introduction + * of custom configurations and Budibase AI + * @param inputs - automation inputs from the OpenAI automation step. + */ +async function legacyOpenAIPrompt(inputs: OpenAIStepInputs) { + const openai = new OpenAI({ + apiKey: env.OPENAI_API_KEY, + }) + + const completion = await openai.chat.completions.create({ + model: inputs.model, + messages: [ + { + role: "user", + content: inputs.prompt, + }, + ], + }) + return completion?.choices[0]?.message?.content +} + export async function run({ inputs, }: { inputs: OpenAIStepInputs }): Promise { - if (!env.OPENAI_API_KEY) { - return { - success: false, - response: - "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", - } - } - if (inputs.prompt == null) { return { success: false, @@ -83,20 +98,24 @@ export async function run({ } try { - const openai = new OpenAI({ - apiKey: env.OPENAI_API_KEY, - }) + let response + const customConfigsEnabled = await pro.features.isAICustomConfigsEnabled() + const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const completion = await openai.chat.completions.create({ - model: inputs.model, - messages: [ - { - role: "user", - content: inputs.prompt, - }, - ], - }) - const response = completion?.choices[0]?.message?.content + if (budibaseAIEnabled || customConfigsEnabled) { + const llm = await pro.ai.LargeLanguageModel.forCurrentTenant(inputs.model) + response = await llm.run(inputs.prompt) + } else { + // fallback to the default that uses the environment variable for backwards compat + if (!env.OPENAI_API_KEY) { + return { + success: false, + response: + "OpenAI API Key not configured - please add the OPENAI_API_KEY environment variable.", + } + } + response = await legacyOpenAIPrompt(inputs) + } return { response, diff --git a/packages/server/src/automations/tests/openai.spec.ts b/packages/server/src/automations/tests/openai.spec.ts index 3a5a57475a..6b5e1fbb6a 100644 --- a/packages/server/src/automations/tests/openai.spec.ts +++ b/packages/server/src/automations/tests/openai.spec.ts @@ -4,6 +4,7 @@ import { withEnv as withCoreEnv, setEnv as setCoreEnv, } from "@budibase/backend-core" +import * as pro from "@budibase/pro" jest.mock("openai", () => ({ OpenAI: jest.fn().mockImplementation(() => ({ @@ -22,7 +23,23 @@ jest.mock("openai", () => ({ }, })), })) +jest.mock("@budibase/pro", () => ({ + ...jest.requireActual("@budibase/pro"), + ai: { + LargeLanguageModel: { + forCurrentTenant: jest.fn().mockImplementation(() => ({ + init: jest.fn(), + run: jest.fn(), + })), + }, + }, + features: { + isAICustomConfigsEnabled: jest.fn(), + isBudibaseAIEnabled: jest.fn(), + }, +})) +const mockedPro = jest.mocked(pro) const mockedOpenAI = OpenAI as jest.MockedClass const OPENAI_PROMPT = "What is the meaning of life?" @@ -41,6 +58,7 @@ describe("test the openai action", () => { afterEach(() => { resetEnv() + jest.clearAllMocks() }) afterAll(_afterAll) @@ -94,4 +112,25 @@ describe("test the openai action", () => { ) expect(res.success).toBeFalsy() }) + + it("should ensure that the pro AI module is called when the budibase AI features are enabled", async () => { + jest.spyOn(pro.features, "isBudibaseAIEnabled").mockResolvedValue(true) + jest.spyOn(pro.features, "isAICustomConfigsEnabled").mockResolvedValue(true) + + const prompt = "What is the meaning of life?" + await runStep("OPENAI", { + model: "gpt-4o-mini", + prompt, + }) + + expect(pro.ai.LargeLanguageModel.forCurrentTenant).toHaveBeenCalledWith( + "gpt-4o-mini" + ) + + const llmInstance = + mockedPro.ai.LargeLanguageModel.forCurrentTenant.mock.results[0].value + // init does not appear to be called currently + // expect(llmInstance.init).toHaveBeenCalled() + expect(llmInstance.run).toHaveBeenCalledWith(prompt) + }) }) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6012ff7789..831528f84d 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -581,16 +581,15 @@ export class GoogleSheetsIntegration implements DatasourcePlus { rows = await sheet.getRows() } - if (hasFilters && query.paginate) { - rows = rows.slice(offset, offset + limit) - } - const headerValues = sheet.headerValues - let response = rows.map(row => - this.buildRowObject(headerValues, row.toObject(), row.rowNumber) + this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber) ) response = dataFilters.runQuery(response, query.filters || {}) + if (hasFilters && query.paginate) { + response = response.slice(offset, offset + limit) + } + if (query.sort) { if (Object.keys(query.sort).length !== 1) { console.warn("Googlesheets does not support multiple sorting", { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index dcf4a61b50..34be1c0c6c 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, + Row, SourceName, Table, TableSourceType, @@ -598,4 +599,193 @@ describe("Google Sheets Integration", () => { ) }) }) + + describe("search", () => { + let table: Table + + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Foo", + }, + { + name: "Bar", + }, + { + name: "Baz", + }, + ], + }) + }) + + it("should be able to find rows with equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + equal: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with not equals filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEqual: { + name: "Foo", + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should be able to find rows with empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + empty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(0) + }) + + it("should be able to find rows with not empty filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + notEmpty: { + name: null, + }, + }, + }) + + expect(response.rows).toHaveLength(3) + }) + + it("should be able to find rows with one of filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + oneOf: { + name: ["Foo", "Bar"], + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Foo") + expect(response.rows[1].name).toEqual("Bar") + }) + + it("should be able to find rows with fuzzy filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + fuzzy: { + name: "oo", + }, + }, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows[0].name).toEqual("Foo") + }) + + it("should be able to find rows with range filter", async () => { + const response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { + range: { + name: { + low: "A", + high: "C", + }, + }, + }, + }) + + expect(response.rows).toHaveLength(2) + expect(response.rows[0].name).toEqual("Bar") + expect(response.rows[1].name).toEqual("Baz") + }) + + it("should paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Unique value!`, + })), + }) + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 50 }, () => ({ + name: `Non-unique value!`, + })), + }) + + let response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + }) + let rows: Row[] = response.rows + + while (response.hasNextPage) { + response = await config.api.row.search(table._id!, { + tableId: table._id!, + query: { equal: { name: "Unique value!" } }, + paginate: true, + limit: 10, + bookmark: response.bookmark, + }) + + expect(response.rows.length).toBeLessThanOrEqual(10) + rows = rows.concat(response.rows) + } + + // Make sure we only get rows matching the query. + expect(rows.length).toEqual(50) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining( + Array.from({ length: 50 }, () => "Unique value!") + ) + ) + + // Make sure all of the rows have a unique ID. + const ids = Object.keys( + rows.reduce((acc, row) => { + acc[row._id!] = true + return acc + }, {}) + ) + expect(ids.length).toEqual(50) + }) + }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 4747f5f9bf..4b9445ebca 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -440,6 +440,8 @@ export class GoogleSheetsMock { endColumnIndex: 0, }) + sheet.properties.gridProperties.rowCount = sheet.data[0].rowData.length + return { spreadsheetId: this.spreadsheet.spreadsheetId, tableRange: range, diff --git a/packages/types/src/documents/global/config.ts b/packages/types/src/documents/global/config.ts index 8d64b49ee9..33f7e10584 100644 --- a/packages/types/src/documents/global/config.ts +++ b/packages/types/src/documents/global/config.ts @@ -111,7 +111,7 @@ export interface SCIMInnerConfig { export interface SCIMConfig extends Config {} -type AIProvider = "OpenAI" | "Anthropic" | "AzureOpenAI" | "Custom" +export type AIProvider = "OpenAI" | "Anthropic" | "TogetherAI" | "Custom" export interface AIInnerConfig { [key: string]: { diff --git a/packages/worker/src/api/controllers/global/configs.ts b/packages/worker/src/api/controllers/global/configs.ts index 70b2279f6c..e6e80ff3a5 100644 --- a/packages/worker/src/api/controllers/global/configs.ts +++ b/packages/worker/src/api/controllers/global/configs.ts @@ -253,6 +253,7 @@ export async function save(ctx: UserCtx) { if (existingConfig) { await verifyAIConfig(config, existingConfig) } + await pro.quotas.updateCustomAIConfigCount(Object.keys(config).length) break } } catch (err: any) { @@ -334,32 +335,6 @@ function enrichOIDCLogos(oidcLogos: OIDCLogosConfig) { ) } -async function enrichAIConfig(aiConfig: AIConfig) { - // Strip out the API Keys from the response so they don't show in the UI - for (const key in aiConfig.config) { - if (aiConfig.config[key].apiKey) { - aiConfig.config[key].apiKey = PASSWORD_REPLACEMENT - } - } - - // Return the Budibase AI data source as part of the response if licensing allows - const budibaseAIEnabled = await pro.features.isBudibaseAIEnabled() - const defaultConfigExists = Object.keys(aiConfig.config).some( - key => aiConfig.config[key].isDefault - ) - if (budibaseAIEnabled) { - aiConfig.config["budibase_ai"] = { - provider: "OpenAI", - active: true, - isDefault: !defaultConfigExists, - defaultModel: env.BUDIBASE_AI_DEFAULT_MODEL || "", - name: "Budibase AI", - } - } - - return aiConfig -} - export async function find(ctx: UserCtx) { try { // Find the config with the most granular scope based on context @@ -372,7 +347,13 @@ export async function find(ctx: UserCtx) { } if (type === ConfigType.AI) { - await enrichAIConfig(scopedConfig) + await pro.sdk.ai.enrichAIConfig(scopedConfig) + // Strip out the API Keys from the response so they don't show in the UI + for (const key in scopedConfig.config) { + if (scopedConfig.config[key].apiKey) { + scopedConfig.config[key].apiKey = PASSWORD_REPLACEMENT + } + } } ctx.body = scopedConfig } else { diff --git a/packages/worker/src/api/controllers/global/tests/configs.spec.ts b/packages/worker/src/api/controllers/global/tests/configs.spec.ts index 3ff6a5298c..9091f29247 100644 --- a/packages/worker/src/api/controllers/global/tests/configs.spec.ts +++ b/packages/worker/src/api/controllers/global/tests/configs.spec.ts @@ -1,4 +1,3 @@ -import * as pro from "@budibase/pro" import { verifyAIConfig } from "../configs" import { TestConfiguration, structures } from "../../../../tests" import { AIInnerConfig } from "@budibase/types" @@ -35,55 +34,6 @@ describe("Global configs controller", () => { }) }) - it("Should return the default BB AI config when the feature is turned on", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(true)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - budibase_ai: { - provider: "OpenAI", - active: true, - isDefault: true, - name: "Budibase AI", - defaultModel: "", - }, - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - - it("Should not not return the default Budibase AI config when on self host", async () => { - jest - .spyOn(pro.features, "isBudibaseAIEnabled") - .mockImplementation(() => Promise.resolve(false)) - const data = structures.configs.ai() - await config.api.configs.saveConfig(data) - const response = await config.api.configs.getAIConfig() - - expect(response.body.config).toEqual({ - ai: { - active: true, - apiKey: "--secret-value--", - baseUrl: "https://api.example.com", - defaultModel: "gpt4", - isDefault: false, - name: "Test", - provider: "OpenAI", - }, - }) - }) - it("Should not update existing secrets when updating an existing AI Config", async () => { const data = structures.configs.ai() await config.api.configs.saveConfig(data) diff --git a/yarn.lock b/yarn.lock index 34c9cb15d9..cd850e833d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -33,6 +33,19 @@ "@jridgewell/gen-mapping" "^0.3.5" "@jridgewell/trace-mapping" "^0.3.24" +"@anthropic-ai/sdk@^0.27.3": + version "0.27.3" + resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.27.3.tgz#592cdd873c85ffab9589ae6f2e250cbf150e1475" + integrity sha512-IjLt0gd3L4jlOfilxVXTifn42FnVffMgDC04RJK1KDZpmkBWLv0XC92MVVmkxrFZNS/7l3xWgP/I3nqtX1sQHw== + dependencies: + "@types/node" "^18.11.18" + "@types/node-fetch" "^2.6.4" + abort-controller "^3.0.0" + agentkeepalive "^4.2.1" + form-data-encoder "1.7.2" + formdata-node "^4.3.2" + node-fetch "^2.6.7" + "@apidevtools/json-schema-ref-parser@^9.0.6": version "9.1.2" resolved "https://registry.yarnpkg.com/@apidevtools/json-schema-ref-parser/-/json-schema-ref-parser-9.1.2.tgz#8ff5386b365d4c9faa7c8b566ff16a46a577d9b8" @@ -2053,7 +2066,7 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@2.32.6": +"@budibase/backend-core@2.32.5": version "0.0.0" dependencies: "@budibase/nano" "10.1.5" @@ -2134,14 +2147,14 @@ through2 "^2.0.0" "@budibase/pro@npm:@budibase/pro@latest": - version "2.32.6" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.6.tgz#02ddef737ee8f52dafd8fab8f8f277dfc89cd33f" - integrity sha512-+XEv4JtMvUKZWyllcw+iFOh44zxsoJLmUdShu4bAjj5zXWgElF6LjFpK51IrQzM6xKfQxn7N2vmxu7175u5dDQ== + version "2.32.5" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.32.5.tgz#2beecf566da972a92200faddc97bc152ea2bbdea" + integrity sha512-afrklI2A8P7pfl/3KxysqO2Sjr0l2yQ1+jyuouEZliEklLxV8AFlzrODr4V2SK3J8E1xk8wG5ztYQS2uT7TnuA== dependencies: - "@budibase/backend-core" "2.32.6" - "@budibase/shared-core" "2.32.6" - "@budibase/string-templates" "2.32.6" - "@budibase/types" "2.32.6" + "@budibase/backend-core" "2.32.5" + "@budibase/shared-core" "2.32.5" + "@budibase/string-templates" "2.32.5" + "@budibase/types" "2.32.5" "@koa/router" "8.0.8" bull "4.10.1" dd-trace "5.2.0" @@ -2153,13 +2166,13 @@ scim-patch "^0.8.1" scim2-parse-filter "^0.2.8" -"@budibase/shared-core@2.32.6": +"@budibase/shared-core@2.32.5": version "0.0.0" dependencies: "@budibase/types" "0.0.0" cron-validate "1.4.5" -"@budibase/string-templates@2.32.6": +"@budibase/string-templates@2.32.5": version "0.0.0" dependencies: "@budibase/handlebars-helpers" "^0.13.2" @@ -2167,7 +2180,7 @@ handlebars "^4.7.8" lodash.clonedeep "^4.5.0" -"@budibase/types@2.32.6": +"@budibase/types@2.32.5": version "0.0.0" dependencies: scim-patch "^0.8.1" @@ -6117,6 +6130,11 @@ resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.7.tgz#63bb7d067db107cc1e457c303bc25d511febf6cb" integrity sha512-FGa1F62FT09qcrueBA6qYTrJPVDzah9a+493+o2PCXsesWHIn27G98TsSMs3WPNbZIEj4+VJf6saSFpvD+3Zsw== +"@types/qs@^6.9.15": + version "6.9.16" + resolved "https://registry.yarnpkg.com/@types/qs/-/qs-6.9.16.tgz#52bba125a07c0482d26747d5d4947a64daf8f794" + integrity sha512-7i+zxXdPD0T4cKDuxCUXJ4wHcsJLwENa6Z3dCu8cfCK743OGy5Nu1RmAGqDPsoTDINVEcdXKRvR/zre+P2Ku1A== + "@types/range-parser@*": version "1.2.4" resolved "https://registry.yarnpkg.com/@types/range-parser/-/range-parser-1.2.4.tgz#cd667bcfdd025213aafb7ca5915a932590acdcdc" @@ -17097,19 +17115,20 @@ open@^8.0.0, open@^8.4.0, open@~8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" -openai@^4.52.1: - version "4.52.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.52.1.tgz#44acc362a844fa2927b0cfa1fb70fb51e388af65" - integrity sha512-kv2hevAWZZ3I/vd2t8znGO2rd8wkowncsfcYpo8i+wU9ML+JEcdqiViANXXjWWGjIhajFNixE6gOY1fEgqILAg== +openai@4.59.0: + version "4.59.0" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.59.0.tgz#3961d11a9afb5920e1bd475948a87969e244fc08" + integrity sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4" + "@types/qs" "^6.9.15" abort-controller "^3.0.0" agentkeepalive "^4.2.1" form-data-encoder "1.7.2" formdata-node "^4.3.2" node-fetch "^2.6.7" - web-streams-polyfill "^3.2.1" + qs "^6.10.3" openapi-response-validator@^9.2.0: version "9.3.1" @@ -20767,7 +20786,16 @@ string-similarity@^4.0.4: resolved "https://registry.yarnpkg.com/string-similarity/-/string-similarity-4.0.4.tgz#42d01ab0b34660ea8a018da8f56a3309bb8b2a5b" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0": + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + +"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.2, string-width@^4.2.3: version "4.2.3" resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -20858,7 +20886,7 @@ stringify-object@^3.2.1: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: +"strip-ansi-cjs@npm:strip-ansi@^6.0.1": version "6.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -20872,6 +20900,13 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + strip-ansi@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2" @@ -22599,11 +22634,6 @@ web-streams-polyfill@4.0.0-beta.3: resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38" integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug== -web-streams-polyfill@^3.2.1: - version "3.3.3" - resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz#2073b91a2fdb1fbfbd401e7de0ac9f8214cecb4b" - integrity sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw== - web-vitals@^4.0.1: version "4.2.3" resolved "https://registry.yarnpkg.com/web-vitals/-/web-vitals-4.2.3.tgz#270c4baecfbc6ec6fc15da1989e465e5f9b94fb7" @@ -22832,7 +22862,7 @@ worker-farm@1.7.0: dependencies: errno "~0.1.7" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": version "7.0.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -22850,6 +22880,15 @@ wrap-ansi@^5.1.0: string-width "^3.0.0" strip-ansi "^5.0.0" +wrap-ansi@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43" + integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== + dependencies: + ansi-styles "^4.0.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"