diff --git a/.eslintignore b/.eslintignore index f2c53c2fdc..94984a446f 100644 --- a/.eslintignore +++ b/.eslintignore @@ -12,4 +12,5 @@ packages/sdk/sdk packages/account-portal/packages/server/build packages/account-portal/packages/ui/.routify packages/account-portal/packages/ui/build -**/*.ivm.bundle.js \ No newline at end of file +**/*.ivm.bundle.js +packages/server/build/oldClientVersions/**/** diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 94a001ba96..030ad6578e 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -66,7 +66,8 @@ jobs: # Run build all the projects - name: Build run: | - yarn build + yarn build:oss + yarn build:account-portal # Check the types of the projects built via esbuild - name: Check types run: | @@ -161,7 +162,7 @@ jobs: docker pull postgres:16.1-bullseye docker pull mongo:7.0-jammy docker pull mariadb:lts - docker pull testcontainers/ryuk:0.3.0 + docker pull testcontainers/ryuk:0.5.1 docker pull budibase/couchdb - run: yarn --frozen-lockfile @@ -231,27 +232,34 @@ jobs: echo "pro_commit=$pro_commit" echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT" echo "base_commit=$base_commit" - echo "base_commit=$base_commit" >> "$GITHUB_OUTPUT" + + base_commit_excluding_merges=$(git log --no-merges -n 1 --format=format:%H $base_commit) + echo "base_commit_excluding_merges=$base_commit_excluding_merges" + echo "base_commit_excluding_merges=$base_commit_excluding_merges" >> "$GITHUB_OUTPUT" else echo "Nothing to do - branch to branch merge." fi - - name: Check submodule merged to base branch - if: ${{ steps.get_pro_commits.outputs.base_commit != '' }} - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const submoduleCommit = '${{ steps.get_pro_commits.outputs.pro_commit }}'; - const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}'; + - name: Check submodule merged and latest on base branch + if: ${{ steps.get_pro_commits.outputs.base_commit_excluding_merges != '' }} + run: | + cd packages/pro + base_commit_excluding_merges='${{ steps.get_pro_commits.outputs.base_commit_excluding_merges }}' + pro_commit='${{ steps.get_pro_commits.outputs.pro_commit }}' - if (submoduleCommit !== baseCommit) { - console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.'); - console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md') - process.exit(1); - } else { - console.log('All good, the submodule had been merged and setup correctly!') - } + any_commit=$(git log --no-merges $base_commit_excluding_merges...$pro_commit) + + if [ -n "$any_commit" ]; then + echo $any_commit + + echo "An error occurred: " + echo 'Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}" branch.' + echo 'Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/master/docs/getting_started.md' + + exit 1 + else + echo 'All good, the submodule had been merged and setup correctly!' + fi check-accountportal-submodule: runs-on: ubuntu-latest @@ -264,7 +272,15 @@ jobs: token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }} fetch-depth: 0 - - name: Check account portal commit + - uses: dorny/paths-filter@v3 + id: changes + with: + filters: | + src: + - packages/account-portal/** + + - if: steps.changes.outputs.src == 'true' + name: Check account portal commit id: get_accountportal_commits run: | cd packages/account-portal diff --git a/.gitignore b/.gitignore index 8861a14d20..661c60e95e 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,9 @@ packages/server/runtime_apps/ bb-airgapped.tar.gz *.iml +packages/server/build/oldClientVersions/**/* +packages/builder/src/components/deploy/clientVersions.json + # Logs logs *.log diff --git a/.vscode/launch.json b/.vscode/launch.json index cfd8d7b155..2fda61345b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,4 +1,3 @@ - { // Use IntelliSense to learn about possible attributes. // Hover to view descriptions of existing attributes. @@ -20,6 +19,13 @@ "runtimeArgs": ["--nolazy", "-r", "ts-node/register/transpile-only"], "args": ["${workspaceFolder}/packages/worker/src/index.ts"], "cwd": "${workspaceFolder}/packages/worker" + }, + { + "type": "chrome", + "request": "launch", + "name": "Launch Chrome against localhost", + "url": "http://localhost:10000", + "webRoot": "${workspaceFolder}" } ], "compounds": [ diff --git a/globalSetup.ts b/globalSetup.ts new file mode 100644 index 0000000000..4cb542a3c3 --- /dev/null +++ b/globalSetup.ts @@ -0,0 +1,25 @@ +import { GenericContainer, Wait } from "testcontainers" + +export default async function setup() { + await new GenericContainer("budibase/couchdb") + .withExposedPorts(5984) + .withEnvironment({ + COUCHDB_PASSWORD: "budibase", + COUCHDB_USER: "budibase", + }) + .withCopyContentToContainer([ + { + content: ` + [log] + level = warn + `, + target: "/opt/couchdb/etc/local.d/test-couchdb.ini", + }, + ]) + .withWaitStrategy( + Wait.forSuccessfulCommand( + "curl http://budibase:budibase@localhost:5984/_up" + ).withStartupTimeout(20000) + ) + .start() +} diff --git a/jestTestcontainersConfigGenerator.js b/jestTestcontainersConfigGenerator.js deleted file mode 100644 index 1e39ed771f..0000000000 --- a/jestTestcontainersConfigGenerator.js +++ /dev/null @@ -1,16 +0,0 @@ -module.exports = () => { - return { - couchdb: { - image: "budibase/couchdb", - ports: [5984], - env: { - COUCHDB_PASSWORD: "budibase", - COUCHDB_USER: "budibase", - }, - wait: { - type: "ports", - timeout: 20000, - } - } - } -} diff --git a/lerna.json b/lerna.json index 9cffdba08a..bacdcb782f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.22.12", + "version": "2.22.13", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/package.json b/package.json index 7de22ab456..32693a0b6f 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,11 @@ }, "scripts": { "preinstall": "node scripts/syncProPackage.js", + "get-past-client-version": "node scripts/getPastClientVersion.js", "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn build && yarn dev", "build": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream", + "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", + "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", "check:types": "lerna run check:types", "build:sdk": "lerna run --stream build:sdk", diff --git a/packages/account-portal b/packages/account-portal index f5b467b6b1..63ce32bca8 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit f5b467b6b1c55c48847545db41be7b1c035e167a +Subproject commit 63ce32bca871f0a752323f5f7ebb5ec16bbbacc3 diff --git a/packages/backend-core/jest-testcontainers-config.js b/packages/backend-core/jest-testcontainers-config.js deleted file mode 100644 index 8ac0f0cd9d..0000000000 --- a/packages/backend-core/jest-testcontainers-config.js +++ /dev/null @@ -1,8 +0,0 @@ -const { join } = require("path") -require("dotenv").config({ - path: join(__dirname, "..", "..", "hosting", ".env"), -}) - -const jestTestcontainersConfigGenerator = require("../../jestTestcontainersConfigGenerator") - -module.exports = jestTestcontainersConfigGenerator() diff --git a/packages/backend-core/jest.config.ts b/packages/backend-core/jest.config.ts index 3f1065ead2..c944b0d7e1 100644 --- a/packages/backend-core/jest.config.ts +++ b/packages/backend-core/jest.config.ts @@ -1,8 +1,8 @@ import { Config } from "@jest/types" const baseConfig: Config.InitialProjectOptions = { - preset: "@trendyol/jest-testcontainers", setupFiles: ["./tests/jestEnv.ts"], + globalSetup: "./../../globalSetup.ts", setupFilesAfterEnv: ["./tests/jestSetup.ts"], transform: { "^.+\\.ts?$": "@swc/jest", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index fe56780982..030fec8728 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -60,7 +60,6 @@ "@shopify/jest-koa-mocks": "5.1.1", "@swc/core": "1.3.71", "@swc/jest": "0.2.27", - "@trendyol/jest-testcontainers": "^2.1.1", "@types/chance": "1.1.3", "@types/cookies": "0.7.8", "@types/jest": "29.5.5", diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index 9c960d76dd..735c2fa86e 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -27,7 +27,7 @@ class Replication { return resolve(info) }) .on("error", function (err) { - throw new Error(`Replication Error: ${err}`) + throw err }) }) } diff --git a/packages/backend-core/src/events/events.ts b/packages/backend-core/src/events/events.ts index f02b9fdf32..92b81553b0 100644 --- a/packages/backend-core/src/events/events.ts +++ b/packages/backend-core/src/events/events.ts @@ -1,4 +1,4 @@ -import { Event } from "@budibase/types" +import { Event, Identity } from "@budibase/types" import { processors } from "./processors" import identification from "./identification" import * as backfill from "./backfill" @@ -7,12 +7,19 @@ import { publishAsyncEvent } from "./asyncEvents" export const publishEvent = async ( event: Event, properties: any, - timestamp?: string | number + timestamp?: string | number, + identityOverride?: Identity ) => { // in future this should use async events via a distributed queue. - const identity = await identification.getCurrentIdentity() + const identity = + identityOverride || (await identification.getCurrentIdentity()) + + // Backfilling is get from the user cache, but when we override the identity cache is not available. Overrides are + // normally performed in automatic actions or operations in async flows (BPM) where the user session is not available. + const backfilling = identityOverride + ? false + : await backfill.isBackfillingEvent(event) - const backfilling = await backfill.isBackfillingEvent(event) // no backfill - send the event and exit if (!backfilling) { // send off async events if required diff --git a/packages/backend-core/src/events/publishers/account.ts b/packages/backend-core/src/events/publishers/account.ts index d337e404ef..99767962dd 100644 --- a/packages/backend-core/src/events/publishers/account.ts +++ b/packages/backend-core/src/events/publishers/account.ts @@ -5,13 +5,19 @@ import { AccountCreatedEvent, AccountDeletedEvent, AccountVerifiedEvent, + Identity, } from "@budibase/types" -async function created(account: Account) { +async function created(account: Account, identityOverride?: Identity) { const properties: AccountCreatedEvent = { tenantId: account.tenantId, } - await publishEvent(Event.ACCOUNT_CREATED, properties) + await publishEvent( + Event.ACCOUNT_CREATED, + properties, + undefined, + identityOverride + ) } async function deleted(account: Account) { diff --git a/packages/backend-core/src/platform/users.ts b/packages/backend-core/src/platform/users.ts index 6f030afb7c..ccaad76b19 100644 --- a/packages/backend-core/src/platform/users.ts +++ b/packages/backend-core/src/platform/users.ts @@ -20,7 +20,7 @@ export async function lookupTenantId(userId: string) { return user.tenantId } -async function getUserDoc(emailOrId: string): Promise { +export async function getUserDoc(emailOrId: string): Promise { const db = getPlatformDB() return db.get(emailOrId) } @@ -79,6 +79,17 @@ async function addUserDoc(emailOrId: string, newDocFn: () => PlatformUser) { } } +export async function addSsoUser( + ssoId: string, + email: string, + userId: string, + tenantId: string +) { + return addUserDoc(ssoId, () => + newUserSsoIdDoc(ssoId, email, userId, tenantId) + ) +} + export async function addUser( tenantId: string, userId: string, @@ -91,9 +102,7 @@ export async function addUser( ] if (ssoId) { - promises.push( - addUserDoc(ssoId, () => newUserSsoIdDoc(ssoId, email, userId, tenantId)) - ) + promises.push(addSsoUser(ssoId, email, userId, tenantId)) } await Promise.all(promises) diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 136cb4b8ad..04d3264e6f 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -500,13 +500,13 @@ export class UserDB { static async createAdminUser( email: string, - password: string, tenantId: string, + password?: string, opts?: CreateAdminUserOpts ) { const user: User = { email: email, - password: password, + password, createdAt: Date.now(), roles: {}, builder: { diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 7da6cbc777..5d4f5a3c11 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -1,80 +1,58 @@ +import { DatabaseImpl } from "../../../src/db" import { execSync } from "child_process" -let dockerPsResult: string | undefined - -function formatDockerPsResult(serverName: string, port: number) { - const lines = dockerPsResult?.split("\n") - let first = true - if (!lines) { - return null - } - for (let line of lines) { - if (first) { - first = false - continue - } - let toLookFor = serverName.split("-service")[0] - if (!line.includes(toLookFor)) { - continue - } - const regex = new RegExp(`0.0.0.0:([0-9]*)->${port}`, "g") - const found = line.match(regex) - if (found) { - return found[0].split(":")[1].split("->")[0] - } - } - return null +interface ContainerInfo { + Command: string + CreatedAt: string + ID: string + Image: string + Labels: string + LocalVolumes: string + Mounts: string + Names: string + Networks: string + Ports: string + RunningFor: string + Size: string + State: string + Status: string } -function getTestContainerSettings( - serverName: string, - key: string -): string | null { - const entry = Object.entries(global).find( - ([k]) => - k.includes(`${serverName.toUpperCase()}`) && - k.includes(`${key.toUpperCase()}`) - ) - if (!entry) { - return null - } - return entry[1] +function getTestcontainers(): ContainerInfo[] { + return execSync("docker ps --format json") + .toString() + .split("\n") + .filter(x => x.length > 0) + .map(x => JSON.parse(x) as ContainerInfo) + .filter(x => x.Labels.includes("org.testcontainers=true")) } -function getContainerInfo(containerName: string, port: number) { - let assignedPort = getTestContainerSettings( - containerName.toUpperCase(), - `PORT_${port}` - ) - if (!dockerPsResult) { - try { - const outputBuffer = execSync("docker ps") - dockerPsResult = outputBuffer.toString("utf8") - } catch (err) { - //no-op - } - } - const possiblePort = formatDockerPsResult(containerName, port) - if (possiblePort) { - assignedPort = possiblePort - } - const host = getTestContainerSettings(containerName.toUpperCase(), "IP") - return { - port: assignedPort, - host, - url: host && assignedPort && `http://${host}:${assignedPort}`, - } +function getContainerByImage(image: string) { + return getTestcontainers().find(x => x.Image.startsWith(image)) } -function getCouchConfig() { - return getContainerInfo("couchdb", 5984) +function getExposedPort(container: ContainerInfo, port: number) { + const match = container.Ports.match(new RegExp(`0.0.0.0:(\\d+)->${port}/tcp`)) + if (!match) { + return undefined + } + return parseInt(match[1]) } export function setupEnv(...envs: any[]) { - const couch = getCouchConfig() + const couch = getContainerByImage("budibase/couchdb") + if (!couch) { + throw new Error("CouchDB container not found") + } + + const couchPort = getExposedPort(couch, 5984) + if (!couchPort) { + throw new Error("CouchDB port not found") + } + const configs = [ - { key: "COUCH_DB_PORT", value: couch.port }, - { key: "COUCH_DB_URL", value: couch.url }, + { key: "COUCH_DB_PORT", value: `${couchPort}` }, + { key: "COUCH_DB_URL", value: `http://localhost:${couchPort}` }, ] for (const config of configs.filter(x => !!x.value)) { @@ -82,4 +60,7 @@ export function setupEnv(...envs: any[]) { env._set(config.key, config.value) } } + + // @ts-expect-error + DatabaseImpl.nano = undefined } diff --git a/packages/backend-core/tests/jestEnv.ts b/packages/backend-core/tests/jestEnv.ts index c2047118ec..2c797c9fff 100644 --- a/packages/backend-core/tests/jestEnv.ts +++ b/packages/backend-core/tests/jestEnv.ts @@ -4,3 +4,7 @@ process.env.NODE_ENV = "jest" process.env.MOCK_REDIS = "1" process.env.LOG_LEVEL = process.env.LOG_LEVEL || "error" process.env.REDIS_PASSWORD = "budibase" +process.env.COUCH_DB_PASSWORD = "budibase" +process.env.COUCH_DB_USER = "budibase" +process.env.API_ENCRYPTION_KEY = "testsecret" +process.env.JWT_SECRET = "testsecret" diff --git a/packages/bbui/rollup.config.js b/packages/bbui/rollup.config.js index e285d548d6..da274e0ba5 100644 --- a/packages/bbui/rollup.config.js +++ b/packages/bbui/rollup.config.js @@ -12,6 +12,13 @@ export default { format: "esm", file: "dist/bbui.es.js", }, + onwarn(warning, warn) { + // suppress eval warnings + if (warning.code === "EVAL") { + return + } + warn(warning) + }, plugins: [ resolve(), commonjs(), diff --git a/packages/builder/.vscode/launch.json b/packages/builder/.vscode/launch.json deleted file mode 100644 index f6b35a0b63..0000000000 --- a/packages/builder/.vscode/launch.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 - "version": "0.2.0", - "configurations": [ - { - "type": "chrome", - "request": "launch", - "name": "Launch Chrome against localhost", - "url": "http://localhost:3000", - "webRoot": "${workspaceFolder}" - } - ] -} \ No newline at end of file diff --git a/packages/builder/.vscode/settings.json b/packages/builder/.vscode/settings.json deleted file mode 100644 index 7148fd3b98..0000000000 --- a/packages/builder/.vscode/settings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "javascript.format.enable": false, - "svelte.plugin.svelte.format.enable": false, - "html.format.enable": false, - "json.format.enable": false, - "editor.trimAutoWhitespace": false, - "sass.format.deleteWhitespace": false -} \ No newline at end of file diff --git a/packages/builder/src/components/common/bindings/BindingPanel.svelte b/packages/builder/src/components/common/bindings/BindingPanel.svelte index 21d389357f..10d95a3e7e 100644 --- a/packages/builder/src/components/common/bindings/BindingPanel.svelte +++ b/packages/builder/src/components/common/bindings/BindingPanel.svelte @@ -371,6 +371,7 @@ diff --git a/packages/builder/src/components/deploy/VersionModal.svelte b/packages/builder/src/components/deploy/VersionModal.svelte index 0944512fe8..316a981325 100644 --- a/packages/builder/src/components/deploy/VersionModal.svelte +++ b/packages/builder/src/components/deploy/VersionModal.svelte @@ -1,4 +1,5 @@ @@ -108,7 +133,26 @@ {:else} -