From 12b69dc37d034126f0bb873ba2ecba21443f5f70 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 2 Jun 2023 15:06:19 +0100 Subject: [PATCH 01/10] Remove --configuration=production --- .github/workflows/release-develop.yml | 2 +- .github/workflows/release-master.yml | 2 +- packages/server/package.json | 2 +- packages/worker/package.json | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 503fc53194..9bb37d7356 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -60,7 +60,7 @@ jobs: node scripts/syncLocalDependencies.js $version echo "Syncing yarn workspace" yarn - - run: yarn build --configuration=production + - run: yarn build - run: yarn build:sdk - name: Publish budibase packages to NPM diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml index a4e679e7bf..a105d8a712 100644 --- a/.github/workflows/release-master.yml +++ b/.github/workflows/release-master.yml @@ -54,7 +54,7 @@ jobs: echo "Syncing yarn workspace" yarn - run: yarn lint - - run: yarn build --configuration=production + - run: yarn build - run: yarn build:sdk - name: Publish budibase packages to NPM diff --git a/packages/server/package.json b/packages/server/package.json index c73bba0f16..b1a3bf25e2 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,7 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production", + "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build", "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION", "build:docs": "node ./scripts/docs/generate.js open", "run:docker": "node dist/index.js", diff --git a/packages/worker/package.json b/packages/worker/package.json index a90a4eb683..f798aee4a0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -19,7 +19,7 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "predocker": "yarn build --configuration=production", + "predocker": "yarn build", "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon", From 511559a7205cda233727a57b3037413ab778619a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 2 Jun 2023 15:46:44 +0100 Subject: [PATCH 02/10] Use predocker --- package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index ffb3fa775a..56f015f8c0 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,7 @@ "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker", "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built", - "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", "test": "lerna run --stream test --stream", "lint:eslint": "eslint packages && eslint qa-core", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"", @@ -62,7 +62,7 @@ "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "build:specs": "lerna run --stream specs", "build:docker": "lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", - "build:docker:pre": "lerna run --stream build && lerna run --stream predocker", + "build:docker:pre": "yarn build && lerna run --stream predocker", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", "build:docker:selfhost": "lerna run --stream build:docker && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh latest && cd -", "build:docker:develop": "node scripts/pinVersions && lerna run --stream build:docker && yarn build:docker:proxy && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh develop && cd -", From fe9a220717029f757fe29cc8dac08d26409fca9a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Fri, 2 Jun 2023 15:49:06 +0100 Subject: [PATCH 03/10] Add yarn.lock --- packages/server/Dockerfile | 1 + packages/server/package.json | 2 +- packages/worker/Dockerfile | 1 + packages/worker/package.json | 4 ++-- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 5880a84326..151e78657f 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -27,6 +27,7 @@ COPY scripts/integrations/oracle/ scripts/integrations/oracle/ RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh COPY package.json . +COPY dist/yarn.lock . RUN yarn install --frozen-lockfile --production=true # Remove unneeded data from file system to reduce image size RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \ diff --git a/packages/server/package.json b/packages/server/package.json index b1a3bf25e2..6e74de6afa 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -18,7 +18,7 @@ "test": "bash scripts/test.sh", "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit", "test:watch": "jest --watch", - "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build", + "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build && cp ../../yarn.lock ./dist/", "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION", "build:docs": "node ./scripts/docs/generate.js open", "run:docker": "node dist/index.js", diff --git a/packages/worker/Dockerfile b/packages/worker/Dockerfile index a80ae00af1..6261c5ea1b 100644 --- a/packages/worker/Dockerfile +++ b/packages/worker/Dockerfile @@ -13,6 +13,7 @@ RUN yarn global add pm2 COPY package.json . +COPY dist/yarn.lock . RUN yarn install --frozen-lockfile --production=true # Remove unneeded data from file system to reduce image size RUN apk del .gyp \ diff --git a/packages/worker/package.json b/packages/worker/package.json index f798aee4a0..f07cb011a0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -19,7 +19,7 @@ "run:docker": "node dist/index.js", "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js", "run:docker:cluster": "pm2-runtime start pm2.config.js", - "predocker": "yarn build", + "predocker": "yarn build && cp ../../yarn.lock ./dist/", "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION", "dev:stack:init": "node ./scripts/dev/manage.js init", "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon", @@ -104,4 +104,4 @@ "update-dotenv": "1.1.1" }, "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc" -} +} \ No newline at end of file From d1af3f930e1988e3ba553cf30cbef30953a4e76d Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 14:34:08 +0100 Subject: [PATCH 04/10] Add yarn.lock in single image docker --- hosting/single/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hosting/single/Dockerfile b/hosting/single/Dockerfile index 64a6b01365..141d5122d3 100644 --- a/hosting/single/Dockerfile +++ b/hosting/single/Dockerfile @@ -11,11 +11,13 @@ RUN chmod +x /cleanup.sh # build server WORKDIR /app ADD packages/server . +COPY dist/yarn.lock . RUN yarn install --frozen-lockfile --production=true && /cleanup.sh # build worker WORKDIR /worker ADD packages/worker . +COPY dist/yarn.lock . RUN yarn install --frozen-lockfile --production=true && /cleanup.sh FROM budibase/couchdb From 500424c651d0407767c37167615a1fd3d400d330 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 6 Jun 2023 15:29:17 +0000 Subject: [PATCH 05/10] Bump version to 2.7.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 338a4590e7..e8e20f0534 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.7.1", + "version": "2.7.2", "npmClient": "yarn", "packages": [ "packages/backend-core", From deeba8bae34008b39313121abe191510361e4ed1 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 6 Jun 2023 22:06:34 +0100 Subject: [PATCH 06/10] Revert "Merge branch 'develop' into master" This reverts commit 6f29c978e05f92a119cf7f0b4dae9c229e55ebd2, reversing changes made to 1c6ff73c19afd38fb2172e1bf19e84276d58f0cd. --- .../middleware/passport/datasource/google.ts | 35 +-- .../_components/GoogleButton.svelte | 14 +- .../modals/GoogleDatasourceConfigModal.svelte | 107 ++------ .../builder/app/[application]/data/new.svelte | 23 +- .../portal/settings/auth/google.svelte | 235 ------------------ .../builder/portal/settings/auth/index.svelte | 154 +++++++++++- .../server/src/api/controllers/datasource.ts | 14 +- .../server/src/integrations/googlesheets.ts | 20 +- .../worker/src/api/controllers/global/auth.ts | 1 + 9 files changed, 212 insertions(+), 391 deletions(-) delete mode 100644 packages/builder/src/pages/builder/portal/settings/auth/google.svelte diff --git a/packages/backend-core/src/middleware/passport/datasource/google.ts b/packages/backend-core/src/middleware/passport/datasource/google.ts index 2f91e01d9a..6fd4e9ff32 100644 --- a/packages/backend-core/src/middleware/passport/datasource/google.ts +++ b/packages/backend-core/src/middleware/passport/datasource/google.ts @@ -1,10 +1,10 @@ import * as google from "../sso/google" import { Cookie } from "../../../constants" import { clearCookie, getCookie } from "../../../utils" +import { doWithDB } from "../../../db" import * as configs from "../../../configs" -import { BBContext, SSOProfile } from "@budibase/types" +import { BBContext, Database, SSOProfile } from "@budibase/types" import { ssoSaveUserNoOp } from "../sso/sso" -import { cache, utils } from "../../../" const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy type Passport = { @@ -36,8 +36,8 @@ export async function preAuth( ssoSaveUserNoOp ) - if (!ctx.query.appId) { - ctx.throw(400, "appId query param not present.") + if (!ctx.query.appId || !ctx.query.datasourceId) { + ctx.throw(400, "appId and datasourceId query params not present.") } return passport.authenticate(strategy, { @@ -69,7 +69,7 @@ export async function postAuth( ( accessToken: string, refreshToken: string, - _profile: SSOProfile, + profile: SSOProfile, done: Function ) => { clearCookie(ctx, Cookie.DatasourceAuth) @@ -79,16 +79,23 @@ export async function postAuth( { successRedirect: "/", failureRedirect: "/error" }, async (err: any, tokens: string[]) => { const baseUrl = `/builder/app/${authStateCookie.appId}/data` - - const id = utils.newid() - await cache.store( - `datasource:creation:${authStateCookie.appId}:google:${id}`, - { - tokens, + // update the DB for the datasource with all the user info + await doWithDB(authStateCookie.appId, async (db: Database) => { + let datasource + try { + datasource = await db.get(authStateCookie.datasourceId) + } catch (err: any) { + if (err.status === 404) { + ctx.redirect(baseUrl) + } } - ) - - ctx.redirect(`${baseUrl}/new?continue_google_setup=${id}`) + if (!datasource.config) { + datasource.config = {} + } + datasource.config.auth = { type: "google", ...tokens } + await db.put(datasource) + ctx.redirect(`${baseUrl}/datasource/${authStateCookie.datasourceId}`) + }) } )(ctx, next) } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte index ceb8fd7f4b..b7d70d88b7 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleButton.svelte @@ -3,6 +3,8 @@ import { store } from "builderStore" import { auth } from "stores/portal" + export let preAuthStep + export let datasource export let disabled export let samePage @@ -13,8 +15,18 @@ class:disabled {disabled} on:click={async () => { + let ds = datasource let appId = $store.appId - const url = `/api/global/auth/${tenantId}/datasource/google?appId=${appId}` + if (!ds) { + const resp = await preAuthStep() + if (resp.datasource && resp.appId) { + ds = resp.datasource + appId = resp.appId + } else { + ds = resp + } + } + const url = `/api/global/auth/${tenantId}/datasource/google?datasourceId=${ds._id}&appId=${appId}` if (samePage) { window.location = url } else { diff --git a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte index 7b4808967d..0783a9fe53 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/modals/GoogleDatasourceConfigModal.svelte @@ -1,110 +1,43 @@ - {#if step === GoogleDatasouceConfigStep.AUTH} - - {#if isGoogleConfigured === true} - - Authenticate with your google account to use the {integrationName} integration. - - - {:else if isGoogleConfigured === false} + + {#if isGoogleConfigured === true} + Google authentication is not enabled, please complete Google SSO - configuration.Authenticate with your google account to use the {IntegrationNames[ + datasource.type + ]} integration. - Configure Google SSO - {/if} - {/if} - {#if step === GoogleDatasouceConfigStep.SET_URL} - - Add the URL of the sheet you want to connect. - - (isValid = e.detail)} - /> + save(datasource, true)} /> + {:else if isGoogleConfigured === false} + Google authentication is not enabled, please complete Google SSO + configuration. + Configure Google SSO {/if} diff --git a/packages/builder/src/pages/builder/app/[application]/data/new.svelte b/packages/builder/src/pages/builder/app/[application]/data/new.svelte index 8ff974112b..fedaf013da 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/new.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/new.svelte @@ -17,7 +17,6 @@ import IntegrationIcon from "components/backend/DatasourceNavigator/IntegrationIcon.svelte" import ICONS from "components/backend/DatasourceNavigator/icons/index.js" import FontAwesomeIcon from "components/common/FontAwesomeIcon.svelte" - import { onMount } from "svelte" let internalTableModal let externalDatasourceModal @@ -130,19 +129,9 @@ return integrationsArray } - let continueGoogleSetup - onMount(() => { - const urlParams = new URLSearchParams(window.location.search) - continueGoogleSetup = urlParams.get("continue_google_setup") - }) - const fetchIntegrations = async () => { const unsortedIntegrations = await API.getIntegrations() integrations = sortIntegrations(unsortedIntegrations) - - if (continueGoogleSetup) { - handleIntegrationSelect(IntegrationTypes.GOOGLE_SHEETS) - } } $: fetchIntegrations() @@ -152,17 +141,9 @@ - { - continueGoogleSetup = null - }} -> + {#if integration?.auth?.type === "google"} - + {:else} {/if} diff --git a/packages/builder/src/pages/builder/portal/settings/auth/google.svelte b/packages/builder/src/pages/builder/portal/settings/auth/google.svelte deleted file mode 100644 index 82ab13cc8e..0000000000 --- a/packages/builder/src/pages/builder/portal/settings/auth/google.svelte +++ /dev/null @@ -1,235 +0,0 @@ - - -{#if providers.google} - - - -
- - Google -
- - - To allow users to authenticate using their Google accounts, fill out the - fields below. Read the documentation for more information. - - - - {#each GoogleConfigFields.Google as field} -
- -
-
- -
- {#if field.copyButton} -
copyToClipboard(field.placeholder)} - > - -
- {/if} -
-
- {/each} -
- - -
- -
- -
-{/if} - - diff --git a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte index 36cf5c13a8..38f5e0788b 100644 --- a/packages/builder/src/pages/builder/portal/settings/auth/index.svelte +++ b/packages/builder/src/pages/builder/portal/settings/auth/index.svelte @@ -1,4 +1,5 @@