diff --git a/.vscode/settings.json b/.vscode/settings.json index d471924fe0..4838a4fd89 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -3,5 +3,17 @@ "editor.codeActionsOnSave": { "source.fixAll": true }, - "editor.defaultFormatter": "svelte.svelte-vscode" + "editor.defaultFormatter": "svelte.svelte-vscode", + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[javascript]": { + "editor.defaultFormatter": "esbenp.prettier-vscode" + }, + "debug.javascript.terminalOptions": { + "skipFiles": [ + "${workspaceFolder}/packages/backend-core/node_modules/**", + "/**" + ] + }, } diff --git a/lerna.json b/lerna.json index ed68735461..4990d54aca 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 1a2ee15375..d2b3d3e3a0 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -20,7 +20,7 @@ "test:watch": "jest --watchAll" }, "dependencies": { - "@budibase/types": "^1.0.212-alpha.12", + "@budibase/types": "^1.0.212-alpha.14", "@techpass/passport-openidconnect": "0.3.2", "aws-sdk": "2.1030.0", "bcrypt": "5.0.1", @@ -36,6 +36,7 @@ "passport-google-oauth": "2.0.0", "passport-jwt": "4.0.0", "passport-local": "1.0.0", + "passport-oauth2-refresh": "^2.1.0", "posthog-node": "1.3.0", "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", diff --git a/packages/backend-core/src/auth.js b/packages/backend-core/src/auth.js index b13cd932c6..b6d6a2027f 100644 --- a/packages/backend-core/src/auth.js +++ b/packages/backend-core/src/auth.js @@ -2,6 +2,9 @@ const passport = require("koa-passport") const LocalStrategy = require("passport-local").Strategy const JwtStrategy = require("passport-jwt").Strategy const { getGlobalDB } = require("./tenancy") +const refresh = require("passport-oauth2-refresh") +const { Configs } = require("./constants") +const { getScopedConfig } = require("./db/utils") const { jwt, local, @@ -12,6 +15,7 @@ const { tenancy, appTenancy, authError, + ssoCallbackUrl, csrf, internalApi, } = require("./middleware") @@ -34,6 +38,122 @@ passport.deserializeUser(async (user, done) => { } }) +async function refreshOIDCAccessToken(db, chosenConfig, refreshToken) { + const callbackUrl = await oidc.getCallbackUrl(db, chosenConfig) + let enrichedConfig + let strategy + + try { + enrichedConfig = await oidc.fetchStrategyConfig(chosenConfig, callbackUrl) + if (!enrichedConfig) { + throw new Error("OIDC Config contents invalid") + } + strategy = await oidc.strategyFactory(enrichedConfig) + } catch (err) { + console.error(err) + throw new Error("Could not refresh OAuth Token") + } + + refresh.use(strategy, { + setRefreshOAuth2() { + return strategy._getOAuth2Client(enrichedConfig) + }, + }) + + return new Promise(resolve => { + refresh.requestNewAccessToken( + Configs.OIDC, + refreshToken, + (err, accessToken, refreshToken, params) => { + resolve({ err, accessToken, refreshToken, params }) + } + ) + }) +} + +async function refreshGoogleAccessToken(db, config, refreshToken) { + let callbackUrl = await google.getCallbackUrl(db, config) + + let strategy + try { + strategy = await google.strategyFactory(config, callbackUrl) + } catch (err) { + console.error(err) + throw new Error("Error constructing OIDC refresh strategy", err) + } + + refresh.use(strategy) + + return new Promise(resolve => { + refresh.requestNewAccessToken( + Configs.GOOGLE, + refreshToken, + (err, accessToken, refreshToken, params) => { + resolve({ err, accessToken, refreshToken, params }) + } + ) + }) +} + +async function refreshOAuthToken(refreshToken, configType, configId) { + const db = getGlobalDB() + + const config = await getScopedConfig(db, { + type: configType, + group: {}, + }) + + let chosenConfig = {} + let refreshResponse + if (configType === Configs.OIDC) { + // configId - retrieved from cookie. + chosenConfig = config.configs.filter(c => c.uuid === configId)[0] + if (!chosenConfig) { + throw new Error("Invalid OIDC configuration") + } + refreshResponse = await refreshOIDCAccessToken( + db, + chosenConfig, + refreshToken + ) + } else { + chosenConfig = config + refreshResponse = await refreshGoogleAccessToken( + db, + chosenConfig, + refreshToken + ) + } + + return refreshResponse +} + +async function updateUserOAuth(userId, oAuthConfig) { + const details = { + accessToken: oAuthConfig.accessToken, + refreshToken: oAuthConfig.refreshToken, + } + + try { + const db = getGlobalDB() + const dbUser = await db.get(userId) + + //Do not overwrite the refresh token if a valid one is not provided. + if (typeof details.refreshToken !== "string") { + delete details.refreshToken + } + + dbUser.oauth2 = { + ...dbUser.oauth2, + ...details, + } + + await db.put(dbUser) + } catch (e) { + console.error("Could not update OAuth details for current user", e) + } +} + module.exports = { buildAuthMiddleware: authenticated, passport, @@ -46,4 +166,7 @@ module.exports = { authError, buildCsrfMiddleware: csrf, internalApi, + refreshOAuthToken, + updateUserOAuth, + ssoCallbackUrl, } diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts index 4910899565..ba3f1dd3e9 100644 --- a/packages/backend-core/src/db/utils.ts +++ b/packages/backend-core/src/db/utils.ts @@ -387,7 +387,9 @@ export const getScopedFullConfig = async function ( if (type === Configs.SETTINGS) { if (scopedConfig && scopedConfig.doc) { // overrides affected by environment variables - scopedConfig.doc.config.platformUrl = await getPlatformUrl() + scopedConfig.doc.config.platformUrl = await getPlatformUrl({ + tenantAware: true, + }) scopedConfig.doc.config.analyticsEnabled = await events.analytics.enabled() } else { @@ -396,7 +398,7 @@ export const getScopedFullConfig = async function ( doc: { _id: generateConfigID({ type, user, workspace }), config: { - platformUrl: await getPlatformUrl(), + platformUrl: await getPlatformUrl({ tenantAware: true }), analyticsEnabled: await events.analytics.enabled(), }, }, diff --git a/packages/backend-core/src/middleware/authenticated.js b/packages/backend-core/src/middleware/authenticated.js index 9c35336dda..4e6e0b7ba2 100644 --- a/packages/backend-core/src/middleware/authenticated.js +++ b/packages/backend-core/src/middleware/authenticated.js @@ -94,7 +94,6 @@ module.exports = ( user = await getUser(userId, session.tenantId) } user.csrfToken = session.csrfToken - delete user.password authenticated = true } catch (err) { error = err @@ -128,6 +127,8 @@ module.exports = ( } if (!user && tenantId) { user = { tenantId } + } else { + delete user.password } // be explicit if (authenticated !== true) { diff --git a/packages/backend-core/src/middleware/index.js b/packages/backend-core/src/middleware/index.js index 6c4c0d8883..1721d56a3c 100644 --- a/packages/backend-core/src/middleware/index.js +++ b/packages/backend-core/src/middleware/index.js @@ -2,7 +2,7 @@ const jwt = require("./passport/jwt") const local = require("./passport/local") const google = require("./passport/google") const oidc = require("./passport/oidc") -const { authError } = require("./passport/utils") +const { authError, ssoCallbackUrl } = require("./passport/utils") const authenticated = require("./authenticated") const auditLog = require("./auditLog") const tenancy = require("./tenancy") @@ -20,6 +20,7 @@ module.exports = { tenancy, authError, internalApi, + ssoCallbackUrl, datasource: { google: datasourceGoogle, }, diff --git a/packages/backend-core/src/middleware/passport/google.js b/packages/backend-core/src/middleware/passport/google.js index 858029ca80..7419974cd7 100644 --- a/packages/backend-core/src/middleware/passport/google.js +++ b/packages/backend-core/src/middleware/passport/google.js @@ -1,6 +1,7 @@ const GoogleStrategy = require("passport-google-oauth").OAuth2Strategy - +const { ssoCallbackUrl } = require("./utils") const { authenticateThirdParty } = require("./third-party-common") +const { Configs } = require("../../../constants") const buildVerifyFn = saveUserFn => { return (accessToken, refreshToken, profile, done) => { @@ -57,5 +58,10 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { ) } } + +exports.getCallbackUrl = async function (db, config) { + return ssoCallbackUrl(db, config, Configs.GOOGLE) +} + // expose for testing exports.buildVerifyFn = buildVerifyFn diff --git a/packages/backend-core/src/middleware/passport/local.js b/packages/backend-core/src/middleware/passport/local.js index 445893b1df..b955d29102 100644 --- a/packages/backend-core/src/middleware/passport/local.js +++ b/packages/backend-core/src/middleware/passport/local.js @@ -55,6 +55,7 @@ exports.authenticate = async function (ctx, email, password, done) { if (await compare(password, dbUser.password)) { const sessionId = newid() const tenantId = getTenantId() + await createASession(dbUser._id, { sessionId, tenantId }) dbUser.token = jwt.sign( diff --git a/packages/backend-core/src/middleware/passport/oidc.js b/packages/backend-core/src/middleware/passport/oidc.js index 1e93e20b1c..20dbd4669b 100644 --- a/packages/backend-core/src/middleware/passport/oidc.js +++ b/packages/backend-core/src/middleware/passport/oidc.js @@ -1,6 +1,8 @@ const fetch = require("node-fetch") const OIDCStrategy = require("@techpass/passport-openidconnect").Strategy const { authenticateThirdParty } = require("./third-party-common") +const { ssoCallbackUrl } = require("./utils") +const { Configs } = require("../../../constants") const buildVerifyFn = saveUserFn => { /** @@ -89,11 +91,24 @@ function validEmail(value) { * from couchDB rather than environment variables, using this factory is necessary for dynamically configuring passport. * @returns Dynamically configured Passport OIDC Strategy */ -exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { +exports.strategyFactory = async function (config, saveUserFn) { try { - const { clientID, clientSecret, configUrl } = config + const verify = buildVerifyFn(saveUserFn) + const strategy = new OIDCStrategy(config, verify) + strategy.name = "oidc" + return strategy + } catch (err) { + console.error(err) + throw new Error("Error constructing OIDC authentication strategy", err) + } +} + +exports.fetchStrategyConfig = async function (enrichedConfig, callbackUrl) { + try { + const { clientID, clientSecret, configUrl } = enrichedConfig if (!clientID || !clientSecret || !callbackUrl || !configUrl) { + //check for remote config and all required elements throw new Error( "Configuration invalid. Must contain clientID, clientSecret, callbackUrl and configUrl" ) @@ -109,24 +124,24 @@ exports.strategyFactory = async function (config, callbackUrl, saveUserFn) { const body = await response.json() - const verify = buildVerifyFn(saveUserFn) - return new OIDCStrategy( - { - issuer: body.issuer, - authorizationURL: body.authorization_endpoint, - tokenURL: body.token_endpoint, - userInfoURL: body.userinfo_endpoint, - clientID: clientID, - clientSecret: clientSecret, - callbackURL: callbackUrl, - }, - verify - ) + return { + issuer: body.issuer, + authorizationURL: body.authorization_endpoint, + tokenURL: body.token_endpoint, + userInfoURL: body.userinfo_endpoint, + clientID: clientID, + clientSecret: clientSecret, + callbackURL: callbackUrl, + } } catch (err) { console.error(err) - throw new Error("Error constructing OIDC authentication strategy", err) + throw new Error("Error constructing OIDC authentication configuration", err) } } +exports.getCallbackUrl = async function (db, config) { + return ssoCallbackUrl(db, config, Configs.OIDC) +} + // expose for testing exports.buildVerifyFn = buildVerifyFn diff --git a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js index c5e9fe0034..c00ab2ea7d 100644 --- a/packages/backend-core/src/middleware/passport/tests/oidc.spec.js +++ b/packages/backend-core/src/middleware/passport/tests/oidc.spec.js @@ -48,8 +48,8 @@ describe("oidc", () => { it("should create successfully create an oidc strategy", async () => { const oidc = require("../oidc") - - await oidc.strategyFactory(oidcConfig, callbackUrl) + const enrichedConfig = await oidc.fetchStrategyConfig(oidcConfig, callbackUrl) + await oidc.strategyFactory(enrichedConfig, callbackUrl) expect(mockFetch).toHaveBeenCalledWith(oidcConfig.configUrl) diff --git a/packages/backend-core/src/middleware/passport/utils.js b/packages/backend-core/src/middleware/passport/utils.js index cbb93bfa3b..217130cd6d 100644 --- a/packages/backend-core/src/middleware/passport/utils.js +++ b/packages/backend-core/src/middleware/passport/utils.js @@ -1,3 +1,7 @@ +const { isMultiTenant, getTenantId } = require("../../tenancy") +const { getScopedConfig } = require("../../db/utils") +const { Configs } = require("../../constants") + /** * Utility to handle authentication errors. * @@ -5,6 +9,7 @@ * @param {*} message Message that will be returned in the response body * @param {*} err (Optional) error that will be logged */ + exports.authError = function (done, message, err = null) { return done( err, @@ -12,3 +17,21 @@ exports.authError = function (done, message, err = null) { { message: message } ) } + +exports.ssoCallbackUrl = async (db, config, type) => { + // incase there is a callback URL from before + if (config && config.callbackURL) { + return config.callbackURL + } + const publicConfig = await getScopedConfig(db, { + type: Configs.SETTINGS, + }) + + let callbackUrl = `/api/global/auth` + if (isMultiTenant()) { + callbackUrl += `/${getTenantId()}` + } + callbackUrl += `/${type}/callback` + + return `${publicConfig.platformUrl}${callbackUrl}` +} diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index e40cddc468..7d4d422631 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -291,6 +291,11 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@budibase/types@^1.0.206": + version "1.0.208" + resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.208.tgz#c45cb494fb5b85229e15a34c6ac1805bae5be867" + integrity sha512-zKIHg6TGK+soVxMNZNrGypP3DCrd3jhlUQEFeQ+rZR6/tCue1G74bjzydY5FjnLEsXeLH1a0hkS5HulTmvQ2bA== + "@istanbuljs/load-nyc-config@^1.0.0": version "1.1.0" resolved "https://registry.yarnpkg.com/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz#fd3db1d59ecf7cf121e80650bb86712f9b55eced" @@ -4123,6 +4128,11 @@ passport-oauth1@1.x.x: passport-strategy "1.x.x" utils-merge "1.x.x" +passport-oauth2-refresh@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/passport-oauth2-refresh/-/passport-oauth2-refresh-2.1.0.tgz#c31cd133826383f5539d16ad8ab4f35ca73ce4a4" + integrity sha512-4ML7ooCESCqiTgdDBzNUFTBcPR8zQq9iM6eppEUGMMvLdsjqRL93jKwWm4Az3OJcI+Q2eIVyI8sVRcPFvxcF/A== + passport-oauth2@1.x.x: version "1.6.1" resolved "https://registry.yarnpkg.com/passport-oauth2/-/passport-oauth2-1.6.1.tgz#c5aee8f849ce8bd436c7f81d904a3cd1666f181b" diff --git a/packages/bbui/package.json b/packages/bbui/package.json index f6d29b874c..472430e18f 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.212-alpha.12", + "@budibase/string-templates": "^1.0.212-alpha.14", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/bbui/src/Form/Combobox.svelte b/packages/bbui/src/Form/Combobox.svelte index 83927b05db..343af559cb 100644 --- a/packages/bbui/src/Form/Combobox.svelte +++ b/packages/bbui/src/Form/Combobox.svelte @@ -40,5 +40,6 @@ on:change={onChange} on:pick on:type + on:blur /> diff --git a/packages/bbui/src/Form/Core/Combobox.svelte b/packages/bbui/src/Form/Core/Combobox.svelte index 2a4bac4a2c..2835b3cd40 100644 --- a/packages/bbui/src/Form/Core/Combobox.svelte +++ b/packages/bbui/src/Form/Core/Combobox.svelte @@ -52,7 +52,10 @@ {id} type="text" on:focus={() => (focus = true)} - on:blur={() => (focus = false)} + on:blur={() => { + focus = false + dispatch("blur") + }} on:change={onType} value={value || ""} placeholder={placeholder || ""} diff --git a/packages/builder/cypress/integration/appPublishWorkflow.spec.js b/packages/builder/cypress/integration/appPublishWorkflow.spec.js index a431051075..e65c01c1b6 100644 --- a/packages/builder/cypress/integration/appPublishWorkflow.spec.js +++ b/packages/builder/cypress/integration/appPublishWorkflow.spec.js @@ -85,12 +85,12 @@ filterTests(['all'], () => { cy.get(interact.APP_TABLE_APP_NAME).click({ force: true }) }) - cy.get(interact.DEPLOYMENT_TOP_NAV).click() - cy.get(interact.PUBLISH_POPOVER_ACTION).click({ force: true }) - cy.get(interact.UNPUBLISH_MODAL) - .within(() => { - cy.get(interact.CONFIRM_WRAP_BUTTON).click({ force: true }) - }) + cy.get(interact.DEPLOYMENT_TOP_GLOBE).should("exist").click({ force: true }) + + cy.get("[data-cy='publish-popover-menu']") + .within(() => { + cy.get(interact.PUBLISH_POPOVER_ACTION).click({ force: true }) + }) cy.get(interact.UNPUBLISH_MODAL).should("be.visible") .within(() => { diff --git a/packages/builder/cypress/integration/createApp.spec.js b/packages/builder/cypress/integration/createApp.spec.js index 097b70db30..00c875e4fa 100644 --- a/packages/builder/cypress/integration/createApp.spec.js +++ b/packages/builder/cypress/integration/createApp.spec.js @@ -52,13 +52,6 @@ filterTests(['smoke', 'all'], () => { // Start create app process. If apps already exist, click second button cy.get(interact.CREATE_APP_BUTTON, { timeout: 1000 }).click({ force: true }) - cy.request(`${Cypress.config().baseUrl}/api/applications?status=all`) - .its("body") - .then(val => { - if (val.length > 0) { - cy.get(interact.CREATE_APP_BUTTON).click({ force: true }) - } - }) const appName = "Cypress Tests" cy.get(interact.SPECTRUM_MODAL).within(() => { diff --git a/packages/builder/package.json b/packages/builder/package.json index c1b342f320..227d21b059 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "license": "GPL-3.0", "private": true, "scripts": { @@ -69,10 +69,10 @@ } }, "dependencies": { - "@budibase/bbui": "^1.0.212-alpha.12", - "@budibase/client": "^1.0.212-alpha.12", - "@budibase/frontend-core": "^1.0.212-alpha.12", - "@budibase/string-templates": "^1.0.212-alpha.12", + "@budibase/bbui": "^1.0.212-alpha.14", + "@budibase/client": "^1.0.212-alpha.14", + "@budibase/frontend-core": "^1.0.212-alpha.14", + "@budibase/string-templates": "^1.0.212-alpha.14", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index 8cbc629291..234f83d7cc 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -49,6 +49,95 @@ export const getBindableProperties = (asset, componentId) => { ] } +/** + * Gets all rest bindable data fields + */ +export const getRestBindings = () => { + const userBindings = getUserBindings() + return [...userBindings, ...getAuthBindings()] +} + +/** + * Gets all rest bindable auth fields + */ +export const getAuthBindings = () => { + let bindings = [] + const safeUser = makePropSafe("user") + const safeOAuth2 = makePropSafe("oauth2") + const safeAccessToken = makePropSafe("accessToken") + + const authBindings = [ + { + runtime: `${safeUser}.${safeOAuth2}.${safeAccessToken}`, + readable: `Current User.OAuthToken`, + key: "accessToken", + }, + ] + + bindings = Object.keys(authBindings).map(key => { + const fieldBinding = authBindings[key] + return { + type: "context", + runtimeBinding: fieldBinding.runtime, + readableBinding: fieldBinding.readable, + fieldSchema: { type: "string", name: fieldBinding.key }, + providerId: "user", + } + }) + return bindings +} + +/** + * Utility - convert a key/value map to an array of custom 'context' bindings + * @param {object} valueMap Key/value pairings + * @param {string} prefix A contextual string prefix/path for a user readable binding + * @return {object[]} An array containing readable/runtime binding objects + */ +export const toBindingsArray = (valueMap, prefix) => { + if (!valueMap) { + return [] + } + return Object.keys(valueMap).reduce((acc, binding) => { + if (!binding || !valueMap[binding]) { + return acc + } + acc.push({ + type: "context", + runtimeBinding: binding, + readableBinding: `${prefix}.${binding}`, + }) + return acc + }, []) +} + +/** + * Utility - coverting a map of readable bindings to runtime + */ +export const readableToRuntimeMap = (bindings, ctx) => { + if (!bindings || !ctx) { + return {} + } + return Object.keys(ctx).reduce((acc, key) => { + let parsedQuery = readableToRuntimeBinding(bindings, ctx[key]) + acc[key] = parsedQuery + return acc + }, {}) +} + +/** + * Utility - coverting a map of runtime bindings to readable + */ +export const runtimeToReadableMap = (bindings, ctx) => { + if (!bindings || !ctx) { + return {} + } + return Object.keys(ctx).reduce((acc, key) => { + let parsedQuery = runtimeToReadableBinding(bindings, ctx[key]) + acc[key] = parsedQuery + return acc + }, {}) +} + /** * Gets the bindable properties exposed by a certain component. */ @@ -298,7 +387,6 @@ const getUserBindings = () => { providerId: "user", }) }) - return bindings } diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte index 2c8b699849..0165d83dcb 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/RestExtraConfigForm.svelte @@ -10,11 +10,31 @@ import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import RestAuthenticationBuilder from "./auth/RestAuthenticationBuilder.svelte" import ViewDynamicVariables from "./variables/ViewDynamicVariables.svelte" + import { + getRestBindings, + readableToRuntimeBinding, + runtimeToReadableMap, + } from "builderStore/dataBinding" + import { cloneDeep } from "lodash/fp" export let datasource export let queries let addHeader + + let parsedHeaders = runtimeToReadableMap( + getRestBindings(), + cloneDeep(datasource?.config?.defaultHeaders) + ) + + const onDefaultHeaderUpdate = headers => { + const flatHeaders = cloneDeep(headers).reduce((acc, entry) => { + acc[entry.name] = readableToRuntimeBinding(getRestBindings(), entry.value) + return acc + }, {}) + + datasource.config.defaultHeaders = flatHeaders + } @@ -30,9 +50,10 @@ onDefaultHeaderUpdate(evt.detail)} noAddButton + bindings={getRestBindings()} />
addHeader.addEntry()}> diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte index 7f0cc7357b..b754f878ce 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationModal.svelte @@ -2,6 +2,8 @@ import { onMount } from "svelte" import { ModalContent, Layout, Select, Body, Input } from "@budibase/bbui" import { AUTH_TYPE_LABELS, AUTH_TYPES } from "./authTypes" + import BindableCombobox from "components/common/bindings/BindableCombobox.svelte" + import { getAuthBindings } from "builderStore/dataBinding" export let configs export let currentConfig @@ -203,11 +205,23 @@ /> {/if} {#if form.type === AUTH_TYPES.BEARER} - (blurred.bearer.token = true)} + value={form.bearer.token} + bindings={getAuthBindings()} + on:change={e => { + form.bearer.token = e.detail + console.log(e.detail) + onFieldChange() + }} + on:blur={() => { + blurred.bearer.token = true + onFieldChange() + }} + allowJS={false} + placeholder="Token" + appendBindingsAsOptions={true} + drawerEnabled={false} error={blurred.bearer.token ? errors.bearer.token : null} /> {/if} diff --git a/packages/builder/src/components/common/bindings/BindableCombobox.svelte b/packages/builder/src/components/common/bindings/BindableCombobox.svelte new file mode 100644 index 0000000000..1e44a55736 --- /dev/null +++ b/packages/builder/src/components/common/bindings/BindableCombobox.svelte @@ -0,0 +1,68 @@ + + +
+ onChange(e.detail, false)} + on:pick={e => onChange(e.detail, true)} + on:blur={() => dispatch("blur")} + {placeholder} + options={allOptions} + {error} + /> +
+ + diff --git a/packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte b/packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte index 44f88e841a..9033844dd0 100644 --- a/packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte +++ b/packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte @@ -18,6 +18,7 @@ export let options export let allowJS = true export let appendBindingsAsOptions = true + export let error const dispatch = createEventDispatcher() let bindingDrawer @@ -59,8 +60,10 @@ value={isJS ? "(JavaScript function)" : readableValue} on:type={e => onChange(e.detail, false)} on:pick={e => onChange(e.detail, true)} + on:blur={() => dispatch("blur")} {placeholder} options={allOptions} + {error} /> {#if !disabled}
{/if}
+ Add the objects on the left to enrich your text. diff --git a/packages/builder/src/components/integration/KeyValueBuilder.svelte b/packages/builder/src/components/integration/KeyValueBuilder.svelte index 39eca0955b..9b46bc0364 100644 --- a/packages/builder/src/components/integration/KeyValueBuilder.svelte +++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte @@ -11,6 +11,7 @@ } from "@budibase/bbui" import { createEventDispatcher } from "svelte" import { lowercase } from "helpers" + import DrawerBindableInput from "components/common/bindings/DrawerBindableInput.svelte" let dispatch = createEventDispatcher() @@ -30,6 +31,7 @@ export let tooltip export let menuItems export let showMenu = false + export let bindings = [] let fields = Object.entries(object || {}).map(([name, value]) => ({ name, @@ -108,6 +110,16 @@ /> {#if options} onBindingChange(binding.name, evt.detail)} + value={runtimeToReadableBinding(bindings, binding.default)} /> {#if bindable} - +{#key $params.selectedDatasource} + +{/key} diff --git a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte index 2baa6aab41..6a798f0178 100644 --- a/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte +++ b/packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte @@ -40,13 +40,39 @@ import { cloneDeep } from "lodash/fp" import { RawRestBodyTypes } from "constants/backend" + import { + getRestBindings, + toBindingsArray, + runtimeToReadableBinding, + readableToRuntimeBinding, + runtimeToReadableMap, + readableToRuntimeMap, + } from "builderStore/dataBinding" + let query, datasource let breakQs = {}, - bindings = {} + requestBindings = {} let saveId, url let response, schema, enabledHeaders let authConfigId let dynamicVariables, addVariableModal, varBinding + let restBindings = getRestBindings() + + $: staticVariables = datasource?.config?.staticVariables || {} + + $: customRequestBindings = toBindingsArray(requestBindings, "Binding") + $: dynamicRequestBindings = toBindingsArray(dynamicVariables, "Dynamic") + $: dataSourceStaticBindings = toBindingsArray( + staticVariables, + "Datasource.Static" + ) + + $: mergedBindings = [ + ...restBindings, + ...customRequestBindings, + ...dynamicRequestBindings, + ...dataSourceStaticBindings, + ] $: datasourceType = datasource?.source $: integrationInfo = $integrations[datasourceType] @@ -63,8 +89,10 @@ Object.keys(schema || {}).length !== 0 || Object.keys(query?.schema || {}).length !== 0 + $: runtimeUrlQueries = readableToRuntimeMap(mergedBindings, breakQs) + function getSelectedQuery() { - return cloneDeep( + const cloneQuery = cloneDeep( $queries.list.find(q => q._id === $queries.selected) || { datasourceId: $params.selectedDatasource, parameters: [], @@ -76,6 +104,7 @@ queryVerb: "read", } ) + return cloneQuery } function checkQueryName(inputUrl = null) { @@ -89,7 +118,9 @@ if (!base) { return base } - const qs = restUtils.buildQueryString(qsObj) + const qs = restUtils.buildQueryString( + runtimeToReadableMap(mergedBindings, qsObj) + ) let newUrl = base if (base.includes("?")) { newUrl = base.split("?")[0] @@ -98,14 +129,21 @@ } function buildQuery() { - const newQuery = { ...query } - const queryString = restUtils.buildQueryString(breakQs) + const newQuery = cloneDeep(query) + const queryString = restUtils.buildQueryString(runtimeUrlQueries) + + newQuery.parameters = restUtils.keyValueToQueryParameters(requestBindings) + newQuery.fields.requestBody = + typeof newQuery.fields.requestBody === "object" + ? readableToRuntimeMap(mergedBindings, newQuery.fields.requestBody) + : readableToRuntimeBinding(mergedBindings, newQuery.fields.requestBody) + newQuery.fields.path = url.split("?")[0] newQuery.fields.queryString = queryString newQuery.fields.authConfigId = authConfigId newQuery.fields.disabledHeaders = restUtils.flipHeaderState(enabledHeaders) newQuery.schema = restUtils.fieldsToSchema(schema) - newQuery.parameters = restUtils.keyValueToQueryParameters(bindings) + return newQuery } @@ -120,6 +158,13 @@ datasource.config.dynamicVariables = rebuildVariables(saveId) datasource = await datasources.save(datasource) } + prettifyQueryRequestBody( + query, + requestBindings, + dynamicVariables, + staticVariables, + restBindings + ) } catch (err) { notifications.error(`Error saving query`) } @@ -127,7 +172,7 @@ async function runQuery() { try { - response = await queries.preview(buildQuery(query)) + response = await queries.preview(buildQuery()) if (response.rows.length === 0) { notifications.info("Request did not return any data") } else { @@ -236,6 +281,36 @@ } } + const prettifyQueryRequestBody = ( + query, + requestBindings, + dynamicVariables, + staticVariables, + restBindings + ) => { + let customRequestBindings = toBindingsArray(requestBindings, "Binding") + let dynamicRequestBindings = toBindingsArray(dynamicVariables, "Dynamic") + let dataSourceStaticBindings = toBindingsArray( + staticVariables, + "Datasource.Static" + ) + + const prettyBindings = [ + ...restBindings, + ...customRequestBindings, + ...dynamicRequestBindings, + ...dataSourceStaticBindings, + ] + + //Parse the body here as now all bindings have been updated. + if (query?.fields?.requestBody) { + query.fields.requestBody = + typeof query.fields.requestBody === "object" + ? runtimeToReadableMap(prettyBindings, query.fields.requestBody) + : runtimeToReadableBinding(prettyBindings, query.fields.requestBody) + } + } + onMount(async () => { query = getSelectedQuery() @@ -250,6 +325,8 @@ const datasourceUrl = datasource?.config.url const qs = query?.fields.queryString breakQs = restUtils.breakQueryString(qs) + breakQs = runtimeToReadableMap(mergedBindings, breakQs) + const path = query.fields.path if ( datasourceUrl && @@ -260,7 +337,7 @@ } url = buildUrl(query.fields.path, breakQs) schema = restUtils.schemaToFields(query.schema) - bindings = restUtils.queryParametersToKeyValue(query.parameters) + requestBindings = restUtils.queryParametersToKeyValue(query.parameters) authConfigId = getAuthConfigId() if (!query.fields.disabledHeaders) { query.fields.disabledHeaders = {} @@ -291,6 +368,14 @@ query.fields.pagination = {} } dynamicVariables = getDynamicVariables(datasource, query._id) + + prettifyQueryRequestBody( + query, + requestBindings, + dynamicVariables, + staticVariables, + restBindings + ) }) @@ -344,16 +429,26 @@ - + diff --git a/packages/cli/package.json b/packages/cli/package.json index 80914ff41e..980a74081f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 707ab74a2b..59d376c611 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.212-alpha.12", - "@budibase/frontend-core": "^1.0.212-alpha.12", - "@budibase/string-templates": "^1.0.212-alpha.12", + "@budibase/bbui": "^1.0.212-alpha.14", + "@budibase/frontend-core": "^1.0.212-alpha.14", + "@budibase/string-templates": "^1.0.212-alpha.14", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/client/src/components/app/forms/validation.js b/packages/client/src/components/app/forms/validation.js index c98ca8467a..2a5e3c4227 100644 --- a/packages/client/src/components/app/forms/validation.js +++ b/packages/client/src/components/app/forms/validation.js @@ -278,6 +278,9 @@ const notEqualHandler = (value, rule) => { // Evaluates a regex constraint const regexHandler = (value, rule) => { const regex = parseType(rule.value, "string") + if (!value) { + value = "" + } return new RegExp(regex).test(value) } diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 8a4bdcf87e..2349dafcbf 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.212-alpha.12", + "@budibase/bbui": "^1.0.212-alpha.14", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/__mocks__/node-fetch.ts b/packages/server/__mocks__/node-fetch.ts index 350680691b..1a7015fa52 100644 --- a/packages/server/__mocks__/node-fetch.ts +++ b/packages/server/__mocks__/node-fetch.ts @@ -15,6 +15,15 @@ module FetchMock { }, }, json: async () => { + //x-www-form-encoded body is a URLSearchParams + //The call to stringify it leaves it blank + if (body?.opts?.body instanceof URLSearchParams) { + const paramArray = Array.from(body.opts.body.entries()) + body.opts.body = paramArray.reduce((acc: any, pair: any) => { + acc[pair[0]] = pair[1] + return acc + }, {}) + } return body }, } diff --git a/packages/server/package.json b/packages/server/package.json index 6fa60da9c9..eeca111d6d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -77,11 +77,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "^1.0.212-alpha.12", - "@budibase/client": "^1.0.212-alpha.12", - "@budibase/pro": "1.0.212-alpha.12", - "@budibase/string-templates": "^1.0.212-alpha.12", - "@budibase/types": "^1.0.212-alpha.12", + "@budibase/backend-core": "^1.0.212-alpha.14", + "@budibase/client": "^1.0.212-alpha.14", + "@budibase/pro": "1.0.212-alpha.14", + "@budibase/string-templates": "^1.0.212-alpha.14", + "@budibase/types": "^1.0.212-alpha.14", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 2abd83140a..ce6eeda7c7 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -8,6 +8,8 @@ import { QUERY_THREAD_TIMEOUT } from "../../../environment" import { getAppDB } from "@budibase/backend-core/context" import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" +import { getCookie } from "@budibase/backend-core/utils" +import { Cookies, Configs } from "@budibase/backend-core/constants" const Runner = new Thread(ThreadType.QUERY, { timeoutMs: QUERY_THREAD_TIMEOUT || 10000, @@ -110,6 +112,21 @@ export async function find(ctx: any) { ctx.body = query } +//Required to discern between OIDC OAuth config entries +function getOAuthConfigCookieId(ctx: any) { + if (ctx.user.providerType === Configs.OIDC) { + return getCookie(ctx, Cookies.OIDC_CONFIG) + } +} + +function getAuthConfig(ctx: any) { + const authCookie = getCookie(ctx, Cookies.Auth) + let authConfigCtx: any = {} + authConfigCtx["configId"] = getOAuthConfigCookieId(ctx) + authConfigCtx["sessionId"] = authCookie ? authCookie.sessionId : null + return authConfigCtx +} + export async function preview(ctx: any) { const db = getAppDB() @@ -119,6 +136,8 @@ export async function preview(ctx: any) { // this stops dynamic variables from calling the same query const { fields, parameters, queryVerb, transformer, queryId } = query + const authConfigCtx: any = getAuthConfig(ctx) + try { const runFn = () => Runner.run({ @@ -129,8 +148,11 @@ export async function preview(ctx: any) { parameters, transformer, queryId, + ctx: { + user: ctx.user, + auth: { ...authConfigCtx }, + }, }) - const { rows, keys, info, extra } = await quotas.addQuery(runFn) await events.query.previewed(datasource, query) ctx.body = { @@ -150,6 +172,8 @@ async function execute(ctx: any, opts = { rowsOnly: false }) { const query = await db.get(ctx.params.queryId) const datasource = await db.get(query.datasourceId) + const authConfigCtx: any = getAuthConfig(ctx) + const enrichedParameters = ctx.request.body.parameters || {} // make sure parameters are fully enriched with defaults if (query && query.parameters) { @@ -172,6 +196,10 @@ async function execute(ctx: any, opts = { rowsOnly: false }) { parameters: enrichedParameters, transformer: query.transformer, queryId: ctx.params.queryId, + ctx: { + user: ctx.user, + auth: { ...authConfigCtx }, + }, }) const { rows, pagination, extra } = await quotas.addQuery(runFn) diff --git a/packages/server/src/api/routes/tests/query.spec.js b/packages/server/src/api/routes/tests/query.spec.js index 5dacda3505..273bdb9993 100644 --- a/packages/server/src/api/routes/tests/query.spec.js +++ b/packages/server/src/api/routes/tests/query.spec.js @@ -346,4 +346,170 @@ describe("/queries", () => { expect(contents).toBe(null) }) }) + + describe("Current User Request Mapping", () => { + + async function previewGet(datasource, fields, params) { + return config.previewQuery(request, config, datasource, fields, params) + } + + async function previewPost(datasource, fields, params) { + return config.previewQuery(request, config, datasource, fields, params, "create") + } + + it("should parse global and query level header mappings", async () => { + const userDetails = config.getUserDetails() + + const datasource = await config.restDatasource({ + defaultHeaders: { + "test": "headerVal", + "emailHdr": "{{[user].[email]}}" + } + }) + const res = await previewGet(datasource, { + path: "www.google.com", + queryString: "email={{[user].[email]}}", + headers: { + queryHdr : "{{[user].[firstName]}}", + secondHdr : "1234" + } + }) + + const parsedRequest = JSON.parse(res.body.extra.raw) + expect(parsedRequest.opts.headers).toEqual({ + "test": "headerVal", + "emailHdr": userDetails.email, + "queryHdr": userDetails.firstName, + "secondHdr" : "1234" + }) + expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email) + }) + + it("should bind the current user to query parameters", async () => { + const userDetails = config.getUserDetails() + + const datasource = await config.restDatasource() + + const res = await previewGet(datasource, { + path: "www.google.com", + queryString: "test={{myEmail}}&testName={{myName}}&testParam={{testParam}}", + }, { + "myEmail" : "{{[user].[email]}}", + "myName" : "{{[user].[firstName]}}", + "testParam" : "1234" + }) + + expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email + + "&testName=" + userDetails.firstName + "&testParam=1234") + }) + + it("should bind the current user the request body - plain text", async () => { + const userDetails = config.getUserDetails() + const datasource = await config.restDatasource() + + const res = await previewPost(datasource, { + path: "www.google.com", + queryString: "testParam={{testParam}}", + requestBody: "This is plain text and this is my email: {{[user].[email]}}. This is a test param: {{testParam}}", + bodyType: "text" + }, { + "testParam" : "1234" + }) + + const parsedRequest = JSON.parse(res.body.extra.raw) + expect(parsedRequest.opts.body).toEqual(`This is plain text and this is my email: ${userDetails.email}. This is a test param: 1234`) + expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234") + }) + + it("should bind the current user the request body - json", async () => { + const userDetails = config.getUserDetails() + const datasource = await config.restDatasource() + + const res = await previewPost(datasource, { + path: "www.google.com", + queryString: "testParam={{testParam}}", + requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}", + bodyType: "json" + }, { + "testParam" : "1234", + "userRef" : "{{[user].[firstName]}}" + }) + + const parsedRequest = JSON.parse(res.body.extra.raw) + const test = `{"email":"${userDetails.email}","queryCode":1234,"userRef":"${userDetails.firstName}"}` + expect(parsedRequest.opts.body).toEqual(test) + expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234") + }) + + it("should bind the current user the request body - xml", async () => { + const userDetails = config.getUserDetails() + const datasource = await config.restDatasource() + + const res = await previewPost(datasource, { + path: "www.google.com", + queryString: "testParam={{testParam}}", + requestBody: " {{[user].[email]}} {{testParam}} " + + "{{userId}} testing ", + bodyType: "xml" + }, { + "testParam" : "1234", + "userId" : "{{[user].[firstName]}}" + }) + + const parsedRequest = JSON.parse(res.body.extra.raw) + const test = ` ${userDetails.email} 1234 ${userDetails.firstName} testing ` + + expect(parsedRequest.opts.body).toEqual(test) + expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234") + }) + + it("should bind the current user the request body - form-data", async () => { + const userDetails = config.getUserDetails() + const datasource = await config.restDatasource() + + const res = await previewPost(datasource, { + path: "www.google.com", + queryString: "testParam={{testParam}}", + requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}", + bodyType: "form" + }, { + "testParam" : "1234", + "userRef" : "{{[user].[firstName]}}" + }) + + const parsedRequest = JSON.parse(res.body.extra.raw) + + const emailData = parsedRequest.opts.body._streams[1] + expect(emailData).toEqual(userDetails.email) + + const queryCodeData = parsedRequest.opts.body._streams[4] + expect(queryCodeData).toEqual("1234") + + const userRef = parsedRequest.opts.body._streams[7] + expect(userRef).toEqual(userDetails.firstName) + + expect(res.body.rows[0].url).toEqual("http://www.google.com?testParam=1234") + }) + + it("should bind the current user the request body - encoded", async () => { + const userDetails = config.getUserDetails() + const datasource = await config.restDatasource() + + const res = await previewPost(datasource, { + path: "www.google.com", + queryString: "testParam={{testParam}}", + requestBody: "{\"email\":\"{{[user].[email]}}\",\"queryCode\":{{testParam}},\"userRef\":\"{{userRef}}\"}", + bodyType: "encoded" + }, { + "testParam" : "1234", + "userRef" : "{{[user].[firstName]}}" + }) + const parsedRequest = JSON.parse(res.body.extra.raw) + + expect(parsedRequest.opts.body.email).toEqual(userDetails.email) + expect(parsedRequest.opts.body.queryCode).toEqual("1234") + expect(parsedRequest.opts.body.userRef).toEqual(userDetails.firstName) + }) + + }); }) diff --git a/packages/server/src/integrations/queries/sql.ts b/packages/server/src/integrations/queries/sql.ts index cf71f2ee2a..271a414d44 100644 --- a/packages/server/src/integrations/queries/sql.ts +++ b/packages/server/src/integrations/queries/sql.ts @@ -9,7 +9,9 @@ export function enrichQueryFields( parameters = {} ) { const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {} - + if (!fields || !parameters) { + return enrichedQuery + } // enrich the fields with dynamic parameters for (let key of Object.keys(fields)) { if (fields[key] == null) { diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 13956e5994..9cc8e1a841 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -287,7 +287,7 @@ module RestModule { input.body = form break case BodyTypes.XML: - if (object != null) { + if (object != null && Object.keys(object).length) { string = new XmlBuilder().buildObject(object) } input.body = string diff --git a/packages/server/src/integrations/tests/rest.spec.js b/packages/server/src/integrations/tests/rest.spec.js index 8f3c7f7f58..0bb1e3a75d 100644 --- a/packages/server/src/integrations/tests/rest.spec.js +++ b/packages/server/src/integrations/tests/rest.spec.js @@ -155,12 +155,27 @@ describe("REST Integration", () => { expect(output.headers["Content-Type"]).toEqual("application/json") }) - it("should allow XML", () => { + it("should allow raw XML", () => { + const output = config.integration.addBody("xml", "12", {}) + expect(output.body.includes("1")).toEqual(true) + expect(output.body.includes("2")).toEqual(true) + expect(output.headers["Content-Type"]).toEqual("application/xml") + }) + + it("should allow a valid js object and parse the contents to xml", () => { const output = config.integration.addBody("xml", input, {}) expect(output.body.includes("1")).toEqual(true) expect(output.body.includes("2")).toEqual(true) expect(output.headers["Content-Type"]).toEqual("application/xml") }) + + it("should allow a valid json string and parse the contents to xml", () => { + const output = config.integration.addBody("xml", JSON.stringify(input), {}) + expect(output.body.includes("1")).toEqual(true) + expect(output.body.includes("2")).toEqual(true) + expect(output.headers["Content-Type"]).toEqual("application/xml") + }) + }) describe("response", () => { diff --git a/packages/server/src/migrations/tests/index.spec.ts b/packages/server/src/migrations/tests/index.spec.ts index faaefe3d61..72bf2ff1a9 100644 --- a/packages/server/src/migrations/tests/index.spec.ts +++ b/packages/server/src/migrations/tests/index.spec.ts @@ -93,8 +93,24 @@ describe("migrations", () => { await clearMigrations() const appId = config.prodAppId const roles = { [appId]: "role_12345" } - await config.createUser(undefined, undefined, false, true, roles) // admin only - await config.createUser(undefined, undefined, false, false, roles) // non admin non builder + await config.createUser( + undefined, + undefined, + undefined, + undefined, + false, + true, + roles + ) // admin only + await config.createUser( + undefined, + undefined, + undefined, + undefined, + false, + false, + roles + ) // non admin non builder await config.createTable() await config.createRow() await config.createRow() diff --git a/packages/server/src/module.d.ts b/packages/server/src/module.d.ts index 843267fba8..4c0e13586a 100644 --- a/packages/server/src/module.d.ts +++ b/packages/server/src/module.d.ts @@ -8,4 +8,5 @@ declare module "@budibase/backend-core/constants" declare module "@budibase/backend-core/auth" declare module "@budibase/backend-core/sessions" declare module "@budibase/backend-core/encryption" +declare module "@budibase/backend-core/utils" declare module "@budibase/backend-core/redis" diff --git a/packages/server/src/tests/utilities/TestConfiguration.js b/packages/server/src/tests/utilities/TestConfiguration.js index fc4d302c63..baa4ec13b8 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.js +++ b/packages/server/src/tests/utilities/TestConfiguration.js @@ -28,6 +28,8 @@ const { encrypt } = require("@budibase/backend-core/encryption") const GLOBAL_USER_ID = "us_uuid1" const EMAIL = "babs@babs.com" +const FIRSTNAME = "Barbara" +const LASTNAME = "Barbington" const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306" class TestConfiguration { @@ -59,6 +61,15 @@ class TestConfiguration { return this.prodAppId } + getUserDetails() { + return { + globalId: GLOBAL_USER_ID, + email: EMAIL, + firstName: FIRSTNAME, + lastName: LASTNAME, + } + } + async doInContext(appId, task) { if (!appId) { appId = this.appId @@ -118,6 +129,8 @@ class TestConfiguration { // USER / AUTH async globalUser({ id = GLOBAL_USER_ID, + firstName = FIRSTNAME, + lastName = LASTNAME, builder = true, admin = false, email = EMAIL, @@ -135,6 +148,8 @@ class TestConfiguration { ...existing, roles: roles || {}, tenantId: TENANT_ID, + firstName, + lastName, } await createASession(id, { sessionId: "sessionid", @@ -161,6 +176,8 @@ class TestConfiguration { async createUser( id = null, + firstName = FIRSTNAME, + lastName = LASTNAME, email = EMAIL, builder = true, admin = false, @@ -169,6 +186,8 @@ class TestConfiguration { const globalId = !id ? `us_${Math.random()}` : `us_${id}` const resp = await this.globalUser({ id: globalId, + firstName, + lastName, email, builder, admin, @@ -520,14 +539,14 @@ class TestConfiguration { // QUERY - async previewQuery(request, config, datasource, fields) { + async previewQuery(request, config, datasource, fields, params, verb) { return request .post(`/api/queries/preview`) .send({ datasourceId: datasource._id, - parameters: {}, + parameters: params || {}, fields, - queryVerb: "read", + queryVerb: verb || "read", name: datasource.name, }) .set(config.defaultHeaders()) diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index ec9d9a6fa6..e85fde970e 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -4,6 +4,12 @@ const ScriptRunner = require("../utilities/scriptRunner") const { integrations } = require("../integrations") const { processStringSync } = require("@budibase/string-templates") const { doInAppContext, getAppDB } = require("@budibase/backend-core/context") +const { + refreshOAuthToken, + updateUserOAuth, +} = require("@budibase/backend-core/auth") +const { getGlobalIDFromUserMetadataID } = require("../db/utils") + const { isSQL } = require("../integrations/utils") const { enrichQueryFields, @@ -21,29 +27,56 @@ class QueryRunner { this.queryId = input.queryId this.noRecursiveQuery = flags.noRecursiveQuery this.cachedVariables = [] + // Additional context items for enrichment + this.ctx = input.ctx // allows the response from a query to be stored throughout this // execution so that if it needs to be re-used for another variable // it can be this.queryResponse = {} this.hasRerun = false + this.hasRefreshedOAuth = false } async execute() { let { datasource, fields, queryVerb, transformer } = this + const Integration = integrations[datasource.source] if (!Integration) { throw "Integration type does not exist." } + + if (datasource.config.authConfigs) { + datasource.config.authConfigs = datasource.config.authConfigs.map( + config => { + return enrichQueryFields(config, this.ctx) + } + ) + } + const integration = new Integration(datasource.config) // pre-query, make sure datasource variables are added to parameters const parameters = await this.addDatasourceVariables() + + // Enrich the parameters with the addition context items. + // 'user' is now a reserved variable key in mapping parameters + const enrichedParameters = enrichQueryFields(parameters, this.ctx) + const enrichedContext = { ...enrichedParameters, ...this.ctx } + + // Parse global headers + if (datasource.config.defaultHeaders) { + datasource.config.defaultHeaders = enrichQueryFields( + datasource.config.defaultHeaders, + enrichedContext + ) + } + let query // handle SQL injections by interpolating the variables if (isSQL(datasource)) { - query = interpolateSQL(fields, parameters, integration) + query = interpolateSQL(fields, enrichedParameters, integration) } else { - query = enrichQueryFields(fields, parameters) + query = enrichQueryFields(fields, enrichedContext) } // Add pagination values for REST queries @@ -67,20 +100,30 @@ class QueryRunner { if (transformer) { const runner = new ScriptRunner(transformer, { data: rows, - params: parameters, + params: enrichedParameters, }) rows = runner.execute() } // if the request fails we retry once, invalidating the cached value - if ( - info && - info.code >= 400 && - this.cachedVariables.length > 0 && - !this.hasRerun - ) { + if (info && info.code >= 400 && !this.hasRerun) { + if ( + this.ctx.user?.provider && + info.code === 401 && + !this.hasRefreshedOAuth + ) { + // Attempt to refresh the access token from the provider + this.hasRefreshedOAuth = true + const authResponse = await this.refreshOAuth2(this.ctx) + + if (!authResponse || authResponse.err) { + // In this event the user may have oAuth issues that + // could require re-authenticating with their provider. + throw new Error("OAuth2 access token could not be refreshed") + } + } + this.hasRerun = true - // invalidate the cache value await threadUtils.invalidateDynamicVariables(this.cachedVariables) return this.execute() } @@ -126,6 +169,31 @@ class QueryRunner { ).execute() } + async refreshOAuth2(ctx) { + const { oauth2, providerType, _id } = ctx.user + const { configId } = ctx.auth + + if (!providerType || !oauth2?.refreshToken) { + console.error("No refresh token found for authenticated user") + return + } + + const resp = await refreshOAuthToken( + oauth2.refreshToken, + providerType, + configId + ) + + // Refresh session flow. Should be in same location as refreshOAuthToken + // There are several other properties available in 'resp' + if (!resp.error) { + const globalUserId = getGlobalIDFromUserMetadataID(_id) + await updateUserOAuth(globalUserId, resp) + } + + return resp + } + async getDynamicVariable(variable) { let { parameters } = this const queryId = variable.queryId, diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index 753956b2fa..15e244715f 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 44411fe107..9897b5dc8f 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index a0927a502a..dcab9fb1a0 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.212-alpha.12", + "version": "1.0.212-alpha.14", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -34,10 +34,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.212-alpha.12", - "@budibase/pro": "1.0.212-alpha.12", - "@budibase/string-templates": "^1.0.212-alpha.12", - "@budibase/types": "^1.0.212-alpha.12", + "@budibase/backend-core": "^1.0.212-alpha.14", + "@budibase/pro": "1.0.212-alpha.14", + "@budibase/string-templates": "^1.0.212-alpha.14", + "@budibase/types": "^1.0.212-alpha.14", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2", diff --git a/packages/worker/src/api/controllers/global/auth.ts b/packages/worker/src/api/controllers/global/auth.ts index 896a7c48de..dc96554cb2 100644 --- a/packages/worker/src/api/controllers/global/auth.ts +++ b/packages/worker/src/api/controllers/global/auth.ts @@ -1,49 +1,22 @@ const core = require("@budibase/backend-core") -const { getScopedConfig } = require("@budibase/backend-core/db") -const { google } = require("@budibase/backend-core/middleware") -const { oidc } = require("@budibase/backend-core/middleware") const { Configs, EmailTemplatePurpose } = require("../../../constants") const { sendEmail, isEmailConfigured } = require("../../../utilities/email") const { setCookie, getCookie, clearCookie, hash, platformLogout } = core.utils const { Cookies, Headers } = core.constants -const { passport } = core.auth +const { passport, ssoCallbackUrl, google, oidc } = core.auth const { checkResetPasswordCode } = require("../../../utilities/redis") -const { - getGlobalDB, - getTenantId, - isMultiTenant, -} = require("@budibase/backend-core/tenancy") +const { getGlobalDB } = require("@budibase/backend-core/tenancy") const env = require("../../../environment") import { events, users as usersCore, context } from "@budibase/backend-core" import { users } from "../../../sdk" import { User } from "@budibase/types" -const ssoCallbackUrl = async (config: any, type: any) => { - // incase there is a callback URL from before - if (config && config.callbackURL) { - return config.callbackURL - } - - const db = getGlobalDB() - const publicConfig = await getScopedConfig(db, { - type: Configs.SETTINGS, - }) - - let callbackUrl = `/api/global/auth` - if (isMultiTenant()) { - callbackUrl += `/${getTenantId()}` - } - callbackUrl += `/${type}/callback` - - return `${publicConfig.platformUrl}${callbackUrl}` -} - export const googleCallbackUrl = async (config: any) => { - return ssoCallbackUrl(config, "google") + return ssoCallbackUrl(getGlobalDB(), config, "google") } export const oidcCallbackUrl = async (config: any) => { - return ssoCallbackUrl(config, "oidc") + return ssoCallbackUrl(getGlobalDB(), config, "oidc") } async function authInternal(ctx: any, user: any, err = null, info = null) { @@ -198,6 +171,8 @@ export const googlePreAuth = async (ctx: any, next: any) => { return passport.authenticate(strategy, { scope: ["profile", "email"], + accessType: "offline", + prompt: "consent", })(ctx, next) } @@ -224,7 +199,7 @@ export const googleAuth = async (ctx: any, next: any) => { )(ctx, next) } -async function oidcStrategyFactory(ctx: any, configId: any) { +export const oidcStrategyFactory = async (ctx: any, configId: any) => { const db = getGlobalDB() const config = await core.db.getScopedConfig(db, { type: Configs.OIDC, @@ -234,7 +209,12 @@ async function oidcStrategyFactory(ctx: any, configId: any) { const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0] let callbackUrl = await exports.oidcCallbackUrl(chosenConfig) - return oidc.strategyFactory(chosenConfig, callbackUrl, users.save) + //Remote Config + const enrichedConfig = await oidc.fetchStrategyConfig( + chosenConfig, + callbackUrl + ) + return oidc.strategyFactory(enrichedConfig, users.save) } /** @@ -249,7 +229,7 @@ export const oidcPreAuth = async (ctx: any, next: any) => { return passport.authenticate(strategy, { // required 'openid' scope is added by oidc strategy factory - scope: ["profile", "email"], + scope: ["profile", "email", "offline_access"], //auth0 offline_access scope required for the refresh token behaviour. })(ctx, next) } diff --git a/packages/worker/src/api/routes/tests/auth.spec.js b/packages/worker/src/api/routes/tests/auth.spec.js index 03b00d8312..165ecd0f4a 100644 --- a/packages/worker/src/api/routes/tests/auth.spec.js +++ b/packages/worker/src/api/routes/tests/auth.spec.js @@ -77,27 +77,30 @@ describe("/api/global/auth", () => { describe("oidc", () => { const auth = require("@budibase/backend-core/auth") - // mock the oidc strategy implementation and return value - let strategyFactory = jest.fn() - let mockStrategyReturn = jest.fn() - strategyFactory.mockReturnValue(mockStrategyReturn) - auth.oidc.strategyFactory = strategyFactory - const passportSpy = jest.spyOn(auth.passport, "authenticate") let oidcConf let chosenConfig let configId + // mock the oidc strategy implementation and return value + let strategyFactory = jest.fn() + let mockStrategyReturn = jest.fn() + let mockStrategyConfig = jest.fn() + auth.oidc.fetchStrategyConfig = mockStrategyConfig + + strategyFactory.mockReturnValue(mockStrategyReturn) + auth.oidc.strategyFactory = strategyFactory + beforeEach(async () => { oidcConf = await config.saveOIDCConfig() chosenConfig = oidcConf.config.configs[0] configId = chosenConfig.uuid + mockStrategyConfig.mockReturnValue(chosenConfig) }) afterEach(() => { expect(strategyFactory).toBeCalledWith( chosenConfig, - `http://localhost:10000/api/global/auth/${TENANT_ID}/oidc/callback`, expect.any(Function) ) }) @@ -107,7 +110,7 @@ describe("/api/global/auth", () => { await request.get(`/api/global/auth/${TENANT_ID}/oidc/configs/${configId}`) expect(passportSpy).toBeCalledWith(mockStrategyReturn, { - scope: ["profile", "email"], + scope: ["profile", "email", "offline_access"] }) expect(passportSpy.mock.calls.length).toBe(1); })