diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index 92392457d6..f7d15b3880 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -77,6 +77,7 @@ export const StaticDatabases = { apiKeys: "apikeys", usageQuota: "usage_quota", licenseInfo: "license_info", + environmentVariables: "environmentvariables", }, }, // contains information about tenancy and so on diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index f0ccdb97a8..02b7713764 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -1,17 +1,14 @@ import { AsyncLocalStorage } from "async_hooks" +import { ContextMap } from "./mainContext" export default class Context { - static storage = new AsyncLocalStorage>() + static storage = new AsyncLocalStorage() - static run(context: Record, func: any) { + static run(context: ContextMap, func: any) { return Context.storage.run(context, () => func()) } - static get(): Record { - return Context.storage.getStore() as Record - } - - static set(context: Record) { - Context.storage.enterWith(context) + static get(): ContextMap { + return Context.storage.getStore() as ContextMap } } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index c44ec4e767..9884d25d5a 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -16,6 +16,7 @@ export type ContextMap = { tenantId?: string appId?: string identity?: IdentityContext + environmentVariables?: Record } let TEST_APP_ID: string | null = null @@ -75,7 +76,7 @@ export function getTenantIDFromAppID(appId: string) { } } -function updateContext(updates: ContextMap) { +function updateContext(updates: ContextMap): ContextMap { let context: ContextMap try { context = Context.get() @@ -120,15 +121,23 @@ export async function doInTenant( return newContext(updates, task) } -export async function doInAppContext(appId: string, task: any): Promise { - if (!appId) { +export async function doInAppContext( + appId: string | null, + task: any +): Promise { + if (!appId && !env.isTest()) { throw new Error("appId is required") } - const tenantId = getTenantIDFromAppID(appId) - const updates: ContextMap = { appId } - if (tenantId) { - updates.tenantId = tenantId + let updates: ContextMap + if (!appId) { + updates = { appId: "" } + } else { + const tenantId = getTenantIDFromAppID(appId) + updates = { appId } + if (tenantId) { + updates.tenantId = tenantId + } } return newContext(updates, task) } @@ -189,25 +198,25 @@ export const getProdAppId = () => { return conversions.getProdAppID(appId) } -export function updateTenantId(tenantId?: string) { - let context: ContextMap = updateContext({ - tenantId, - }) - Context.set(context) +export function doInEnvironmentContext( + values: Record, + task: any +) { + if (!values) { + throw new Error("Must supply environment variables.") + } + const updates = { + environmentVariables: values, + } + return newContext(updates, task) } -export function updateAppId(appId: string) { - let context: ContextMap = updateContext({ - appId, - }) - try { - Context.set(context) - } catch (err) { - if (env.isTest()) { - TEST_APP_ID = appId - } else { - throw err - } +export function getEnvironmentVariables() { + const context = Context.get() + if (!context.environmentVariables) { + return null + } else { + return context.environmentVariables } } diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 91556ddcd6..d742ca1cc9 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -37,6 +37,7 @@ const environment = { }, JS_BCRYPT: process.env.JS_BCRYPT, JWT_SECRET: process.env.JWT_SECRET, + ENCRYPTION_KEY: process.env.ENCRYPTION_KEY, COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", COUCH_DB_USERNAME: process.env.COUCH_DB_USER, COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD, diff --git a/packages/backend-core/src/events/publishers/environmentVariable.ts b/packages/backend-core/src/events/publishers/environmentVariable.ts new file mode 100644 index 0000000000..d28e259b82 --- /dev/null +++ b/packages/backend-core/src/events/publishers/environmentVariable.ts @@ -0,0 +1,38 @@ +import { + Event, + EnvironmentVariableCreatedEvent, + EnvironmentVariableDeletedEvent, + EnvironmentVariableUpgradePanelOpenedEvent, +} from "@budibase/types" +import { publishEvent } from "../events" + +async function created(name: string, environments: string[]) { + const properties: EnvironmentVariableCreatedEvent = { + name, + environments, + } + await publishEvent(Event.ENVIRONMENT_VARIABLE_CREATED, properties) +} + +async function deleted(name: string) { + const properties: EnvironmentVariableDeletedEvent = { + name, + } + await publishEvent(Event.ENVIRONMENT_VARIABLE_DELETED, properties) +} + +async function upgradePanelOpened(userId: string) { + const properties: EnvironmentVariableUpgradePanelOpenedEvent = { + userId, + } + await publishEvent( + Event.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED, + properties + ) +} + +export default { + created, + deleted, + upgradePanelOpened, +} diff --git a/packages/backend-core/src/events/publishers/index.ts b/packages/backend-core/src/events/publishers/index.ts index 2316785ed7..34e47b2990 100644 --- a/packages/backend-core/src/events/publishers/index.ts +++ b/packages/backend-core/src/events/publishers/index.ts @@ -20,3 +20,4 @@ export { default as backfill } from "./backfill" export { default as group } from "./group" export { default as plugin } from "./plugin" export { default as backup } from "./backup" +export { default as environmentVariable } from "./environmentVariable" diff --git a/packages/backend-core/src/security/encryption.ts b/packages/backend-core/src/security/encryption.ts index a9006f302d..d0707cb850 100644 --- a/packages/backend-core/src/security/encryption.ts +++ b/packages/backend-core/src/security/encryption.ts @@ -2,19 +2,45 @@ import crypto from "crypto" import env from "../environment" const ALGO = "aes-256-ctr" -const SECRET = env.JWT_SECRET const SEPARATOR = "-" const ITERATIONS = 10000 const RANDOM_BYTES = 16 const STRETCH_LENGTH = 32 +export enum SecretOption { + JWT = "jwt", + ENCRYPTION = "encryption", +} + +function getSecret(secretOption: SecretOption): string { + let secret, secretName + switch (secretOption) { + case SecretOption.ENCRYPTION: + secret = env.ENCRYPTION_KEY + secretName = "ENCRYPTION_KEY" + break + case SecretOption.JWT: + default: + secret = env.JWT_SECRET + secretName = "JWT_SECRET" + break + } + if (!secret) { + throw new Error(`Secret "${secretName}" has not been set in environment.`) + } + return secret +} + function stretchString(string: string, salt: Buffer) { return crypto.pbkdf2Sync(string, salt, ITERATIONS, STRETCH_LENGTH, "sha512") } -export function encrypt(input: string) { +export function encrypt( + input: string, + secretOption: SecretOption = SecretOption.JWT +) { const salt = crypto.randomBytes(RANDOM_BYTES) - const stretched = stretchString(SECRET!, salt) + const stretched = stretchString(getSecret(secretOption), salt) const cipher = crypto.createCipheriv(ALGO, stretched, salt) const base = cipher.update(input) const final = cipher.final() @@ -22,10 +48,13 @@ export function encrypt(input: string) { return `${salt.toString("hex")}${SEPARATOR}${encrypted}` } -export function decrypt(input: string) { +export function decrypt( + input: string, + secretOption: SecretOption = SecretOption.JWT +) { const [salt, encrypted] = input.split(SEPARATOR) const saltBuffer = Buffer.from(salt, "hex") - const stretched = stretchString(SECRET!, saltBuffer) + const stretched = stretchString(getSecret(secretOption), saltBuffer) const decipher = crypto.createDecipheriv(ALGO, stretched, saltBuffer) const base = decipher.update(Buffer.from(encrypted, "hex")) const final = decipher.final() diff --git a/packages/bbui/src/Form/Core/EnvDropdown.svelte b/packages/bbui/src/Form/Core/EnvDropdown.svelte new file mode 100644 index 0000000000..f51704248b --- /dev/null +++ b/packages/bbui/src/Form/Core/EnvDropdown.svelte @@ -0,0 +1,282 @@ + + +
+
+ + + +
+ {#if open} +
+
    + {#if !environmentVariablesEnabled} +
    + Upgrade your plan to get environment variables +
    + {:else if variables.length} +
    + {#each variables as variable, idx} +
  • handleVarSelect(variable.name)} + > + +
    + {variable.name} + +
    + +
    +
  • + {/each} +
    + {:else} +
    + You don't have any environment variables yet +
    + {/if} +
+ + {#if environmentVariablesEnabled} +
showModal()} class="add-variable"> + +
Add Variable
+
+ {:else} +
handleUpgradePanel()} class="add-variable"> + +
Upgrade plan
+
+ {/if} +
+ {/if} +
+ + diff --git a/packages/bbui/src/Form/EnvDropdown.svelte b/packages/bbui/src/Form/EnvDropdown.svelte new file mode 100644 index 0000000000..33924af0a5 --- /dev/null +++ b/packages/bbui/src/Form/EnvDropdown.svelte @@ -0,0 +1,52 @@ + + + + + diff --git a/packages/bbui/src/index.js b/packages/bbui/src/index.js index b56aa597ad..482226ed88 100644 --- a/packages/bbui/src/index.js +++ b/packages/bbui/src/index.js @@ -27,6 +27,7 @@ export { default as RadioGroup } from "./Form/RadioGroup.svelte" export { default as Checkbox } from "./Form/Checkbox.svelte" export { default as InputDropdown } from "./Form/InputDropdown.svelte" export { default as PickerDropdown } from "./Form/PickerDropdown.svelte" +export { default as EnvDropdown } from "./Form/EnvDropdown.svelte" export { default as DetailSummary } from "./DetailSummary/DetailSummary.svelte" export { default as Popover } from "./Popover/Popover.svelte" export { default as ProgressBar } from "./ProgressBar/ProgressBar.svelte" diff --git a/packages/builder/src/builderStore/dataBinding.js b/packages/builder/src/builderStore/dataBinding.js index 6de73d487c..b8fa22e51b 100644 --- a/packages/builder/src/builderStore/dataBinding.js +++ b/packages/builder/src/builderStore/dataBinding.js @@ -21,6 +21,7 @@ import { import { TableNames } from "../constants" import { JSONUtils } from "@budibase/frontend-core" import ActionDefinitions from "components/design/settings/controls/ButtonActionEditor/manifest.json" +import { environment, licensing } from "stores/portal" // Regex to match all instances of template strings const CAPTURE_VAR_INSIDE_TEMPLATE = /{{([^}]+)}}/g @@ -53,8 +54,13 @@ export const getBindableProperties = (asset, componentId) => { * Gets all rest bindable data fields */ export const getRestBindings = () => { + const environmentVariablesEnabled = get(licensing).environmentVariablesEnabled const userBindings = getUserBindings() - return [...userBindings, ...getAuthBindings()] + return [ + ...userBindings, + ...getAuthBindings(), + ...(environmentVariablesEnabled ? getEnvironmentBindings() : []), + ] } /** @@ -89,6 +95,20 @@ export const getAuthBindings = () => { return bindings } +export const getEnvironmentBindings = () => { + let envVars = get(environment).variables + return envVars.map(variable => { + return { + type: "context", + runtimeBinding: `env.${makePropSafe(variable.name)}`, + readableBinding: `env.${variable.name}`, + category: "Environment", + icon: "Key", + display: { type: "string", name: variable.name }, + } + }) +} + /** * Utility - convert a key/value map to an array of custom 'context' bindings * @param {object} valueMap Key/value pairings diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index a73db5648b..5806189caa 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -18,6 +18,7 @@ import { automationStore } from "builderStore" import { tables } from "stores/backend" + import { environment, licensing } from "stores/portal" import WebhookDisplay from "../Shared/WebhookDisplay.svelte" import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte" import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte" @@ -33,6 +34,7 @@ import { Utils } from "@budibase/frontend-core" import { TriggerStepID, ActionStepID } from "constants/backend/automations" import { cloneDeep } from "lodash/fp" + import { onMount } from "svelte" export let block export let testData @@ -166,6 +168,24 @@ ) } + // Environment bindings + if ($licensing.environmentVariablesEnabled) { + bindings = bindings.concat( + $environment.variables.map(variable => { + return { + label: `env.${variable.name}`, + path: `env.${variable.name}`, + icon: "Key", + category: "Environment", + display: { + type: "string", + name: variable.name, + }, + } + }) + ) + } + return bindings } @@ -196,6 +216,14 @@ onChange({ detail: tempFilters }, defKey) drawer.hide() } + + onMount(async () => { + try { + await environment.loadVariables() + } catch (error) { + console.error(error) + } + })
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte index 6b35f3313f..211a6512a2 100644 --- a/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte +++ b/packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte @@ -6,17 +6,26 @@ Toggle, Button, TextArea, + Modal, + EnvDropdown, Accordion, + notifications, } from "@budibase/bbui" import KeyValueBuilder from "components/integration/KeyValueBuilder.svelte" import { capitalise } from "helpers" import { IntegrationTypes } from "constants/backend" import { createValidationStore } from "helpers/validation/yup" - import { createEventDispatcher } from "svelte" + import { createEventDispatcher, onMount } from "svelte" + import { environment, licensing, auth } from "stores/portal" + import CreateEditVariableModal from "components/portal/environment/CreateEditVariableModal.svelte" export let datasource export let schema export let creating + + let createVariableModal + let selectedKey + const validation = createValidationStore() const dispatch = createEventDispatcher() @@ -70,6 +79,37 @@ .filter(el => filter(el)) .map(([key]) => key) } + + async function save(data) { + try { + await environment.createVariable(data) + config[selectedKey] = `{{ env.${data.name} }}` + createVariableModal.hide() + } catch (err) { + notifications.error(`Failed to create variable: ${err.message}`) + } + } + + function showModal(configKey) { + selectedKey = configKey + createVariableModal.show() + } + + async function handleUpgradePanel() { + await environment.upgradePanelOpened() + $licensing.goToUpgradePage() + } + + onMount(async () => { + try { + await environment.loadVariables() + if ($auth.user) { + await licensing.init() + } + } catch (err) { + console.error(err) + } + })
@@ -134,11 +174,15 @@ {:else}
- showModal(configKey)} + variables={$environment.variables} type={schema[configKey].type} on:change bind:value={config[configKey]} error={$validation.errors[configKey]} + environmentVariablesEnabled={$licensing.environmentVariablesEnabled} + {handleUpgradePanel} />
{/if} @@ -146,6 +190,10 @@
+ + + + diff --git a/packages/builder/src/stores/backend/datasources.js b/packages/builder/src/stores/backend/datasources.js index bbbf86a41d..ed84bb8ee9 100644 --- a/packages/builder/src/stores/backend/datasources.js +++ b/packages/builder/src/stores/backend/datasources.js @@ -28,7 +28,7 @@ export function createDatasourcesStore() { })) } - const updateDatasource = async response => { + const updateDatasource = response => { const { datasource, error } = response store.update(state => { const currentIdx = state.list.findIndex(ds => ds._id === datasource._id) @@ -52,7 +52,7 @@ export function createDatasourcesStore() { datasourceId: datasource?._id, tablesFilter, }) - return await updateDatasource(response) + return updateDatasource(response) } const save = async (body, fetchSchema = false) => { diff --git a/packages/builder/src/stores/portal/environment.js b/packages/builder/src/stores/portal/environment.js new file mode 100644 index 0000000000..458eea27be --- /dev/null +++ b/packages/builder/src/stores/portal/environment.js @@ -0,0 +1,68 @@ +import { writable } from "svelte/store" +import { API } from "api" +import { Constants } from "@budibase/frontend-core" + +export function createEnvironmentStore() { + const { subscribe, update } = writable({ + variables: [], + status: {}, + }) + + async function checkStatus() { + const status = await API.checkEnvironmentVariableStatus() + update(store => { + store.status = status + return store + }) + } + + async function loadVariables() { + const envVars = await API.fetchEnvironmentVariables() + const mappedVars = envVars.variables.map(name => ({ name })) + update(store => { + store.variables = mappedVars + return store + }) + } + + async function createVariable(data) { + await API.createEnvironmentVariable(data) + let mappedVar = { name: data.name } + update(store => { + store.variables = [mappedVar, ...store.variables] + return store + }) + } + + async function deleteVariable(varName) { + await API.deleteEnvironmentVariable(varName) + update(store => { + store.variables = store.variables.filter( + envVar => envVar.name !== varName + ) + return store + }) + } + + async function updateVariable(data) { + await API.updateEnvironmentVariable(data) + } + + async function upgradePanelOpened() { + await API.publishEvent( + Constants.EventPublishType.ENV_VAR_UPGRADE_PANEL_OPENED + ) + } + + return { + subscribe, + checkStatus, + loadVariables, + createVariable, + deleteVariable, + updateVariable, + upgradePanelOpened, + } +} + +export const environment = createEnvironmentStore() diff --git a/packages/builder/src/stores/portal/index.js b/packages/builder/src/stores/portal/index.js index 9a27604034..8d704a5e15 100644 --- a/packages/builder/src/stores/portal/index.js +++ b/packages/builder/src/stores/portal/index.js @@ -11,4 +11,5 @@ export { groups } from "./groups" export { plugins } from "./plugins" export { backups } from "./backups" export { overview } from "./overview" +export { environment } from "./environment" export { menu } from "./menu" diff --git a/packages/builder/src/stores/portal/licensing.js b/packages/builder/src/stores/portal/licensing.js index 59a1622c9f..6f5c80e03c 100644 --- a/packages/builder/src/stores/portal/licensing.js +++ b/packages/builder/src/stores/portal/licensing.js @@ -60,6 +60,9 @@ export const createLicensingStore = () => { const backupsEnabled = license.features.includes( Constants.Features.BACKUPS ) + const environmentVariablesEnabled = license.features.includes( + Constants.Features.ENVIRONMENT_VARIABLES + ) store.update(state => { return { @@ -68,6 +71,7 @@ export const createLicensingStore = () => { isFreePlan, groupsEnabled, backupsEnabled, + environmentVariablesEnabled, } }) }, diff --git a/packages/builder/src/stores/portal/menu.js b/packages/builder/src/stores/portal/menu.js index 29fd6144f5..8eea36c08c 100644 --- a/packages/builder/src/stores/portal/menu.js +++ b/packages/builder/src/stores/portal/menu.js @@ -50,6 +50,10 @@ export const menu = derived([admin, auth], ([$admin, $auth]) => { title: "Organisation", href: "/builder/portal/settings/organisation", }, + { + title: "Environment", + href: "/builder/portal/settings/environment", + }, ] if (!$admin.cloud) { settingsSubPages.push({ diff --git a/packages/frontend-core/src/api/environmentVariables.js b/packages/frontend-core/src/api/environmentVariables.js new file mode 100644 index 0000000000..badd93ad69 --- /dev/null +++ b/packages/frontend-core/src/api/environmentVariables.js @@ -0,0 +1,36 @@ +export const buildEnvironmentVariableEndpoints = API => ({ + checkEnvironmentVariableStatus: async () => { + return await API.get({ + url: `/api/env/variables/status`, + }) + }, + + /** + * Fetches a list of environment variables + */ + fetchEnvironmentVariables: async () => { + return await API.get({ + url: `/api/env/variables`, + json: false, + }) + }, + + createEnvironmentVariable: async data => { + return await API.post({ + url: `/api/env/variables`, + body: data, + }) + }, + deleteEnvironmentVariable: async varName => { + return await API.delete({ + url: `/api/env/variables/${varName}`, + }) + }, + + updateEnvironmentVariable: async data => { + return await API.patch({ + url: `/api/env/variables/${data.name}`, + body: data, + }) + }, +}) diff --git a/packages/frontend-core/src/api/events.js b/packages/frontend-core/src/api/events.js new file mode 100644 index 0000000000..3f17722d3e --- /dev/null +++ b/packages/frontend-core/src/api/events.js @@ -0,0 +1,13 @@ +export const buildEventEndpoints = API => ({ + /** + * Publish a specific event to the backend. + */ + publishEvent: async eventType => { + return await API.post({ + url: `/api/global/event/publish`, + body: { + type: eventType, + }, + }) + }, +}) diff --git a/packages/frontend-core/src/api/index.js b/packages/frontend-core/src/api/index.js index 9a40b21351..e2935b416b 100644 --- a/packages/frontend-core/src/api/index.js +++ b/packages/frontend-core/src/api/index.js @@ -26,7 +26,8 @@ import { buildLicensingEndpoints } from "./licensing" import { buildGroupsEndpoints } from "./groups" import { buildPluginEndpoints } from "./plugins" import { buildBackupsEndpoints } from "./backups" - +import { buildEnvironmentVariableEndpoints } from "./environmentVariables" +import { buildEventEndpoints } from "./events" const defaultAPIClientConfig = { /** * Certain definitions can't change at runtime for client apps, such as the @@ -247,5 +248,7 @@ export const createAPIClient = config => { ...buildGroupsEndpoints(API), ...buildPluginEndpoints(API), ...buildBackupsEndpoints(API), + ...buildEnvironmentVariableEndpoints(API), + ...buildEventEndpoints(API), } } diff --git a/packages/frontend-core/src/constants.js b/packages/frontend-core/src/constants.js index ff9f4628d0..3a16013df2 100644 --- a/packages/frontend-core/src/constants.js +++ b/packages/frontend-core/src/constants.js @@ -114,6 +114,7 @@ export const ApiVersion = "1" export const Features = { USER_GROUPS: "userGroups", BACKUPS: "appBackups", + ENVIRONMENT_VARIABLES: "environmentVariables", } // Role IDs @@ -174,3 +175,7 @@ export const Themes = [ base: "darkest", }, ] + +export const EventPublishType = { + ENV_VAR_UPGRADE_PANEL_OPENED: "environment_variable_upgrade_panel_opened", +} diff --git a/packages/server/__mocks__/node-fetch.ts b/packages/server/__mocks__/node-fetch.ts index 44d1e54a32..78071dc8af 100644 --- a/packages/server/__mocks__/node-fetch.ts +++ b/packages/server/__mocks__/node-fetch.ts @@ -1,5 +1,7 @@ +// @ts-ignore import fs from "fs" module FetchMock { + // @ts-ignore const fetch = jest.requireActual("node-fetch") let failCount = 0 diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 80f21cd043..b8566bbf4c 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -29,6 +29,7 @@ async function init() { ACCOUNT_PORTAL_URL: "http://localhost:10001", ACCOUNT_PORTAL_API_KEY: "budibase", JWT_SECRET: "testsecret", + ENCRYPTION_KEY: "testsecret", REDIS_PASSWORD: "budibase", MINIO_ACCESS_KEY: "budibase", MINIO_SECRET_KEY: "budibase", diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 06e7dc8a57..33f5a53894 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -112,12 +112,11 @@ function checkAppName( } } -async function createInstance(template: any, includeSampleData: boolean) { - const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null - const baseAppId = generateAppID(tenantId) - const appId = generateDevAppID(baseAppId) - await context.updateAppId(appId) - +async function createInstance( + appId: string, + template: any, + includeSampleData: boolean +) { const db = context.getAppDB() await db.put({ _id: "_design/database", @@ -250,82 +249,90 @@ async function performAppCreate(ctx: BBContext) { instanceConfig.file = ctx.request.files.templateFile } const includeSampleData = isQsTrue(ctx.request.body.sampleData) - const instance = await createInstance(instanceConfig, includeSampleData) - const appId = instance._id - const db = context.getAppDB() + const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null + const appId = generateDevAppID(generateAppID(tenantId)) - let newApplication: App = { - _id: DocumentType.APP_METADATA, - _rev: undefined, - appId, - type: "app", - version: packageJson.version, - componentLibraries: ["@budibase/standard-components"], - name: name, - url: url, - template: templateKey, - instance, - tenantId: tenancy.getTenantId(), - updatedAt: new Date().toISOString(), - createdAt: new Date().toISOString(), - status: AppStatus.DEV, - navigation: { - navigation: "Top", - title: name, - navWidth: "Large", - navBackground: "var(--spectrum-global-color-gray-100)", - links: [ - { - url: "/home", - text: "Home", - }, - ], - }, - theme: "spectrum--light", - customTheme: { - buttonBorderRadius: "16px", - }, - } + return await context.doInAppContext(appId, async () => { + const instance = await createInstance( + appId, + instanceConfig, + includeSampleData + ) + const db = context.getAppDB() - // If we used a template or imported an app there will be an existing doc. - // Fetch and migrate some metadata from the existing app. - try { - const existing: App = await db.get(DocumentType.APP_METADATA) - const keys: (keyof App)[] = [ - "_rev", - "navigation", - "theme", - "customTheme", - "icon", - ] - keys.forEach(key => { - if (existing[key]) { - // @ts-ignore - newApplication[key] = existing[key] - } - }) - - // Migrate navigation settings and screens if required - if (existing) { - const navigation = await migrateAppNavigation() - if (navigation) { - newApplication.navigation = navigation - } + let newApplication: App = { + _id: DocumentType.APP_METADATA, + _rev: undefined, + appId, + type: "app", + version: packageJson.version, + componentLibraries: ["@budibase/standard-components"], + name: name, + url: url, + template: templateKey, + instance, + tenantId: tenancy.getTenantId(), + updatedAt: new Date().toISOString(), + createdAt: new Date().toISOString(), + status: AppStatus.DEV, + navigation: { + navigation: "Top", + title: name, + navWidth: "Large", + navBackground: "var(--spectrum-global-color-gray-100)", + links: [ + { + url: "/home", + text: "Home", + }, + ], + }, + theme: "spectrum--light", + customTheme: { + buttonBorderRadius: "16px", + }, } - } catch (err) { - // Nothing to do - } - const response = await db.put(newApplication, { force: true }) - newApplication._rev = response.rev + // If we used a template or imported an app there will be an existing doc. + // Fetch and migrate some metadata from the existing app. + try { + const existing: App = await db.get(DocumentType.APP_METADATA) + const keys: (keyof App)[] = [ + "_rev", + "navigation", + "theme", + "customTheme", + "icon", + ] + keys.forEach(key => { + if (existing[key]) { + // @ts-ignore + newApplication[key] = existing[key] + } + }) - /* istanbul ignore next */ - if (!env.isTest()) { - await createApp(appId) - } + // Migrate navigation settings and screens if required + if (existing) { + const navigation = await migrateAppNavigation() + if (navigation) { + newApplication.navigation = navigation + } + } + } catch (err) { + // Nothing to do + } - await cache.app.invalidateAppMetadata(appId, newApplication) - return newApplication + const response = await db.put(newApplication, { force: true }) + newApplication._rev = response.rev + + /* istanbul ignore next */ + if (!env.isTest()) { + await createApp(appId) + } + + await cache.app.invalidateAppMetadata(appId, newApplication) + return newApplication + }) } async function creationEvents(request: any, app: App) { diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index d47629ed3e..469cd29b56 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -12,9 +12,11 @@ import { getIntegration } from "../../integrations" import { getDatasourceAndQuery } from "./row/utils" import { invalidateDynamicVariables } from "../../threads/utils" import { db as dbCore, context, events } from "@budibase/backend-core" -import { BBContext, Datasource, Row } from "@budibase/types" +import { UserCtx, Datasource, Row } from "@budibase/types" +import sdk from "../../sdk" +import { mergeConfigs } from "../../sdk/app/datasources/datasources" -export async function fetch(ctx: BBContext) { +export async function fetch(ctx: UserCtx) { // Get internal tables const db = context.getAppDB() const internalTables = await db.allDocs( @@ -43,25 +45,23 @@ export async function fetch(ctx: BBContext) { ) ).rows.map(row => row.doc) - const allDatasources = [bbInternalDb, ...datasources] + const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([ + bbInternalDb, + ...datasources, + ]) for (let datasource of allDatasources) { - if (datasource.config && datasource.config.auth) { - // strip secrets from response so they don't show in the network request - delete datasource.config.auth - } - if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { - datasource.entities = internal[datasource._id] + datasource.entities = internal[datasource._id!] } } ctx.body = [bbInternalDb, ...datasources] } -export async function buildSchemaFromDb(ctx: BBContext) { +export async function buildSchemaFromDb(ctx: UserCtx) { const db = context.getAppDB() - const datasource = await db.get(ctx.params.datasourceId) + const datasource = await sdk.datasources.get(ctx.params.datasourceId) const tablesFilter = ctx.request.body.tablesFilter let { tables, error } = await buildSchemaHelper(datasource) @@ -146,11 +146,11 @@ async function invalidateVariables( await invalidateDynamicVariables(toInvalidate) } -export async function update(ctx: BBContext) { +export async function update(ctx: UserCtx) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId - let datasource = await db.get(datasourceId) - const auth = datasource.config.auth + let datasource = await sdk.datasources.get(datasourceId) + const auth = datasource.config?.auth await invalidateVariables(datasource, ctx.request.body) const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE @@ -159,10 +159,13 @@ export async function update(ctx: BBContext) { ? { name: ctx.request.body?.name } : ctx.request.body - datasource = { ...datasource, ...dataSourceBody } + datasource = { + ...datasource, + ...sdk.datasources.mergeConfigs(dataSourceBody, datasource), + } if (auth && !ctx.request.body.auth) { // don't strip auth config from DB - datasource.config.auth = auth + datasource.config!.auth = auth } const response = await db.put(datasource) @@ -179,10 +182,12 @@ export async function update(ctx: BBContext) { ctx.status = 200 ctx.message = "Datasource saved successfully." - ctx.body = { datasource } + ctx.body = { + datasource: await sdk.datasources.removeSecretSingle(datasource), + } } -export async function save(ctx: BBContext) { +export async function save(ctx: UserCtx) { const db = context.getAppDB() const plus = ctx.request.body.datasource.plus const fetchSchema = ctx.request.body.fetchSchema @@ -213,7 +218,9 @@ export async function save(ctx: BBContext) { } } - const response: any = { datasource } + const response: any = { + datasource: await sdk.datasources.removeSecretSingle(datasource), + } if (schemaError) { response.error = schemaError } @@ -251,11 +258,11 @@ async function destroyInternalTablesBySourceId(datasourceId: string) { } } -export async function destroy(ctx: BBContext) { +export async function destroy(ctx: UserCtx) { const db = context.getAppDB() const datasourceId = ctx.params.datasourceId - const datasource = await db.get(datasourceId) + const datasource = await sdk.datasources.get(datasourceId) // Delete all queries for the datasource if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { @@ -279,13 +286,14 @@ export async function destroy(ctx: BBContext) { ctx.status = 200 } -export async function find(ctx: BBContext) { +export async function find(ctx: UserCtx) { const database = context.getAppDB() - ctx.body = await database.get(ctx.params.datasourceId) + const datasource = await database.get(ctx.params.datasourceId) + ctx.body = await sdk.datasources.removeSecretSingle(datasource) } // dynamic query functionality -export async function query(ctx: BBContext) { +export async function query(ctx: UserCtx) { const queryJson = ctx.request.body try { ctx.body = await getDatasourceAndQuery(queryJson) @@ -313,7 +321,7 @@ function updateError(error: any, newError: any, tables: string[]) { async function buildSchemaHelper(datasource: Datasource) { const Connector = await getIntegration(datasource.source) - + datasource = await sdk.datasources.enrich(datasource) // Connect to the DB and build the schema const connector = new Connector(datasource.config) await connector.buildSchema(datasource._id, datasource.entities) diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts index 0aaf8bb3dd..fd4842ea75 100644 --- a/packages/server/src/api/controllers/query/index.ts +++ b/packages/server/src/api/controllers/query/index.ts @@ -7,6 +7,8 @@ import { invalidateDynamicVariables } from "../../../threads/utils" import env from "../../../environment" import { quotas } from "@budibase/pro" import { events, context, utils, constants } from "@budibase/backend-core" +import sdk from "../../../sdk" +import { QueryEvent } from "../../../threads/definitions" const Runner = new Thread(ThreadType.QUERY, { timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000, @@ -81,7 +83,7 @@ export async function save(ctx: any) { const db = context.getAppDB() const query = ctx.request.body - const datasource = await db.get(query.datasourceId) + const datasource = await sdk.datasources.get(query.datasourceId) let eventFn if (!query._id) { @@ -126,9 +128,9 @@ function getAuthConfig(ctx: any) { } export async function preview(ctx: any) { - const db = context.getAppDB() - - const datasource = await db.get(ctx.request.body.datasourceId) + const { datasource, envVars } = await sdk.datasources.getWithEnvVars( + ctx.request.body.datasourceId + ) const query = ctx.request.body // preview may not have a queryId as it hasn't been saved, but if it does // this stops dynamic variables from calling the same query @@ -137,20 +139,22 @@ export async function preview(ctx: any) { const authConfigCtx: any = getAuthConfig(ctx) try { - const runFn = () => - Runner.run({ - appId: ctx.appId, - datasource, - queryVerb, - fields, - parameters, - transformer, - queryId, - ctx: { - user: ctx.user, - auth: { ...authConfigCtx }, - }, - }) + const inputs: QueryEvent = { + appId: ctx.appId, + datasource, + queryVerb, + fields, + parameters, + transformer, + queryId, + // have to pass down to the thread runner - can't put into context now + environmentVariables: envVars, + ctx: { + user: ctx.user, + auth: { ...authConfigCtx }, + }, + } + const runFn = () => Runner.run(inputs) const { rows, keys, info, extra } = await quotas.addQuery(runFn, { datasourceId: datasource._id, @@ -201,7 +205,9 @@ async function execute( const db = context.getAppDB() const query = await db.get(ctx.params.queryId) - const datasource = await db.get(query.datasourceId) + const { datasource, envVars } = await sdk.datasources.getWithEnvVars( + query.datasourceId + ) let authConfigCtx: any = {} if (!opts.isAutomation) { @@ -219,21 +225,23 @@ async function execute( // call the relevant CRUD method on the integration class try { - const runFn = () => - Runner.run({ - appId: ctx.appId, - datasource, - queryVerb: query.queryVerb, - fields: query.fields, - pagination: ctx.request.body.pagination, - parameters: enrichedParameters, - transformer: query.transformer, - queryId: ctx.params.queryId, - ctx: { - user: ctx.user, - auth: { ...authConfigCtx }, - }, - }) + const inputs: QueryEvent = { + appId: ctx.appId, + datasource, + queryVerb: query.queryVerb, + fields: query.fields, + pagination: ctx.request.body.pagination, + parameters: enrichedParameters, + transformer: query.transformer, + queryId: ctx.params.queryId, + // have to pass down to the thread runner - can't put into context now + environmentVariables: envVars, + ctx: { + user: ctx.user, + auth: { ...authConfigCtx }, + }, + } + const runFn = () => Runner.run(inputs) const { rows, pagination, extra } = await quotas.addQuery(runFn, { datasourceId: datasource._id, @@ -266,18 +274,18 @@ export async function executeV2( const removeDynamicVariables = async (queryId: any) => { const db = context.getAppDB() const query = await db.get(queryId) - const datasource = await db.get(query.datasourceId) - const dynamicVariables = datasource.config.dynamicVariables + const datasource = await sdk.datasources.get(query.datasourceId) + const dynamicVariables = datasource.config?.dynamicVariables as any[] if (dynamicVariables) { // delete dynamic variables from the datasource - datasource.config.dynamicVariables = dynamicVariables.filter( + datasource.config!.dynamicVariables = dynamicVariables!.filter( (dv: any) => dv.queryId !== queryId ) await db.put(datasource) // invalidate the deleted variables - const variablesToDelete = dynamicVariables.filter( + const variablesToDelete = dynamicVariables!.filter( (dv: any) => dv.queryId === queryId ) await invalidateDynamicVariables(variablesToDelete) @@ -289,7 +297,7 @@ export async function destroy(ctx: any) { const queryId = ctx.params.queryId await removeDynamicVariables(queryId) const query = await db.get(queryId) - const datasource = await db.get(query.datasourceId) + const datasource = await sdk.datasources.get(query.datasourceId) await db.remove(ctx.params.queryId, ctx.params.revId) ctx.message = `Query deleted.` ctx.status = 200 diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 6085921423..81b698fedd 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -25,6 +25,7 @@ import { cloneDeep } from "lodash/fp" import { processFormulas, processDates } from "../../../utilities/rowProcessor" import { context } from "@budibase/backend-core" import { removeKeyNumbering } from "./utils" +import sdk from "../../../sdk" export interface ManyRelationship { tableId?: string @@ -664,8 +665,7 @@ export class ExternalRequest { throw "Unable to run without a table name" } if (!this.datasource) { - const db = context.getAppDB() - this.datasource = await db.get(datasourceId) + this.datasource = await sdk.datasources.get(datasourceId!) if (!this.datasource || !this.datasource.entities) { throw "No tables found, fetch tables before query." } diff --git a/packages/server/src/api/controllers/row/external.ts b/packages/server/src/api/controllers/row/external.ts index 83564564b8..ed14eb6929 100644 --- a/packages/server/src/api/controllers/row/external.ts +++ b/packages/server/src/api/controllers/row/external.ts @@ -19,6 +19,7 @@ import { Table, Datasource, } from "@budibase/types" +import sdk from "../../../sdk" export async function handleRequest( operation: Operation, @@ -179,10 +180,9 @@ export async function validate(ctx: BBContext) { export async function exportRows(ctx: BBContext) { const { datasourceId } = breakExternalTableId(ctx.params.tableId) - const db = context.getAppDB() const format = ctx.query.format const { columns } = ctx.request.body - const datasource = await db.get(datasourceId) + const datasource = await sdk.datasources.get(datasourceId!) if (!datasource || !datasource.entities) { ctx.throw(400, "Datasource has not been configured for plus API.") } @@ -225,8 +225,7 @@ export async function fetchEnrichedRow(ctx: BBContext) { const id = ctx.params.rowId const tableId = ctx.params.tableId const { datasourceId, tableName } = breakExternalTableId(tableId) - const db = context.getAppDB() - const datasource: Datasource = await db.get(datasourceId) + const datasource: Datasource = await sdk.datasources.get(datasourceId!) if (!tableName) { ctx.throw(400, "Unable to find table.") } diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index 60aa8a6b7a..aa58ca6055 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -8,6 +8,7 @@ export { removeKeyNumbering } from "../../../integrations/base/utils" const validateJs = require("validate.js") const { cloneDeep } = require("lodash/fp") import { Ctx } from "@budibase/types" +import sdk from "../../../sdk" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { @@ -21,8 +22,7 @@ validateJs.extend(validateJs.validators.datetime, { export async function getDatasourceAndQuery(json: any) { const datasourceId = json.endpoint.datasourceId - const db = context.getAppDB() - const datasource = await db.get(datasourceId) + const datasource = await sdk.datasources.get(datasourceId) return makeExternalQuery(datasource, json) } diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index 1d2bdba32a..c58e1a374d 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -1,20 +1,21 @@ require("svelte/register") -const send = require("koa-send") -const { resolve, join } = require("../../../utilities/centralPath") +import { resolve, join } from "../../../utilities/centralPath" const uuid = require("uuid") import { ObjectStoreBuckets } from "../../../constants" -const { processString } = require("@budibase/string-templates") -const { +import { processString } from "@budibase/string-templates" +import { loadHandlebarsFile, NODE_MODULES_PATH, TOP_LEVEL_PATH, -} = require("../../../utilities/fileSystem") +} from "../../../utilities/fileSystem" import env from "../../../environment" -const { DocumentType } = require("../../../db/utils") -const { context, objectStore, utils } = require("@budibase/backend-core") -const AWS = require("aws-sdk") -const fs = require("fs") +import { DocumentType } from "../../../db/utils" +import { context, objectStore, utils } from "@budibase/backend-core" +import AWS from "aws-sdk" +import fs from "fs" +import sdk from "../../../sdk" +const send = require("koa-send") async function prepareUpload({ s3Key, bucket, metadata, file }: any) { const response = await objectStore.upload({ @@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) { title: appInfo.name, production: env.isProd(), appId, - clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version), + clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version), usedPlugins: plugins, }) @@ -135,7 +136,7 @@ export const serveBuilderPreview = async function (ctx: any) { let appId = context.getAppId() const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`) ctx.body = await processString(previewHbs, { - clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version), + clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version), }) } else { // just return the app info for jest to assert on @@ -150,13 +151,11 @@ export const serveClientLibrary = async function (ctx: any) { } export const getSignedUploadURL = async function (ctx: any) { - const database = context.getAppDB() - // Ensure datasource is valid let datasource try { const { datasourceId } = ctx.params - datasource = await database.get(datasourceId) + datasource = await sdk.datasources.get(datasourceId, { enriched: true }) if (!datasource) { ctx.throw(400, "The specified datasource could not be found") } @@ -172,8 +171,8 @@ export const getSignedUploadURL = async function (ctx: any) { // Determine type of datasource and generate signed URL let signedUrl let publicUrl - const awsRegion = datasource?.config?.region || "eu-west-1" - if (datasource.source === "S3") { + const awsRegion = (datasource?.config?.region || "eu-west-1") as string + if (datasource?.source === "S3") { const { bucket, key } = ctx.request.body || {} if (!bucket || !key) { ctx.throw(400, "bucket and key values are required") @@ -182,8 +181,8 @@ export const getSignedUploadURL = async function (ctx: any) { try { const s3 = new AWS.S3({ region: awsRegion, - accessKeyId: datasource?.config?.accessKeyId, - secretAccessKey: datasource?.config?.secretAccessKey, + accessKeyId: datasource?.config?.accessKeyId as string, + secretAccessKey: datasource?.config?.secretAccessKey as string, apiVersion: "2006-03-01", signatureVersion: "v4", }) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index dbe09f59c1..0869daef46 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -219,7 +219,7 @@ export async function save(ctx: BBContext) { } const db = context.getAppDB() - const datasource = await db.get(datasourceId) + const datasource = await sdk.datasources.get(datasourceId) if (!datasource.entities) { datasource.entities = {} } @@ -322,15 +322,17 @@ export async function destroy(ctx: BBContext) { const datasourceId = getDatasourceId(tableToDelete) const db = context.getAppDB() - const datasource = await db.get(datasourceId) + const datasource = await sdk.datasources.get(datasourceId!) const tables = datasource.entities const operation = Operation.DELETE_TABLE - await makeTableRequest(datasource, operation, tableToDelete, tables) + if (tables) { + await makeTableRequest(datasource, operation, tableToDelete, tables) + cleanupRelationships(tableToDelete, tables) + delete tables[tableToDelete.name] + datasource.entities = tables + } - cleanupRelationships(tableToDelete, tables) - - delete datasource.entities[tableToDelete.name] await db.put(datasource) return tableToDelete diff --git a/packages/server/src/api/controllers/webhook.ts b/packages/server/src/api/controllers/webhook.ts index f877110646..c3fc3892d3 100644 --- a/packages/server/src/api/controllers/webhook.ts +++ b/packages/server/src/api/controllers/webhook.ts @@ -39,60 +39,62 @@ export async function destroy(ctx: BBContext) { } export async function buildSchema(ctx: BBContext) { - await context.updateAppId(ctx.params.instance) - const db = context.getAppDB() - const webhook = (await db.get(ctx.params.id)) as Webhook - webhook.bodySchema = toJsonSchema(ctx.request.body) - // update the automation outputs - if (webhook.action.type === WebhookActionType.AUTOMATION) { - let automation = (await db.get(webhook.action.target)) as Automation - const autoOutputs = automation.definition.trigger.schema.outputs - let properties = webhook.bodySchema.properties - // reset webhook outputs - autoOutputs.properties = { - body: autoOutputs.properties.body, - } - for (let prop of Object.keys(properties)) { - autoOutputs.properties[prop] = { - type: properties[prop].type, - description: AUTOMATION_DESCRIPTION, + await context.doInAppContext(ctx.params.instance, async () => { + const db = context.getAppDB() + const webhook = (await db.get(ctx.params.id)) as Webhook + webhook.bodySchema = toJsonSchema(ctx.request.body) + // update the automation outputs + if (webhook.action.type === WebhookActionType.AUTOMATION) { + let automation = (await db.get(webhook.action.target)) as Automation + const autoOutputs = automation.definition.trigger.schema.outputs + let properties = webhook.bodySchema.properties + // reset webhook outputs + autoOutputs.properties = { + body: autoOutputs.properties.body, } + for (let prop of Object.keys(properties)) { + autoOutputs.properties[prop] = { + type: properties[prop].type, + description: AUTOMATION_DESCRIPTION, + } + } + await db.put(automation) } - await db.put(automation) - } - ctx.body = await db.put(webhook) + ctx.body = await db.put(webhook) + }) } export async function trigger(ctx: BBContext) { const prodAppId = dbCore.getProdAppID(ctx.params.instance) - await context.updateAppId(prodAppId) - try { - const db = context.getAppDB() - const webhook = (await db.get(ctx.params.id)) as Webhook - // validate against the schema - if (webhook.bodySchema) { - validate(ctx.request.body, webhook.bodySchema) - } - const target = await db.get(webhook.action.target) - if (webhook.action.type === WebhookActionType.AUTOMATION) { - // trigger with both the pure request and then expand it - // incase the user has produced a schema to bind to - await triggers.externalTrigger(target, { - body: ctx.request.body, - ...ctx.request.body, - appId: prodAppId, - }) - } - ctx.status = 200 - ctx.body = { - message: "Webhook trigger fired successfully", - } - } catch (err: any) { - if (err.status === 404) { + await context.doInAppContext(prodAppId, async () => { + try { + const db = context.getAppDB() + const webhook = (await db.get(ctx.params.id)) as Webhook + // validate against the schema + if (webhook.bodySchema) { + validate(ctx.request.body, webhook.bodySchema) + } + const target = await db.get(webhook.action.target) + if (webhook.action.type === WebhookActionType.AUTOMATION) { + // trigger with both the pure request and then expand it + // incase the user has produced a schema to bind to + await triggers.externalTrigger(target, { + body: ctx.request.body, + ...ctx.request.body, + appId: prodAppId, + }) + } ctx.status = 200 ctx.body = { - message: "Application not deployed yet.", + message: "Webhook trigger fired successfully", + } + } catch (err: any) { + if (err.status === 404) { + ctx.status = 200 + ctx.body = { + message: "Application not deployed yet.", + } } } - } + }) } diff --git a/packages/server/src/api/routes/index.ts b/packages/server/src/api/routes/index.ts index 8d6b1754a7..9dc119ec83 100644 --- a/packages/server/src/api/routes/index.ts +++ b/packages/server/src/api/routes/index.ts @@ -33,6 +33,7 @@ export { default as publicRoutes } from "./public" const appBackupRoutes = pro.appBackups const scheduleRoutes = pro.schedules +const environmentVariableRoutes = pro.environmentVariables export const mainRoutes: Router[] = [ appBackupRoutes, @@ -63,6 +64,7 @@ export const mainRoutes: Router[] = [ migrationRoutes, pluginRoutes, scheduleRoutes, + environmentVariableRoutes, // these need to be handled last as they still use /api/:tableId // this could be breaking as koa may recognise other routes as this tableRoutes, diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index f4462b3595..6d1cd206c6 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -3,7 +3,7 @@ import * as rowController from "../controllers/row" import authorized from "../../middleware/authorized" import { paramResource, paramSubResource } from "../../middleware/resourceId" import { permissions } from "@budibase/backend-core" -const { internalSearchValidator } = require("./utils/validators") +import { internalSearchValidator } from "./utils/validators" const { PermissionType, PermissionLevel } = permissions const router: Router = new Router() diff --git a/packages/server/src/api/routes/tests/datasource.spec.ts b/packages/server/src/api/routes/tests/datasource.spec.ts index 66888023c4..cad8009dae 100644 --- a/packages/server/src/api/routes/tests/datasource.spec.ts +++ b/packages/server/src/api/routes/tests/datasource.spec.ts @@ -2,7 +2,8 @@ jest.mock("pg") import * as setup from "./utilities" import { checkBuilderEndpoint } from "./utilities/TestFunctions" import { checkCacheForDynamicVariable } from "../../../threads/utils" -import { events } from "@budibase/backend-core" +import { context, events } from "@budibase/backend-core" +import sdk from "../../../sdk" let { basicDatasource } = setup.structures const pg = require("pg") @@ -184,4 +185,37 @@ describe("/datasources", () => { }) }) }) + + describe("check secret replacement", () => { + async function makeDatasource() { + datasource = basicDatasource() + datasource.datasource.config.password = "testing" + const res = await request + .post(`/api/datasources`) + .send(datasource) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + return res.body.datasource + } + + it("should save a datasource with password", async () => { + const datasource = await makeDatasource() + expect(datasource.config.password).toBe("--secret-value--") + }) + + it("should not the password on update with the --secret-value--", async () => { + const datasource = await makeDatasource() + await request + .put(`/api/datasources/${datasource._id}`) + .send(datasource) + .set(config.defaultHeaders()) + .expect("Content-Type", /json/) + .expect(200) + await context.doInAppContext(config.getAppId(), async () => { + const dbDatasource: any = await sdk.datasources.get(datasource._id) + expect(dbDatasource.config.password).toBe("testing") + }) + }) + }) }) diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index d4168c020f..79cbe6e4ac 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -24,6 +24,7 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] + env?: Record trigger: any } diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index fac8403c49..b3fbb22bde 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -1,6 +1,6 @@ import { bootstrap } from "global-agent" const fixPath = require("fix-path") -const { checkDevelopmentEnvironment } = require("./utilities/fileSystem") +import { checkDevelopmentEnvironment } from "./utilities/fileSystem" function runServer() { // this will shutdown the system if development environment not ready diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 7435b28141..4f31e37744 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -1,10 +1,12 @@ import { QueryJson, Datasource } from "@budibase/types" -const { getIntegration } = require("../index") +import { getIntegration } from "../index" +import sdk from "../../sdk" export async function makeExternalQuery( datasource: Datasource, json: QueryJson ) { + datasource = await sdk.datasources.enrich(datasource) const Integration = await getIntegration(datasource.source) // query is the opinionated function if (Integration.prototype.query) { diff --git a/packages/server/src/integrations/base/sqlTable.ts b/packages/server/src/integrations/base/sqlTable.ts index 2ab610fecd..58b2b7a3d1 100644 --- a/packages/server/src/integrations/base/sqlTable.ts +++ b/packages/server/src/integrations/base/sqlTable.ts @@ -3,7 +3,7 @@ import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types" import { breakExternalTableId } from "../utils" import SchemaBuilder = Knex.SchemaBuilder import CreateTableBuilder = Knex.CreateTableBuilder -const { FieldTypes, RelationshipTypes } = require("../../constants") +import { FieldTypes, RelationshipTypes } from "../../constants" function generateSchema( schema: CreateTableBuilder, diff --git a/packages/server/src/integrations/dynamodb.ts b/packages/server/src/integrations/dynamodb.ts index bf9a0c3b52..28c1c7b52b 100644 --- a/packages/server/src/integrations/dynamodb.ts +++ b/packages/server/src/integrations/dynamodb.ts @@ -5,8 +5,8 @@ import { IntegrationBase, } from "@budibase/types" -const AWS = require("aws-sdk") -const { AWS_REGION } = require("../db/dynamoClient") +import AWS from "aws-sdk" +import { AWS_REGION } from "../db/dynamoClient" interface DynamoDBConfig { region: string @@ -182,7 +182,7 @@ class DynamoDBIntegration implements IntegrationBase { return response } - async describe(query: { table: string }) { + async describe(query: { table: string }): Promise { const params = { TableName: query.table, } diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 1cd9a356b3..b3881c63a9 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -16,7 +16,7 @@ import { finaliseExternalTables, } from "./utils" import dayjs from "dayjs" -const { NUMBER_REGEX } = require("../utilities") +import { NUMBER_REGEX } from "../utilities" import Sql from "./base/sql" import { MySQLColumn } from "./base/types" diff --git a/packages/server/src/integrations/queries/sql.ts b/packages/server/src/integrations/queries/sql.ts index 6e66114ec2..6d42117d7d 100644 --- a/packages/server/src/integrations/queries/sql.ts +++ b/packages/server/src/integrations/queries/sql.ts @@ -1,55 +1,10 @@ -import { findHBSBlocks, processStringSync } from "@budibase/string-templates" +import { findHBSBlocks } from "@budibase/string-templates" import { DatasourcePlus } from "@budibase/types" +import sdk from "../../sdk" const CONST_CHAR_REGEX = new RegExp("'[^']*'", "g") -export function enrichQueryFields( - fields: { [key: string]: any }, - parameters = {} -) { - const enrichedQuery: { [key: string]: any } = Array.isArray(fields) ? [] : {} - if (!fields || !parameters) { - return enrichedQuery - } - // enrich the fields with dynamic parameters - for (let key of Object.keys(fields)) { - if (fields[key] == null) { - continue - } - if (typeof fields[key] === "object") { - // enrich nested fields object - enrichedQuery[key] = enrichQueryFields(fields[key], parameters) - } else if (typeof fields[key] === "string") { - // enrich string value as normal - enrichedQuery[key] = processStringSync(fields[key], parameters, { - noEscaping: true, - noHelpers: true, - escapeNewlines: true, - }) - } else { - enrichedQuery[key] = fields[key] - } - } - if ( - enrichedQuery.json || - enrichedQuery.customData || - enrichedQuery.requestBody - ) { - try { - enrichedQuery.json = JSON.parse( - enrichedQuery.json || - enrichedQuery.customData || - enrichedQuery.requestBody - ) - } catch (err) { - // no json found, ignore - } - delete enrichedQuery.customData - } - return enrichedQuery -} - -export function interpolateSQL( +export async function interpolateSQL( fields: { [key: string]: any }, parameters: { [key: string]: any }, integration: DatasourcePlus @@ -90,7 +45,7 @@ export function interpolateSQL( else if (listRegexMatch) { arrays.push(binding) // determine the length of the array - const value = enrichQueryFields([binding], parameters)[0] + const value = (await sdk.queries.enrichContext([binding], parameters))[0] .split(",") .map((val: string) => val.trim()) // build a string like ($1, $2, $3) @@ -109,7 +64,7 @@ export function interpolateSQL( } // replicate the knex structure fields.sql = sql - fields.bindings = enrichQueryFields(variables, parameters) + fields.bindings = await sdk.queries.enrichContext(variables, parameters) // check for arrays in the data let updated: string[] = [] for (let i = 0; i < variables.length; i++) { diff --git a/packages/server/src/integrations/rest.ts b/packages/server/src/integrations/rest.ts index 08d2337593..f13c14e504 100644 --- a/packages/server/src/integrations/rest.ts +++ b/packages/server/src/integrations/rest.ts @@ -16,11 +16,11 @@ import { import { get } from "lodash" import * as https from "https" import qs from "querystring" -const fetch = require("node-fetch") -const { formatBytes } = require("../utilities") -const { performance } = require("perf_hooks") -const FormData = require("form-data") -const { URLSearchParams } = require("url") +import fetch from "node-fetch" +import { formatBytes } from "../utilities" +import { performance } from "perf_hooks" +import FormData from "form-data" +import { URLSearchParams } from "url" const BodyTypes = { NONE: "none", @@ -204,12 +204,12 @@ class RestIntegration implements IntegrationBase { // Append page number or cursor param if configured if (pageParam && paginationValues.page != null) { - params.append(pageParam, paginationValues.page) + params.append(pageParam, paginationValues.page as string) } // Append page size param if configured if (sizeParam && paginationValues.limit != null) { - params.append(sizeParam, paginationValues.limit) + params.append(sizeParam, String(paginationValues.limit)) } // Prepend query string with pagination params @@ -280,7 +280,7 @@ class RestIntegration implements IntegrationBase { case BodyTypes.ENCODED: const params = new URLSearchParams() for (let [key, value] of Object.entries(object)) { - params.append(key, value) + params.append(key, value as string) } addPaginationToBody((key: string, value: any) => { params.append(key, value) diff --git a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts index a5cb203e0f..51f6c2d301 100644 --- a/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts +++ b/packages/server/src/migrations/functions/usageQuotas/tests/syncRows.spec.ts @@ -2,7 +2,7 @@ import TestConfig from "../../../../tests/utilities/TestConfiguration" import * as syncRows from "../syncRows" import { quotas } from "@budibase/pro" import { QuotaUsageType, StaticQuotaName } from "@budibase/types" -const { db: dbCore } = require("@budibase/backend-core") +import { db as dbCore, context } from "@budibase/backend-core" describe("syncRows", () => { let config = new TestConfig(false) @@ -24,13 +24,17 @@ describe("syncRows", () => { // app 1 const app1 = config.app - await config.createTable() - await config.createRow() + await context.doInAppContext(app1.appId, async () => { + await config.createTable() + await config.createRow() + }) // app 2 const app2 = await config.createApp("second-app") - await config.createTable() - await config.createRow() - await config.createRow() + await context.doInAppContext(app2.appId, async () => { + await config.createTable() + await config.createRow() + await config.createRow() + }) // migrate await syncRows.run() diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts new file mode 100644 index 0000000000..7a324b86f9 --- /dev/null +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -0,0 +1,98 @@ +import { context } from "@budibase/backend-core" +import { processObjectSync, findHBSBlocks } from "@budibase/string-templates" +import { + Datasource, + DatasourceFieldType, + Integration, + PASSWORD_REPLACEMENT, +} from "@budibase/types" +import { cloneDeep } from "lodash/fp" +import { getEnvironmentVariables } from "../../utils" +import { getDefinitions } from "../../../integrations" + +const ENV_VAR_PREFIX = "env." +const USER_PREFIX = "user" + +async function enrichDatasourceWithValues(datasource: Datasource) { + const cloned = cloneDeep(datasource) + const env = await getEnvironmentVariables() + const processed = processObjectSync(cloned, { env }, { onlyFound: true }) + return { + datasource: processed as Datasource, + envVars: env as Record, + } +} + +export async function enrich(datasource: Datasource) { + const { datasource: response } = await enrichDatasourceWithValues(datasource) + return response +} + +export async function get( + datasourceId: string, + opts?: { enriched: boolean } +): Promise { + const appDb = context.getAppDB() + const datasource = await appDb.get(datasourceId) + if (opts?.enriched) { + return (await enrichDatasourceWithValues(datasource)).datasource + } else { + return datasource + } +} + +export async function getWithEnvVars(datasourceId: string) { + const appDb = context.getAppDB() + const datasource = await appDb.get(datasourceId) + return enrichDatasourceWithValues(datasource) +} + +export async function removeSecrets(datasources: Datasource[]) { + const definitions = await getDefinitions() + for (let datasource of datasources) { + const schema = definitions[datasource.source] + if (datasource.config) { + // strip secrets from response, so they don't show in the network request + if (datasource.config.auth) { + delete datasource.config.auth + } + // remove passwords + for (let key of Object.keys(datasource.config)) { + if (typeof datasource.config[key] !== "string") { + continue + } + const blocks = findHBSBlocks(datasource.config[key] as string) + const usesEnvVars = + blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null + if ( + !usesEnvVars && + schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD + ) { + datasource.config[key] = PASSWORD_REPLACEMENT + } + } + } + } + return datasources +} + +export async function removeSecretSingle(datasource: Datasource) { + return (await removeSecrets([datasource]))[0] +} + +export function mergeConfigs(update: Datasource, old: Datasource) { + if (!update.config) { + return update + } + for (let [key, value] of Object.entries(update.config)) { + if (value !== PASSWORD_REPLACEMENT) { + continue + } + if (old.config?.[key]) { + update.config[key] = old.config?.[key] + } else { + delete update.config[key] + } + } + return update +} diff --git a/packages/server/src/sdk/app/datasources/index.ts b/packages/server/src/sdk/app/datasources/index.ts new file mode 100644 index 0000000000..1ce6b0e689 --- /dev/null +++ b/packages/server/src/sdk/app/datasources/index.ts @@ -0,0 +1,5 @@ +import * as datasources from "./datasources" + +export default { + ...datasources, +} diff --git a/packages/server/src/sdk/app/queries/index.ts b/packages/server/src/sdk/app/queries/index.ts new file mode 100644 index 0000000000..4228fd9d57 --- /dev/null +++ b/packages/server/src/sdk/app/queries/index.ts @@ -0,0 +1,5 @@ +import * as queries from "./queries" + +export default { + ...queries, +} diff --git a/packages/server/src/sdk/app/queries/queries.ts b/packages/server/src/sdk/app/queries/queries.ts new file mode 100644 index 0000000000..ca74eb44b5 --- /dev/null +++ b/packages/server/src/sdk/app/queries/queries.ts @@ -0,0 +1,50 @@ +import { getEnvironmentVariables } from "../../utils" +import { processStringSync } from "@budibase/string-templates" + +export async function enrichContext( + fields: Record, + inputs = {} +): Promise> { + const enrichedQuery: Record = Array.isArray(fields) ? [] : {} + if (!fields || !inputs) { + return enrichedQuery + } + const env = await getEnvironmentVariables() + const parameters = { ...inputs, env } + // enrich the fields with dynamic parameters + for (let key of Object.keys(fields)) { + if (fields[key] == null) { + continue + } + if (typeof fields[key] === "object") { + // enrich nested fields object + enrichedQuery[key] = await enrichContext(fields[key], parameters) + } else if (typeof fields[key] === "string") { + // enrich string value as normal + enrichedQuery[key] = processStringSync(fields[key], parameters, { + noEscaping: true, + noHelpers: true, + escapeNewlines: true, + }) + } else { + enrichedQuery[key] = fields[key] + } + } + if ( + enrichedQuery.json || + enrichedQuery.customData || + enrichedQuery.requestBody + ) { + try { + enrichedQuery.json = JSON.parse( + enrichedQuery.json || + enrichedQuery.customData || + enrichedQuery.requestBody + ) + } catch (err) { + // no json found, ignore + } + delete enrichedQuery.customData + } + return enrichedQuery +} diff --git a/packages/server/src/sdk/app/tables/index.ts b/packages/server/src/sdk/app/tables/index.ts index 98c7fc6a8b..6bb09ae845 100644 --- a/packages/server/src/sdk/app/tables/index.ts +++ b/packages/server/src/sdk/app/tables/index.ts @@ -6,6 +6,7 @@ import { isSQL, } from "../../../integrations/utils" import { Table, Database } from "@budibase/types" +import datasources from "../datasources" async function getAllInternalTables(db?: Database): Promise { if (!db) { @@ -23,9 +24,11 @@ async function getAllInternalTables(db?: Database): Promise { })) } -async function getAllExternalTables(datasourceId: any): Promise { +async function getAllExternalTables( + datasourceId: any +): Promise> { const db = context.getAppDB() - const datasource = await db.get(datasourceId) + const datasource = await datasources.get(datasourceId, { enriched: true }) if (!datasource || !datasource.entities) { throw "Datasource is not configured fully." } @@ -44,7 +47,7 @@ async function getTable(tableId: any): Promise { const db = context.getAppDB() if (isExternalTable(tableId)) { let { datasourceId, tableName } = breakExternalTableId(tableId) - const datasource = await db.get(datasourceId) + const datasource = await datasources.get(datasourceId!) const table = await getExternalTable(datasourceId, tableName) return { ...table, sql: isSQL(datasource) } } else { diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 2ee40992c8..294c99a12c 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -2,6 +2,8 @@ import { default as backups } from "./app/backups" import { default as tables } from "./app/tables" import { default as automations } from "./app/automations" import { default as applications } from "./app/applications" +import { default as datasources } from "./app/datasources" +import { default as queries } from "./app/queries" import { default as rows } from "./app/rows" import { default as users } from "./users" @@ -12,6 +14,8 @@ const sdk = { applications, rows, users, + datasources, + queries, } // default export for TS diff --git a/packages/server/src/sdk/utils/index.ts b/packages/server/src/sdk/utils/index.ts new file mode 100644 index 0000000000..03599412a6 --- /dev/null +++ b/packages/server/src/sdk/utils/index.ts @@ -0,0 +1,16 @@ +import { environmentVariables } from "@budibase/pro" +import { context, db as dbCore } from "@budibase/backend-core" +import { AppEnvironment } from "@budibase/types" + +export async function getEnvironmentVariables() { + let envVars = context.getEnvironmentVariables() + if (!envVars) { + const appId = context.getAppId() + const appEnv = dbCore.isDevAppID(appId) + ? AppEnvironment.DEVELOPMENT + : AppEnvironment.PRODUCTION + + envVars = await environmentVariables.fetchValues(appEnv) + } + return envVars +} diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 7ac20e14ef..34382eb48f 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -364,20 +364,23 @@ class TestConfiguration { // create dev app // clear any old app this.appId = null - // @ts-ignore - await context.updateAppId(null) - this.app = await this._req({ name: appName }, null, controllers.app.create) - this.appId = this.app.appId - // @ts-ignore - await context.updateAppId(this.appId) + await context.doInAppContext(null, async () => { + this.app = await this._req( + { name: appName }, + null, + controllers.app.create + ) + this.appId = this.app.appId + }) + return await context.doInAppContext(this.appId, async () => { + // create production app + this.prodApp = await this.publish() - // create production app - this.prodApp = await this.publish() + this.allApps.push(this.prodApp) + this.allApps.push(this.app) - this.allApps.push(this.prodApp) - this.allApps.push(this.app) - - return this.app + return this.app + }) } async publish() { diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index d765ef8472..91105a2194 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -16,7 +16,6 @@ import { storeLog } from "../automations/logging" import { Automation, AutomationStep, AutomationStatus } from "@budibase/types" import { LoopStep, - LoopStepType, LoopInput, TriggerOutput, AutomationContext, @@ -26,6 +25,7 @@ import { WorkerCallback } from "./definitions" import { context, logging } from "@budibase/backend-core" import { processObject } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" +import * as sdkUtils from "../sdk/utils" import env from "../environment" const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId @@ -225,6 +225,8 @@ class Orchestrator { } async execute() { + // this will retrieve from context created at start of thread + this._context.env = await sdkUtils.getEnvironmentVariables() let automation = this._automation let stopped = false let loopStep: AutomationStep | undefined = undefined @@ -478,7 +480,11 @@ export const removeStalled = async (job: Job) => { throw new Error("Unable to execute, event doesn't contain app ID.") } await context.doInAppContext(appId, async () => { - const automationOrchestrator = new Orchestrator(job) - await automationOrchestrator.stopCron("stalled") + const envVars = await sdkUtils.getEnvironmentVariables() + // put into automation thread for whole context + await context.doInEnvironmentContext(envVars, async () => { + const automationOrchestrator = new Orchestrator(job) + await automationOrchestrator.stopCron("stalled") + }) }) } diff --git a/packages/server/src/threads/definitions.ts b/packages/server/src/threads/definitions.ts index 3da69d3640..2cf5d8066c 100644 --- a/packages/server/src/threads/definitions.ts +++ b/packages/server/src/threads/definitions.ts @@ -1,3 +1,5 @@ +import { EnvironmentVariablesDecrypted } from "@budibase/types" + export type WorkerCallback = (error: any, response?: any) => void export interface QueryEvent { @@ -9,6 +11,7 @@ export interface QueryEvent { pagination?: any transformer: any queryId: string + environmentVariables?: Record ctx?: any } diff --git a/packages/server/src/threads/query.ts b/packages/server/src/threads/query.ts index 3d6428dd56..682ab77f41 100644 --- a/packages/server/src/threads/query.ts +++ b/packages/server/src/threads/query.ts @@ -6,13 +6,11 @@ import { getIntegration } from "../integrations" import { processStringSync } from "@budibase/string-templates" import { context, cache, auth } from "@budibase/backend-core" import { getGlobalIDFromUserMetadataID } from "../db/utils" +import sdk from "../sdk" import { cloneDeep } from "lodash/fp" -const { isSQL } = require("../integrations/utils") -const { - enrichQueryFields, - interpolateSQL, -} = require("../integrations/queries/sql") +import { isSQL } from "../integrations/utils" +import { interpolateSQL } from "../integrations/queries/sql" class QueryRunner { datasource: any @@ -62,10 +60,11 @@ class QueryRunner { } if (datasourceClone.config.authConfigs) { - datasourceClone.config.authConfigs = - datasourceClone.config.authConfigs.map((config: any) => { - return enrichQueryFields(config, this.ctx) - }) + const updatedConfigs = [] + for (let config of datasourceClone.config.authConfigs) { + updatedConfigs.push(await sdk.queries.enrichContext(config, this.ctx)) + } + datasourceClone.config.authConfigs = updatedConfigs } const integration = new Integration(datasourceClone.config) @@ -75,12 +74,15 @@ class QueryRunner { // Enrich the parameters with the addition context items. // 'user' is now a reserved variable key in mapping parameters - const enrichedParameters = enrichQueryFields(parameters, this.ctx) + const enrichedParameters = await sdk.queries.enrichContext( + parameters, + this.ctx + ) const enrichedContext = { ...enrichedParameters, ...this.ctx } // Parse global headers if (datasourceClone.config.defaultHeaders) { - datasourceClone.config.defaultHeaders = enrichQueryFields( + datasourceClone.config.defaultHeaders = await sdk.queries.enrichContext( datasourceClone.config.defaultHeaders, enrichedContext ) @@ -89,9 +91,9 @@ class QueryRunner { let query // handle SQL injections by interpolating the variables if (isSQL(datasourceClone)) { - query = interpolateSQL(fieldsClone, enrichedParameters, integration) + query = await interpolateSQL(fieldsClone, enrichedParameters, integration) } else { - query = enrichQueryFields(fieldsClone, enrichedContext) + query = await sdk.queries.enrichContext(fieldsClone, enrichedContext) } // Add pagination values for REST queries @@ -166,7 +168,9 @@ class QueryRunner { async runAnotherQuery(queryId: string, parameters: any) { const db = context.getAppDB() const query = await db.get(queryId) - const datasource = await db.get(query.datasourceId) + const datasource = await sdk.datasources.get(query.datasourceId, { + enriched: true, + }) return new QueryRunner( { datasource, @@ -280,7 +284,7 @@ class QueryRunner { } export function execute(input: QueryEvent, callback: WorkerCallback) { - context.doInAppContext(input.appId!, async () => { + const run = async () => { const Runner = new QueryRunner(input) try { const response = await Runner.execute() @@ -288,5 +292,14 @@ export function execute(input: QueryEvent, callback: WorkerCallback) { } catch (err) { callback(err) } + } + context.doInAppContext(input.appId!, async () => { + if (input.environmentVariables) { + return context.doInEnvironmentContext(input.environmentVariables, () => { + return run() + }) + } else { + return run() + } }) } diff --git a/packages/server/src/utilities/fileSystem/filesystem.ts b/packages/server/src/utilities/fileSystem/filesystem.ts index 48810ffc47..086eaa835a 100644 --- a/packages/server/src/utilities/fileSystem/filesystem.ts +++ b/packages/server/src/utilities/fileSystem/filesystem.ts @@ -1,10 +1,10 @@ import { PathLike } from "fs" -const { budibaseTempDir } = require("../budibaseDir") -const fs = require("fs") -const { join } = require("path") -const uuid = require("uuid/v4") +import fs from "fs" +import { budibaseTempDir } from "../budibaseDir" +import { join } from "path" import env from "../../environment" import tar from "tar" +const uuid = require("uuid/v4") export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..") @@ -112,6 +112,7 @@ export const sendTempFile = (fileContents: any) => { * allows a centralised location to check logic is all good. */ export const readFileSync = (filepath: PathLike, options = "utf8") => { + // @ts-ignore return fs.readFileSync(filepath, options) } @@ -147,6 +148,7 @@ export const findFileRec = (startPath: PathLike, filter: string): any => { const files = fs.readdirSync(startPath) for (let i = 0, len = files.length; i < len; i++) { + // @ts-ignore const filename = join(startPath, files[i]) const stat = fs.lstatSync(filename) diff --git a/packages/server/src/utilities/fileSystem/plugin.ts b/packages/server/src/utilities/fileSystem/plugin.ts index ea1742faba..0bc2b7de44 100644 --- a/packages/server/src/utilities/fileSystem/plugin.ts +++ b/packages/server/src/utilities/fileSystem/plugin.ts @@ -1,8 +1,7 @@ import { Plugin } from "@budibase/types" - -const { budibaseTempDir } = require("../budibaseDir") -const fs = require("fs") -const { join } = require("path") +import { budibaseTempDir } from "../budibaseDir" +import fs from "fs" +import { join } from "path" import { objectStore } from "@budibase/backend-core" const DATASOURCE_PATH = join(budibaseTempDir(), "datasource") diff --git a/packages/server/src/utilities/fileSystem/template.ts b/packages/server/src/utilities/fileSystem/template.ts index 17a959007e..87e0b44e5c 100644 --- a/packages/server/src/utilities/fileSystem/template.ts +++ b/packages/server/src/utilities/fileSystem/template.ts @@ -1,5 +1,5 @@ -const fs = require("fs") -const { join } = require("path") +import fs from "fs" +import { join } from "path" import { ObjectStoreBuckets } from "../../constants" import { objectStore } from "@budibase/backend-core" diff --git a/packages/string-templates/src/helpers/index.js b/packages/string-templates/src/helpers/index.js index f04fa58399..bed3d0c3e3 100644 --- a/packages/string-templates/src/helpers/index.js +++ b/packages/string-templates/src/helpers/index.js @@ -32,11 +32,15 @@ const HELPERS = [ // javascript helper new Helper(HelperFunctionNames.JS, processJS, false), // this help is applied to all statements - new Helper(HelperFunctionNames.ALL, (value, { __opts }) => { + new Helper(HelperFunctionNames.ALL, (value, inputs) => { + const { __opts } = inputs if (isObject(value)) { return new SafeString(JSON.stringify(value)) } // null/undefined values produce bad results + if (__opts && __opts.onlyFound && value == null) { + return __opts.input + } if (value == null || typeof value !== "string") { return value == null ? "" : value } diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index ab7afc1dcc..26f44369b0 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -146,16 +146,31 @@ module.exports.processStringSync = (string, context, opts) => { if (typeof string !== "string") { throw "Cannot process non-string types." } - try { - const template = createTemplate(string, opts) + function process(stringPart) { + const template = createTemplate(stringPart, opts) const now = Math.floor(Date.now() / 1000) * 1000 return processors.postprocess( template({ now: new Date(now).toISOString(), - __opts: opts, + __opts: { + ...opts, + input: stringPart, + }, ...context, }) ) + } + try { + if (opts && opts.onlyFound) { + const blocks = exports.findHBSBlocks(string) + for (let block of blocks) { + const outcome = process(block) + string = string.replace(block, outcome) + } + return string + } else { + return process(string) + } } catch (err) { return input } diff --git a/packages/string-templates/test/basic.spec.js b/packages/string-templates/test/basic.spec.js index 8dd1aeb394..3346a81ddd 100644 --- a/packages/string-templates/test/basic.spec.js +++ b/packages/string-templates/test/basic.spec.js @@ -221,3 +221,9 @@ describe("check find hbs blocks function", () => { }) }) +describe("should leave HBS blocks if not found using option", () => { + it("should replace one, leave one", async () => { + const output = await processString("{{ a }}, {{ b }}", { b: 1 }, { onlyFound: true }) + expect(output).toBe("{{ a }}, 1") + }) +}) diff --git a/packages/types/src/api/web/app/backup.ts b/packages/types/src/api/web/app/backup.ts index f16f8416ae..c9a8d07f5e 100644 --- a/packages/types/src/api/web/app/backup.ts +++ b/packages/types/src/api/web/app/backup.ts @@ -12,6 +12,11 @@ export interface CreateAppBackupRequest { name: string } +export interface CreateAppBackupResponse { + backupId: string + message: string +} + export interface UpdateAppBackupRequest { name: string } diff --git a/packages/types/src/api/web/global/environmentVariables.ts b/packages/types/src/api/web/global/environmentVariables.ts new file mode 100644 index 0000000000..6f339eb821 --- /dev/null +++ b/packages/types/src/api/web/global/environmentVariables.ts @@ -0,0 +1,18 @@ +export interface StatusEnvironmentVariableResponse { + encryptionKeyAvailable: boolean +} + +export interface CreateEnvironmentVariableRequest { + name: string + production: string + development: string +} + +export interface UpdateEnvironmentVariableRequest { + production: string + development: string +} + +export interface GetEnvironmentVariablesResponse { + variables: string[] +} diff --git a/packages/types/src/api/web/global/events.ts b/packages/types/src/api/web/global/events.ts new file mode 100644 index 0000000000..907cfc74aa --- /dev/null +++ b/packages/types/src/api/web/global/events.ts @@ -0,0 +1,7 @@ +export enum EventPublishType { + ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable_upgrade_panel_opened", +} + +export interface PostEventPublishRequest { + type: EventPublishType +} diff --git a/packages/types/src/api/web/global/index.ts b/packages/types/src/api/web/global/index.ts new file mode 100644 index 0000000000..415ed55ab1 --- /dev/null +++ b/packages/types/src/api/web/global/index.ts @@ -0,0 +1,2 @@ +export * from "./environmentVariables" +export * from "./events" diff --git a/packages/types/src/api/web/index.ts b/packages/types/src/api/web/index.ts index 1dbe22aa46..9688a89c7b 100644 --- a/packages/types/src/api/web/index.ts +++ b/packages/types/src/api/web/index.ts @@ -3,3 +3,4 @@ export * from "./user" export * from "./errors" export * from "./schedule" export * from "./app" +export * from "./global" diff --git a/packages/types/src/documents/app/datasource.ts b/packages/types/src/documents/app/datasource.ts index efdc2ca1bd..a37e3d3ddc 100644 --- a/packages/types/src/documents/app/datasource.ts +++ b/packages/types/src/documents/app/datasource.ts @@ -8,7 +8,7 @@ export interface Datasource extends Document { source: SourceName // the config is defined by the schema config?: { - [key: string]: string | number | boolean + [key: string]: string | number | boolean | any[] } plus?: boolean entities?: { diff --git a/packages/types/src/documents/global/environmentVariables.ts b/packages/types/src/documents/global/environmentVariables.ts new file mode 100644 index 0000000000..f7255d8583 --- /dev/null +++ b/packages/types/src/documents/global/environmentVariables.ts @@ -0,0 +1,20 @@ +import { Document } from "../document" + +export interface EnvironmentVariablesDoc extends Document { + variables: string +} + +export type EnvironmentVariableValue = { + production: string + development: string +} + +// what comes out of the "variables" when it is decrypted +export type EnvironmentVariablesDecrypted = Record< + string, + EnvironmentVariableValue +> + +export interface EnvironmentVariablesDocDecrypted extends Document { + variables: EnvironmentVariablesDecrypted +} diff --git a/packages/types/src/documents/global/index.ts b/packages/types/src/documents/global/index.ts index 40a30ee477..11ce7513f2 100644 --- a/packages/types/src/documents/global/index.ts +++ b/packages/types/src/documents/global/index.ts @@ -5,3 +5,4 @@ export * from "./plugin" export * from "./quotas" export * from "./schedule" export * from "./templates" +export * from "./environmentVariables" diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 1847725c9d..605b431d9e 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -1,5 +1,7 @@ import { Table } from "../documents" +export const PASSWORD_REPLACEMENT = "--secret-value--" + export enum Operation { CREATE = "CREATE", READ = "READ", diff --git a/packages/types/src/sdk/environmentVariables.ts b/packages/types/src/sdk/environmentVariables.ts new file mode 100644 index 0000000000..0fe01de1bb --- /dev/null +++ b/packages/types/src/sdk/environmentVariables.ts @@ -0,0 +1,4 @@ +export enum AppEnvironment { + PRODUCTION = "production", + DEVELOPMENT = "development", +} diff --git a/packages/types/src/sdk/events/environmentVariable.ts b/packages/types/src/sdk/events/environmentVariable.ts new file mode 100644 index 0000000000..0b81d78df8 --- /dev/null +++ b/packages/types/src/sdk/events/environmentVariable.ts @@ -0,0 +1,14 @@ +import { BaseEvent } from "./event" + +export interface EnvironmentVariableCreatedEvent extends BaseEvent { + name: string + environments: string[] +} + +export interface EnvironmentVariableDeletedEvent extends BaseEvent { + name: string +} + +export interface EnvironmentVariableUpgradePanelOpenedEvent extends BaseEvent { + userId: string +} diff --git a/packages/types/src/sdk/events/event.ts b/packages/types/src/sdk/events/event.ts index 2c197e7738..5d1f30883c 100644 --- a/packages/types/src/sdk/events/event.ts +++ b/packages/types/src/sdk/events/event.ts @@ -172,6 +172,11 @@ export enum Event { // BACKUP APP_BACKUP_RESTORED = "app:backup:restored", APP_BACKUP_TRIGGERED = "app:backup:triggered", + + // ENVIRONMENT VARIABLE + ENVIRONMENT_VARIABLE_CREATED = "environment_variable:created", + ENVIRONMENT_VARIABLE_DELETED = "environment_variable:deleted", + ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED = "environment_variable:upgrade_panel_opened", } // properties added at the final stage of the event pipeline diff --git a/packages/types/src/sdk/events/index.ts b/packages/types/src/sdk/events/index.ts index 5abc30f5b9..009d9beac4 100644 --- a/packages/types/src/sdk/events/index.ts +++ b/packages/types/src/sdk/events/index.ts @@ -21,3 +21,4 @@ export * from "./identification" export * from "./userGroup" export * from "./plugin" export * from "./backup" +export * from "./environmentVariable" diff --git a/packages/types/src/sdk/index.ts b/packages/types/src/sdk/index.ts index e5512b062b..f8f9d9cb97 100644 --- a/packages/types/src/sdk/index.ts +++ b/packages/types/src/sdk/index.ts @@ -11,3 +11,4 @@ export * from "./locks" export * from "./db" export * from "./middleware" export * from "./featureFlag" +export * from "./environmentVariables" diff --git a/packages/worker/src/api/controllers/global/events.ts b/packages/worker/src/api/controllers/global/events.ts new file mode 100644 index 0000000000..758fb41b54 --- /dev/null +++ b/packages/worker/src/api/controllers/global/events.ts @@ -0,0 +1,17 @@ +import { + UserCtx, + PostEventPublishRequest, + EventPublishType, +} from "@budibase/types" +import { events } from "@budibase/backend-core" + +export async function publish(ctx: UserCtx) { + switch (ctx.request.body.type) { + case EventPublishType.ENVIRONMENT_VARIABLE_UPGRADE_PANEL_OPENED: + await events.environmentVariable.upgradePanelOpened(ctx.user._id!) + break + default: + ctx.throw(400, "Invalid publish event type.") + } + ctx.status = 200 +} diff --git a/packages/worker/src/api/routes/global/events.ts b/packages/worker/src/api/routes/global/events.ts new file mode 100644 index 0000000000..6fe8088761 --- /dev/null +++ b/packages/worker/src/api/routes/global/events.ts @@ -0,0 +1,8 @@ +import Router from "@koa/router" +import * as controller from "../../controllers/global/events" + +const router: Router = new Router() + +router.post("/api/global/event/publish", controller.publish) + +export default router diff --git a/packages/worker/src/api/routes/index.ts b/packages/worker/src/api/routes/index.ts index f0d4911771..3aa9422238 100644 --- a/packages/worker/src/api/routes/index.ts +++ b/packages/worker/src/api/routes/index.ts @@ -7,6 +7,7 @@ import templateRoutes from "./global/templates" import emailRoutes from "./global/email" import authRoutes from "./global/auth" import roleRoutes from "./global/roles" +import eventRoutes from "./global/events" import environmentRoutes from "./system/environment" import tenantsRoutes from "./system/tenants" import statusRoutes from "./system/status" @@ -34,4 +35,5 @@ export const routes: Router[] = [ migrationRoutes, accountRoutes, restoreRoutes, + eventRoutes, ]