diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 0881d2528d..8e235532cf 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -11,6 +11,7 @@ on: branches: - master - develop + workflow_dispatch: env: BRANCH: ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/release-develop.yml b/.github/workflows/release-develop.yml index 87e4f0988a..7ee02ccdd1 100644 --- a/.github/workflows/release-develop.yml +++ b/.github/workflows/release-develop.yml @@ -14,6 +14,7 @@ on: - 'yarn.lock' - 'package.json' - 'yarn.lock' + workflow_dispatch: env: POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }} @@ -26,6 +27,11 @@ jobs: runs-on: ubuntu-latest steps: + - name: Fail if branch is not develop + if: github.ref != 'refs/heads/develop' + run: | + echo "Ref is not develop, you must run this job from develop." + exit 1 - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3fe81dbcd1..359ad4467b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,6 +14,7 @@ on: - 'yarn.lock' - 'package.json' - 'yarn.lock' + workflow_dispatch: env: POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }} @@ -27,6 +28,11 @@ jobs: runs-on: ubuntu-latest steps: + - name: Fail if branch is not master + if: github.ref != 'refs/heads/master' + run: | + echo "Ref is not master, you must run this job from master." + exit 1 - uses: actions/checkout@v2 - uses: actions/setup-node@v1 with: diff --git a/.github/workflows/smoke_test.yaml b/.github/workflows/smoke_test.yaml index b26d0386fc..79c97b69af 100644 --- a/.github/workflows/smoke_test.yaml +++ b/.github/workflows/smoke_test.yaml @@ -32,7 +32,9 @@ jobs: uses: cypress-io/github-action@v2 with: install: false - command: yarn test:e2e:ci + command: yarn test:e2e:ci:record + env: + CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} # TODO: upload recordings to s3 # - name: Configure AWS Credentials diff --git a/charts/budibase/templates/app-service-deployment.yaml b/charts/budibase/templates/app-service-deployment.yaml index c80cfa2ecc..3bbe718d73 100644 --- a/charts/budibase/templates/app-service-deployment.yaml +++ b/charts/budibase/templates/app-service-deployment.yaml @@ -34,6 +34,7 @@ spec: {{ else }} value: http://{{ .Release.Name }}-svc-couchdb:{{ .Values.services.couchdb.port }} {{ end }} + {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: secretKeyRef: @@ -44,6 +45,7 @@ spec: secretKeyRef: name: {{ template "couchdb.fullname" . }} key: adminPassword + {{ end }} - name: ENABLE_ANALYTICS value: {{ .Values.globals.enableAnalytics | quote }} - name: INTERNAL_API_KEY @@ -112,6 +114,8 @@ spec: value: {{ .Values.globals.google.secret | quote }} - name: AUTOMATION_MAX_ITERATIONS value: {{ .Values.globals.automationMaxIterations | quote }} + - name: EXCLUDE_QUOTAS_TENANTS + value: {{ .Values.globals.excludeQuotasTenants | quote }} image: budibase/apps:{{ .Values.globals.appVersion }} imagePullPolicy: Always diff --git a/charts/budibase/templates/worker-service-deployment.yaml b/charts/budibase/templates/worker-service-deployment.yaml index c2180aca2b..15ff05e214 100644 --- a/charts/budibase/templates/worker-service-deployment.yaml +++ b/charts/budibase/templates/worker-service-deployment.yaml @@ -29,6 +29,7 @@ spec: - env: - name: CLUSTER_PORT value: {{ .Values.services.worker.port | quote }} + {{ if .Values.services.couchdb.enabled }} - name: COUCH_DB_USER valueFrom: secretKeyRef: @@ -39,6 +40,7 @@ spec: secretKeyRef: name: {{ template "couchdb.fullname" . }} key: adminPassword + {{ end }} - name: COUCH_DB_URL {{ if .Values.services.couchdb.url }} value: {{ .Values.services.couchdb.url }} diff --git a/lerna.json b/lerna.json index c9aeb85fbc..708450c773 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "1.0.130-alpha.0", + "version": "1.0.142", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/package.json b/package.json index 727104d830..fb6d9da990 100644 --- a/package.json +++ b/package.json @@ -47,6 +47,7 @@ "lint:fix": "yarn run lint:fix:prettier && yarn run lint:fix:eslint", "test:e2e": "lerna run cy:test --stream", "test:e2e:ci": "lerna run cy:ci --stream", + "test:e2e:ci:record": "lerna run cy:ci:record --stream", "build:specs": "lerna run specs", "build:docker": "lerna run build:docker && npm run build:docker:proxy:compose && cd hosting/scripts/linux/ && ./release-to-docker-hub.sh $BUDIBASE_RELEASE_VERSION && cd -", "build:docker:proxy": "docker build hosting/proxy -t proxy-service", diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 370f529ed7..d13cb2f50b 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "description": "Budibase backend core libraries used in server and worker", "main": "src/index.js", "author": "Budibase", diff --git a/packages/backend-core/src/environment.js b/packages/backend-core/src/environment.js index ae454eb023..f628e899ad 100644 --- a/packages/backend-core/src/environment.js +++ b/packages/backend-core/src/environment.js @@ -6,6 +6,10 @@ function isTest() { ) } +function isDev() { + return process.env.NODE_ENV !== "production" +} + module.exports = { JWT_SECRET: process.env.JWT_SECRET, COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005", @@ -38,6 +42,7 @@ module.exports = { process.env.GLOBAL_CLOUD_BUCKET_NAME || "prod-budi-tenant-uploads", USE_COUCH: process.env.USE_COUCH || true, isTest, + isDev, _set(key, value) { process.env[key] = value module.exports[key] = value diff --git a/packages/bbui/package.json b/packages/bbui/package.json index ec35c0c0b7..6dfaaf6975 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "^1.2.1", - "@budibase/string-templates": "^1.0.130-alpha.0", + "@budibase/string-templates": "^1.0.142", "@spectrum-css/actionbutton": "^1.0.1", "@spectrum-css/actiongroup": "^1.0.1", "@spectrum-css/avatar": "^3.0.2", diff --git a/packages/bbui/src/Table/DateTimeRenderer.svelte b/packages/bbui/src/Table/DateTimeRenderer.svelte index 5d856968e7..f4b0821069 100644 --- a/packages/bbui/src/Table/DateTimeRenderer.svelte +++ b/packages/bbui/src/Table/DateTimeRenderer.svelte @@ -2,17 +2,22 @@ import dayjs from "dayjs" export let value + export let schema // adding the 0- will turn a string like 00:00:00 into a valid ISO // date, but will make actual ISO dates invalid $: time = new Date(`0-${value}`) - $: isTime = !isNaN(time) + $: isTimeOnly = !isNaN(time) || schema?.timeOnly + $: isDateOnly = schema?.dateOnly + $: format = isTimeOnly + ? "HH:mm:ss" + : isDateOnly + ? "MMMM D YYYY" + : "MMMM D YYYY, HH:mm"
- {dayjs(isTime ? time : value).format( - isTime ? "HH:mm:ss" : "MMMM D YYYY, HH:mm" - )} + {dayjs(isTimeOnly ? time : value).format(format)}
diff --git a/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte new file mode 100644 index 0000000000..c30e8fc2ee --- /dev/null +++ b/packages/builder/src/components/backend/DatasourceNavigator/_components/GoogleSignIn.svelte @@ -0,0 +1,145 @@ + + + + + btn_google_dark_normal_ios + Created with Sketch. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte index 1441d3834b..95a5f54e32 100644 --- a/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte +++ b/packages/builder/src/components/design/PropertiesPanel/PropertyControls/FilterEditor/FilterDrawer.svelte @@ -49,6 +49,10 @@ filters = [...filters, duplicate] } + const getSchema = filter => { + return schemaFields.find(field => field.name === filter.field) + } + const onFieldChange = (expression, field) => { // Update the field type expression.type = enrichedSchemaFields.find(x => x.name === field)?.type @@ -150,7 +154,12 @@ bind:value={filter.value} /> {:else if filter.type === "datetime"} - + {:else} {/if} diff --git a/packages/cli/package.json b/packages/cli/package.json index c9653d6a7e..2d9fbadcd7 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { diff --git a/packages/client/package.json b/packages/client/package.json index 06a0b3f6c4..ee8dfa93e2 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "^1.0.130-alpha.0", - "@budibase/frontend-core": "^1.0.130-alpha.0", - "@budibase/string-templates": "^1.0.130-alpha.0", + "@budibase/bbui": "^1.0.142", + "@budibase/frontend-core": "^1.0.142", + "@budibase/string-templates": "^1.0.142", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte index 54ad1f3a80..b8815279dd 100644 --- a/packages/client/src/components/app/dynamic-filter/FilterModal.svelte +++ b/packages/client/src/components/app/dynamic-filter/FilterModal.svelte @@ -88,6 +88,10 @@ const schema = schemaFields.find(x => x.name === field) return schema?.constraints?.inclusion || [] } + + const getSchema = filter => { + return schemaFields.find(field => field.name === filter.field) + }
@@ -134,7 +138,12 @@ bind:value={filter.value} /> {:else if filter.type === "datetime"} - + {:else} {/if} diff --git a/packages/client/src/components/app/forms/Field.svelte b/packages/client/src/components/app/forms/Field.svelte index 3ebfc5084f..b267f6caff 100644 --- a/packages/client/src/components/app/forms/Field.svelte +++ b/packages/client/src/components/app/forms/Field.svelte @@ -44,7 +44,6 @@ fieldApi = value?.fieldApi fieldSchema = value?.fieldSchema }) - onDestroy(() => unsubscribe?.()) // Determine label class from position $: labelClass = labelPos === "above" ? "" : `spectrum-FieldLabel--${labelPos}` @@ -52,6 +51,11 @@ const updateLabel = e => { builderStore.actions.updateProp("label", e.target.textContent) } + + onDestroy(() => { + fieldApi?.deregister() + unsubscribe?.() + }) diff --git a/packages/client/src/components/app/forms/FormStep.svelte b/packages/client/src/components/app/forms/FormStep.svelte index 4441f515ee..22972c5c48 100644 --- a/packages/client/src/components/app/forms/FormStep.svelte +++ b/packages/client/src/components/app/forms/FormStep.svelte @@ -22,7 +22,7 @@ if ( formContext && $builderStore.inBuilder && - $componentStore.selectedComponentPath?.includes($component.id) + $componentStore?.selectedComponentPath?.includes($component.id) ) { formContext.formApi.setStep(step) } diff --git a/packages/client/src/components/app/forms/InnerForm.svelte b/packages/client/src/components/app/forms/InnerForm.svelte index 99dcbf4d5e..752bc9a2eb 100644 --- a/packages/client/src/components/app/forms/InnerForm.svelte +++ b/packages/client/src/components/app/forms/InnerForm.svelte @@ -329,6 +329,17 @@ } } + // We don't want to actually remove the field state when deregistering, just + // remove any errors and validation + const deregister = () => { + const fieldInfo = getField(field) + fieldInfo.update(state => { + state.fieldState.validator = null + state.fieldState.error = null + return state + }) + } + // Updates the disabled state of a certain field const setDisabled = fieldDisabled => { const fieldInfo = getField(field) @@ -348,6 +359,7 @@ reset, updateValidation, setDisabled, + deregister, validate: () => { // Validate the field by force setting the same value again const { fieldState } = get(getField(field)) diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index dc172cbabd..85aaa58655 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "^1.0.130-alpha.0", + "@budibase/bbui": "^1.0.142", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/server/package.json b/packages/server/package.json index beee8b436b..878bf46e50 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -68,10 +68,10 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "^10.0.3", - "@budibase/backend-core": "^1.0.130-alpha.0", - "@budibase/client": "^1.0.130-alpha.0", - "@budibase/pro": "1.0.130-alpha.0", - "@budibase/string-templates": "^1.0.130-alpha.0", + "@budibase/backend-core": "^1.0.142", + "@budibase/client": "^1.0.142", + "@budibase/pro": "1.0.142", + "@budibase/string-templates": "^1.0.142", "@bull-board/api": "^3.7.0", "@bull-board/koa": "^3.7.0", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/automations/automationUtils.js b/packages/server/src/automations/automationUtils.js index 0d858741dd..a3bcae0cee 100644 --- a/packages/server/src/automations/automationUtils.js +++ b/packages/server/src/automations/automationUtils.js @@ -86,3 +86,15 @@ exports.substituteLoopStep = (hbsString, substitute) => { return hbsString } + +exports.stringSplit = value => { + if (value == null) { + return [] + } + if (value.split("\n").length > 1) { + value = value.split("\n") + } else { + value = value.split(",") + } + return value +} diff --git a/packages/server/src/integrations/microsoftSqlServer.ts b/packages/server/src/integrations/microsoftSqlServer.ts index 1b37b5df9a..a7b9906481 100644 --- a/packages/server/src/integrations/microsoftSqlServer.ts +++ b/packages/server/src/integrations/microsoftSqlServer.ts @@ -242,12 +242,10 @@ module MSSQLModule { if (typeof name !== "string") { continue } - const type: string = convertSqlType(def.DATA_TYPE) - schema[name] = { autocolumn: !!autoColumns.find((col: string) => col === name), name: name, - type, + ...convertSqlType(def.DATA_TYPE), } } tables[tableName] = { diff --git a/packages/server/src/integrations/mysql.ts b/packages/server/src/integrations/mysql.ts index 065a1b2333..4fe996a019 100644 --- a/packages/server/src/integrations/mysql.ts +++ b/packages/server/src/integrations/mysql.ts @@ -15,6 +15,7 @@ import { } from "./utils" import { DatasourcePlus } from "./base/datasourcePlus" import dayjs from "dayjs" +import { FieldTypes } from "../constants" const { NUMBER_REGEX } = require("../utilities") module MySQLModule { @@ -101,7 +102,7 @@ module MySQLModule { } // if not a number, see if it is a date - important to do in this order as any // integer will be considered a valid date - else if (dayjs(binding).isValid()) { + else if (/^\d/.test(binding) && dayjs(binding).isValid()) { bindings[i] = dayjs(binding).toDate() } } @@ -151,20 +152,24 @@ module MySQLModule { async internalQuery( query: SqlQuery, - connect: boolean = true + opts: { connect?: boolean; disableCoercion?: boolean } = { + connect: true, + disableCoercion: false, + } ): Promise { try { - if (connect) { + if (opts?.connect) { await this.connect() } + const baseBindings = query.bindings || [] + const bindings = opts?.disableCoercion + ? baseBindings + : bindingTypeCoerce(baseBindings) // Node MySQL is callback based, so we must wrap our call in a promise - const response = await this.client.query( - query.sql, - bindingTypeCoerce(query.bindings || []) - ) + const response = await this.client.query(query.sql, bindings) return response[0] } finally { - if (connect) { + if (opts?.connect) { await this.disconnect() } } @@ -179,7 +184,7 @@ module MySQLModule { // get the tables first const tablesResp = await this.internalQuery( { sql: "SHOW TABLES;" }, - false + { connect: false } ) const tableNames = tablesResp.map( (obj: any) => @@ -191,7 +196,7 @@ module MySQLModule { const schema: TableSchema = {} const descResp = await this.internalQuery( { sql: `DESCRIBE \`${tableName}\`;` }, - false + { connect: false } ) for (let column of descResp) { const columnName = column.Field @@ -211,8 +216,8 @@ module MySQLModule { schema[columnName] = { name: columnName, autocolumn: isAuto, - type: convertSqlType(column.Type), constraints, + ...convertSqlType(column.Type), } } if (!tables[tableName]) { @@ -254,7 +259,8 @@ module MySQLModule { async query(json: QueryJson) { await this.connect() try { - const queryFn = (query: any) => this.internalQuery(query, false) + const queryFn = (query: any) => + this.internalQuery(query, { connect: false, disableCoercion: true }) return await this.queryWithReturning(json, queryFn) } finally { await this.disconnect() diff --git a/packages/server/src/integrations/oracle.ts b/packages/server/src/integrations/oracle.ts index 7cb7ba88cf..94e51694c1 100644 --- a/packages/server/src/integrations/oracle.ts +++ b/packages/server/src/integrations/oracle.ts @@ -279,9 +279,9 @@ module OracleModule { ) } - private internalConvertType(column: OracleColumn): string { + private internalConvertType(column: OracleColumn): { type: string } { if (this.isBooleanType(column)) { - return FieldTypes.BOOLEAN + return { type: FieldTypes.BOOLEAN } } return convertSqlType(column.type) @@ -328,7 +328,7 @@ module OracleModule { fieldSchema = { autocolumn: OracleIntegration.isAutoColumn(oracleColumn), name: columnName, - type: this.internalConvertType(oracleColumn), + ...this.internalConvertType(oracleColumn), } table.schema[columnName] = fieldSchema } diff --git a/packages/server/src/integrations/postgres.ts b/packages/server/src/integrations/postgres.ts index 6301f4dcaf..1da0ab31af 100644 --- a/packages/server/src/integrations/postgres.ts +++ b/packages/server/src/integrations/postgres.ts @@ -225,7 +225,6 @@ module PostgresModule { } } - const type: string = convertSqlType(column.data_type) const identity = !!( column.identity_generation || column.identity_start || @@ -240,7 +239,7 @@ module PostgresModule { tables[tableName].schema[columnName] = { autocolumn: isAuto, name: columnName, - type, + ...convertSqlType(column.data_type), } } diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils.ts index 326b213bc7..7e4efad84f 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils.ts @@ -35,6 +35,9 @@ const SQL_DATE_TYPE_MAP = { date: FieldTypes.DATETIME, } +const SQL_DATE_ONLY_TYPES = ["date"] +const SQL_TIME_ONLY_TYPES = ["time"] + const SQL_STRING_TYPE_MAP = { varchar: FieldTypes.STRING, char: FieldTypes.STRING, @@ -85,9 +88,9 @@ export function breakExternalTableId(tableId: string | undefined) { return {} } const parts = tableId.split(DOUBLE_SEPARATOR) - let tableName = parts.pop() + let datasourceId = parts.shift() // if they need joined - let datasourceId = parts.join(DOUBLE_SEPARATOR) + let tableName = parts.join(DOUBLE_SEPARATOR) return { datasourceId, tableName } } @@ -137,12 +140,20 @@ export function breakRowIdField(_id: string | { _id: string }): any[] { } export function convertSqlType(type: string) { + let foundType = FieldTypes.STRING + const lcType = type.toLowerCase() for (let [external, internal] of Object.entries(SQL_TYPE_MAP)) { - if (type.toLowerCase().includes(external)) { - return internal + if (lcType.includes(external)) { + foundType = internal + break } } - return FieldTypes.STRING + const schema: any = { type: foundType } + if (foundType === FieldTypes.DATETIME) { + schema.dateOnly = SQL_DATE_ONLY_TYPES.includes(lcType) + schema.timeOnly = SQL_TIME_ONLY_TYPES.includes(lcType) + } + return schema } export function getSqlQuery(query: SqlQuery | string): SqlQuery { diff --git a/packages/server/src/threads/automation.js b/packages/server/src/threads/automation.js index db462f5a8d..98c45e4af3 100644 --- a/packages/server/src/threads/automation.js +++ b/packages/server/src/threads/automation.js @@ -100,10 +100,10 @@ class Orchestrator { let automation = this._automation const app = await this.getApp() let stopped = false - let loopStep + let loopStep = null let stepCount = 0 - let loopStepNumber + let loopStepNumber = null let loopSteps = [] for (let step of automation.definition.steps) { stepCount++ @@ -117,15 +117,17 @@ class Orchestrator { if (loopStep) { input = await processObject(loopStep.inputs, this._context) } - let iterations = loopStep ? input.binding.length : 1 + let iterations = loopStep + ? Array.isArray(input.binding) + ? input.binding.length + : automationUtils.stringSplit(input.binding).length + : 1 let iterationCount = 0 for (let index = 0; index < iterations; index++) { let originalStepInput = cloneDeep(step.inputs) // Handle if the user has set a max iteration count or if it reaches the max limit set by us if (loopStep) { - // lets first of all handle the input - // if the input is array then use it, if it is a string then split it on every new line let newInput = await processObject( loopStep.inputs, cloneDeep(this._context) @@ -134,9 +136,6 @@ class Orchestrator { newInput, loopStep.schema.inputs ) - this._context.steps[loopStepNumber] = { - currentItem: newInput.binding[index], - } let tempOutput = { items: loopSteps, iterations: iterationCount } if ( @@ -154,6 +153,20 @@ class Orchestrator { break } + let item + if ( + typeof loopStep.inputs.binding === "string" && + loopStep.inputs.option === "String" + ) { + item = automationUtils.stringSplit(newInput.binding) + } else { + item = loopStep.inputs.binding + } + + this._context.steps[loopStepNumber] = { + currentItem: item[index], + } + // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it // Pretty hacky because we need to account for the row object for (let [key, value] of Object.entries(originalStepInput)) { @@ -178,7 +191,6 @@ class Orchestrator { } } } - if ( index === parseInt(env.AUTOMATION_MAX_ITERATIONS) || index === loopStep.inputs.iterations @@ -192,10 +204,25 @@ class Orchestrator { break } + let isFailure = false if ( - this._context.steps[loopStepNumber]?.currentItem === - loopStep.inputs.failure + typeof this._context.steps[loopStepNumber]?.currentItem === "object" ) { + isFailure = Object.keys( + this._context.steps[loopStepNumber].currentItem + ).some(value => { + return ( + this._context.steps[loopStepNumber].currentItem[value] === + loopStep.inputs.failure + ) + }) + } else { + isFailure = + this._context.steps[loopStepNumber]?.currentItem === + loopStep.inputs.failure + } + + if (isFailure) { this.updateContextAndOutput(loopStepNumber, step, tempOutput, { status: AutomationErrors.FAILURE_CONDITION, success: false, @@ -286,18 +313,16 @@ class Orchestrator { module.exports = (input, callback) => { const appId = input.data.event.appId - doInAppContext(appId, () => { + doInAppContext(appId, async () => { const automationOrchestrator = new Orchestrator( input.data.automation, input.data.event ) - automationOrchestrator - .execute() - .then(response => { - callback(null, response) - }) - .catch(err => { - callback(err) - }) + try { + const response = await automationOrchestrator.execute() + callback(null, response) + } catch (err) { + callback(err) + } }) } diff --git a/packages/server/src/threads/query.js b/packages/server/src/threads/query.js index 71994a7244..ec9d9a6fa6 100644 --- a/packages/server/src/threads/query.js +++ b/packages/server/src/threads/query.js @@ -191,14 +191,13 @@ class QueryRunner { } module.exports = (input, callback) => { - doInAppContext(input.appId, () => { + doInAppContext(input.appId, async () => { const Runner = new QueryRunner(input) - Runner.execute() - .then(response => { - callback(null, response) - }) - .catch(err => { - callback(err) - }) + try { + const response = await Runner.execute() + callback(null, response) + } catch (err) { + callback(err) + } }) } diff --git a/packages/server/src/utilities/fileSystem/index.js b/packages/server/src/utilities/fileSystem/index.js index 8a02afc5b3..e2a76c49a1 100644 --- a/packages/server/src/utilities/fileSystem/index.js +++ b/packages/server/src/utilities/fileSystem/index.js @@ -2,7 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir") const fs = require("fs") const { join } = require("path") const uuid = require("uuid/v4") -const { doWithDB } = require("@budibase/backend-core/db") +const { + doWithDB, + dangerousGetDB, + closeDB, +} = require("@budibase/backend-core/db") const { ObjectStoreBuckets } = require("../../constants") const { upload, @@ -151,14 +155,18 @@ exports.streamBackup = async appId => { * @return {*} either a readable stream or a string */ exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => { - return doWithDB(dbName, async db => { - // Stream the dump if required - if (stream) { - const memStream = new MemoryStream() - db.dump(memStream, { filter }) - return memStream - } + // streaming a DB dump is a bit more complicated, can't close DB + if (stream) { + const db = dangerousGetDB(dbName) + const memStream = new MemoryStream() + memStream.on("end", async () => { + await closeDB(db) + }) + db.dump(memStream, { filter }) + return memStream + } + return doWithDB(dbName, async db => { // Write the dump to file if required if (exportName) { const path = join(budibaseTempDir(), exportName) diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index 552e983961..35e9ce1387 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1014,10 +1014,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.130-alpha.0": - version "1.0.130-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.130-alpha.0.tgz#f6da46473f52d3e513a5eb7f352ae3bde6e61b78" - integrity sha512-DPuqEN8/OHFWPpUcfofjQ33lijsAKGDKc0DEF0QgLJOp2kMtBJa80tGjzTxccCZaFkCC2P5p+8kkMxKZQ6vYdA== +"@budibase/backend-core@1.0.138": + version "1.0.138" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0" + integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -1091,12 +1091,12 @@ svelte-flatpickr "^3.2.3" svelte-portal "^1.0.0" -"@budibase/pro@1.0.130-alpha.0": - version "1.0.130-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.130-alpha.0.tgz#ddae5dc39992d7f1c4f021d1e4f5effff687a183" - integrity sha512-R8DvPQ6hpLChOSp0BONwrCHOgYrYDO4r2tF8KzEg4uycZ+jbaKQJs2lZ8wdkCbXGjAHIcRNt+4RKUfa9r2epJw== +"@budibase/pro@1.0.138": + version "1.0.138" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544" + integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA== dependencies: - "@budibase/backend-core" "1.0.130-alpha.0" + "@budibase/backend-core" "1.0.138" node-fetch "^2.6.1" "@budibase/standard-components@^0.9.139": diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index a2a1f05c6d..676c625a2c 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/string-templates/src/index.js b/packages/string-templates/src/index.js index 6e464c27c4..f4feceac4b 100644 --- a/packages/string-templates/src/index.js +++ b/packages/string-templates/src/index.js @@ -70,7 +70,7 @@ function createTemplate(string, opts) { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {Promise} The structure input, as fully updated as possible. */ module.exports.processObject = async (object, context, opts) => { @@ -101,7 +101,7 @@ module.exports.processObject = async (object, context, opts) => { * then nothing will occur. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {Promise} The enriched string, all templates should have been replaced if they can be. */ module.exports.processString = async (string, context, opts) => { @@ -115,7 +115,7 @@ module.exports.processString = async (string, context, opts) => { * @param {object|array} object The input structure which is to be recursed, it is important to note that * if the structure contains any cycles then this will fail. * @param {object} context The context that handlebars should fill data from. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {object|array} The structure input, as fully updated as possible. */ module.exports.processObjectSync = (object, context, opts) => { @@ -136,7 +136,7 @@ module.exports.processObjectSync = (object, context, opts) => { * then nothing will occur. This is a pure sync call and therefore does not have the full functionality of the async call. * @param {string} string The template string which is the filled from the context object. * @param {object} context An object of information which will be used to enrich the string. - * @param {object|undefined} opts optional - specify some options for processing. + * @param {object|undefined} [opts] optional - specify some options for processing. * @returns {string} The enriched string, all templates should have been replaced if they can be. */ module.exports.processStringSync = (string, context, opts) => { @@ -194,7 +194,7 @@ module.exports.makePropSafe = property => { /** * Checks whether or not a template string contains totally valid syntax (simply tries running it) * @param string The string to test for valid syntax - this may contain no templates and will be considered valid. - * @param opts optional - specify some options for processing. + * @param [opts] optional - specify some options for processing. * @returns {boolean} Whether or not the input string is valid. */ module.exports.isValid = (string, opts) => { @@ -205,6 +205,7 @@ module.exports.isValid = (string, opts) => { "array", "cannot read property", "undefined", + "json at position 0", ] // this is a portion of a specific string always output by handlebars in the case of a syntax error const invalidCases = [`expecting '`] diff --git a/packages/string-templates/test/helpers.spec.js b/packages/string-templates/test/helpers.spec.js index 0d39660d59..17e6876bba 100644 --- a/packages/string-templates/test/helpers.spec.js +++ b/packages/string-templates/test/helpers.spec.js @@ -360,6 +360,13 @@ describe("Test the literal helper", () => { }) }) +describe("Test that JSONpase helper", () => { + it("should state that the JSONparse helper is valid", async () => { + const output = isValid(`{{ JSONparse input }}`) + expect(output).toBe(true) + }) +}) + describe("Cover a few complex use cases", () => { it("should allow use of three different collection helpers", async () => { const output = await processString( diff --git a/packages/worker/package.json b/packages/worker/package.json index 7ac3a44e0c..e886a707f1 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "1.0.130-alpha.0", + "version": "1.0.142", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -31,9 +31,9 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "^1.0.130-alpha.0", - "@budibase/pro": "1.0.130-alpha.0", - "@budibase/string-templates": "^1.0.130-alpha.0", + "@budibase/backend-core": "^1.0.142", + "@budibase/pro": "1.0.142", + "@budibase/string-templates": "^1.0.142", "@koa/router": "^8.0.0", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "^0.3.0", diff --git a/packages/worker/yarn.lock b/packages/worker/yarn.lock index 57b402d43b..a2a29f5f77 100644 --- a/packages/worker/yarn.lock +++ b/packages/worker/yarn.lock @@ -293,10 +293,10 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== -"@budibase/backend-core@1.0.130-alpha.0": - version "1.0.130-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.130-alpha.0.tgz#f6da46473f52d3e513a5eb7f352ae3bde6e61b78" - integrity sha512-DPuqEN8/OHFWPpUcfofjQ33lijsAKGDKc0DEF0QgLJOp2kMtBJa80tGjzTxccCZaFkCC2P5p+8kkMxKZQ6vYdA== +"@budibase/backend-core@1.0.138": + version "1.0.138" + resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.138.tgz#5297d6cf5b9ec8c15f0a6df4c7d8273b8ac900f0" + integrity sha512-1qN/5urKX8bBXwEz266Z94rco8dTI7VqIh75m8ZcqrAfoUpjvZJS76gZxfc5U/QWPwrgVFnLtYvnEjaLbGEflg== dependencies: "@techpass/passport-openidconnect" "^0.3.0" aws-sdk "^2.901.0" @@ -321,12 +321,12 @@ uuid "^8.3.2" zlib "^1.0.5" -"@budibase/pro@1.0.130-alpha.0": - version "1.0.130-alpha.0" - resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.130-alpha.0.tgz#ddae5dc39992d7f1c4f021d1e4f5effff687a183" - integrity sha512-R8DvPQ6hpLChOSp0BONwrCHOgYrYDO4r2tF8KzEg4uycZ+jbaKQJs2lZ8wdkCbXGjAHIcRNt+4RKUfa9r2epJw== +"@budibase/pro@1.0.138": + version "1.0.138" + resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.138.tgz#cacbebe5ce93eb533af62a794a638944c2c61544" + integrity sha512-4ABlUZvl2h8sd8awJATf3KJeoFWV/8SoqdbKiH1ICdUcM/6dad7nhbJ15QqJL+Uuh/+mN2yEbr8V6Un2+yF+CA== dependencies: - "@budibase/backend-core" "1.0.130-alpha.0" + "@budibase/backend-core" "1.0.138" node-fetch "^2.6.1" "@cspotcode/source-map-consumer@0.8.0":