diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 7f1e08601a..eb11627758 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -179,6 +179,13 @@ jobs: - run: yarn --frozen-lockfile - name: Test server + env: + DD_CIVISIBILITY_AGENTLESS_ENABLED: true + DD_API_KEY: "${{ secrets.DATADOG_API_KEY }}" + DD_SITE: "datadoghq.eu" + NODE_OPTIONS: "-r dd-trace/ci/init" + DD_ENV: "ci" + DD_SERVICE: "budibase/packages/server" run: | if ${{ env.USE_NX_AFFECTED }}; then yarn test --scope=@budibase/server --since=${{ env.NX_BASE_BRANCH }} diff --git a/.gitignore b/.gitignore index b68ddd975f..32d1416f4a 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,8 @@ bb-airgapped.tar.gz packages/server/build/oldClientVersions/**/* packages/builder/src/components/deploy/clientVersions.json +packages/server/src/integrations/tests/utils/*.lock + # Logs logs *.log diff --git a/charts/budibase/values.yaml b/charts/budibase/values.yaml index 58bd889857..4e80be7322 100644 --- a/charts/budibase/values.yaml +++ b/charts/budibase/values.yaml @@ -641,7 +641,7 @@ couchdb: # @ignore repository: budibase/couchdb # @ignore - tag: v3.2.1 + tag: v3.3.3 # @ignore pullPolicy: Always diff --git a/lerna.json b/lerna.json index 8b1fa18d23..70d6a683e0 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.28.3", + "version": "2.28.6", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/packages/account-portal b/packages/account-portal index a03225549e..247f56d455 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit a03225549e3ce61f43d0da878da162e08941b939 +Subproject commit 247f56d455abbd64da17d865275ed978f577549f diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index ed79a14340..4db63ad695 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -8,6 +8,7 @@ import { DatabaseOpts, DatabasePutOpts, DatabaseQueryOpts, + DBError, Document, isDocument, RowResponse, @@ -41,7 +42,7 @@ function buildNano(couchInfo: { url: string; cookie: string }) { type DBCall = () => Promise -class CouchDBError extends Error { +class CouchDBError extends Error implements DBError { status: number statusCode: number reason: string diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 1e7da2f9a2..e58660a889 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -93,15 +93,21 @@ function isApps() { return environment.SERVICE_TYPE === ServiceType.APPS } +function isQA() { + return environment.BUDIBASE_ENVIRONMENT === "QA" +} + const environment = { isTest, isJest, isDev, isWorker, isApps, + isQA, isProd: () => { return !isDev() }, + BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT, JS_BCRYPT: process.env.JS_BCRYPT, JWT_SECRET: process.env.JWT_SECRET, JWT_SECRET_FALLBACK: process.env.JWT_SECRET_FALLBACK, @@ -120,6 +126,7 @@ const environment = { REDIS_CLUSTERED: process.env.REDIS_CLUSTERED, MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY, MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY, + AWS_SESSION_TOKEN: process.env.AWS_SESSION_TOKEN, AWS_REGION: process.env.AWS_REGION, MINIO_URL: process.env.MINIO_URL, MINIO_ENABLED: process.env.MINIO_ENABLED || 1, diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index de94e3968b..68b1b10ec2 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -101,6 +101,11 @@ export function ObjectStore( } } + // for AWS Credentials using temporary session token + if (!env.MINIO_ENABLED && env.AWS_SESSION_TOKEN) { + config.sessionToken = env.AWS_SESSION_TOKEN + } + // custom S3 is in use i.e. minio if (env.MINIO_URL) { if (opts.presigning && env.MINIO_ENABLED) { diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index 333accc985..62b971f9f5 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -63,12 +63,12 @@ class InMemoryQueue implements Partial { * Same callback API as Bull, each callback passed to this will consume messages as they are * available. Please note this is a queue service, not a notification service, so each * consumer will receive different messages. - * @param func The callback function which will return a "Job", the same * as the Bull API, within this job the property "data" contains the JSON message. Please * note this is incredibly limited compared to Bull as in reality the Job would contain * a lot more information about the queue and current status of Bull cluster. */ - async process(func: any) { + async process(concurrencyOrFunc: number | any, func?: any) { + func = typeof concurrencyOrFunc === "number" ? func : concurrencyOrFunc this._emitter.on("message", async () => { if (this._messages.length <= 0) { return diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 1838eed92f..f633d0885e 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -21,6 +21,7 @@ let cleanupInterval: NodeJS.Timeout async function cleanup() { for (let queue of QUEUES) { await queue.clean(CLEANUP_PERIOD_MS, "completed") + await queue.clean(CLEANUP_PERIOD_MS, "failed") } } diff --git a/packages/backend-core/src/tenancy/db.ts b/packages/backend-core/src/tenancy/db.ts index 10477a8579..f2e4705fa8 100644 --- a/packages/backend-core/src/tenancy/db.ts +++ b/packages/backend-core/src/tenancy/db.ts @@ -1,6 +1,16 @@ import { getDB } from "../db/db" import { getGlobalDBName } from "../context" +import { TenantInfo } from "@budibase/types" export function getTenantDB(tenantId: string) { return getDB(getGlobalDBName(tenantId)) } + +export async function saveTenantInfo(tenantInfo: TenantInfo) { + const db = getTenantDB(tenantInfo.tenantId) + // save the tenant info to db + return await db.put({ + _id: "tenant_info", + ...tenantInfo, + }) +} diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 8faab53d90..7e120bc9f8 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -2,6 +2,7 @@ import TableSelector from "./TableSelector.svelte" import FieldSelector from "./FieldSelector.svelte" import SchemaSetup from "./SchemaSetup.svelte" + import RowSelector from "./RowSelector.svelte" import { Button, Select, @@ -14,6 +15,8 @@ DatePicker, DrawerContent, Helpers, + Toggle, + Divider, } from "@budibase/bbui" import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte" import { automationStore, selectedAutomation, tables } from "stores/builder" @@ -40,7 +43,8 @@ EditorModes, } from "components/common/CodeEditor" import FilterBuilder from "components/design/settings/controls/FilterEditor/FilterBuilder.svelte" - import { LuceneUtils, Utils, memo } from "@budibase/frontend-core" + import { QueryUtils, Utils, memo } from "@budibase/frontend-core" + import { getSchemaForDatasourcePlus, getEnvironmentBindings, @@ -129,6 +133,7 @@ $: customStepLayouts($memoBlock, schemaProperties) const customStepLayouts = block => { + console.log("BUILDING", inputData["row"]) if ( rowSteps.includes(block.stepId) || (rowTriggers.includes(block.stepId) && isTestModal) @@ -256,7 +261,6 @@ }).schema delete request._tableId } - try { if (isTestModal) { let newTestData = { schema } @@ -489,7 +493,7 @@ } function saveFilters(key) { - const filters = LuceneUtils.buildLuceneQuery(tempFilters) + const filters = QueryUtils.buildQuery(tempFilters) onChange({ [key]: filters, @@ -639,6 +643,24 @@
+ {JSON.stringify(inputData)} +
+ { + // DEAN - review this + onChange({ + row: { [key]: "" }, //null + meta: { + [key]: e.detail, + }, + }) + }} + /> +
+
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte index 5b75bbeb5f..8b9cacf2cb 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelector.svelte @@ -18,6 +18,7 @@ import { capitalise } from "helpers" import { memo } from "@budibase/frontend-core" import PropField from "./PropField.svelte" + import { cloneDeep, isPlainObject, mergeWith } from "lodash" const dispatch = createEventDispatcher() @@ -42,21 +43,29 @@ let customPopover let popoverAnchor let editableRow = {} - let columns = new Set() - // Avoid unnecessary updates + //?? + let editableMeta = {} + let editableFields = {} + // let columns = new Set() + + // Avoid unnecessary updates - DEAN double check after refactor $: memoStore.set({ row, meta, }) - // Legacy support $: fields = $memoStore?.meta?.fields - $: if ($memoStore?.meta?.columns) { - columns = new Set(meta?.columns) + $: if ($memoStore?.meta?.fields) { + editableFields = cloneDeep($memoStore?.meta?.fields) } + // Needs to go now... entirely + // $: if ($memoStore?.meta?.columns) { + // columns = new Set(meta?.columns) + // } + $: parsedBindings = bindings.map(binding => { let clone = Object.assign({}, binding) clone.icon = "ShareAndroid" @@ -73,59 +82,62 @@ schemaFields = Object.entries(table?.schema ?? {}) .filter(entry => { const [key, field] = entry - return field.type !== "formula" && !field.autocolumn + return field.type !== "formula" && !field.autocolumn // DEAN - revise autocolumn exclusion for testmodal }) .sort( ([, schemaA], [, schemaB]) => (schemaA.type === "attachment") - (schemaB.type === "attachment") ) - // Parse out any unused data. - if ($memoStore?.meta?.columns) { - for (const column of meta?.columns) { - if (!(column in table?.schema)) { - columns.delete(column) - } + // Parse out any data not in the schema. + for (const column in editableFields) { + if (!(column in table?.schema)) { + delete editableFields[column] } - columns = new Set(columns) + } + editableFields = editableFields + } + + // Go through the table schema and build out the editable content + // schemaFields.forEach(entry => { + for (const entry of schemaFields) { + const [key, fieldSchema] = entry + if ($memoStore?.row?.[key] && !editableRow?.[key]) { + editableRow = { + ...editableRow, + [key]: $memoStore?.row[key], + } + } + + // Legacy + if (editableFields[key]?.clearRelationships) { + const emptyField = coerce( + !$memoStore?.row.hasOwnProperty(key) ? "" : $memoStore?.row[key], + fieldSchema.type + ) + + // remove this and place the field in the editable row. + delete editableFields[key]?.clearRelationships + + // Default the field + editableRow = { + ...editableRow, + [key]: emptyField, + } + + console.log("DEAN EMPTY - clearRelationships", emptyField) } } - if (columns.size) { - for (const key of columns) { - const entry = schemaFields.find(entry => { - const [fieldKey] = entry - return fieldKey == key - }) - - if (entry) { - const [_, fieldSchema] = entry - editableRow = { - ...editableRow, - [key]: coerce( - !(key in $memoStore?.row) ? "" : $memoStore?.row[key], - fieldSchema.type - ), - } - } - } - } else { - schemaFields.forEach(entry => { - const [key] = entry - if ($memoStore?.row?.[key] && !editableRow?.[key]) { - editableRow = { - ...editableRow, - [key]: $memoStore?.row[key], - } - columns.add(key) - } - }) - columns = new Set(columns) - } + // Possible to go through the automation fields schema? + console.log("ACTUAL ROW", row) + console.log("EDITABLE FIELDS", editableFields) + console.log("EDITABLE ROW", editableRow) } // Legacy - add explicitly cleared relationships to the request. - $: if (schemaFields?.length && fields) { + // DEAN - review this + $: if (schemaFields?.length && fields && false) { // Meta fields processing. Object.keys(fields).forEach(key => { if (fields[key]?.clearRelationships) { @@ -181,87 +193,121 @@ return value } - const onChange = u => { - const update = { - _tableId: tableId, - row: { ...$memoStore.row }, - meta: { ...$memoStore.meta }, - ...u, + const onChange = update => { + const customizer = (objValue, srcValue, key) => { + if (isPlainObject(objValue) && isPlainObject(srcValue)) { + const result = mergeWith({}, objValue, srcValue, customizer) + let outcome = Object.keys(result).reduce((acc, key) => { + if (result[key] !== null) { + acc[key] = result[key] + } else { + console.log(key + " is null", objValue) + } + return acc + }, {}) + return outcome + } + return srcValue } - dispatch("change", update) - } - const fieldUpdate = (e, field) => { - const update = { - row: { - ...$memoStore?.row, - [field]: e.detail, + const result = mergeWith( + {}, + { + row: editableRow, + meta: { + fields: editableFields, + }, }, - } - onChange(update) + update, + customizer + ) + console.log("Row Selector - MERGED", result) + dispatch("change", result) } -{#if columns.size} - {#each schemaFields as [field, schema]} - {#if !schema.autocolumn && columns.has(field)} - -
- {#if isTestModal} +{#each schemaFields || [] as [field, schema]} + {#if !schema.autocolumn && editableFields.hasOwnProperty(field)} + +
+ {#if isTestModal} + + {:else} + { + onChange({ + row: { + [field]: e.detail.row[field], + }, + }) + }} + {bindings} + allowJS={true} + updateOnChange={false} + drawerLeft="260px" + > { + console.log("RowSelectorTypes > RowSelector > ", change) + onChange(change) + }} /> - {:else} - fieldUpdate(e, field)} - {bindings} - allowJS={true} - updateOnChange={false} - drawerLeft="260px" - > - - - {/if} - { - columns.delete(field) - const update = { ...editableRow } - delete update[field] - onChange({ row: update, meta: { columns: Array.from(columns) } }) - }} - /> -
- - {/if} - {/each} -{/if} + + {/if} + { + // Clear row data + const update = { ...editableRow } + update[field] = null + // delete update[field] + + // Clear any related metadata + // delete editableFields[field] + // editableFields[field] = null + console.log("REMOVE STATE", { + row: update, + meta: { fields: { ...editableFields, [field]: null } }, + }) + onChange({ + row: update, + meta: { fields: { ...editableFields, [field]: null } }, + }) + }} + /> +
+ + {/if} +{/each} {#if table && schemaFields}
{ - if (columns.has(field)) { - columns.delete(field) + if (editableFields.hasOwnProperty(field)) { + editableFields[field] = null } else { - columns.add(field) + editableFields[field] = {} } - onChange({ meta: { columns: Array.from(columns) } }) + onChange({ meta: { fields: editableFields } }) }} > - import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui" + import { + Select, + DatePicker, + Multiselect, + TextArea, + Toggle, + } from "@budibase/bbui" import { FieldType } from "@budibase/types" import LinkedRowSelector from "components/common/LinkedRowSelector.svelte" import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte" @@ -12,9 +18,12 @@ export let field export let schema export let value + export let meta export let bindings export let isTestModal + $: console.log(field + "VALUE???", value[field]) + $: parsedBindings = bindings.map(binding => { let clone = Object.assign({}, binding) clone.icon = "ShareAndroid" @@ -33,38 +42,54 @@ function handleAttachmentParams(keyValueObj) { let params = {} - - if ( - (schema.type === FieldType.ATTACHMENT_SINGLE || - schema.type === FieldType.SIGNATURE_SINGLE) && - Object.keys(keyValueObj).length === 0 - ) { - return [] + // DEAN - review this + if (!keyValueObj) { + return null } + if (!Array.isArray(keyValueObj) && keyValueObj) { keyValueObj = [keyValueObj] } if (keyValueObj.length) { for (let param of keyValueObj) { - params[param.url] = param.filename + params[param.url || ""] = param.filename || "" } } + console.log("handleAttachmentParams ", params) return params } {#if schemaHasOptions(schema) && schema.type !== "array"} onChange(e, field)} + on:change={e => + onChange({ + row: { + [field]: e.detail, + }, + })} value={value[field]} options={[ { label: "True", value: "true" }, @@ -75,10 +100,23 @@ onChange(e, field)} + on:change={e => + onChange({ + row: { + [field]: e.detail, + }, + })} /> {:else if schema.type === "longform"} -