From abaa40a2723c2bcbf65d96aaaceb294b5d14e5e4 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 10 Sep 2024 17:09:42 +0100 Subject: [PATCH 01/27] automation steps using names --- .../SetupPanel/AutomationBlockSetup.svelte | 40 ++++++--- .../src/helpers/automations/nameHelpers.js | 85 +++++++++++++++++++ .../builder/src/stores/builder/automations.js | 43 ++++++++-- .../server/src/definitions/automations.ts | 1 + packages/server/src/threads/automation.ts | 31 ++++++- 5 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 packages/builder/src/helpers/automations/nameHelpers.js diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 6c4865b539..1bd65c82f8 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -49,6 +49,7 @@ import { getSchemaForDatasourcePlus, getEnvironmentBindings, + runtimeToReadableBinding, } from "dataBinding" import { TriggerStepID, ActionStepID } from "constants/backend/automations" import { onMount } from "svelte" @@ -595,9 +596,13 @@ let loopBlockCount = 0 const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => { if (!name) return - const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock) + const runtimeBinding = determineRuntimeBinding( + name, + idx, + isLoopBlock, + bindingName + ) const categoryName = determineCategoryName(idx, isLoopBlock, bindingName) - bindings.push( createBindingObject( name, @@ -613,7 +618,7 @@ ) } - const determineRuntimeBinding = (name, idx, isLoopBlock) => { + const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => { let runtimeName /* Begin special cases for generating custom schemas based on triggers */ @@ -634,12 +639,18 @@ } /* End special cases for generating custom schemas based on triggers */ + let hasUserDefinedName = + automation.stepNames?.[allSteps[idx - loopBlockCount]]?.id if (isLoopBlock) { runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { - runtimeName = `steps[${idx - loopBlockCount}].${name}` + runtimeName = hasUserDefinedName + ? `stepsByName.${bindingName}.${name}` + : `steps[${idx - loopBlockCount}].${name}` } else { - runtimeName = `steps.${idx - loopBlockCount}.${name}` + runtimeName = hasUserDefinedName + ? `stepsByName.${bindingName}.${name}` + : `steps.${idx - loopBlockCount}.${name}` } return idx === 0 ? `trigger.${name}` : runtimeName } @@ -666,11 +677,12 @@ const field = Object.values(FIELDS).find( field => field.type === value.type && field.subtype === value.subtype ) - + console.log(bindingName) return { - readableBinding: bindingName - ? `${bindingName}.${name}` - : runtimeBinding, + readableBinding: + bindingName && !isLoopBlock + ? `steps.${bindingName}.${name}` + : runtimeBinding, runtimeBinding, type: value.type, description: value.description, @@ -690,8 +702,15 @@ allSteps[idx]?.stepId === ActionStepID.LOOP && allSteps.some(x => x.blockToLoop === block.id) let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {} + if (wasLoopBlock) { + break + } let bindingName = - automation.stepNames?.[allSteps[idx - loopBlockCount].id] + automation.stepNames?.[allSteps[idx - loopBlockCount].id] || + !isLoopBlock + ? allSteps[idx]?.name + : allSteps[idx - 1]?.name + console.log(idx == 4 && bindingName) if (isLoopBlock) { schema = { @@ -746,7 +765,6 @@ addBinding(name, value, icon, idx, isLoopBlock, bindingName) ) } - return bindings } diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js new file mode 100644 index 0000000000..da5c999b2f --- /dev/null +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -0,0 +1,85 @@ +import { AutomationActionStepId } from "@budibase/types" + +export const updateBindingsInInputs = (inputs, oldName, newName) => { + if (typeof inputs === "string") { + return inputs.replace( + new RegExp(`stepsByName\\.${oldName}\\.`, "g"), + `stepsByName.${newName}.` + ) + } + + if (Array.isArray(inputs)) { + return inputs.map(item => updateBindingsInInputs(item, oldName, newName)) + } + + if (typeof inputs === "object" && inputs !== null) { + const updatedInputs = {} + for (const [key, value] of Object.entries(inputs)) { + updatedInputs[key] = updateBindingsInInputs(value, oldName, newName) + } + return updatedInputs + } + + return inputs +} + +export const updateBindingsInSteps = (steps, oldName, newName) => { + return steps.map(step => { + const updatedStep = { + ...step, + inputs: updateBindingsInInputs(step.inputs, oldName, newName), + } + + // Handle branch steps + if ("branches" in updatedStep.inputs) { + updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({ + ...branch, + condition: updateBindingsInInputs(branch.condition, oldName, newName), + })) + + if (updatedStep.inputs.children) { + for (const [key, childSteps] of Object.entries( + updatedStep.inputs.children + )) { + updatedStep.inputs.children[key] = updateBindingsInSteps( + childSteps, + oldName, + newName + ) + } + } + } + + return updatedStep + }) +} + +export const getNewStepName = (automation, step) => { + const baseName = step.name + + const countExistingSteps = steps => { + return steps.reduce((count, currentStep) => { + if (currentStep.name && currentStep.name.startsWith(baseName)) { + count++ + } + if ( + currentStep.stepId === AutomationActionStepId.BRANCH && + currentStep.inputs && + currentStep.inputs.children + ) { + Object.values(currentStep.inputs.children).forEach(branchSteps => { + count += countExistingSteps(branchSteps) + }) + } + return count + }, 0) + } + + const existingCount = countExistingSteps(automation.definition.steps) + + if (existingCount === 0) { + return baseName + } + + return `${baseName} ${existingCount + 1}` +} diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js index fdb0991911..eaf565c4f2 100644 --- a/packages/builder/src/stores/builder/automations.js +++ b/packages/builder/src/stores/builder/automations.js @@ -6,6 +6,10 @@ import { createHistoryStore } from "stores/builder/history" import { notifications } from "@budibase/bbui" import { updateReferencesInObject } from "dataBinding" import { AutomationTriggerStepId } from "@budibase/types" +import { + updateBindingsInSteps, + getNewStepName, +} from "helpers/automations/nameHelpers" const initialAutomationState = { automations: [], @@ -275,13 +279,17 @@ const automationActions = store => ({ await store.actions.save(newAutomation) }, constructBlock(type, stepId, blockDefinition) { - return { + let newName + const newStep = { ...blockDefinition, inputs: blockDefinition.inputs || {}, stepId, type, id: generate(), } + newName = getNewStepName(get(selectedAutomation), newStep) + newStep.name = newName + return newStep }, addBlockToAutomation: async (block, blockIdx) => { const automation = get(selectedAutomation) @@ -301,15 +309,36 @@ const automationActions = store => ({ saveAutomationName: async (blockId, name) => { const automation = get(selectedAutomation) let newAutomation = cloneDeep(automation) - if (!automation) { + if (!newAutomation) { return } - newAutomation.definition.stepNames = { - ...newAutomation.definition.stepNames, - [blockId]: name.trim(), - } - await store.actions.save(newAutomation) + const stepIndex = newAutomation.definition.steps.findIndex( + step => step.id === blockId + ) + + if (stepIndex !== -1) { + const oldName = newAutomation.definition.steps[stepIndex].name + const newName = name.trim() + + // Update stepNames + newAutomation.definition.stepNames = { + ...newAutomation.definition.stepNames, + [blockId]: newName, + } + + // Update the name in the step itself + newAutomation.definition.steps[stepIndex].name = newName + + // Update bindings in all steps + newAutomation.definition.steps = updateBindingsInSteps( + newAutomation.definition.steps, + oldName, + newName + ) + + await store.actions.save(newAutomation) + } }, deleteAutomationName: async blockId => { const automation = get(selectedAutomation) diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index e84eecea51..6488e604e9 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,6 +15,7 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] + stepsByName?: Record env?: Record trigger: any } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index f374ff159a..3e5ac32b0a 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -89,7 +89,12 @@ class Orchestrator { delete triggerOutput.appId delete triggerOutput.metadata // step zero is never used as the template string is zero indexed for customer facing - this.context = { steps: [{}], trigger: triggerOutput } + this.context = { + steps: [{}], + stepsByName: {}, + trigger: triggerOutput, + } + this.automation = automation // create an emitter which has the chain count for this automation run in it, so it can block // excessive chaining if required @@ -449,6 +454,11 @@ class Orchestrator { outputs: tempOutput, inputs: steps[stepToLoopIndex].inputs, }) + console.log(this.context) + + const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id + this.context.stepsByName![stepName] = tempOutput + console.log(this.context) this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") @@ -542,8 +552,11 @@ class Orchestrator { loopIteration ) } + console.log(this.context) + const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await processObject(originalStepInput, this.context) + let inputs = await this.testProcesss(originalStepInput, this.context) + inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) const outputs = await stepFn({ @@ -570,6 +583,18 @@ class Orchestrator { return null } + private async testProcesss(inputs: any, context: any) { + const processContext = { + ...context, + steps: { + ...context.steps, + ...context.stepsByName, + }, + } + + return processObject(inputs, processContext) + } + private handleStepOutput( step: AutomationStep, outputs: any, @@ -587,6 +612,8 @@ class Orchestrator { } else { this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs) this.context.steps[this.context.steps.length] = outputs + const stepName = step.name || step.id + this.context.stepsByName![stepName] = outputs } } } From bf98d61ea6fbab5f9f8a51bf1cc4b930abd79259 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:35:46 +0100 Subject: [PATCH 02/27] add tests for new binding update code --- .../SetupPanel/AutomationBlockSetup.svelte | 3 - .../src/helpers/automations/nameHelpers.js | 59 ++++-- .../src/helpers/tests/nameHelpers.spec.js | 177 ++++++++++++++++++ .../builder/src/stores/builder/automations.js | 6 +- 4 files changed, 224 insertions(+), 21 deletions(-) create mode 100644 packages/builder/src/helpers/tests/nameHelpers.spec.js diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 1bd65c82f8..1e2e409243 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -49,7 +49,6 @@ import { getSchemaForDatasourcePlus, getEnvironmentBindings, - runtimeToReadableBinding, } from "dataBinding" import { TriggerStepID, ActionStepID } from "constants/backend/automations" import { onMount } from "svelte" @@ -677,7 +676,6 @@ const field = Object.values(FIELDS).find( field => field.type === value.type && field.subtype === value.subtype ) - console.log(bindingName) return { readableBinding: bindingName && !isLoopBlock @@ -710,7 +708,6 @@ !isLoopBlock ? allSteps[idx]?.name : allSteps[idx - 1]?.name - console.log(idx == 4 && bindingName) if (isLoopBlock) { schema = { diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js index da5c999b2f..b863796a54 100644 --- a/packages/builder/src/helpers/automations/nameHelpers.js +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -1,40 +1,71 @@ import { AutomationActionStepId } from "@budibase/types" -export const updateBindingsInInputs = (inputs, oldName, newName) => { +export const updateBindingsInInputs = (inputs, oldName, newName, stepIndex) => { if (typeof inputs === "string") { - return inputs.replace( - new RegExp(`stepsByName\\.${oldName}\\.`, "g"), - `stepsByName.${newName}.` - ) + return inputs + .replace( + new RegExp(`stepsByName\\.${oldName}\\.`, "g"), + `stepsByName.${newName}.` + ) + .replace( + new RegExp(`steps\\.${stepIndex}\\.`, "g"), + `stepsByName.${newName}.` + ) } if (Array.isArray(inputs)) { - return inputs.map(item => updateBindingsInInputs(item, oldName, newName)) + return inputs.map(item => + updateBindingsInInputs(item, oldName, newName, stepIndex) + ) } if (typeof inputs === "object" && inputs !== null) { const updatedInputs = {} for (const [key, value] of Object.entries(inputs)) { - updatedInputs[key] = updateBindingsInInputs(value, oldName, newName) + const updatedKey = updateBindingsInInputs( + key, + oldName, + newName, + stepIndex + ) + updatedInputs[updatedKey] = updateBindingsInInputs( + value, + oldName, + newName, + stepIndex + ) } return updatedInputs } - return inputs } -export const updateBindingsInSteps = (steps, oldName, newName) => { +export const updateBindingsInSteps = ( + steps, + oldName, + newName, + changedStepIndex +) => { return steps.map(step => { const updatedStep = { ...step, - inputs: updateBindingsInInputs(step.inputs, oldName, newName), + inputs: updateBindingsInInputs( + step.inputs, + oldName, + newName, + changedStepIndex + ), } - // Handle branch steps if ("branches" in updatedStep.inputs) { updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({ ...branch, - condition: updateBindingsInInputs(branch.condition, oldName, newName), + condition: updateBindingsInInputs( + branch.condition, + oldName, + newName, + changedStepIndex + ), })) if (updatedStep.inputs.children) { @@ -44,7 +75,8 @@ export const updateBindingsInSteps = (steps, oldName, newName) => { updatedStep.inputs.children[key] = updateBindingsInSteps( childSteps, oldName, - newName + newName, + changedStepIndex ) } } @@ -53,7 +85,6 @@ export const updateBindingsInSteps = (steps, oldName, newName) => { return updatedStep }) } - export const getNewStepName = (automation, step) => { const baseName = step.name diff --git a/packages/builder/src/helpers/tests/nameHelpers.spec.js b/packages/builder/src/helpers/tests/nameHelpers.spec.js new file mode 100644 index 0000000000..1ce2d1987a --- /dev/null +++ b/packages/builder/src/helpers/tests/nameHelpers.spec.js @@ -0,0 +1,177 @@ +import { cloneDeep } from "lodash" +import { + updateBindingsInInputs, + updateBindingsInSteps, +} from "../automations/nameHelpers" +describe("Automation Binding Update Functions", () => { + const sampleAutomation = { + definition: { + steps: [ + { + name: "First Step", + inputs: { + text: "Starting automation", + }, + id: "step1", + }, + { + name: "Second Step", + inputs: { + text: "{{ steps.0.success }} and {{ stepsByName.First Step.message }}", + }, + id: "step2", + }, + { + name: "Branch", + inputs: { + branches: [ + { + name: "branch1", + condition: { + equal: { + "steps.1.success": true, + }, + }, + }, + ], + children: { + branch1: [ + { + name: "Nested Step", + inputs: { + text: "{{ stepsByName.Second Step.message }} and {{ steps.1.success }}", + }, + id: "nestedStep", + }, + ], + }, + }, + id: "branchStep", + }, + ], + stepNames: { + step1: "First Step", + step2: "Second Step", + branchStep: "Branch", + }, + }, + } + + it("updateBindingsInInputs updates string bindings correctly", () => { + const input = "{{ stepsByName.oldName.success }} and {{ steps.1.message }}" + const result = updateBindingsInInputs(input, "oldName", "newName", 1) + expect(result).toBe( + "{{ stepsByName.newName.success }} and {{ stepsByName.newName.message }}" + ) + }) + + it("updateBindingsInInputs handles nested objects", () => { + const input = { + text: "{{ stepsByName.oldName.success }}", + nested: { + value: "{{ steps.1.message }}", + }, + } + const result = updateBindingsInInputs(input, "oldName", "newName", 1) + expect(result).toEqual({ + text: "{{ stepsByName.newName.success }}", + nested: { + value: "{{ stepsByName.newName.message }}", + }, + }) + }) + + it("updateBindingsInSteps updates bindings in all steps", () => { + const steps = cloneDeep(sampleAutomation.definition.steps) + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect(result[1].name).toBe("Second Step") + + expect(result[2].inputs.branches[0].condition.equal).toEqual({ + "stepsByName.Renamed Step.success": true, + }) + + const nestedStepText = result[2].inputs.children.branch1[0].inputs.text + expect(nestedStepText).toBe( + "{{ stepsByName.Renamed Step.message }} and {{ stepsByName.Renamed Step.success }}" + ) + }) + + it("updateBindingsInSteps handles steps with no bindings", () => { + const steps = [ + { + name: "No Binding Step", + inputs: { + text: "Plain text", + }, + id: "noBindingStep", + }, + ] + const result = updateBindingsInSteps(steps, "Old Name", "New Name", 0) + expect(result).toEqual(steps) + }) + + it("updateBindingsInSteps updates bindings in deeply nested branches", () => { + const deeplyNestedStep = { + name: "Deep Branch", + inputs: { + branches: [ + { + name: "deepBranch", + condition: { + equal: { + "stepsByName.Second Step.success": true, + }, + }, + }, + ], + children: { + deepBranch: [ + { + name: "Deep Log", + inputs: { + text: "{{ steps.1.message }}", + }, + }, + ], + }, + }, + } + + const steps = [...sampleAutomation.definition.steps, deeplyNestedStep] + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect( + result[3].inputs.branches[0].condition.equal[ + "stepsByName.Renamed Step.success" + ] + ).toBe(true) + expect(result[3].inputs.children.deepBranch[0].inputs.text).toBe( + "{{ stepsByName.Renamed Step.message }}" + ) + }) + + it("updateBindingsInSteps does not affect unrelated bindings", () => { + const steps = cloneDeep(sampleAutomation.definition.steps) + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect(result[1].inputs.text).toBe( + "{{ steps.0.success }} and {{ stepsByName.First Step.message }}" + ) + }) +}) diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js index eaf565c4f2..6627c67080 100644 --- a/packages/builder/src/stores/builder/automations.js +++ b/packages/builder/src/stores/builder/automations.js @@ -321,20 +321,18 @@ const automationActions = store => ({ const oldName = newAutomation.definition.steps[stepIndex].name const newName = name.trim() - // Update stepNames newAutomation.definition.stepNames = { ...newAutomation.definition.stepNames, [blockId]: newName, } - // Update the name in the step itself newAutomation.definition.steps[stepIndex].name = newName - // Update bindings in all steps newAutomation.definition.steps = updateBindingsInSteps( newAutomation.definition.steps, oldName, - newName + newName, + stepIndex ) await store.actions.save(newAutomation) From 2d4ac7fced1121dd49768426038372bfbed55e04 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:40:06 +0100 Subject: [PATCH 03/27] remove logs --- packages/server/src/threads/automation.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 3e5ac32b0a..24833d0db9 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -454,11 +454,9 @@ class Orchestrator { outputs: tempOutput, inputs: steps[stepToLoopIndex].inputs, }) - console.log(this.context) const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id this.context.stepsByName![stepName] = tempOutput - console.log(this.context) this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") @@ -552,7 +550,6 @@ class Orchestrator { loopIteration ) } - console.log(this.context) const stepFn = await this.getStepFunctionality(step.stepId) let inputs = await this.testProcesss(originalStepInput, this.context) From 0c11924027595cdd12d2534f00cae0d1ab8d2f7e Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:40:52 +0100 Subject: [PATCH 04/27] refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 048c37ecd9..c24374879d 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 048c37ecd921340614bf61a76a774aaa46176569 +Subproject commit c24374879d2b61516fabc24d7404e7da235be05e diff --git a/packages/pro b/packages/pro index ec1d2bda75..a4d1d15d9c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c +Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758 From 27b65e22efac6e78692a7c7990473bf59dc71dca Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:54:42 +0100 Subject: [PATCH 05/27] fix error with updating existinh name count --- packages/builder/src/helpers/automations/nameHelpers.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js index b863796a54..ad18213ab0 100644 --- a/packages/builder/src/helpers/automations/nameHelpers.js +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -105,9 +105,10 @@ export const getNewStepName = (automation, step) => { return count }, 0) } - - const existingCount = countExistingSteps(automation.definition.steps) - + let existingCount = 0 + if (automation?.definition) { + existingCount = countExistingSteps(automation.definition.steps) + } if (existingCount === 0) { return baseName } From e4918aea60752f69c032db921379d85e11b50063 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:55:31 +0100 Subject: [PATCH 06/27] refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index c24374879d..048c37ecd9 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c24374879d2b61516fabc24d7404e7da235be05e +Subproject commit 048c37ecd921340614bf61a76a774aaa46176569 diff --git a/packages/pro b/packages/pro index a4d1d15d9c..ec1d2bda75 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758 +Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c From 60dd500ecbf569dd70924332e39eb2738f360763 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 15:12:50 +0100 Subject: [PATCH 07/27] rename func --- packages/server/src/threads/automation.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 24833d0db9..288524b099 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -552,7 +552,10 @@ class Orchestrator { } const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await this.testProcesss(originalStepInput, this.context) + let inputs = await this.addContextAndProcess( + originalStepInput, + this.context + ) inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) @@ -580,7 +583,7 @@ class Orchestrator { return null } - private async testProcesss(inputs: any, context: any) { + private async addContextAndProcess(inputs: any, context: any) { const processContext = { ...context, steps: { From d4d4f8a1501bf02fa45cc70f388642355328ce65 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 15:24:49 +0100 Subject: [PATCH 08/27] ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 7899d07904..cd64894684 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 7899d07904d89d48954dd500da7b5dec32b781dd +Subproject commit cd648946847aade0ac1f53a1498596670c8a0aee From a831a4bf077b9bc95f824058abebbf8e3d9fdaa9 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 12 Sep 2024 14:58:53 +0100 Subject: [PATCH 09/27] tests to cover automation naming --- .../tests/scenarios/looping.spec.ts | 24 ++ .../tests/scenarios/scenarios.spec.ts | 309 ++++++++++-------- .../tests/utilities/AutomationTestBuilder.ts | 54 +-- .../documents/app/automation/automation.ts | 1 + 4 files changed, 233 insertions(+), 155 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 9bc382a187..f229610542 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -242,4 +242,28 @@ describe("Loop automations", () => { expect(results.steps[1].outputs.message).toContain("- 3") expect(results.steps[3].outputs.message).toContain("- 3") }) + + it("should use automation names to loop with", async () => { + const builder = createAutomationBuilder({ + name: "Test Trigger with Loop and Create Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .loop( + { + option: LoopStepType.ARRAY, + binding: [1, 2, 3], + }, + "FirstLoopStep" + ) + .serverLog({ text: "Message {{loop.currentItem}}" }, "FirstLoopLog") + .serverLog( + { text: "{{steps.FirstLoopLog.iterations}}" }, + "FirstLoopIterationLog" + ) + .run() + + expect(results.steps[1].outputs.message).toContain("- 3") + }) }) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 40d6094525..398ef20100 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -49,151 +49,188 @@ describe("Automation Scenarios", () => { }, }) }) - }) - it("should trigger an automation which querys the database", async () => { - const table = await config.createTable() - const row = { - name: "Test Row", - description: "original description", - tableId: table._id, - } - await config.createRow(row) - await config.createRow(row) - const builder = createAutomationBuilder({ - name: "Test Row Save and Create", + it("should trigger an automation which querys the database", async () => { + const table = await config.createTable() + const row = { + name: "Test Row", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(1) + expect(results.steps[0].outputs.rows).toHaveLength(2) }) - const results = await builder - .appAction({ fields: {} }) - .queryRows({ - tableId: table._id!, + it("should trigger an automation which querys the database then deletes a row", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", }) - .run() - expect(results.steps).toHaveLength(1) - expect(results.steps[0].outputs.rows).toHaveLength(2) - }) + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.1.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() - it("should trigger an automation which querys the database then deletes a row", async () => { - const table = await config.createTable() - const row = { - name: "DFN", - description: "original description", - tableId: table._id, - } - await config.createRow(row) - await config.createRow(row) - const builder = createAutomationBuilder({ - name: "Test Row Save and Create", + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) }) - const results = await builder - .appAction({ fields: {} }) - .queryRows({ - tableId: table._id!, - }) - .deleteRow({ - tableId: table._id!, - id: "{{ steps.1.rows.0._id }}", - }) - .queryRows({ - tableId: table._id!, - }) - .run() - - expect(results.steps).toHaveLength(3) - expect(results.steps[1].outputs.success).toBeTruthy() - expect(results.steps[2].outputs.rows).toHaveLength(1) - }) - - it("should query an external database for some data then insert than into an internal table", async () => { - const { datasource, client } = await setup.setupTestDatasource( - config, - DatabaseName.MYSQL - ) - - const newTable = await config.createTable({ - name: "table", - type: "table", - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - age: { - name: "age", - type: FieldType.NUMBER, - constraints: { - presence: true, - }, - }, - }, - }) - - const tableName = await setup.createTestTable(client, { - name: { type: "string" }, - age: { type: "number" }, - }) - - const rows = [ - { name: "Joe", age: 20 }, - { name: "Bob", age: 25 }, - { name: "Paul", age: 30 }, - ] - - await setup.insertTestData(client, tableName, rows) - - const query = await setup.saveTestQuery( - config, - client, - tableName, - datasource - ) - - const builder = createAutomationBuilder({ - name: "Test external query and save", - }) - - const results = await builder - .appAction({ - fields: {}, - }) - .executeQuery({ - query: { - queryId: query._id!, - }, - }) - .loop({ - option: LoopStepType.ARRAY, - binding: "{{ steps.1.response }}", - }) - .createRow({ - row: { - name: "{{ loop.currentItem.name }}", - age: "{{ loop.currentItem.age }}", - tableId: newTable._id!, - }, - }) - .queryRows({ - tableId: newTable._id!, - }) - .run() - - expect(results.steps).toHaveLength(3) - - expect(results.steps[1].outputs.iterations).toBe(3) - expect(results.steps[1].outputs.items).toHaveLength(3) - - expect(results.steps[2].outputs.rows).toHaveLength(3) - - rows.forEach(expectedRow => { - expect(results.steps[2].outputs.rows).toEqual( - expect.arrayContaining([expect.objectContaining(expectedRow)]) + it("should query an external database for some data then insert than into an internal table", async () => { + const { datasource, client } = await setup.setupTestDatasource( + config, + DatabaseName.MYSQL ) + + const newTable = await config.createTable({ + name: "table", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const tableName = await setup.createTestTable(client, { + name: { type: "string" }, + age: { type: "number" }, + }) + + const rows = [ + { name: "Joe", age: 20 }, + { name: "Bob", age: 25 }, + { name: "Paul", age: 30 }, + ] + + await setup.insertTestData(client, tableName, rows) + + const query = await setup.saveTestQuery( + config, + client, + tableName, + datasource + ) + + const builder = createAutomationBuilder({ + name: "Test external query and save", + }) + + const results = await builder + .appAction({ + fields: {}, + }) + .executeQuery({ + query: { + queryId: query._id!, + }, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.response }}", + }) + .createRow({ + row: { + name: "{{ loop.currentItem.name }}", + age: "{{ loop.currentItem.age }}", + tableId: newTable._id!, + }, + }) + .queryRows({ + tableId: newTable._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[1].outputs.iterations).toBe(3) + expect(results.steps[1].outputs.items).toHaveLength(3) + + expect(results.steps[2].outputs.rows).toHaveLength(3) + + rows.forEach(expectedRow => { + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining([expect.objectContaining(expectedRow)]) + ) + }) + }) + }) + + describe("Name Based Automations", () => { + it("should fetch and delete a rpw using automation naming", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Query and Delete Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + }, + "InitialQueryStep" + ) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.InitialQueryStep.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) }) }) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index f477efabe4..34c25d7004 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -57,21 +57,27 @@ type BranchConfig = { class BaseStepBuilder { protected steps: AutomationStep[] = [] + protected stepNames: { [key: string]: string } = {} protected step( stepId: TStep, stepSchema: Omit, - inputs: AutomationStepInputs + inputs: AutomationStepInputs, + stepName?: string ): this { + const id = uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, - id: uuidv4(), + id, stepId, + name: stepName || stepSchema.name, }) + if (stepName) { + this.stepNames[id] = stepName + } return this } - protected addBranchStep(branchConfig: BranchConfig): void { const branchStepInputs: BranchStepInputs = { branches: [] as Branch[], @@ -99,66 +105,74 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs): this { + createRow(inputs: CreateRowStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, - inputs + inputs, + stepName ) } - updateRow(inputs: UpdateRowStepInputs): this { + updateRow(inputs: UpdateRowStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, - inputs + inputs, + stepName ) } - deleteRow(inputs: DeleteRowStepInputs): this { + deleteRow(inputs: DeleteRowStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, - inputs + inputs, + stepName ) } - sendSmtpEmail(inputs: SmtpEmailStepInputs): this { + sendSmtpEmail(inputs: SmtpEmailStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, - inputs + inputs, + stepName ) } - executeQuery(inputs: ExecuteQueryStepInputs): this { + executeQuery(inputs: ExecuteQueryStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, - inputs + inputs, + stepName ) } - queryRows(inputs: QueryRowsStepInputs): this { + queryRows(inputs: QueryRowsStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, - inputs + inputs, + stepName ) } - loop(inputs: LoopStepInputs): this { + loop(inputs: LoopStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, - inputs + inputs, + stepName ) } - serverLog(input: ServerLogStepInputs): this { + serverLog(input: ServerLogStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, - input + input, + stepName ) } } @@ -186,6 +200,7 @@ class AutomationBuilder extends BaseStepBuilder { definition: { steps: [], trigger: {} as AutomationTrigger, + stepNames: {}, }, type: "automation", appId: options.appId ?? setup.getConfig().getAppId(), @@ -268,6 +283,7 @@ class AutomationBuilder extends BaseStepBuilder { build(): Automation { this.automationConfig.definition.steps = this.steps + this.automationConfig.definition.stepNames = this.stepNames return this.automationConfig } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 72f8a1aa7c..78d22c787f 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -124,6 +124,7 @@ export interface Automation extends Document { definition: { steps: AutomationStep[] trigger: AutomationTrigger + stepNames?: Record } screenId?: string uiTree?: any From 7d6cce20e972b3fa9ef1edb988c0afba53f3119b Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 12 Sep 2024 21:02:50 +0100 Subject: [PATCH 10/27] fix getAvailableBindings again, aka my personal hell --- .../SetupPanel/AutomationBlockSetup.svelte | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 1e2e409243..be7286a79e 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -700,14 +700,12 @@ allSteps[idx]?.stepId === ActionStepID.LOOP && allSteps.some(x => x.blockToLoop === block.id) let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {} - if (wasLoopBlock) { - break + if (allSteps[idx]?.name.includes("Looping")) { + isLoopBlock = true + loopBlockCount++ } let bindingName = - automation.stepNames?.[allSteps[idx - loopBlockCount].id] || - !isLoopBlock - ? allSteps[idx]?.name - : allSteps[idx - 1]?.name + automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name if (isLoopBlock) { schema = { @@ -756,7 +754,7 @@ if (wasLoopBlock) { loopBlockCount++ - continue + schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } Object.entries(schema).forEach(([name, value]) => addBinding(name, value, icon, idx, isLoopBlock, bindingName) From aaa7c9162f6063bac88dec0d93e8556015c08ea6 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 11:59:18 +0100 Subject: [PATCH 11/27] account Portal --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index cd64894684..723eff3807 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit cd648946847aade0ac1f53a1498596670c8a0aee +Subproject commit 723eff3807fa24fa30de69f9f556910e190a29d1 From 7e767e408955687f8841acb6fb68e9b34e44efd7 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 12:25:39 +0100 Subject: [PATCH 12/27] some pr comments --- .../tests/scenarios/looping.spec.ts | 9 +++-- .../tests/scenarios/scenarios.spec.ts | 2 +- .../tests/utilities/AutomationTestBuilder.ts | 38 +++++++++++-------- .../documents/app/automation/automation.ts | 1 + 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index f229610542..6dde05ddb8 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -255,12 +255,15 @@ describe("Loop automations", () => { option: LoopStepType.ARRAY, binding: [1, 2, 3], }, - "FirstLoopStep" + { stepName: "FirstLoopStep" } + ) + .serverLog( + { text: "Message {{loop.currentItem}}" }, + { stepName: "FirstLoopLog" } ) - .serverLog({ text: "Message {{loop.currentItem}}" }, "FirstLoopLog") .serverLog( { text: "{{steps.FirstLoopLog.iterations}}" }, - "FirstLoopIterationLog" + { stepName: "FirstLoopIterationLog" } ) .run() diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 398ef20100..49c05984fe 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -217,7 +217,7 @@ describe("Automation Scenarios", () => { { tableId: table._id!, }, - "InitialQueryStep" + { stepName: "InitialQueryStep" } ) .deleteRow({ tableId: table._id!, diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 34c25d7004..7528ea0f2e 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -105,74 +105,80 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs, stepName?: string): this { + createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, inputs, - stepName + opts?.stepName ) } - updateRow(inputs: UpdateRowStepInputs, stepName?: string): this { + updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, inputs, - stepName + opts?.stepName ) } - deleteRow(inputs: DeleteRowStepInputs, stepName?: string): this { + deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, inputs, - stepName + opts?.stepName ) } - sendSmtpEmail(inputs: SmtpEmailStepInputs, stepName?: string): this { + sendSmtpEmail( + inputs: SmtpEmailStepInputs, + opts?: { stepName?: string } + ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, inputs, - stepName + opts?.stepName ) } - executeQuery(inputs: ExecuteQueryStepInputs, stepName?: string): this { + executeQuery( + inputs: ExecuteQueryStepInputs, + opts?: { stepName?: string } + ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, inputs, - stepName + opts?.stepName ) } - queryRows(inputs: QueryRowsStepInputs, stepName?: string): this { + queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, inputs, - stepName + opts?.stepName ) } - loop(inputs: LoopStepInputs, stepName?: string): this { + loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, inputs, - stepName + opts?.stepName ) } - serverLog(input: ServerLogStepInputs, stepName?: string): this { + serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, input, - stepName + opts?.stepName ) } } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 78d22c787f..effe99a328 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -124,6 +124,7 @@ export interface Automation extends Document { definition: { steps: AutomationStep[] trigger: AutomationTrigger + // stepNames is used to lookup step names from their correspnding step ID. stepNames?: Record } screenId?: string From 04daa423cfe326aba255de37bd4bb5b6d042fc0e Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:03:37 +0100 Subject: [PATCH 13/27] small fix for edge case --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index be7286a79e..eb85cb19ba 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,6 +62,7 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" + import { name } from "helpers/validation/yup/app" export let block export let testData @@ -638,8 +639,7 @@ } /* End special cases for generating custom schemas based on triggers */ - let hasUserDefinedName = - automation.stepNames?.[allSteps[idx - loopBlockCount]]?.id + let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id] if (isLoopBlock) { runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { @@ -756,10 +756,12 @@ loopBlockCount++ schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } - Object.entries(schema).forEach(([name, value]) => + Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) - ) + }) + console.log(bindings) } + console.log(bindings) return bindings } From 59549b74b5bc615317cb4994f876808e285c67ef Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:04:13 +0100 Subject: [PATCH 14/27] js scripting edge case --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index eb85cb19ba..fe9710164b 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -644,7 +644,7 @@ runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { runtimeName = hasUserDefinedName - ? `stepsByName.${bindingName}.${name}` + ? `stepsByName[${bindingName}].${name}` : `steps[${idx - loopBlockCount}].${name}` } else { runtimeName = hasUserDefinedName From 70b9d085162a9b8394a450bd0b80e46a44e2d4e7 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:10:38 +0100 Subject: [PATCH 15/27] lint --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index fe9710164b..af67ae8d22 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,7 +62,6 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" - import { name } from "helpers/validation/yup/app" export let block export let testData @@ -759,9 +758,7 @@ Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) }) - console.log(bindings) } - console.log(bindings) return bindings } From 8222e1df3fd546c926d414144f7ad1fc98ffaec2 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:33:01 +0100 Subject: [PATCH 16/27] refs --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 723eff3807..25c6ed45c8 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 723eff3807fa24fa30de69f9f556910e190a29d1 +Subproject commit 25c6ed45c826f27b1ee7a7d1460dc1a386c6c471 From 9d6fc54a992e2ee0fe389a519d6d158d88f5995d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 16:12:07 +0100 Subject: [PATCH 17/27] Adding function parameter limit control for different SQL DBs, every DB has different limits with Postgres being the lowest at 100. We need to fix for wide tables which are related. --- packages/backend-core/src/sql/sql.ts | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b5243f856..b0e7b30067 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core" import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any -const MAX_SQS_RELATIONSHIP_FIELDS = 63 function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS @@ -877,6 +876,22 @@ class InternalBuilder { return `'${unaliased}'${separator}${tableField}` } + maxFunctionParameters() { + // functions like say json_build_object() in SQL have a limit as to how many can be performed + // before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as + // for JSON column building as part of relationships. We also have a default limit to avoid very complex + // functions being built - it is likely this is not necessary or the best way to do it. + switch (this.client) { + case SqlClient.SQL_LITE: + return 127 + case SqlClient.POSTGRES: + return 100 + // other DBs don't have a limit, but set some sort of limit + default: + return 200 + } + } + addJsonRelationships( query: Knex.QueryBuilder, fromTable: string, @@ -908,12 +923,10 @@ class InternalBuilder { let relationshipFields = fields.filter( field => field.split(".")[0] === toAlias ) - if (this.client === SqlClient.SQL_LITE) { - relationshipFields = relationshipFields.slice( - 0, - MAX_SQS_RELATIONSHIP_FIELDS - ) - } + relationshipFields = relationshipFields.slice( + 0, + Math.floor(this.maxFunctionParameters() / 2) + ) const fieldList: string = relationshipFields .map(field => this.buildJsonField(field)) .join(",") From 68a710699d6ab8fa978d9ee46c0020814b966845 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 18:09:01 +0100 Subject: [PATCH 18/27] Getting external DBs to correctly handle when too many fields. --- packages/backend-core/src/sql/sql.ts | 15 +++++-- .../api/controllers/row/ExternalRequest.ts | 13 +++--- .../src/api/routes/tests/search.spec.ts | 42 +++++++++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b0e7b30067..ff32026814 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -899,7 +899,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint } = this.query + const { resource, tableAliases: aliases, endpoint, meta } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -914,15 +914,22 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } + const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, }) - let relationshipFields = fields.filter( - field => field.split(".")[0] === toAlias - ) + const requiredFields = [ + ...(relatedTable?.primary || []), + relatedTable?.primaryDisplay, + ] + let relationshipFields = fields + .filter(field => field.split(".")[0] === toAlias) + // sort the required fields to first in the list + .sort(field => (requiredFields.includes(field) ? 1 : -1)) + relationshipFields = relationshipFields.slice( 0, Math.floor(this.maxFunctionParameters() / 2) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index ac2d1e8c39..9c9bd0b284 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -12,6 +12,7 @@ import { OneToManyRelationshipFieldMetadata, Operation, PaginationJson, + QueryJson, RelationshipFieldMetadata, Row, SearchFilters, @@ -161,7 +162,6 @@ export class ExternalRequest { private readonly tableId: string private datasource?: Datasource private tables: { [key: string]: Table } = {} - private tableList: Table[] constructor(operation: T, tableId: string, datasource?: Datasource) { this.operation = operation @@ -170,7 +170,6 @@ export class ExternalRequest { if (datasource && datasource.entities) { this.tables = datasource.entities } - this.tableList = Object.values(this.tables) } private prepareFilters( @@ -301,7 +300,6 @@ export class ExternalRequest { throw "No tables found, fetch tables before query." } this.tables = this.datasource.entities - this.tableList = Object.values(this.tables) } return { tables: this.tables, datasource: this.datasource } } @@ -463,7 +461,7 @@ export class ExternalRequest { breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] - if (!lookupField || !row[lookupField]) { + if (!lookupField || !row?.[lookupField] == null) { continue } const endpoint = getEndpoint(relatedTableId, Operation.READ) @@ -631,7 +629,8 @@ export class ExternalRequest { const { datasource: ds } = await this.retrieveMetadata(datasourceId) datasource = ds } - const table = this.tables[tableName] + const tables = this.tables + const table = tables[tableName] let isSql = isSQL(datasource) if (!table) { throw new Error( @@ -686,7 +685,7 @@ export class ExternalRequest { ) { throw "Deletion must be filtered" } - let json = { + let json: QueryJson = { endpoint: { datasourceId: datasourceId!, entityId: tableName, @@ -715,7 +714,7 @@ export class ExternalRequest { }, meta: { table, - id: config.id, + tables: tables, }, } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c9e7f4c3c5..0b0802bab2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3080,4 +3080,46 @@ describe.each([ }).toHaveLength(4) }) }) + + isSql && + describe("max related columns", () => { + let relatedRows: Row[] + + beforeAll(async () => { + const relatedSchema: TableSchema = {} + const row: Row = {} + for (let i = 0; i < 100; i++) { + const name = `column${i}` + relatedSchema[name] = { name, type: FieldType.NUMBER } + row[name] = i + } + const relatedTable = await createTable(relatedSchema) + table = await createTable({ + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTable._id!, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }) + relatedRows = await Promise.all([ + config.api.row.save(relatedTable._id!, row), + ]) + await config.api.row.save(table._id!, { + name: "foo", + related1: [relatedRows[0]._id], + }) + }) + + it("retrieve the row with relationships", async () => { + await expectQuery({}).toContainExactly([ + { + name: "foo", + related1: [{ _id: relatedRows[0]._id }], + }, + ]) + }) + }) }) From 63c0d9afb8cf6eb21a5b84734aa2c5b97534b883 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 18:27:53 +0100 Subject: [PATCH 19/27] Sorting the field list to make sure we have the important fields at the top (if known). --- packages/backend-core/src/sql/sql.ts | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index ff32026814..55f71d76b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -55,6 +55,20 @@ function getRelationshipLimit() { return envLimit || 500 } +function prioritisedArraySort(toSort: string[], priorities: string[]) { + return toSort.sort((a, b) => { + const aPriority = priorities.find(field => field && a.endsWith(field)) + const bPriority = priorities.find(field => field && b.endsWith(field)) + if (aPriority && !bPriority) { + return -1 + } + if (!aPriority && bPriority) { + return 1 + } + return a.localeCompare(b) + }) +} + function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( @@ -924,11 +938,12 @@ class InternalBuilder { const requiredFields = [ ...(relatedTable?.primary || []), relatedTable?.primaryDisplay, - ] - let relationshipFields = fields - .filter(field => field.split(".")[0] === toAlias) - // sort the required fields to first in the list - .sort(field => (requiredFields.includes(field) ? 1 : -1)) + ].filter(field => field) as string[] + // sort the required fields to first in the list, so they don't get sliced out + let relationshipFields = prioritisedArraySort( + fields.filter(field => field.split(".")[0] === toAlias), + requiredFields + ) relationshipFields = relationshipFields.slice( 0, From ec400dee6f73b510d5b96378abb66f7877cda5cb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 19:20:58 +0100 Subject: [PATCH 20/27] Fixing test cases. --- packages/server/src/integrations/tests/sql.spec.ts | 4 ++-- packages/server/src/integrations/tests/sqlAlias.spec.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index cf5b88b0fd..5a505fbb40 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -162,7 +162,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) @@ -170,7 +170,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 9548499c65..fc5af4238c 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -63,7 +63,7 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")` ) ), }) @@ -95,7 +95,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) @@ -113,7 +113,7 @@ describe("Captures of real examples", () => { bindings: [...filters, relationshipLimit, relationshipLimit], sql: multiline( `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) - select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc` ), From 614132eb0462a071cbbcc6f0c867e589e1502108 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 16 Sep 2024 20:40:02 +0000 Subject: [PATCH 21/27] Bump version to 2.32.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a9c50ea4b5..c710d888c7 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.4", + "version": "2.32.5", "npmClient": "yarn", "packages": [ "packages/*", From 5e245d11e5963774b8c16f478624c41e031abcb7 Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 17 Sep 2024 09:20:23 +0100 Subject: [PATCH 22/27] Bump account portal to latest --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 25c6ed45c8..905773d708 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 25c6ed45c826f27b1ee7a7d1460dc1a386c6c471 +Subproject commit 905773d70854a43c6ef2461c7a49671bff56fedc From da122d2ac157bc26ba69bfdec32a184469033539 Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 18 Sep 2024 14:55:09 +0100 Subject: [PATCH 23/27] Add table ID to row deletion requests that only contain string IDs --- packages/server/src/api/controllers/row/index.ts | 2 +- packages/server/src/api/routes/tests/row.spec.ts | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/server/src/api/controllers/row/index.ts b/packages/server/src/api/controllers/row/index.ts index cd85f57982..2e5785157d 100644 --- a/packages/server/src/api/controllers/row/index.ts +++ b/packages/server/src/api/controllers/row/index.ts @@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx) { const { tableId } = utils.getSourceId(ctx) const processedRows = request.rows.map(row => { - let processedRow: Row = typeof row == "string" ? { _id: row } : row + let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row return !processedRow._rev ? addRev(fixRow(processedRow, ctx.params), tableId) : fixRow(processedRow, ctx.params) diff --git a/packages/server/src/api/routes/tests/row.spec.ts b/packages/server/src/api/routes/tests/row.spec.ts index 9790703806..dc03a21d6d 100644 --- a/packages/server/src/api/routes/tests/row.spec.ts +++ b/packages/server/src/api/routes/tests/row.spec.ts @@ -1138,6 +1138,18 @@ describe.each([ await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) }) + it("should be able to delete a row with ID only", async () => { + const createdRow = await config.api.row.save(table._id!, {}) + const rowUsage = await getRowUsage() + + const res = await config.api.row.bulkDelete(table._id!, { + rows: [createdRow._id!], + }) + expect(res[0]._id).toEqual(createdRow._id) + expect(res[0].tableId).toEqual(table._id!) + await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage) + }) + it("should be able to bulk delete rows, including a row that doesn't exist", async () => { const createdRow = await config.api.row.save(table._id!, {}) const createdRow2 = await config.api.row.save(table._id!, {}) From be5a4f5d9763122f38d5c083cd4ccd70944f5f0b Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 10:11:24 +0100 Subject: [PATCH 24/27] Require a flag to be set to query PostHog for feature flags. --- packages/backend-core/src/environment.ts | 1 + packages/backend-core/src/features/index.ts | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 6bef6efeb3..0ffe9966b1 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -143,6 +143,7 @@ const environment = { POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN, POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com", + POSTHOG_FEATURE_FLAGS: process.env.POSTHOG_FEATURE_FLAGS, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN, diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index 5b6ea4ca92..fda95c6553 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -6,7 +6,12 @@ import tracer from "dd-trace" let posthog: PostHog | undefined export function init(opts?: PostHogOptions) { - if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) { + if ( + env.POSTHOG_TOKEN && + env.POSTHOG_API_HOST && + !env.SELF_HOSTED && + env.POSTHOG_FEATURE_FLAGS + ) { console.log("initializing posthog client...") posthog = new PostHog(env.POSTHOG_TOKEN, { host: env.POSTHOG_API_HOST, From 6d7cffa43e269ffd1672e4e7de169cf45b2d69f4 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 10:15:18 +0100 Subject: [PATCH 25/27] Make flag name more accurate. --- packages/backend-core/src/environment.ts | 2 +- packages/backend-core/src/features/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/environment.ts b/packages/backend-core/src/environment.ts index 0ffe9966b1..2ab8c550cc 100644 --- a/packages/backend-core/src/environment.ts +++ b/packages/backend-core/src/environment.ts @@ -143,7 +143,7 @@ const environment = { POSTHOG_TOKEN: process.env.POSTHOG_TOKEN, POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN, POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com", - POSTHOG_FEATURE_FLAGS: process.env.POSTHOG_FEATURE_FLAGS, + POSTHOG_FEATURE_FLAGS_ENABLED: process.env.POSTHOG_FEATURE_FLAGS_ENABLED, ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS, TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS, CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN, diff --git a/packages/backend-core/src/features/index.ts b/packages/backend-core/src/features/index.ts index fda95c6553..0765d09036 100644 --- a/packages/backend-core/src/features/index.ts +++ b/packages/backend-core/src/features/index.ts @@ -10,7 +10,7 @@ export function init(opts?: PostHogOptions) { env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED && - env.POSTHOG_FEATURE_FLAGS + env.POSTHOG_FEATURE_FLAGS_ENABLED ) { console.log("initializing posthog client...") posthog = new PostHog(env.POSTHOG_TOKEN, { From 74effbb55b51da624bee420c5ef054ee93fbd31a Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 19 Sep 2024 11:41:10 +0100 Subject: [PATCH 26/27] Fix tests. --- packages/backend-core/src/features/tests/features.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/backend-core/src/features/tests/features.spec.ts b/packages/backend-core/src/features/tests/features.spec.ts index d092585cc6..01c9bfa3c6 100644 --- a/packages/backend-core/src/features/tests/features.spec.ts +++ b/packages/backend-core/src/features/tests/features.spec.ts @@ -148,6 +148,7 @@ describe("feature flags", () => { const env: Partial = { TENANT_FEATURE_FLAGS: environmentFlags, SELF_HOSTED: false, + POSTHOG_FEATURE_FLAGS_ENABLED: "true", } if (posthogFlags) { From 5d945b90ff8a650239eac13b5d9a2b7f54c6c973 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Thu, 19 Sep 2024 12:58:57 +0000 Subject: [PATCH 27/27] Bump version to 2.32.6 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index c710d888c7..d695869907 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.5", + "version": "2.32.6", "npmClient": "yarn", "packages": [ "packages/*",