CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-09-24 21:31:17 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' of github.com:budibase/budibase into budi-8637-googlesheets-issues-automations-row-actions-and-filtering-1

Browse source
This commit is contained in:
Sam Rose 2024-09-19 14:07:35 +01:00
commit 40e654f515
No known key found for this signature in database
21 changed files with 747 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lerna.json
View file

@ -1,6 +1,6 @@
{
"$schema": "node_modules/lerna/schemas/lerna-schema.json",
"version": "2.32.4",
"version": "2.32.6",
"npmClient": "yarn",
"packages": [
"packages/*",

1
packages/backend-core/src/environment.ts
View file

@ -143,6 +143,7 @@ const environment = {
POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,
POSTHOG_PERSONAL_TOKEN: process.env.POSTHOG_PERSONAL_TOKEN,
POSTHOG_API_HOST: process.env.POSTHOG_API_HOST || "https://us.i.posthog.com",
POSTHOG_FEATURE_FLAGS_ENABLED: process.env.POSTHOG_FEATURE_FLAGS_ENABLED,
ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
TENANT_FEATURE_FLAGS: process.env.TENANT_FEATURE_FLAGS,
CLOUDFRONT_CDN: process.env.CLOUDFRONT_CDN,

7
packages/backend-core/src/features/index.ts
View file

@ -6,7 +6,12 @@ import tracer from "dd-trace"
let posthog: PostHog | undefined
export function init(opts?: PostHogOptions) {
if (env.POSTHOG_TOKEN && env.POSTHOG_API_HOST && !env.SELF_HOSTED) {
if (
env.POSTHOG_TOKEN &&
env.POSTHOG_API_HOST &&
!env.SELF_HOSTED &&
env.POSTHOG_FEATURE_FLAGS_ENABLED
) {
console.log("initializing posthog client...")
posthog = new PostHog(env.POSTHOG_TOKEN, {
host: env.POSTHOG_API_HOST,

1
packages/backend-core/src/features/tests/features.spec.ts
View file

@ -148,6 +148,7 @@ describe("feature flags", () => {
const env: Partial<typeof environment> = {
TENANT_FEATURE_FLAGS: environmentFlags,
SELF_HOSTED: false,
POSTHOG_FEATURE_FLAGS_ENABLED: "true",
}
if (posthogFlags) {

55
packages/backend-core/src/sql/sql.ts
View file

@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core"
import { cloneDeep } from "lodash"
type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
const MAX_SQS_RELATIONSHIP_FIELDS = 63
function getBaseLimit() {
const envLimit = environment.SQL_MAX_ROWS
@ -56,6 +55,20 @@ function getRelationshipLimit() {
return envLimit || 500
}
function prioritisedArraySort(toSort: string[], priorities: string[]) {
return toSort.sort((a, b) => {
const aPriority = priorities.find(field => field && a.endsWith(field))
const bPriority = priorities.find(field => field && b.endsWith(field))
if (aPriority && !bPriority) {
return -1
}
if (!aPriority && bPriority) {
return 1
}
return a.localeCompare(b)
})
}
function getTableName(table?: Table): string | undefined {
// SQS uses the table ID rather than the table name
if (
@ -877,6 +890,22 @@ class InternalBuilder {
return `'${unaliased}'${separator}${tableField}`
}
maxFunctionParameters() {
// functions like say json_build_object() in SQL have a limit as to how many can be performed
// before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as
// for JSON column building as part of relationships. We also have a default limit to avoid very complex
// functions being built - it is likely this is not necessary or the best way to do it.
switch (this.client) {
case SqlClient.SQL_LITE:
return 127
case SqlClient.POSTGRES:
return 100
// other DBs don't have a limit, but set some sort of limit
default:
return 200
}
}
addJsonRelationships(
query: Knex.QueryBuilder,
fromTable: string,
@ -884,7 +913,7 @@ class InternalBuilder {
): Knex.QueryBuilder {
const sqlClient = this.client
const knex = this.knex
const { resource, tableAliases: aliases, endpoint } = this.query
const { resource, tableAliases: aliases, endpoint, meta } = this.query
const fields = resource?.fields || []
for (let relationship of relationships) {
const {
@ -899,21 +928,27 @@ class InternalBuilder {
if (!toTable || !fromTable) {
continue
}
const relatedTable = meta.tables?.[toTable]
const toAlias = aliases?.[toTable] || toTable,
fromAlias = aliases?.[fromTable] || fromTable
let toTableWithSchema = this.tableNameWithSchema(toTable, {
alias: toAlias,
schema: endpoint.schema,
})
let relationshipFields = fields.filter(
field => field.split(".")[0] === toAlias
const requiredFields = [
...(relatedTable?.primary || []),
relatedTable?.primaryDisplay,
].filter(field => field) as string[]
// sort the required fields to first in the list, so they don't get sliced out
let relationshipFields = prioritisedArraySort(
fields.filter(field => field.split(".")[0] === toAlias),
requiredFields
)
relationshipFields = relationshipFields.slice(
0,
Math.floor(this.maxFunctionParameters() / 2)
)
if (this.client === SqlClient.SQL_LITE) {
relationshipFields = relationshipFields.slice(
0,
MAX_SQS_RELATIONSHIP_FIELDS
)
}
const fieldList: string = relationshipFields
.map(field => this.buildJsonField(field))
.join(",")

40
packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
View file

@ -595,9 +595,13 @@
let loopBlockCount = 0
const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => {
if (!name) return
const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock)
const runtimeBinding = determineRuntimeBinding(
name,
idx,
isLoopBlock,
bindingName
)
const categoryName = determineCategoryName(idx, isLoopBlock, bindingName)
bindings.push(
createBindingObject(
name,
@ -613,7 +617,7 @@
)
}
const determineRuntimeBinding = (name, idx, isLoopBlock) => {
const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => {
let runtimeName
/* Begin special cases for generating custom schemas based on triggers */
@ -634,12 +638,17 @@
}
/* End special cases for generating custom schemas based on triggers */
let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id]
if (isLoopBlock) {
runtimeName = `loop.${name}`
} else if (block.name.startsWith("JS")) {
runtimeName = `steps[${idx - loopBlockCount}].${name}`
runtimeName = hasUserDefinedName
? `stepsByName[${bindingName}].${name}`
: `steps[${idx - loopBlockCount}].${name}`
} else {
runtimeName = `steps.${idx - loopBlockCount}.${name}`
runtimeName = hasUserDefinedName
? `stepsByName.${bindingName}.${name}`
: `steps.${idx - loopBlockCount}.${name}`
}
return idx === 0 ? `trigger.${name}` : runtimeName
}
@ -666,11 +675,11 @@
const field = Object.values(FIELDS).find(
field => field.type === value.type && field.subtype === value.subtype
)
return {
readableBinding: bindingName
? `${bindingName}.${name}`
: runtimeBinding,
readableBinding:
bindingName && !isLoopBlock
? `steps.${bindingName}.${name}`
: runtimeBinding,
runtimeBinding,
type: value.type,
description: value.description,
@ -690,8 +699,12 @@
allSteps[idx]?.stepId === ActionStepID.LOOP &&
allSteps.some(x => x.blockToLoop === block.id)
let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {}
if (allSteps[idx]?.name.includes("Looping")) {
isLoopBlock = true
loopBlockCount++
}
let bindingName =
automation.stepNames?.[allSteps[idx - loopBlockCount].id]
automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name
if (isLoopBlock) {
schema = {
@ -740,13 +753,12 @@
if (wasLoopBlock) {
loopBlockCount++
continue
schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties)
}
Object.entries(schema).forEach(([name, value]) =>
Object.entries(schema).forEach(([name, value]) => {
addBinding(name, value, icon, idx, isLoopBlock, bindingName)
)
})
}
return bindings
}

117
packages/builder/src/helpers/automations/nameHelpers.js Normal file
View file

@ -0,0 +1,117 @@
import { AutomationActionStepId } from "@budibase/types"
export const updateBindingsInInputs = (inputs, oldName, newName, stepIndex) => {
if (typeof inputs === "string") {
return inputs
.replace(
new RegExp(`stepsByName\\.${oldName}\\.`, "g"),
`stepsByName.${newName}.`
)
.replace(
new RegExp(`steps\\.${stepIndex}\\.`, "g"),
`stepsByName.${newName}.`
)
}
if (Array.isArray(inputs)) {
return inputs.map(item =>
updateBindingsInInputs(item, oldName, newName, stepIndex)
)
}
if (typeof inputs === "object" && inputs !== null) {
const updatedInputs = {}
for (const [key, value] of Object.entries(inputs)) {
const updatedKey = updateBindingsInInputs(
key,
oldName,
newName,
stepIndex
)
updatedInputs[updatedKey] = updateBindingsInInputs(
value,
oldName,
newName,
stepIndex
)
}
return updatedInputs
}
return inputs
}
export const updateBindingsInSteps = (
steps,
oldName,
newName,
changedStepIndex
) => {
return steps.map(step => {
const updatedStep = {
...step,
inputs: updateBindingsInInputs(
step.inputs,
oldName,
newName,
changedStepIndex
),
}
if ("branches" in updatedStep.inputs) {
updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({
...branch,
condition: updateBindingsInInputs(
branch.condition,
oldName,
newName,
changedStepIndex
),
}))
if (updatedStep.inputs.children) {
for (const [key, childSteps] of Object.entries(
updatedStep.inputs.children
)) {
updatedStep.inputs.children[key] = updateBindingsInSteps(
childSteps,
oldName,
newName,
changedStepIndex
)
}
}
}
return updatedStep
})
}
export const getNewStepName = (automation, step) => {
const baseName = step.name
const countExistingSteps = steps => {
return steps.reduce((count, currentStep) => {
if (currentStep.name && currentStep.name.startsWith(baseName)) {
count++
}
if (
currentStep.stepId === AutomationActionStepId.BRANCH &&
currentStep.inputs &&
currentStep.inputs.children
) {
Object.values(currentStep.inputs.children).forEach(branchSteps => {
count += countExistingSteps(branchSteps)
})
}
return count
}, 0)
}
let existingCount = 0
if (automation?.definition) {
existingCount = countExistingSteps(automation.definition.steps)
}
if (existingCount === 0) {
return baseName
}
return `${baseName} ${existingCount + 1}`
}

177
packages/builder/src/helpers/tests/nameHelpers.spec.js Normal file
View file

@ -0,0 +1,177 @@
import { cloneDeep } from "lodash"
import {
updateBindingsInInputs,
updateBindingsInSteps,
} from "../automations/nameHelpers"
describe("Automation Binding Update Functions", () => {
const sampleAutomation = {
definition: {
steps: [
{
name: "First Step",
inputs: {
text: "Starting automation",
},
id: "step1",
},
{
name: "Second Step",
inputs: {
text: "{{ steps.0.success }} and {{ stepsByName.First Step.message }}",
},
id: "step2",
},
{
name: "Branch",
inputs: {
branches: [
{
name: "branch1",
condition: {
equal: {
"steps.1.success": true,
},
},
},
],
children: {
branch1: [
{
name: "Nested Step",
inputs: {
text: "{{ stepsByName.Second Step.message }} and {{ steps.1.success }}",
},
id: "nestedStep",
},
],
},
},
id: "branchStep",
},
],
stepNames: {
step1: "First Step",
step2: "Second Step",
branchStep: "Branch",
},
},
}
it("updateBindingsInInputs updates string bindings correctly", () => {
const input = "{{ stepsByName.oldName.success }} and {{ steps.1.message }}"
const result = updateBindingsInInputs(input, "oldName", "newName", 1)
expect(result).toBe(
"{{ stepsByName.newName.success }} and {{ stepsByName.newName.message }}"
)
})
it("updateBindingsInInputs handles nested objects", () => {
const input = {
text: "{{ stepsByName.oldName.success }}",
nested: {
value: "{{ steps.1.message }}",
},
}
const result = updateBindingsInInputs(input, "oldName", "newName", 1)
expect(result).toEqual({
text: "{{ stepsByName.newName.success }}",
nested: {
value: "{{ stepsByName.newName.message }}",
},
})
})
it("updateBindingsInSteps updates bindings in all steps", () => {
const steps = cloneDeep(sampleAutomation.definition.steps)
const result = updateBindingsInSteps(
steps,
"Second Step",
"Renamed Step",
1
)
expect(result[1].name).toBe("Second Step")
expect(result[2].inputs.branches[0].condition.equal).toEqual({
"stepsByName.Renamed Step.success": true,
})
const nestedStepText = result[2].inputs.children.branch1[0].inputs.text
expect(nestedStepText).toBe(
"{{ stepsByName.Renamed Step.message }} and {{ stepsByName.Renamed Step.success }}"
)
})
it("updateBindingsInSteps handles steps with no bindings", () => {
const steps = [
{
name: "No Binding Step",
inputs: {
text: "Plain text",
},
id: "noBindingStep",
},
]
const result = updateBindingsInSteps(steps, "Old Name", "New Name", 0)
expect(result).toEqual(steps)
})
it("updateBindingsInSteps updates bindings in deeply nested branches", () => {
const deeplyNestedStep = {
name: "Deep Branch",
inputs: {
branches: [
{
name: "deepBranch",
condition: {
equal: {
"stepsByName.Second Step.success": true,
},
},
},
],
children: {
deepBranch: [
{
name: "Deep Log",
inputs: {
text: "{{ steps.1.message }}",
},
},
],
},
},
}
const steps = [...sampleAutomation.definition.steps, deeplyNestedStep]
const result = updateBindingsInSteps(
steps,
"Second Step",
"Renamed Step",
1
)
expect(
result[3].inputs.branches[0].condition.equal[
"stepsByName.Renamed Step.success"
]
).toBe(true)
expect(result[3].inputs.children.deepBranch[0].inputs.text).toBe(
"{{ stepsByName.Renamed Step.message }}"
)
})
it("updateBindingsInSteps does not affect unrelated bindings", () => {
const steps = cloneDeep(sampleAutomation.definition.steps)
const result = updateBindingsInSteps(
steps,
"Second Step",
"Renamed Step",
1
)
expect(result[1].inputs.text).toBe(
"{{ steps.0.success }} and {{ stepsByName.First Step.message }}"
)
})
})

41
packages/builder/src/stores/builder/automations.js
View file

@ -6,6 +6,10 @@ import { createHistoryStore } from "stores/builder/history"
import { notifications } from "@budibase/bbui"
import { updateReferencesInObject } from "dataBinding"
import { AutomationTriggerStepId } from "@budibase/types"
import {
updateBindingsInSteps,
getNewStepName,
} from "helpers/automations/nameHelpers"
const initialAutomationState = {
automations: [],
@ -275,13 +279,17 @@ const automationActions = store => ({
await store.actions.save(newAutomation)
},
constructBlock(type, stepId, blockDefinition) {
return {
let newName
const newStep = {
...blockDefinition,
inputs: blockDefinition.inputs || {},
stepId,
type,
id: generate(),
}
newName = getNewStepName(get(selectedAutomation), newStep)
newStep.name = newName
return newStep
},
addBlockToAutomation: async (block, blockIdx) => {
const automation = get(selectedAutomation)
@ -301,15 +309,34 @@ const automationActions = store => ({
saveAutomationName: async (blockId, name) => {
const automation = get(selectedAutomation)
let newAutomation = cloneDeep(automation)
if (!automation) {
if (!newAutomation) {
return
}
newAutomation.definition.stepNames = {
...newAutomation.definition.stepNames,
[blockId]: name.trim(),
}
await store.actions.save(newAutomation)
const stepIndex = newAutomation.definition.steps.findIndex(
step => step.id === blockId
)
if (stepIndex !== -1) {
const oldName = newAutomation.definition.steps[stepIndex].name
const newName = name.trim()
newAutomation.definition.stepNames = {
...newAutomation.definition.stepNames,
[blockId]: newName,
}
newAutomation.definition.steps[stepIndex].name = newName
newAutomation.definition.steps = updateBindingsInSteps(
newAutomation.definition.steps,
oldName,
newName,
stepIndex
)
await store.actions.save(newAutomation)
}
},
deleteAutomationName: async blockId => {
const automation = get(selectedAutomation)

13
packages/server/src/api/controllers/row/ExternalRequest.ts
View file

@ -12,6 +12,7 @@ import {
OneToManyRelationshipFieldMetadata,
Operation,
PaginationJson,
QueryJson,
RelationshipFieldMetadata,
Row,
SearchFilters,
@ -161,7 +162,6 @@ export class ExternalRequest<T extends Operation> {
private readonly tableId: string
private datasource?: Datasource
private tables: { [key: string]: Table } = {}
private tableList: Table[]
constructor(operation: T, tableId: string, datasource?: Datasource) {
this.operation = operation
@ -170,7 +170,6 @@ export class ExternalRequest<T extends Operation> {
if (datasource && datasource.entities) {
this.tables = datasource.entities
}
this.tableList = Object.values(this.tables)
}
private prepareFilters(
@ -301,7 +300,6 @@ export class ExternalRequest<T extends Operation> {
throw "No tables found, fetch tables before query."
}
this.tables = this.datasource.entities
this.tableList = Object.values(this.tables)
}
return { tables: this.tables, datasource: this.datasource }
}
@ -463,7 +461,7 @@ export class ExternalRequest<T extends Operation> {
breakExternalTableId(relatedTableId)
// @ts-ignore
const linkPrimaryKey = this.tables[relatedTableName].primary[0]
if (!lookupField || !row[lookupField]) {
if (!lookupField || !row?.[lookupField] == null) {
continue
}
const endpoint = getEndpoint(relatedTableId, Operation.READ)
@ -631,7 +629,8 @@ export class ExternalRequest<T extends Operation> {
const { datasource: ds } = await this.retrieveMetadata(datasourceId)
datasource = ds
}
const table = this.tables[tableName]
const tables = this.tables
const table = tables[tableName]
let isSql = isSQL(datasource)
if (!table) {
throw new Error(
@ -686,7 +685,7 @@ export class ExternalRequest<T extends Operation> {
) {
throw "Deletion must be filtered"
}
let json = {
let json: QueryJson = {
endpoint: {
datasourceId: datasourceId!,
entityId: tableName,
@ -715,7 +714,7 @@ export class ExternalRequest<T extends Operation> {
},
meta: {
table,
id: config.id,
tables: tables,
},
}

2
packages/server/src/api/controllers/row/index.ts
View file

@ -138,7 +138,7 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
const { tableId } = utils.getSourceId(ctx)
const processedRows = request.rows.map(row => {
let processedRow: Row = typeof row == "string" ? { _id: row } : row
let processedRow: Row = typeof row == "string" ? { _id: row, tableId } : row
return !processedRow._rev
? addRev(fixRow(processedRow, ctx.params), tableId)
: fixRow(processedRow, ctx.params)

12
packages/server/src/api/routes/tests/row.spec.ts
View file

@ -1138,6 +1138,18 @@ describe.each([
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to delete a row with ID only", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const rowUsage = await getRowUsage()
const res = await config.api.row.bulkDelete(table._id!, {
rows: [createdRow._id!],
})
expect(res[0]._id).toEqual(createdRow._id)
expect(res[0].tableId).toEqual(table._id!)
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
})
it("should be able to bulk delete rows, including a row that doesn't exist", async () => {
const createdRow = await config.api.row.save(table._id!, {})
const createdRow2 = await config.api.row.save(table._id!, {})

42
packages/server/src/api/routes/tests/search.spec.ts
View file

@ -3080,4 +3080,46 @@ describe.each([
}).toHaveLength(4)
})
})
isSql &&
describe("max related columns", () => {
let relatedRows: Row[]
beforeAll(async () => {
const relatedSchema: TableSchema = {}
const row: Row = {}
for (let i = 0; i < 100; i++) {
const name = `column${i}`
relatedSchema[name] = { name, type: FieldType.NUMBER }
row[name] = i
}
const relatedTable = await createTable(relatedSchema)
table = await createTable({
name: { name: "name", type: FieldType.STRING },
related1: {
type: FieldType.LINK,
name: "related1",
fieldName: "main1",
tableId: relatedTable._id!,
relationshipType: RelationshipType.MANY_TO_MANY,
},
})
relatedRows = await Promise.all([
config.api.row.save(relatedTable._id!, row),
])
await config.api.row.save(table._id!, {
name: "foo",
related1: [relatedRows[0]._id],
})
})
it("retrieve the row with relationships", async () => {
await expectQuery({}).toContainExactly([
{
name: "foo",
related1: [{ _id: relatedRows[0]._id }],
},
])
})
})
})

27
packages/server/src/automations/tests/scenarios/looping.spec.ts
View file

@ -242,4 +242,31 @@ describe("Loop automations", () => {
expect(results.steps[1].outputs.message).toContain("- 3")
expect(results.steps[3].outputs.message).toContain("- 3")
})
it("should use automation names to loop with", async () => {
const builder = createAutomationBuilder({
name: "Test Trigger with Loop and Create Row",
})
const results = await builder
.appAction({ fields: {} })
.loop(
{
option: LoopStepType.ARRAY,
binding: [1, 2, 3],
},
{ stepName: "FirstLoopStep" }
)
.serverLog(
{ text: "Message {{loop.currentItem}}" },
{ stepName: "FirstLoopLog" }
)
.serverLog(
{ text: "{{steps.FirstLoopLog.iterations}}" },
{ stepName: "FirstLoopIterationLog" }
)
.run()
expect(results.steps[1].outputs.message).toContain("- 3")
})
})

309
packages/server/src/automations/tests/scenarios/scenarios.spec.ts
View file

@ -49,151 +49,188 @@ describe("Automation Scenarios", () => {
},
})
})
})
it("should trigger an automation which querys the database", async () => {
const table = await config.createTable()
const row = {
name: "Test Row",
description: "original description",
tableId: table._id,
}
await config.createRow(row)
await config.createRow(row)
const builder = createAutomationBuilder({
name: "Test Row Save and Create",
it("should trigger an automation which querys the database", async () => {
const table = await config.createTable()
const row = {
name: "Test Row",
description: "original description",
tableId: table._id,
}
await config.createRow(row)
await config.createRow(row)
const builder = createAutomationBuilder({
name: "Test Row Save and Create",
})
const results = await builder
.appAction({ fields: {} })
.queryRows({
tableId: table._id!,
})
.run()
expect(results.steps).toHaveLength(1)
expect(results.steps[0].outputs.rows).toHaveLength(2)
})
const results = await builder
.appAction({ fields: {} })
.queryRows({
tableId: table._id!,
it("should trigger an automation which querys the database then deletes a row", async () => {
const table = await config.createTable()
const row = {
name: "DFN",
description: "original description",
tableId: table._id,
}
await config.createRow(row)
await config.createRow(row)
const builder = createAutomationBuilder({
name: "Test Row Save and Create",
})
.run()
expect(results.steps).toHaveLength(1)
expect(results.steps[0].outputs.rows).toHaveLength(2)
})
const results = await builder
.appAction({ fields: {} })
.queryRows({
tableId: table._id!,
})
.deleteRow({
tableId: table._id!,
id: "{{ steps.1.rows.0._id }}",
})
.queryRows({
tableId: table._id!,
})
.run()
it("should trigger an automation which querys the database then deletes a row", async () => {
const table = await config.createTable()
const row = {
name: "DFN",
description: "original description",
tableId: table._id,
}
await config.createRow(row)
await config.createRow(row)
const builder = createAutomationBuilder({
name: "Test Row Save and Create",
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.success).toBeTruthy()
expect(results.steps[2].outputs.rows).toHaveLength(1)
})
const results = await builder
.appAction({ fields: {} })
.queryRows({
tableId: table._id!,
})
.deleteRow({
tableId: table._id!,
id: "{{ steps.1.rows.0._id }}",
})
.queryRows({
tableId: table._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.success).toBeTruthy()
expect(results.steps[2].outputs.rows).toHaveLength(1)
})
it("should query an external database for some data then insert than into an internal table", async () => {
const { datasource, client } = await setup.setupTestDatasource(
config,
DatabaseName.MYSQL
)
const newTable = await config.createTable({
name: "table",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const tableName = await setup.createTestTable(client, {
name: { type: "string" },
age: { type: "number" },
})
const rows = [
{ name: "Joe", age: 20 },
{ name: "Bob", age: 25 },
{ name: "Paul", age: 30 },
]
await setup.insertTestData(client, tableName, rows)
const query = await setup.saveTestQuery(
config,
client,
tableName,
datasource
)
const builder = createAutomationBuilder({
name: "Test external query and save",
})
const results = await builder
.appAction({
fields: {},
})
.executeQuery({
query: {
queryId: query._id!,
},
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.response }}",
})
.createRow({
row: {
name: "{{ loop.currentItem.name }}",
age: "{{ loop.currentItem.age }}",
tableId: newTable._id!,
},
})
.queryRows({
tableId: newTable._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.iterations).toBe(3)
expect(results.steps[1].outputs.items).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(3)
rows.forEach(expectedRow => {
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining([expect.objectContaining(expectedRow)])
it("should query an external database for some data then insert than into an internal table", async () => {
const { datasource, client } = await setup.setupTestDatasource(
config,
DatabaseName.MYSQL
)
const newTable = await config.createTable({
name: "table",
type: "table",
schema: {
name: {
name: "name",
type: FieldType.STRING,
constraints: {
presence: true,
},
},
age: {
name: "age",
type: FieldType.NUMBER,
constraints: {
presence: true,
},
},
},
})
const tableName = await setup.createTestTable(client, {
name: { type: "string" },
age: { type: "number" },
})
const rows = [
{ name: "Joe", age: 20 },
{ name: "Bob", age: 25 },
{ name: "Paul", age: 30 },
]
await setup.insertTestData(client, tableName, rows)
const query = await setup.saveTestQuery(
config,
client,
tableName,
datasource
)
const builder = createAutomationBuilder({
name: "Test external query and save",
})
const results = await builder
.appAction({
fields: {},
})
.executeQuery({
query: {
queryId: query._id!,
},
})
.loop({
option: LoopStepType.ARRAY,
binding: "{{ steps.1.response }}",
})
.createRow({
row: {
name: "{{ loop.currentItem.name }}",
age: "{{ loop.currentItem.age }}",
tableId: newTable._id!,
},
})
.queryRows({
tableId: newTable._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.iterations).toBe(3)
expect(results.steps[1].outputs.items).toHaveLength(3)
expect(results.steps[2].outputs.rows).toHaveLength(3)
rows.forEach(expectedRow => {
expect(results.steps[2].outputs.rows).toEqual(
expect.arrayContaining([expect.objectContaining(expectedRow)])
)
})
})
})
describe("Name Based Automations", () => {
it("should fetch and delete a rpw using automation naming", async () => {
const table = await config.createTable()
const row = {
name: "DFN",
description: "original description",
tableId: table._id,
}
await config.createRow(row)
await config.createRow(row)
const builder = createAutomationBuilder({
name: "Test Query and Delete Row",
})
const results = await builder
.appAction({ fields: {} })
.queryRows(
{
tableId: table._id!,
},
{ stepName: "InitialQueryStep" }
)
.deleteRow({
tableId: table._id!,
id: "{{ steps.InitialQueryStep.rows.0._id }}",
})
.queryRows({
tableId: table._id!,
})
.run()
expect(results.steps).toHaveLength(3)
expect(results.steps[1].outputs.success).toBeTruthy()
expect(results.steps[2].outputs.rows).toHaveLength(1)
})
})
})

60
packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts
View file

@ -57,21 +57,27 @@ type BranchConfig = {
class BaseStepBuilder {
protected steps: AutomationStep[] = []
protected stepNames: { [key: string]: string } = {}
protected step<TStep extends AutomationActionStepId>(
stepId: TStep,
stepSchema: Omit<AutomationStep, "id" | "stepId" | "inputs">,
inputs: AutomationStepInputs<TStep>
inputs: AutomationStepInputs<TStep>,
stepName?: string
): this {
const id = uuidv4()
this.steps.push({
...stepSchema,
inputs: inputs as any,
id: uuidv4(),
id,
stepId,
name: stepName || stepSchema.name,
})
if (stepName) {
this.stepNames[id] = stepName
}
return this
}
protected addBranchStep(branchConfig: BranchConfig): void {
const branchStepInputs: BranchStepInputs = {
branches: [] as Branch[],
@ -99,66 +105,80 @@ class BaseStepBuilder {
}
// STEPS
createRow(inputs: CreateRowStepInputs): this {
createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this {
return this.step(
AutomationActionStepId.CREATE_ROW,
BUILTIN_ACTION_DEFINITIONS.CREATE_ROW,
inputs
inputs,
opts?.stepName
)
}
updateRow(inputs: UpdateRowStepInputs): this {
updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this {
return this.step(
AutomationActionStepId.UPDATE_ROW,
BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW,
inputs
inputs,
opts?.stepName
)
}
deleteRow(inputs: DeleteRowStepInputs): this {
deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this {
return this.step(
AutomationActionStepId.DELETE_ROW,
BUILTIN_ACTION_DEFINITIONS.DELETE_ROW,
inputs
inputs,
opts?.stepName
)
}
sendSmtpEmail(inputs: SmtpEmailStepInputs): this {
sendSmtpEmail(
inputs: SmtpEmailStepInputs,
opts?: { stepName?: string }
): this {
return this.step(
AutomationActionStepId.SEND_EMAIL_SMTP,
BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP,
inputs
inputs,
opts?.stepName
)
}
executeQuery(inputs: ExecuteQueryStepInputs): this {
executeQuery(
inputs: ExecuteQueryStepInputs,
opts?: { stepName?: string }
): this {
return this.step(
AutomationActionStepId.EXECUTE_QUERY,
BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY,
inputs
inputs,
opts?.stepName
)
}
queryRows(inputs: QueryRowsStepInputs): this {
queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this {
return this.step(
AutomationActionStepId.QUERY_ROWS,
BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS,
inputs
inputs,
opts?.stepName
)
}
loop(inputs: LoopStepInputs): this {
loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this {
return this.step(
AutomationActionStepId.LOOP,
BUILTIN_ACTION_DEFINITIONS.LOOP,
inputs
inputs,
opts?.stepName
)
}
serverLog(input: ServerLogStepInputs): this {
serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this {
return this.step(
AutomationActionStepId.SERVER_LOG,
BUILTIN_ACTION_DEFINITIONS.SERVER_LOG,
input
input,
opts?.stepName
)
}
}
@ -186,6 +206,7 @@ class AutomationBuilder extends BaseStepBuilder {
definition: {
steps: [],
trigger: {} as AutomationTrigger,
stepNames: {},
},
type: "automation",
appId: options.appId ?? setup.getConfig().getAppId(),
@ -268,6 +289,7 @@ class AutomationBuilder extends BaseStepBuilder {
build(): Automation {
this.automationConfig.definition.steps = this.steps
this.automationConfig.definition.stepNames = this.stepNames
return this.automationConfig
}

1
packages/server/src/definitions/automations.ts
View file

@ -15,6 +15,7 @@ export interface TriggerOutput {
export interface AutomationContext extends AutomationResults {
steps: any[]
stepsByName?: Record<string, any>
env?: Record<string, string>
trigger: any
}

4
packages/server/src/integrations/tests/sql.spec.ts
View file

@ -162,7 +162,7 @@ describe("SQL query builder", () => {
const query = sql._query(generateRelationshipJson({ schema: "production" }))
expect(query).toEqual({
bindings: [limit, relationshipLimit],
sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
})
})
@ -170,7 +170,7 @@ describe("SQL query builder", () => {
const query = sql._query(generateRelationshipJson())
expect(query).toEqual({
bindings: [limit, relationshipLimit],
sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`,
})
})

6
packages/server/src/integrations/tests/sqlAlias.spec.ts
View file

@ -63,7 +63,7 @@ describe("Captures of real examples", () => {
bindings: [primaryLimit, relationshipLimit, relationshipLimit],
sql: expect.stringContaining(
multiline(
`select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")`
`select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")`
)
),
})
@ -95,7 +95,7 @@ describe("Captures of real examples", () => {
sql: expect.stringContaining(
multiline(
`with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1)
select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid"))
select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname"))
from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks"
from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc`
)
@ -113,7 +113,7 @@ describe("Captures of real examples", () => {
bindings: [...filters, relationshipLimit, relationshipLimit],
sql: multiline(
`with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3)
select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid"))
select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname"))
from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid"
where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc`
),

31
packages/server/src/threads/automation.ts
View file

@ -89,7 +89,12 @@ class Orchestrator {
delete triggerOutput.appId
delete triggerOutput.metadata
// step zero is never used as the template string is zero indexed for customer facing
this.context = { steps: [{}], trigger: triggerOutput }
this.context = {
steps: [{}],
stepsByName: {},
trigger: triggerOutput,
}
this.automation = automation
// create an emitter which has the chain count for this automation run in it, so it can block
// excessive chaining if required
@ -451,6 +456,9 @@ class Orchestrator {
outputs: tempOutput,
inputs: steps[stepToLoopIndex].inputs,
})
const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id
this.context.stepsByName![stepName] = tempOutput
this.context.steps[this.context.steps.length] = tempOutput
this.context.steps = this.context.steps.filter(
item => !item.hasOwnProperty.call(item, "currentItem")
@ -555,8 +563,13 @@ class Orchestrator {
loopIteration
)
}
const stepFn = await this.getStepFunctionality(step.stepId)
let inputs = await processObject(originalStepInput, this.context)
let inputs = await this.addContextAndProcess(
originalStepInput,
this.context
)
inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
const outputs = await stepFn({
@ -583,6 +596,18 @@ class Orchestrator {
return null
}
private async addContextAndProcess(inputs: any, context: any) {
const processContext = {
...context,
steps: {
...context.steps,
...context.stepsByName,
},
}
return processObject(inputs, processContext)
}
private handleStepOutput(
step: AutomationStep,
outputs: any,
@ -600,6 +625,8 @@ class Orchestrator {
} else {
this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
this.context.steps[this.context.steps.length] = outputs
const stepName = step.name || step.id
this.context.stepsByName![stepName] = outputs
}
}
}

2
packages/types/src/documents/app/automation/automation.ts
View file

@ -124,6 +124,8 @@ export interface Automation extends Document {
definition: {
steps: AutomationStep[]
trigger: AutomationTrigger
// stepNames is used to lookup step names from their correspnding step ID.
stepNames?: Record<string, string>
}
screenId?: string
uiTree?: any