From abaa40a2723c2bcbf65d96aaaceb294b5d14e5e4 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Tue, 10 Sep 2024 17:09:42 +0100 Subject: [PATCH 01/30] automation steps using names --- .../SetupPanel/AutomationBlockSetup.svelte | 40 ++++++--- .../src/helpers/automations/nameHelpers.js | 85 +++++++++++++++++++ .../builder/src/stores/builder/automations.js | 43 ++++++++-- .../server/src/definitions/automations.ts | 1 + packages/server/src/threads/automation.ts | 31 ++++++- 5 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 packages/builder/src/helpers/automations/nameHelpers.js diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 6c4865b539..1bd65c82f8 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -49,6 +49,7 @@ import { getSchemaForDatasourcePlus, getEnvironmentBindings, + runtimeToReadableBinding, } from "dataBinding" import { TriggerStepID, ActionStepID } from "constants/backend/automations" import { onMount } from "svelte" @@ -595,9 +596,13 @@ let loopBlockCount = 0 const addBinding = (name, value, icon, idx, isLoopBlock, bindingName) => { if (!name) return - const runtimeBinding = determineRuntimeBinding(name, idx, isLoopBlock) + const runtimeBinding = determineRuntimeBinding( + name, + idx, + isLoopBlock, + bindingName + ) const categoryName = determineCategoryName(idx, isLoopBlock, bindingName) - bindings.push( createBindingObject( name, @@ -613,7 +618,7 @@ ) } - const determineRuntimeBinding = (name, idx, isLoopBlock) => { + const determineRuntimeBinding = (name, idx, isLoopBlock, bindingName) => { let runtimeName /* Begin special cases for generating custom schemas based on triggers */ @@ -634,12 +639,18 @@ } /* End special cases for generating custom schemas based on triggers */ + let hasUserDefinedName = + automation.stepNames?.[allSteps[idx - loopBlockCount]]?.id if (isLoopBlock) { runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { - runtimeName = `steps[${idx - loopBlockCount}].${name}` + runtimeName = hasUserDefinedName + ? `stepsByName.${bindingName}.${name}` + : `steps[${idx - loopBlockCount}].${name}` } else { - runtimeName = `steps.${idx - loopBlockCount}.${name}` + runtimeName = hasUserDefinedName + ? `stepsByName.${bindingName}.${name}` + : `steps.${idx - loopBlockCount}.${name}` } return idx === 0 ? `trigger.${name}` : runtimeName } @@ -666,11 +677,12 @@ const field = Object.values(FIELDS).find( field => field.type === value.type && field.subtype === value.subtype ) - + console.log(bindingName) return { - readableBinding: bindingName - ? `${bindingName}.${name}` - : runtimeBinding, + readableBinding: + bindingName && !isLoopBlock + ? `steps.${bindingName}.${name}` + : runtimeBinding, runtimeBinding, type: value.type, description: value.description, @@ -690,8 +702,15 @@ allSteps[idx]?.stepId === ActionStepID.LOOP && allSteps.some(x => x.blockToLoop === block.id) let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {} + if (wasLoopBlock) { + break + } let bindingName = - automation.stepNames?.[allSteps[idx - loopBlockCount].id] + automation.stepNames?.[allSteps[idx - loopBlockCount].id] || + !isLoopBlock + ? allSteps[idx]?.name + : allSteps[idx - 1]?.name + console.log(idx == 4 && bindingName) if (isLoopBlock) { schema = { @@ -746,7 +765,6 @@ addBinding(name, value, icon, idx, isLoopBlock, bindingName) ) } - return bindings } diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js new file mode 100644 index 0000000000..da5c999b2f --- /dev/null +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -0,0 +1,85 @@ +import { AutomationActionStepId } from "@budibase/types" + +export const updateBindingsInInputs = (inputs, oldName, newName) => { + if (typeof inputs === "string") { + return inputs.replace( + new RegExp(`stepsByName\\.${oldName}\\.`, "g"), + `stepsByName.${newName}.` + ) + } + + if (Array.isArray(inputs)) { + return inputs.map(item => updateBindingsInInputs(item, oldName, newName)) + } + + if (typeof inputs === "object" && inputs !== null) { + const updatedInputs = {} + for (const [key, value] of Object.entries(inputs)) { + updatedInputs[key] = updateBindingsInInputs(value, oldName, newName) + } + return updatedInputs + } + + return inputs +} + +export const updateBindingsInSteps = (steps, oldName, newName) => { + return steps.map(step => { + const updatedStep = { + ...step, + inputs: updateBindingsInInputs(step.inputs, oldName, newName), + } + + // Handle branch steps + if ("branches" in updatedStep.inputs) { + updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({ + ...branch, + condition: updateBindingsInInputs(branch.condition, oldName, newName), + })) + + if (updatedStep.inputs.children) { + for (const [key, childSteps] of Object.entries( + updatedStep.inputs.children + )) { + updatedStep.inputs.children[key] = updateBindingsInSteps( + childSteps, + oldName, + newName + ) + } + } + } + + return updatedStep + }) +} + +export const getNewStepName = (automation, step) => { + const baseName = step.name + + const countExistingSteps = steps => { + return steps.reduce((count, currentStep) => { + if (currentStep.name && currentStep.name.startsWith(baseName)) { + count++ + } + if ( + currentStep.stepId === AutomationActionStepId.BRANCH && + currentStep.inputs && + currentStep.inputs.children + ) { + Object.values(currentStep.inputs.children).forEach(branchSteps => { + count += countExistingSteps(branchSteps) + }) + } + return count + }, 0) + } + + const existingCount = countExistingSteps(automation.definition.steps) + + if (existingCount === 0) { + return baseName + } + + return `${baseName} ${existingCount + 1}` +} diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js index fdb0991911..eaf565c4f2 100644 --- a/packages/builder/src/stores/builder/automations.js +++ b/packages/builder/src/stores/builder/automations.js @@ -6,6 +6,10 @@ import { createHistoryStore } from "stores/builder/history" import { notifications } from "@budibase/bbui" import { updateReferencesInObject } from "dataBinding" import { AutomationTriggerStepId } from "@budibase/types" +import { + updateBindingsInSteps, + getNewStepName, +} from "helpers/automations/nameHelpers" const initialAutomationState = { automations: [], @@ -275,13 +279,17 @@ const automationActions = store => ({ await store.actions.save(newAutomation) }, constructBlock(type, stepId, blockDefinition) { - return { + let newName + const newStep = { ...blockDefinition, inputs: blockDefinition.inputs || {}, stepId, type, id: generate(), } + newName = getNewStepName(get(selectedAutomation), newStep) + newStep.name = newName + return newStep }, addBlockToAutomation: async (block, blockIdx) => { const automation = get(selectedAutomation) @@ -301,15 +309,36 @@ const automationActions = store => ({ saveAutomationName: async (blockId, name) => { const automation = get(selectedAutomation) let newAutomation = cloneDeep(automation) - if (!automation) { + if (!newAutomation) { return } - newAutomation.definition.stepNames = { - ...newAutomation.definition.stepNames, - [blockId]: name.trim(), - } - await store.actions.save(newAutomation) + const stepIndex = newAutomation.definition.steps.findIndex( + step => step.id === blockId + ) + + if (stepIndex !== -1) { + const oldName = newAutomation.definition.steps[stepIndex].name + const newName = name.trim() + + // Update stepNames + newAutomation.definition.stepNames = { + ...newAutomation.definition.stepNames, + [blockId]: newName, + } + + // Update the name in the step itself + newAutomation.definition.steps[stepIndex].name = newName + + // Update bindings in all steps + newAutomation.definition.steps = updateBindingsInSteps( + newAutomation.definition.steps, + oldName, + newName + ) + + await store.actions.save(newAutomation) + } }, deleteAutomationName: async blockId => { const automation = get(selectedAutomation) diff --git a/packages/server/src/definitions/automations.ts b/packages/server/src/definitions/automations.ts index e84eecea51..6488e604e9 100644 --- a/packages/server/src/definitions/automations.ts +++ b/packages/server/src/definitions/automations.ts @@ -15,6 +15,7 @@ export interface TriggerOutput { export interface AutomationContext extends AutomationResults { steps: any[] + stepsByName?: Record env?: Record trigger: any } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index f374ff159a..3e5ac32b0a 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -89,7 +89,12 @@ class Orchestrator { delete triggerOutput.appId delete triggerOutput.metadata // step zero is never used as the template string is zero indexed for customer facing - this.context = { steps: [{}], trigger: triggerOutput } + this.context = { + steps: [{}], + stepsByName: {}, + trigger: triggerOutput, + } + this.automation = automation // create an emitter which has the chain count for this automation run in it, so it can block // excessive chaining if required @@ -449,6 +454,11 @@ class Orchestrator { outputs: tempOutput, inputs: steps[stepToLoopIndex].inputs, }) + console.log(this.context) + + const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id + this.context.stepsByName![stepName] = tempOutput + console.log(this.context) this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") @@ -542,8 +552,11 @@ class Orchestrator { loopIteration ) } + console.log(this.context) + const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await processObject(originalStepInput, this.context) + let inputs = await this.testProcesss(originalStepInput, this.context) + inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) const outputs = await stepFn({ @@ -570,6 +583,18 @@ class Orchestrator { return null } + private async testProcesss(inputs: any, context: any) { + const processContext = { + ...context, + steps: { + ...context.steps, + ...context.stepsByName, + }, + } + + return processObject(inputs, processContext) + } + private handleStepOutput( step: AutomationStep, outputs: any, @@ -587,6 +612,8 @@ class Orchestrator { } else { this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs) this.context.steps[this.context.steps.length] = outputs + const stepName = step.name || step.id + this.context.stepsByName![stepName] = outputs } } } From ae1e2dd2f2e916550d7c04aa843691aab6fa1509 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 10 Sep 2024 17:41:33 +0100 Subject: [PATCH 02/30] Add column rename test. --- .../integrations/tests/googlesheets.spec.ts | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 9781b97972..3b8ea49685 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -10,6 +10,7 @@ import { TableSourceType, } from "@budibase/types" import { GoogleSheetsMock } from "./utils/googlesheets" +import exp from "constants" describe("Google Sheets Integration", () => { const config = new TestConfiguration() @@ -299,5 +300,42 @@ describe("Google Sheets Integration", () => { expect(mock.cell("A2")).toEqual("Test Contact Updated") expect(mock.cell("B2")).toEqual("original description updated") }) + + it("should be able to rename a column", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + const { name, ...otherColumns } = table.schema + const renamedTable = await config.api.table.save({ + ...table, + schema: { + ...otherColumns, + renamed: { + ...table.schema.name, + }, + }, + _rename: { + old: "name", + updated: "renamed", + }, + }) + + expect(renamedTable.schema.name).not.toBeDefined() + expect(renamedTable.schema.renamed).toBeDefined() + + expect(mock.cell("A1")).toEqual("renamed") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual("Test Contact") + expect(mock.cell("B2")).toEqual("original description") + expect(mock.cell("A3")).toEqual(null) + expect(mock.cell("B3")).toEqual(null) + + const renamedRow = await config.api.row.get(table._id!, row._id!) + expect(renamedRow.renamed).toEqual("Test Contact") + expect(renamedRow.description).toEqual("original description") + expect(renamedRow.name).not.toBeDefined() + }) }) }) From 69b8661d8a2e404b20be7f881ad05d6c4b7e4ac2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 10 Sep 2024 18:29:11 +0100 Subject: [PATCH 03/30] Working toward delete tests. --- .../integrations/tests/googlesheets.spec.ts | 116 +++++++++ .../integrations/tests/utils/googlesheets.ts | 231 ++++++++++++------ 2 files changed, 273 insertions(+), 74 deletions(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 3b8ea49685..3e7e0cddc4 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, + Row, SourceName, Table, TableSourceType, @@ -337,5 +338,120 @@ describe("Google Sheets Integration", () => { expect(renamedRow.description).toEqual("original description") expect(renamedRow.name).not.toBeDefined() }) + + // TODO: this gets the error "Sheet is not large enough to fit 27 columns. Resize the sheet first." + // eslint-disable-next-line jest/no-commented-out-tests + // it("should be able to add a new column", async () => { + // const updatedTable = await config.api.table.save({ + // ...table, + // schema: { + // ...table.schema, + // newColumn: { + // name: "newColumn", + // type: FieldType.STRING, + // }, + // }, + // }) + + // expect(updatedTable.schema.newColumn).toBeDefined() + + // expect(mock.cell("A1")).toEqual("name") + // expect(mock.cell("B1")).toEqual("description") + // expect(mock.cell("C1")).toEqual("newColumn") + // }) + + it("should be able to delete a column", async () => { + const row = await config.api.row.save(table._id!, { + name: "Test Contact", + description: "original description", + }) + + const updatedTable = await config.api.table.save({ + ...table, + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + + expect(updatedTable.schema.name).toBeDefined() + expect(updatedTable.schema.description).not.toBeDefined() + + // TODO: we don't delete data in deleted columns yet, should we? + // expect(mock.cell("A1")).toEqual("name") + // expect(mock.cell("B1")).toEqual(null) + + const updatedRow = await config.api.row.get(table._id!, row._id!) + expect(updatedRow.name).toEqual("Test Contact") + expect(updatedRow.description).not.toBeDefined() + }) + }) + + describe("delete", () => { + let table: Table + beforeEach(async () => { + table = await config.api.table.save({ + name: "Test Table", + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + description: { + name: "description", + type: FieldType.STRING, + constraints: { + type: "string", + }, + }, + }, + }) + + await config.api.row.bulkImport(table._id!, { + rows: [ + { + name: "Test Contact 1", + description: "original description 1", + }, + { + name: "Test Contact 2", + description: "original description 2", + }, + ], + }) + }) + + it.skip("can delete a table", async () => { + await config.api.table.destroy(table._id!, table._rev!) + expect(mock.cell("A1")).toEqual(null) + expect(mock.cell("B1")).toEqual(null) + }) + + it.skip("can delete a row", async () => { + const rows = await config.api.row.fetch(table._id!) + expect(rows.length).toEqual(2) + + for (const row of rows) { + await config.api.row.delete(table._id!, { _id: row._id! }) + } + + expect(mock.cell("A1")).toEqual("name") + expect(mock.cell("B1")).toEqual("description") + expect(mock.cell("A2")).toEqual(null) + expect(mock.cell("B2")).toEqual(null) + expect(mock.cell("A3")).toEqual(null) + expect(mock.cell("B3")).toEqual(null) + + const emptyRows = await config.api.row.fetch(table._id!) + expect(emptyRows.length).toEqual(0) + }) }) }) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index c58066bee5..90f3b3652c 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -21,6 +21,7 @@ import type { CellFormat, CellPadding, Color, + GridRange, } from "google-spreadsheet/src/lib/types/sheets-types" const BLACK: Color = { red: 0, green: 0, blue: 0 } @@ -88,11 +89,32 @@ interface UpdateValuesResponse { updatedData: ValueRange } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest +interface AddSheetRequest { + properties: WorksheetProperties +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response#AddSheetResponse interface AddSheetResponse { properties: WorksheetProperties } +interface DeleteRangeRequest { + range: GridRange + shiftDimension: WorksheetDimension +} + +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request +interface BatchUpdateRequest { + requests: { + addSheet?: AddSheetRequest + deleteRange?: DeleteRangeRequest + }[] + includeSpreadsheetInResponse: boolean + responseRanges: string[] + responseIncludeGridData: boolean +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/response interface BatchUpdateResponse { spreadsheetId: string @@ -102,23 +124,6 @@ interface BatchUpdateResponse { updatedSpreadsheet: Spreadsheet } -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#AddSheetRequest -interface AddSheetRequest { - properties: WorksheetProperties -} - -interface Request { - addSheet?: AddSheetRequest -} - -// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request -interface BatchUpdateRequest { - requests: Request[] - includeSpreadsheetInResponse: boolean - responseRanges: string[] - responseIncludeGridData: boolean -} - // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/sheets#RowData interface RowData { values: CellData[] @@ -369,13 +374,17 @@ export class GoogleSheetsMock { private handleValueAppend(request: AppendRequest): AppendResponse { const { range, params, body } = request - const { sheet, bottomRight } = this.parseA1Notation(range) + const { sheetId, endRowIndex } = this.parseA1Notation(range) + const sheet = this.getSheetById(sheetId) + if (!sheet) { + throw new Error(`Sheet ${sheetId} not found`) + } const newRows = body.values.map(v => this.valuesToRowData(v)) const toDelete = params.insertDataOption === "INSERT_ROWS" ? newRows.length : 0 - sheet.data[0].rowData.splice(bottomRight.row + 1, toDelete, ...newRows) - sheet.data[0].rowMetadata.splice(bottomRight.row + 1, toDelete, { + sheet.data[0].rowData.splice(endRowIndex + 1, toDelete, ...newRows) + sheet.data[0].rowMetadata.splice(endRowIndex + 1, toDelete, { hiddenByUser: false, hiddenByFilter: false, pixelSize: 100, @@ -384,17 +393,13 @@ export class GoogleSheetsMock { // It's important to give back a correct updated range because the API // library we use makes use of it to assign the correct row IDs to rows. - const updatedRange = this.createA1FromRanges( - sheet, - { - row: bottomRight.row + 1, - column: 0, - }, - { - row: bottomRight.row + newRows.length, - column: 0, - } - ) + const updatedRange = this.createA1({ + sheetId, + startRowIndex: endRowIndex + 1, + startColumnIndex: 0, + endRowIndex: endRowIndex + newRows.length, + endColumnIndex: 0, + }) return { spreadsheetId: this.spreadsheet.spreadsheetId, @@ -438,6 +443,9 @@ export class GoogleSheetsMock { addSheet: this.handleAddSheet(request.addSheet), }) } + if (request.deleteRange) { + this.handleDeleteRange(request.deleteRange) + } } return response @@ -474,12 +482,24 @@ export class GoogleSheetsMock { return { properties: properties as WorksheetProperties } } + private handleDeleteRange(request: DeleteRangeRequest) { + const { range, shiftDimension } = request + + if (shiftDimension !== "ROWS") { + throw new Error("Only row-based deletes are supported") + } + + this.iterateRange(range, (cell, value) => { + cell.userEnteredValue = this.createValue(null) + }) + } + private handleGetSpreadsheet(): Spreadsheet { return this.spreadsheet } private handleValueUpdate(valueRange: ValueRange): UpdateValuesResponse { - this.iterateCells(valueRange, (cell, value) => { + this.iterateValueRange(valueRange, (cell, value) => { cell.userEnteredValue = this.createValue(value) }) @@ -494,7 +514,30 @@ export class GoogleSheetsMock { return response } - private iterateCells( + private iterateRange( + range: Required, + cb: (cell: CellData) => void + ) { + const { + sheetId, + startRowIndex, + endRowIndex, + startColumnIndex, + endColumnIndex, + } = range + + for (let row = startRowIndex; row <= endRowIndex; row++) { + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) + if (!cell) { + throw new Error("Cell not found") + } + cb(cell) + } + } + } + + private iterateValueRange( valueRange: ValueRange, cb: (cell: CellData, value: Value) => void ) { @@ -502,33 +545,46 @@ export class GoogleSheetsMock { throw new Error("Only row-major updates are supported") } - const { sheet, topLeft, bottomRight } = this.parseA1Notation( - valueRange.range - ) - for (let row = topLeft.row; row <= bottomRight.row; row++) { - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) + const { + sheetId, + startColumnIndex, + startRowIndex, + endColumnIndex, + endRowIndex, + } = this.parseA1Notation(valueRange.range) + + for (let row = startRowIndex; row <= endRowIndex; row++) { + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) if (!cell) { throw new Error("Cell not found") } - const value = valueRange.values[row - topLeft.row][col - topLeft.column] + const value = + valueRange.values[row - startRowIndex][col - startColumnIndex] cb(cell, value) } } } private getValueRange(range: string): ValueRange { - const { sheet, topLeft, bottomRight } = this.parseA1Notation(range) + const { + sheetId, + startRowIndex, + endRowIndex, + startColumnIndex, + endColumnIndex, + } = this.parseA1Notation(range) + const valueRange: ValueRange = { range, majorDimension: "ROWS", values: [], } - for (let row = topLeft.row; row <= bottomRight.row; row++) { + for (let row = startRowIndex; row <= endRowIndex; row++) { const values: Value[] = [] - for (let col = topLeft.column; col <= bottomRight.column; col++) { - const cell = this.getCellNumericIndexes(sheet, row, col) + for (let col = startColumnIndex; col <= endColumnIndex; col++) { + const cell = this.getCellNumericIndexes(sheetId, row, col) if (!cell) { throw new Error("Cell not found") } @@ -693,11 +749,9 @@ export class GoogleSheetsMock { } private cellData(cell: string): CellData | undefined { - const { - sheet, - topLeft: { row, column }, - } = this.parseA1Notation(cell) - return this.getCellNumericIndexes(sheet, row, column) + const { sheetId, startColumnIndex, startRowIndex } = + this.parseA1Notation(cell) + return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex) } cell(cell: string): Value | undefined { @@ -709,10 +763,18 @@ export class GoogleSheetsMock { } private getCellNumericIndexes( - sheet: Sheet, + sheet: Sheet | number, row: number, column: number ): CellData | undefined { + if (typeof sheet === "number") { + const foundSheet = this.getSheetById(sheet) + if (!foundSheet) { + return undefined + } + sheet = foundSheet + } + const data = sheet.data[0] const rowData = data.rowData[row] if (!rowData) { @@ -751,11 +813,7 @@ export class GoogleSheetsMock { // "Sheet1!A:B" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 99, column: 1 } } // "Sheet1!1:1" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 0, column: 25 } } // "Sheet1!1:2" -> { topLeft: { row: 0, column: 0 }, bottomRight: { row: 1, column: 25 } } - private parseA1Notation(range: string): { - sheet: Sheet - topLeft: Range - bottomRight: Range - } { + private parseA1Notation(range: string): Required { let sheet: Sheet let rest: string if (!range.includes("!")) { @@ -793,35 +851,54 @@ export class GoogleSheetsMock { parsedBottomRight = parsedTopLeft } - if (parsedTopLeft && parsedTopLeft.row === undefined) { - parsedTopLeft.row = 0 - } - if (parsedTopLeft && parsedTopLeft.column === undefined) { - parsedTopLeft.column = 0 - } - if (parsedBottomRight && parsedBottomRight.row === undefined) { - parsedBottomRight.row = sheet.properties.gridProperties.rowCount - 1 - } - if (parsedBottomRight && parsedBottomRight.column === undefined) { - parsedBottomRight.column = sheet.properties.gridProperties.columnCount - 1 + return this.ensureGridRange({ + sheetId: sheet.properties.sheetId, + startRowIndex: parsedTopLeft.row, + endRowIndex: parsedBottomRight.row, + startColumnIndex: parsedTopLeft.column, + endColumnIndex: parsedBottomRight.column, + }) + } + + private ensureGridRange(range: GridRange): Required { + const sheet = this.getSheetById(range.sheetId) + if (!sheet) { + throw new Error(`Sheet ${range.sheetId} not found`) } return { - sheet, - topLeft: parsedTopLeft as Range, - bottomRight: parsedBottomRight as Range, + sheetId: range.sheetId, + startRowIndex: range.startRowIndex ?? 0, + endRowIndex: + range.endRowIndex ?? sheet.properties.gridProperties.rowCount - 1, + startColumnIndex: range.startColumnIndex ?? 0, + endColumnIndex: + range.endColumnIndex ?? sheet.properties.gridProperties.columnCount - 1, } } - private createA1FromRanges(sheet: Sheet, topLeft: Range, bottomRight: Range) { + private createA1(range: Required) { + const { + sheetId, + startColumnIndex, + startRowIndex, + endColumnIndex, + endRowIndex, + } = range + + const sheet = this.getSheetById(sheetId) + if (!sheet) { + throw new Error(`Sheet ${range.sheetId} not found`) + } + let title = sheet.properties.title if (title.includes(" ")) { title = `'${title}'` } - const topLeftLetter = this.numberToLetter(topLeft.column) - const bottomRightLetter = this.numberToLetter(bottomRight.column) - const topLeftRow = topLeft.row + 1 - const bottomRightRow = bottomRight.row + 1 + const topLeftLetter = this.numberToLetter(startColumnIndex) + const bottomRightLetter = this.numberToLetter(endColumnIndex) + const topLeftRow = startRowIndex + 1 + const bottomRightRow = endRowIndex + 1 return `${title}!${topLeftLetter}${topLeftRow}:${bottomRightLetter}${bottomRightRow}` } @@ -860,4 +937,10 @@ export class GoogleSheetsMock { sheet => sheet.properties.title === name ) } + + private getSheetById(id: number): Sheet | undefined { + return this.spreadsheet.sheets.find( + sheet => sheet.properties.sheetId === id + ) + } } From bf98d61ea6fbab5f9f8a51bf1cc4b930abd79259 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:35:46 +0100 Subject: [PATCH 04/30] add tests for new binding update code --- .../SetupPanel/AutomationBlockSetup.svelte | 3 - .../src/helpers/automations/nameHelpers.js | 59 ++++-- .../src/helpers/tests/nameHelpers.spec.js | 177 ++++++++++++++++++ .../builder/src/stores/builder/automations.js | 6 +- 4 files changed, 224 insertions(+), 21 deletions(-) create mode 100644 packages/builder/src/helpers/tests/nameHelpers.spec.js diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 1bd65c82f8..1e2e409243 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -49,7 +49,6 @@ import { getSchemaForDatasourcePlus, getEnvironmentBindings, - runtimeToReadableBinding, } from "dataBinding" import { TriggerStepID, ActionStepID } from "constants/backend/automations" import { onMount } from "svelte" @@ -677,7 +676,6 @@ const field = Object.values(FIELDS).find( field => field.type === value.type && field.subtype === value.subtype ) - console.log(bindingName) return { readableBinding: bindingName && !isLoopBlock @@ -710,7 +708,6 @@ !isLoopBlock ? allSteps[idx]?.name : allSteps[idx - 1]?.name - console.log(idx == 4 && bindingName) if (isLoopBlock) { schema = { diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js index da5c999b2f..b863796a54 100644 --- a/packages/builder/src/helpers/automations/nameHelpers.js +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -1,40 +1,71 @@ import { AutomationActionStepId } from "@budibase/types" -export const updateBindingsInInputs = (inputs, oldName, newName) => { +export const updateBindingsInInputs = (inputs, oldName, newName, stepIndex) => { if (typeof inputs === "string") { - return inputs.replace( - new RegExp(`stepsByName\\.${oldName}\\.`, "g"), - `stepsByName.${newName}.` - ) + return inputs + .replace( + new RegExp(`stepsByName\\.${oldName}\\.`, "g"), + `stepsByName.${newName}.` + ) + .replace( + new RegExp(`steps\\.${stepIndex}\\.`, "g"), + `stepsByName.${newName}.` + ) } if (Array.isArray(inputs)) { - return inputs.map(item => updateBindingsInInputs(item, oldName, newName)) + return inputs.map(item => + updateBindingsInInputs(item, oldName, newName, stepIndex) + ) } if (typeof inputs === "object" && inputs !== null) { const updatedInputs = {} for (const [key, value] of Object.entries(inputs)) { - updatedInputs[key] = updateBindingsInInputs(value, oldName, newName) + const updatedKey = updateBindingsInInputs( + key, + oldName, + newName, + stepIndex + ) + updatedInputs[updatedKey] = updateBindingsInInputs( + value, + oldName, + newName, + stepIndex + ) } return updatedInputs } - return inputs } -export const updateBindingsInSteps = (steps, oldName, newName) => { +export const updateBindingsInSteps = ( + steps, + oldName, + newName, + changedStepIndex +) => { return steps.map(step => { const updatedStep = { ...step, - inputs: updateBindingsInInputs(step.inputs, oldName, newName), + inputs: updateBindingsInInputs( + step.inputs, + oldName, + newName, + changedStepIndex + ), } - // Handle branch steps if ("branches" in updatedStep.inputs) { updatedStep.inputs.branches = updatedStep.inputs.branches.map(branch => ({ ...branch, - condition: updateBindingsInInputs(branch.condition, oldName, newName), + condition: updateBindingsInInputs( + branch.condition, + oldName, + newName, + changedStepIndex + ), })) if (updatedStep.inputs.children) { @@ -44,7 +75,8 @@ export const updateBindingsInSteps = (steps, oldName, newName) => { updatedStep.inputs.children[key] = updateBindingsInSteps( childSteps, oldName, - newName + newName, + changedStepIndex ) } } @@ -53,7 +85,6 @@ export const updateBindingsInSteps = (steps, oldName, newName) => { return updatedStep }) } - export const getNewStepName = (automation, step) => { const baseName = step.name diff --git a/packages/builder/src/helpers/tests/nameHelpers.spec.js b/packages/builder/src/helpers/tests/nameHelpers.spec.js new file mode 100644 index 0000000000..1ce2d1987a --- /dev/null +++ b/packages/builder/src/helpers/tests/nameHelpers.spec.js @@ -0,0 +1,177 @@ +import { cloneDeep } from "lodash" +import { + updateBindingsInInputs, + updateBindingsInSteps, +} from "../automations/nameHelpers" +describe("Automation Binding Update Functions", () => { + const sampleAutomation = { + definition: { + steps: [ + { + name: "First Step", + inputs: { + text: "Starting automation", + }, + id: "step1", + }, + { + name: "Second Step", + inputs: { + text: "{{ steps.0.success }} and {{ stepsByName.First Step.message }}", + }, + id: "step2", + }, + { + name: "Branch", + inputs: { + branches: [ + { + name: "branch1", + condition: { + equal: { + "steps.1.success": true, + }, + }, + }, + ], + children: { + branch1: [ + { + name: "Nested Step", + inputs: { + text: "{{ stepsByName.Second Step.message }} and {{ steps.1.success }}", + }, + id: "nestedStep", + }, + ], + }, + }, + id: "branchStep", + }, + ], + stepNames: { + step1: "First Step", + step2: "Second Step", + branchStep: "Branch", + }, + }, + } + + it("updateBindingsInInputs updates string bindings correctly", () => { + const input = "{{ stepsByName.oldName.success }} and {{ steps.1.message }}" + const result = updateBindingsInInputs(input, "oldName", "newName", 1) + expect(result).toBe( + "{{ stepsByName.newName.success }} and {{ stepsByName.newName.message }}" + ) + }) + + it("updateBindingsInInputs handles nested objects", () => { + const input = { + text: "{{ stepsByName.oldName.success }}", + nested: { + value: "{{ steps.1.message }}", + }, + } + const result = updateBindingsInInputs(input, "oldName", "newName", 1) + expect(result).toEqual({ + text: "{{ stepsByName.newName.success }}", + nested: { + value: "{{ stepsByName.newName.message }}", + }, + }) + }) + + it("updateBindingsInSteps updates bindings in all steps", () => { + const steps = cloneDeep(sampleAutomation.definition.steps) + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect(result[1].name).toBe("Second Step") + + expect(result[2].inputs.branches[0].condition.equal).toEqual({ + "stepsByName.Renamed Step.success": true, + }) + + const nestedStepText = result[2].inputs.children.branch1[0].inputs.text + expect(nestedStepText).toBe( + "{{ stepsByName.Renamed Step.message }} and {{ stepsByName.Renamed Step.success }}" + ) + }) + + it("updateBindingsInSteps handles steps with no bindings", () => { + const steps = [ + { + name: "No Binding Step", + inputs: { + text: "Plain text", + }, + id: "noBindingStep", + }, + ] + const result = updateBindingsInSteps(steps, "Old Name", "New Name", 0) + expect(result).toEqual(steps) + }) + + it("updateBindingsInSteps updates bindings in deeply nested branches", () => { + const deeplyNestedStep = { + name: "Deep Branch", + inputs: { + branches: [ + { + name: "deepBranch", + condition: { + equal: { + "stepsByName.Second Step.success": true, + }, + }, + }, + ], + children: { + deepBranch: [ + { + name: "Deep Log", + inputs: { + text: "{{ steps.1.message }}", + }, + }, + ], + }, + }, + } + + const steps = [...sampleAutomation.definition.steps, deeplyNestedStep] + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect( + result[3].inputs.branches[0].condition.equal[ + "stepsByName.Renamed Step.success" + ] + ).toBe(true) + expect(result[3].inputs.children.deepBranch[0].inputs.text).toBe( + "{{ stepsByName.Renamed Step.message }}" + ) + }) + + it("updateBindingsInSteps does not affect unrelated bindings", () => { + const steps = cloneDeep(sampleAutomation.definition.steps) + const result = updateBindingsInSteps( + steps, + "Second Step", + "Renamed Step", + 1 + ) + + expect(result[1].inputs.text).toBe( + "{{ steps.0.success }} and {{ stepsByName.First Step.message }}" + ) + }) +}) diff --git a/packages/builder/src/stores/builder/automations.js b/packages/builder/src/stores/builder/automations.js index eaf565c4f2..6627c67080 100644 --- a/packages/builder/src/stores/builder/automations.js +++ b/packages/builder/src/stores/builder/automations.js @@ -321,20 +321,18 @@ const automationActions = store => ({ const oldName = newAutomation.definition.steps[stepIndex].name const newName = name.trim() - // Update stepNames newAutomation.definition.stepNames = { ...newAutomation.definition.stepNames, [blockId]: newName, } - // Update the name in the step itself newAutomation.definition.steps[stepIndex].name = newName - // Update bindings in all steps newAutomation.definition.steps = updateBindingsInSteps( newAutomation.definition.steps, oldName, - newName + newName, + stepIndex ) await store.actions.save(newAutomation) From 2d4ac7fced1121dd49768426038372bfbed55e04 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:40:06 +0100 Subject: [PATCH 05/30] remove logs --- packages/server/src/threads/automation.ts | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 3e5ac32b0a..24833d0db9 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -454,11 +454,9 @@ class Orchestrator { outputs: tempOutput, inputs: steps[stepToLoopIndex].inputs, }) - console.log(this.context) const stepName = steps[stepToLoopIndex].name || steps[stepToLoopIndex].id this.context.stepsByName![stepName] = tempOutput - console.log(this.context) this.context.steps[this.context.steps.length] = tempOutput this.context.steps = this.context.steps.filter( item => !item.hasOwnProperty.call(item, "currentItem") @@ -552,7 +550,6 @@ class Orchestrator { loopIteration ) } - console.log(this.context) const stepFn = await this.getStepFunctionality(step.stepId) let inputs = await this.testProcesss(originalStepInput, this.context) From 0c11924027595cdd12d2534f00cae0d1ab8d2f7e Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:40:52 +0100 Subject: [PATCH 06/30] refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index 048c37ecd9..c24374879d 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 048c37ecd921340614bf61a76a774aaa46176569 +Subproject commit c24374879d2b61516fabc24d7404e7da235be05e diff --git a/packages/pro b/packages/pro index ec1d2bda75..a4d1d15d9c 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c +Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758 From 27b65e22efac6e78692a7c7990473bf59dc71dca Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:54:42 +0100 Subject: [PATCH 07/30] fix error with updating existinh name count --- packages/builder/src/helpers/automations/nameHelpers.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/builder/src/helpers/automations/nameHelpers.js b/packages/builder/src/helpers/automations/nameHelpers.js index b863796a54..ad18213ab0 100644 --- a/packages/builder/src/helpers/automations/nameHelpers.js +++ b/packages/builder/src/helpers/automations/nameHelpers.js @@ -105,9 +105,10 @@ export const getNewStepName = (automation, step) => { return count }, 0) } - - const existingCount = countExistingSteps(automation.definition.steps) - + let existingCount = 0 + if (automation?.definition) { + existingCount = countExistingSteps(automation.definition.steps) + } if (existingCount === 0) { return baseName } From e4918aea60752f69c032db921379d85e11b50063 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 11:55:31 +0100 Subject: [PATCH 08/30] refs --- packages/account-portal | 2 +- packages/pro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/account-portal b/packages/account-portal index c24374879d..048c37ecd9 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit c24374879d2b61516fabc24d7404e7da235be05e +Subproject commit 048c37ecd921340614bf61a76a774aaa46176569 diff --git a/packages/pro b/packages/pro index a4d1d15d9c..ec1d2bda75 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit a4d1d15d9ce6ac3deedb2e42625c90ba32756758 +Subproject commit ec1d2bda756f02c6b4efdee086e4c59b0c2a1b0c From 60dd500ecbf569dd70924332e39eb2738f360763 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 15:12:50 +0100 Subject: [PATCH 09/30] rename func --- packages/server/src/threads/automation.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 24833d0db9..288524b099 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -552,7 +552,10 @@ class Orchestrator { } const stepFn = await this.getStepFunctionality(step.stepId) - let inputs = await this.testProcesss(originalStepInput, this.context) + let inputs = await this.addContextAndProcess( + originalStepInput, + this.context + ) inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs) @@ -580,7 +583,7 @@ class Orchestrator { return null } - private async testProcesss(inputs: any, context: any) { + private async addContextAndProcess(inputs: any, context: any) { const processContext = { ...context, steps: { From d4d4f8a1501bf02fa45cc70f388642355328ce65 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Wed, 11 Sep 2024 15:24:49 +0100 Subject: [PATCH 10/30] ref --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 7899d07904..cd64894684 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 7899d07904d89d48954dd500da7b5dec32b781dd +Subproject commit cd648946847aade0ac1f53a1498596670c8a0aee From a831a4bf077b9bc95f824058abebbf8e3d9fdaa9 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 12 Sep 2024 14:58:53 +0100 Subject: [PATCH 11/30] tests to cover automation naming --- .../tests/scenarios/looping.spec.ts | 24 ++ .../tests/scenarios/scenarios.spec.ts | 309 ++++++++++-------- .../tests/utilities/AutomationTestBuilder.ts | 54 +-- .../documents/app/automation/automation.ts | 1 + 4 files changed, 233 insertions(+), 155 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index 9bc382a187..f229610542 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -242,4 +242,28 @@ describe("Loop automations", () => { expect(results.steps[1].outputs.message).toContain("- 3") expect(results.steps[3].outputs.message).toContain("- 3") }) + + it("should use automation names to loop with", async () => { + const builder = createAutomationBuilder({ + name: "Test Trigger with Loop and Create Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .loop( + { + option: LoopStepType.ARRAY, + binding: [1, 2, 3], + }, + "FirstLoopStep" + ) + .serverLog({ text: "Message {{loop.currentItem}}" }, "FirstLoopLog") + .serverLog( + { text: "{{steps.FirstLoopLog.iterations}}" }, + "FirstLoopIterationLog" + ) + .run() + + expect(results.steps[1].outputs.message).toContain("- 3") + }) }) diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 40d6094525..398ef20100 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -49,151 +49,188 @@ describe("Automation Scenarios", () => { }, }) }) - }) - it("should trigger an automation which querys the database", async () => { - const table = await config.createTable() - const row = { - name: "Test Row", - description: "original description", - tableId: table._id, - } - await config.createRow(row) - await config.createRow(row) - const builder = createAutomationBuilder({ - name: "Test Row Save and Create", + it("should trigger an automation which querys the database", async () => { + const table = await config.createTable() + const row = { + name: "Test Row", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(1) + expect(results.steps[0].outputs.rows).toHaveLength(2) }) - const results = await builder - .appAction({ fields: {} }) - .queryRows({ - tableId: table._id!, + it("should trigger an automation which querys the database then deletes a row", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Row Save and Create", }) - .run() - expect(results.steps).toHaveLength(1) - expect(results.steps[0].outputs.rows).toHaveLength(2) - }) + const results = await builder + .appAction({ fields: {} }) + .queryRows({ + tableId: table._id!, + }) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.1.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() - it("should trigger an automation which querys the database then deletes a row", async () => { - const table = await config.createTable() - const row = { - name: "DFN", - description: "original description", - tableId: table._id, - } - await config.createRow(row) - await config.createRow(row) - const builder = createAutomationBuilder({ - name: "Test Row Save and Create", + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) }) - const results = await builder - .appAction({ fields: {} }) - .queryRows({ - tableId: table._id!, - }) - .deleteRow({ - tableId: table._id!, - id: "{{ steps.1.rows.0._id }}", - }) - .queryRows({ - tableId: table._id!, - }) - .run() - - expect(results.steps).toHaveLength(3) - expect(results.steps[1].outputs.success).toBeTruthy() - expect(results.steps[2].outputs.rows).toHaveLength(1) - }) - - it("should query an external database for some data then insert than into an internal table", async () => { - const { datasource, client } = await setup.setupTestDatasource( - config, - DatabaseName.MYSQL - ) - - const newTable = await config.createTable({ - name: "table", - type: "table", - schema: { - name: { - name: "name", - type: FieldType.STRING, - constraints: { - presence: true, - }, - }, - age: { - name: "age", - type: FieldType.NUMBER, - constraints: { - presence: true, - }, - }, - }, - }) - - const tableName = await setup.createTestTable(client, { - name: { type: "string" }, - age: { type: "number" }, - }) - - const rows = [ - { name: "Joe", age: 20 }, - { name: "Bob", age: 25 }, - { name: "Paul", age: 30 }, - ] - - await setup.insertTestData(client, tableName, rows) - - const query = await setup.saveTestQuery( - config, - client, - tableName, - datasource - ) - - const builder = createAutomationBuilder({ - name: "Test external query and save", - }) - - const results = await builder - .appAction({ - fields: {}, - }) - .executeQuery({ - query: { - queryId: query._id!, - }, - }) - .loop({ - option: LoopStepType.ARRAY, - binding: "{{ steps.1.response }}", - }) - .createRow({ - row: { - name: "{{ loop.currentItem.name }}", - age: "{{ loop.currentItem.age }}", - tableId: newTable._id!, - }, - }) - .queryRows({ - tableId: newTable._id!, - }) - .run() - - expect(results.steps).toHaveLength(3) - - expect(results.steps[1].outputs.iterations).toBe(3) - expect(results.steps[1].outputs.items).toHaveLength(3) - - expect(results.steps[2].outputs.rows).toHaveLength(3) - - rows.forEach(expectedRow => { - expect(results.steps[2].outputs.rows).toEqual( - expect.arrayContaining([expect.objectContaining(expectedRow)]) + it("should query an external database for some data then insert than into an internal table", async () => { + const { datasource, client } = await setup.setupTestDatasource( + config, + DatabaseName.MYSQL ) + + const newTable = await config.createTable({ + name: "table", + type: "table", + schema: { + name: { + name: "name", + type: FieldType.STRING, + constraints: { + presence: true, + }, + }, + age: { + name: "age", + type: FieldType.NUMBER, + constraints: { + presence: true, + }, + }, + }, + }) + + const tableName = await setup.createTestTable(client, { + name: { type: "string" }, + age: { type: "number" }, + }) + + const rows = [ + { name: "Joe", age: 20 }, + { name: "Bob", age: 25 }, + { name: "Paul", age: 30 }, + ] + + await setup.insertTestData(client, tableName, rows) + + const query = await setup.saveTestQuery( + config, + client, + tableName, + datasource + ) + + const builder = createAutomationBuilder({ + name: "Test external query and save", + }) + + const results = await builder + .appAction({ + fields: {}, + }) + .executeQuery({ + query: { + queryId: query._id!, + }, + }) + .loop({ + option: LoopStepType.ARRAY, + binding: "{{ steps.1.response }}", + }) + .createRow({ + row: { + name: "{{ loop.currentItem.name }}", + age: "{{ loop.currentItem.age }}", + tableId: newTable._id!, + }, + }) + .queryRows({ + tableId: newTable._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + + expect(results.steps[1].outputs.iterations).toBe(3) + expect(results.steps[1].outputs.items).toHaveLength(3) + + expect(results.steps[2].outputs.rows).toHaveLength(3) + + rows.forEach(expectedRow => { + expect(results.steps[2].outputs.rows).toEqual( + expect.arrayContaining([expect.objectContaining(expectedRow)]) + ) + }) + }) + }) + + describe("Name Based Automations", () => { + it("should fetch and delete a rpw using automation naming", async () => { + const table = await config.createTable() + const row = { + name: "DFN", + description: "original description", + tableId: table._id, + } + await config.createRow(row) + await config.createRow(row) + const builder = createAutomationBuilder({ + name: "Test Query and Delete Row", + }) + + const results = await builder + .appAction({ fields: {} }) + .queryRows( + { + tableId: table._id!, + }, + "InitialQueryStep" + ) + .deleteRow({ + tableId: table._id!, + id: "{{ steps.InitialQueryStep.rows.0._id }}", + }) + .queryRows({ + tableId: table._id!, + }) + .run() + + expect(results.steps).toHaveLength(3) + expect(results.steps[1].outputs.success).toBeTruthy() + expect(results.steps[2].outputs.rows).toHaveLength(1) }) }) }) diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index f477efabe4..34c25d7004 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -57,21 +57,27 @@ type BranchConfig = { class BaseStepBuilder { protected steps: AutomationStep[] = [] + protected stepNames: { [key: string]: string } = {} protected step( stepId: TStep, stepSchema: Omit, - inputs: AutomationStepInputs + inputs: AutomationStepInputs, + stepName?: string ): this { + const id = uuidv4() this.steps.push({ ...stepSchema, inputs: inputs as any, - id: uuidv4(), + id, stepId, + name: stepName || stepSchema.name, }) + if (stepName) { + this.stepNames[id] = stepName + } return this } - protected addBranchStep(branchConfig: BranchConfig): void { const branchStepInputs: BranchStepInputs = { branches: [] as Branch[], @@ -99,66 +105,74 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs): this { + createRow(inputs: CreateRowStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, - inputs + inputs, + stepName ) } - updateRow(inputs: UpdateRowStepInputs): this { + updateRow(inputs: UpdateRowStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, - inputs + inputs, + stepName ) } - deleteRow(inputs: DeleteRowStepInputs): this { + deleteRow(inputs: DeleteRowStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, - inputs + inputs, + stepName ) } - sendSmtpEmail(inputs: SmtpEmailStepInputs): this { + sendSmtpEmail(inputs: SmtpEmailStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, - inputs + inputs, + stepName ) } - executeQuery(inputs: ExecuteQueryStepInputs): this { + executeQuery(inputs: ExecuteQueryStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, - inputs + inputs, + stepName ) } - queryRows(inputs: QueryRowsStepInputs): this { + queryRows(inputs: QueryRowsStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, - inputs + inputs, + stepName ) } - loop(inputs: LoopStepInputs): this { + loop(inputs: LoopStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, - inputs + inputs, + stepName ) } - serverLog(input: ServerLogStepInputs): this { + serverLog(input: ServerLogStepInputs, stepName?: string): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, - input + input, + stepName ) } } @@ -186,6 +200,7 @@ class AutomationBuilder extends BaseStepBuilder { definition: { steps: [], trigger: {} as AutomationTrigger, + stepNames: {}, }, type: "automation", appId: options.appId ?? setup.getConfig().getAppId(), @@ -268,6 +283,7 @@ class AutomationBuilder extends BaseStepBuilder { build(): Automation { this.automationConfig.definition.steps = this.steps + this.automationConfig.definition.stepNames = this.stepNames return this.automationConfig } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 72f8a1aa7c..78d22c787f 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -124,6 +124,7 @@ export interface Automation extends Document { definition: { steps: AutomationStep[] trigger: AutomationTrigger + stepNames?: Record } screenId?: string uiTree?: any From 7d6cce20e972b3fa9ef1edb988c0afba53f3119b Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Thu, 12 Sep 2024 21:02:50 +0100 Subject: [PATCH 12/30] fix getAvailableBindings again, aka my personal hell --- .../SetupPanel/AutomationBlockSetup.svelte | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 1e2e409243..be7286a79e 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -700,14 +700,12 @@ allSteps[idx]?.stepId === ActionStepID.LOOP && allSteps.some(x => x.blockToLoop === block.id) let schema = cloneDeep(allSteps[idx]?.schema?.outputs?.properties) ?? {} - if (wasLoopBlock) { - break + if (allSteps[idx]?.name.includes("Looping")) { + isLoopBlock = true + loopBlockCount++ } let bindingName = - automation.stepNames?.[allSteps[idx - loopBlockCount].id] || - !isLoopBlock - ? allSteps[idx]?.name - : allSteps[idx - 1]?.name + automation.stepNames?.[allSteps[idx].id] || allSteps[idx].name if (isLoopBlock) { schema = { @@ -756,7 +754,7 @@ if (wasLoopBlock) { loopBlockCount++ - continue + schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } Object.entries(schema).forEach(([name, value]) => addBinding(name, value, icon, idx, isLoopBlock, bindingName) From aaa7c9162f6063bac88dec0d93e8556015c08ea6 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 11:59:18 +0100 Subject: [PATCH 13/30] account Portal --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index cd64894684..723eff3807 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit cd648946847aade0ac1f53a1498596670c8a0aee +Subproject commit 723eff3807fa24fa30de69f9f556910e190a29d1 From 7e767e408955687f8841acb6fb68e9b34e44efd7 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 12:25:39 +0100 Subject: [PATCH 14/30] some pr comments --- .../tests/scenarios/looping.spec.ts | 9 +++-- .../tests/scenarios/scenarios.spec.ts | 2 +- .../tests/utilities/AutomationTestBuilder.ts | 38 +++++++++++-------- .../documents/app/automation/automation.ts | 1 + 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/packages/server/src/automations/tests/scenarios/looping.spec.ts b/packages/server/src/automations/tests/scenarios/looping.spec.ts index f229610542..6dde05ddb8 100644 --- a/packages/server/src/automations/tests/scenarios/looping.spec.ts +++ b/packages/server/src/automations/tests/scenarios/looping.spec.ts @@ -255,12 +255,15 @@ describe("Loop automations", () => { option: LoopStepType.ARRAY, binding: [1, 2, 3], }, - "FirstLoopStep" + { stepName: "FirstLoopStep" } + ) + .serverLog( + { text: "Message {{loop.currentItem}}" }, + { stepName: "FirstLoopLog" } ) - .serverLog({ text: "Message {{loop.currentItem}}" }, "FirstLoopLog") .serverLog( { text: "{{steps.FirstLoopLog.iterations}}" }, - "FirstLoopIterationLog" + { stepName: "FirstLoopIterationLog" } ) .run() diff --git a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts index 398ef20100..49c05984fe 100644 --- a/packages/server/src/automations/tests/scenarios/scenarios.spec.ts +++ b/packages/server/src/automations/tests/scenarios/scenarios.spec.ts @@ -217,7 +217,7 @@ describe("Automation Scenarios", () => { { tableId: table._id!, }, - "InitialQueryStep" + { stepName: "InitialQueryStep" } ) .deleteRow({ tableId: table._id!, diff --git a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts index 34c25d7004..7528ea0f2e 100644 --- a/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts +++ b/packages/server/src/automations/tests/utilities/AutomationTestBuilder.ts @@ -105,74 +105,80 @@ class BaseStepBuilder { } // STEPS - createRow(inputs: CreateRowStepInputs, stepName?: string): this { + createRow(inputs: CreateRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.CREATE_ROW, BUILTIN_ACTION_DEFINITIONS.CREATE_ROW, inputs, - stepName + opts?.stepName ) } - updateRow(inputs: UpdateRowStepInputs, stepName?: string): this { + updateRow(inputs: UpdateRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.UPDATE_ROW, BUILTIN_ACTION_DEFINITIONS.UPDATE_ROW, inputs, - stepName + opts?.stepName ) } - deleteRow(inputs: DeleteRowStepInputs, stepName?: string): this { + deleteRow(inputs: DeleteRowStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.DELETE_ROW, BUILTIN_ACTION_DEFINITIONS.DELETE_ROW, inputs, - stepName + opts?.stepName ) } - sendSmtpEmail(inputs: SmtpEmailStepInputs, stepName?: string): this { + sendSmtpEmail( + inputs: SmtpEmailStepInputs, + opts?: { stepName?: string } + ): this { return this.step( AutomationActionStepId.SEND_EMAIL_SMTP, BUILTIN_ACTION_DEFINITIONS.SEND_EMAIL_SMTP, inputs, - stepName + opts?.stepName ) } - executeQuery(inputs: ExecuteQueryStepInputs, stepName?: string): this { + executeQuery( + inputs: ExecuteQueryStepInputs, + opts?: { stepName?: string } + ): this { return this.step( AutomationActionStepId.EXECUTE_QUERY, BUILTIN_ACTION_DEFINITIONS.EXECUTE_QUERY, inputs, - stepName + opts?.stepName ) } - queryRows(inputs: QueryRowsStepInputs, stepName?: string): this { + queryRows(inputs: QueryRowsStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.QUERY_ROWS, BUILTIN_ACTION_DEFINITIONS.QUERY_ROWS, inputs, - stepName + opts?.stepName ) } - loop(inputs: LoopStepInputs, stepName?: string): this { + loop(inputs: LoopStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.LOOP, BUILTIN_ACTION_DEFINITIONS.LOOP, inputs, - stepName + opts?.stepName ) } - serverLog(input: ServerLogStepInputs, stepName?: string): this { + serverLog(input: ServerLogStepInputs, opts?: { stepName?: string }): this { return this.step( AutomationActionStepId.SERVER_LOG, BUILTIN_ACTION_DEFINITIONS.SERVER_LOG, input, - stepName + opts?.stepName ) } } diff --git a/packages/types/src/documents/app/automation/automation.ts b/packages/types/src/documents/app/automation/automation.ts index 78d22c787f..effe99a328 100644 --- a/packages/types/src/documents/app/automation/automation.ts +++ b/packages/types/src/documents/app/automation/automation.ts @@ -124,6 +124,7 @@ export interface Automation extends Document { definition: { steps: AutomationStep[] trigger: AutomationTrigger + // stepNames is used to lookup step names from their correspnding step ID. stepNames?: Record } screenId?: string From 04daa423cfe326aba255de37bd4bb5b6d042fc0e Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:03:37 +0100 Subject: [PATCH 15/30] small fix for edge case --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index be7286a79e..eb85cb19ba 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,6 +62,7 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" + import { name } from "helpers/validation/yup/app" export let block export let testData @@ -638,8 +639,7 @@ } /* End special cases for generating custom schemas based on triggers */ - let hasUserDefinedName = - automation.stepNames?.[allSteps[idx - loopBlockCount]]?.id + let hasUserDefinedName = automation.stepNames?.[allSteps[idx]?.id] if (isLoopBlock) { runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { @@ -756,10 +756,12 @@ loopBlockCount++ schema = cloneDeep(allSteps[idx - 1]?.schema?.outputs?.properties) } - Object.entries(schema).forEach(([name, value]) => + Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) - ) + }) + console.log(bindings) } + console.log(bindings) return bindings } From 59549b74b5bc615317cb4994f876808e285c67ef Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:04:13 +0100 Subject: [PATCH 16/30] js scripting edge case --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index eb85cb19ba..fe9710164b 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -644,7 +644,7 @@ runtimeName = `loop.${name}` } else if (block.name.startsWith("JS")) { runtimeName = hasUserDefinedName - ? `stepsByName.${bindingName}.${name}` + ? `stepsByName[${bindingName}].${name}` : `steps[${idx - loopBlockCount}].${name}` } else { runtimeName = hasUserDefinedName From 70b9d085162a9b8394a450bd0b80e46a44e2d4e7 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:10:38 +0100 Subject: [PATCH 17/30] lint --- .../automation/SetupPanel/AutomationBlockSetup.svelte | 3 --- 1 file changed, 3 deletions(-) diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index fe9710164b..af67ae8d22 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -62,7 +62,6 @@ } from "@budibase/types" import { FIELDS } from "constants/backend" import PropField from "./PropField.svelte" - import { name } from "helpers/validation/yup/app" export let block export let testData @@ -759,9 +758,7 @@ Object.entries(schema).forEach(([name, value]) => { addBinding(name, value, icon, idx, isLoopBlock, bindingName) }) - console.log(bindings) } - console.log(bindings) return bindings } From 8222e1df3fd546c926d414144f7ad1fc98ffaec2 Mon Sep 17 00:00:00 2001 From: Peter Clement Date: Fri, 13 Sep 2024 16:33:01 +0100 Subject: [PATCH 18/30] refs --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 723eff3807..25c6ed45c8 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 723eff3807fa24fa30de69f9f556910e190a29d1 +Subproject commit 25c6ed45c826f27b1ee7a7d1460dc1a386c6c471 From eaad70d0316bb5078702c97238799e5241592fd9 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 11:59:06 +0100 Subject: [PATCH 19/30] Get table deletion working. --- .../src/api/controllers/table/external.ts | 1 + .../integrations/tests/googlesheets.spec.ts | 7 ++-- .../integrations/tests/utils/googlesheets.ts | 34 +++++++++++++++---- 3 files changed, 31 insertions(+), 11 deletions(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index c3356919c8..94c16f3090 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -74,6 +74,7 @@ export async function destroy(ctx: UserCtx) { builderSocket?.emitDatasourceUpdate(ctx, datasource) return table } catch (err: any) { + throw err if (err instanceof Error) { ctx.throw(400, err.message) } else { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 3e7e0cddc4..9a3cee3881 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -429,13 +429,12 @@ describe("Google Sheets Integration", () => { }) }) - it.skip("can delete a table", async () => { + it("can delete a table", async () => { await config.api.table.destroy(table._id!, table._rev!) - expect(mock.cell("A1")).toEqual(null) - expect(mock.cell("B1")).toEqual(null) + expect(mock.sheet(table.name)).toBeUndefined() }) - it.skip("can delete a row", async () => { + it.only("can delete a row", async () => { const rows = await config.api.row.fetch(table._id!) expect(rows.length).toEqual(2) diff --git a/packages/server/src/integrations/tests/utils/googlesheets.ts b/packages/server/src/integrations/tests/utils/googlesheets.ts index 90f3b3652c..4b17c25b01 100644 --- a/packages/server/src/integrations/tests/utils/googlesheets.ts +++ b/packages/server/src/integrations/tests/utils/googlesheets.ts @@ -104,11 +104,17 @@ interface DeleteRangeRequest { shiftDimension: WorksheetDimension } +// https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request#DeleteSheetRequest +interface DeleteSheetRequest { + sheetId: number +} + // https://developers.google.com/sheets/api/reference/rest/v4/spreadsheets/request interface BatchUpdateRequest { requests: { addSheet?: AddSheetRequest deleteRange?: DeleteRangeRequest + deleteSheet?: DeleteSheetRequest }[] includeSpreadsheetInResponse: boolean responseRanges: string[] @@ -445,6 +451,11 @@ export class GoogleSheetsMock { } if (request.deleteRange) { this.handleDeleteRange(request.deleteRange) + response.replies.push({}) + } + if (request.deleteSheet) { + this.handleDeleteSheet(request.deleteSheet) + response.replies.push({}) } } @@ -489,11 +500,16 @@ export class GoogleSheetsMock { throw new Error("Only row-based deletes are supported") } - this.iterateRange(range, (cell, value) => { + this.iterateRange(range, cell => { cell.userEnteredValue = this.createValue(null) }) } + private handleDeleteSheet(request: DeleteSheetRequest) { + const { sheetId } = request + this.spreadsheet.sheets.splice(sheetId, 1) + } + private handleGetSpreadsheet(): Spreadsheet { return this.spreadsheet } @@ -514,17 +530,14 @@ export class GoogleSheetsMock { return response } - private iterateRange( - range: Required, - cb: (cell: CellData) => void - ) { + private iterateRange(range: GridRange, cb: (cell: CellData) => void) { const { sheetId, startRowIndex, endRowIndex, startColumnIndex, endColumnIndex, - } = range + } = this.ensureGridRange(range) for (let row = startRowIndex; row <= endRowIndex; row++) { for (let col = startColumnIndex; col <= endColumnIndex; col++) { @@ -754,7 +767,7 @@ export class GoogleSheetsMock { return this.getCellNumericIndexes(sheetId, startRowIndex, startColumnIndex) } - cell(cell: string): Value | undefined { + public cell(cell: string): Value | undefined { const cellData = this.cellData(cell) if (!cellData) { return undefined @@ -762,6 +775,13 @@ export class GoogleSheetsMock { return this.cellValue(cellData) } + public sheet(name: string | number): Sheet | undefined { + if (typeof name === "number") { + return this.getSheetById(name) + } + return this.getSheetByName(name) + } + private getCellNumericIndexes( sheet: Sheet | number, row: number, From ddf70415626fad3c0a75baab3e151bb4bbff5848 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 12:06:27 +0100 Subject: [PATCH 20/30] Get row deletion working. --- .../server/src/integrations/tests/googlesheets.spec.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 9a3cee3881..5cb67ae816 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -430,15 +430,19 @@ describe("Google Sheets Integration", () => { }) it("can delete a table", async () => { + expect(mock.sheet(table.name)).toBeDefined() await config.api.table.destroy(table._id!, table._rev!) expect(mock.sheet(table.name)).toBeUndefined() }) - it.only("can delete a row", async () => { + it("can delete a row", async () => { const rows = await config.api.row.fetch(table._id!) expect(rows.length).toEqual(2) - for (const row of rows) { + // Because row IDs in Google Sheets are sequential and determined by the + // actual row in the sheet, deleting a row will shift the row IDs down by + // one. This is why we reverse the rows before deleting them. + for (const row of rows.reverse()) { await config.api.row.delete(table._id!, { _id: row._id! }) } From ea6e2a472ed482a4d7b329f43b32fc39f6eec6e8 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 12:09:04 +0100 Subject: [PATCH 21/30] Remove unused throw. --- packages/server/src/api/controllers/table/external.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/api/controllers/table/external.ts b/packages/server/src/api/controllers/table/external.ts index 94c16f3090..c3356919c8 100644 --- a/packages/server/src/api/controllers/table/external.ts +++ b/packages/server/src/api/controllers/table/external.ts @@ -74,7 +74,6 @@ export async function destroy(ctx: UserCtx) { builderSocket?.emitDatasourceUpdate(ctx, datasource) return table } catch (err: any) { - throw err if (err instanceof Error) { ctx.throw(400, err.message) } else { From d6c8ae8ec44dfa8540339b4b6e3dff0534c0da29 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 12:09:50 +0100 Subject: [PATCH 22/30] Remove unused imports. --- packages/server/src/integrations/tests/googlesheets.spec.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 5cb67ae816..079e418f3b 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,13 +5,11 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, - Row, SourceName, Table, TableSourceType, } from "@budibase/types" import { GoogleSheetsMock } from "./utils/googlesheets" -import exp from "constants" describe("Google Sheets Integration", () => { const config = new TestConfiguration() From 9d6fc54a992e2ee0fe389a519d6d158d88f5995d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 16:12:07 +0100 Subject: [PATCH 23/30] Adding function parameter limit control for different SQL DBs, every DB has different limits with Postgres being the lowest at 100. We need to fix for wide tables which are related. --- packages/backend-core/src/sql/sql.ts | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2b5243f856..b0e7b30067 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -40,7 +40,6 @@ import { dataFilters, helpers } from "@budibase/shared-core" import { cloneDeep } from "lodash" type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any -const MAX_SQS_RELATIONSHIP_FIELDS = 63 function getBaseLimit() { const envLimit = environment.SQL_MAX_ROWS @@ -877,6 +876,22 @@ class InternalBuilder { return `'${unaliased}'${separator}${tableField}` } + maxFunctionParameters() { + // functions like say json_build_object() in SQL have a limit as to how many can be performed + // before a limit is met, this limit exists in Postgres/SQLite. This can be very important, such as + // for JSON column building as part of relationships. We also have a default limit to avoid very complex + // functions being built - it is likely this is not necessary or the best way to do it. + switch (this.client) { + case SqlClient.SQL_LITE: + return 127 + case SqlClient.POSTGRES: + return 100 + // other DBs don't have a limit, but set some sort of limit + default: + return 200 + } + } + addJsonRelationships( query: Knex.QueryBuilder, fromTable: string, @@ -908,12 +923,10 @@ class InternalBuilder { let relationshipFields = fields.filter( field => field.split(".")[0] === toAlias ) - if (this.client === SqlClient.SQL_LITE) { - relationshipFields = relationshipFields.slice( - 0, - MAX_SQS_RELATIONSHIP_FIELDS - ) - } + relationshipFields = relationshipFields.slice( + 0, + Math.floor(this.maxFunctionParameters() / 2) + ) const fieldList: string = relationshipFields .map(field => this.buildJsonField(field)) .join(",") From 26ad987072e13b4956dca68ee44c681aa8e2aee3 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 16:15:09 +0100 Subject: [PATCH 24/30] Fix Google Sheets pagination. --- .../server/src/integrations/googlesheets.ts | 15 +++++--- .../integrations/tests/googlesheets.spec.ts | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 0d766ac1ef..dd9bef84ab 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -551,11 +551,16 @@ export class GoogleSheetsIntegration implements DatasourcePlus { await this.connect() const hasFilters = dataFilters.hasFilters(query.filters) const limit = query.paginate?.limit || 100 - const page: number = - typeof query.paginate?.page === "number" - ? query.paginate.page - : parseInt(query.paginate?.page || "1") - const offset = (page - 1) * limit + let offset = query.paginate?.offset || 0 + + let page = query.paginate?.page + if (typeof page === "string") { + page = parseInt(page) + } + if (page !== undefined) { + offset = page * limit + } + const sheet = this.client.sheetsByTitle[query.sheet] let rows: GoogleSpreadsheetRow[] = [] if (query.paginate && !hasFilters) { diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index 079e418f3b..a1fabc2c04 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,6 +5,7 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, + Row, SourceName, Table, TableSourceType, @@ -208,6 +209,42 @@ describe("Google Sheets Integration", () => { expect(row2.name).toEqual("Test Contact 2") expect(row2.description).toEqual("original description 2") }) + + it("can paginate correctly", async () => { + await config.api.row.bulkImport(table._id!, { + rows: Array.from({ length: 248 }, (_, i) => ({ + name: `${i}`, + description: "", + })), + }) + + let resp = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + paginate: true, + limit: 10, + }) + let rows = resp.rows + + while (resp.hasNextPage) { + resp = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + paginate: true, + limit: 10, + bookmark: resp.bookmark, + }) + rows = rows.concat(resp.rows) + if (rows.length > 250) { + throw new Error("Too many rows returned") + } + } + + expect(rows.length).toEqual(250) + expect(rows.map(row => row.name)).toEqual( + expect.arrayContaining(Array.from({ length: 248 }, (_, i) => `${i}`)) + ) + }) }) describe("update", () => { From 9f21dc88b016ff631f132f1239381ee8c88d0a62 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Mon, 16 Sep 2024 16:18:57 +0100 Subject: [PATCH 25/30] Fix lint. --- packages/server/src/integrations/tests/googlesheets.spec.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/server/src/integrations/tests/googlesheets.spec.ts b/packages/server/src/integrations/tests/googlesheets.spec.ts index a1fabc2c04..62d56bb2c2 100644 --- a/packages/server/src/integrations/tests/googlesheets.spec.ts +++ b/packages/server/src/integrations/tests/googlesheets.spec.ts @@ -5,7 +5,6 @@ import TestConfiguration from "../../tests/utilities/TestConfiguration" import { Datasource, FieldType, - Row, SourceName, Table, TableSourceType, From 68a710699d6ab8fa978d9ee46c0020814b966845 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 18:09:01 +0100 Subject: [PATCH 26/30] Getting external DBs to correctly handle when too many fields. --- packages/backend-core/src/sql/sql.ts | 15 +++++-- .../api/controllers/row/ExternalRequest.ts | 13 +++--- .../src/api/routes/tests/search.spec.ts | 42 +++++++++++++++++++ 3 files changed, 59 insertions(+), 11 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b0e7b30067..ff32026814 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -899,7 +899,7 @@ class InternalBuilder { ): Knex.QueryBuilder { const sqlClient = this.client const knex = this.knex - const { resource, tableAliases: aliases, endpoint } = this.query + const { resource, tableAliases: aliases, endpoint, meta } = this.query const fields = resource?.fields || [] for (let relationship of relationships) { const { @@ -914,15 +914,22 @@ class InternalBuilder { if (!toTable || !fromTable) { continue } + const relatedTable = meta.tables?.[toTable] const toAlias = aliases?.[toTable] || toTable, fromAlias = aliases?.[fromTable] || fromTable let toTableWithSchema = this.tableNameWithSchema(toTable, { alias: toAlias, schema: endpoint.schema, }) - let relationshipFields = fields.filter( - field => field.split(".")[0] === toAlias - ) + const requiredFields = [ + ...(relatedTable?.primary || []), + relatedTable?.primaryDisplay, + ] + let relationshipFields = fields + .filter(field => field.split(".")[0] === toAlias) + // sort the required fields to first in the list + .sort(field => (requiredFields.includes(field) ? 1 : -1)) + relationshipFields = relationshipFields.slice( 0, Math.floor(this.maxFunctionParameters() / 2) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index ac2d1e8c39..9c9bd0b284 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -12,6 +12,7 @@ import { OneToManyRelationshipFieldMetadata, Operation, PaginationJson, + QueryJson, RelationshipFieldMetadata, Row, SearchFilters, @@ -161,7 +162,6 @@ export class ExternalRequest { private readonly tableId: string private datasource?: Datasource private tables: { [key: string]: Table } = {} - private tableList: Table[] constructor(operation: T, tableId: string, datasource?: Datasource) { this.operation = operation @@ -170,7 +170,6 @@ export class ExternalRequest { if (datasource && datasource.entities) { this.tables = datasource.entities } - this.tableList = Object.values(this.tables) } private prepareFilters( @@ -301,7 +300,6 @@ export class ExternalRequest { throw "No tables found, fetch tables before query." } this.tables = this.datasource.entities - this.tableList = Object.values(this.tables) } return { tables: this.tables, datasource: this.datasource } } @@ -463,7 +461,7 @@ export class ExternalRequest { breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] - if (!lookupField || !row[lookupField]) { + if (!lookupField || !row?.[lookupField] == null) { continue } const endpoint = getEndpoint(relatedTableId, Operation.READ) @@ -631,7 +629,8 @@ export class ExternalRequest { const { datasource: ds } = await this.retrieveMetadata(datasourceId) datasource = ds } - const table = this.tables[tableName] + const tables = this.tables + const table = tables[tableName] let isSql = isSQL(datasource) if (!table) { throw new Error( @@ -686,7 +685,7 @@ export class ExternalRequest { ) { throw "Deletion must be filtered" } - let json = { + let json: QueryJson = { endpoint: { datasourceId: datasourceId!, entityId: tableName, @@ -715,7 +714,7 @@ export class ExternalRequest { }, meta: { table, - id: config.id, + tables: tables, }, } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c9e7f4c3c5..0b0802bab2 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -3080,4 +3080,46 @@ describe.each([ }).toHaveLength(4) }) }) + + isSql && + describe("max related columns", () => { + let relatedRows: Row[] + + beforeAll(async () => { + const relatedSchema: TableSchema = {} + const row: Row = {} + for (let i = 0; i < 100; i++) { + const name = `column${i}` + relatedSchema[name] = { name, type: FieldType.NUMBER } + row[name] = i + } + const relatedTable = await createTable(relatedSchema) + table = await createTable({ + name: { name: "name", type: FieldType.STRING }, + related1: { + type: FieldType.LINK, + name: "related1", + fieldName: "main1", + tableId: relatedTable._id!, + relationshipType: RelationshipType.MANY_TO_MANY, + }, + }) + relatedRows = await Promise.all([ + config.api.row.save(relatedTable._id!, row), + ]) + await config.api.row.save(table._id!, { + name: "foo", + related1: [relatedRows[0]._id], + }) + }) + + it("retrieve the row with relationships", async () => { + await expectQuery({}).toContainExactly([ + { + name: "foo", + related1: [{ _id: relatedRows[0]._id }], + }, + ]) + }) + }) }) From 63c0d9afb8cf6eb21a5b84734aa2c5b97534b883 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 18:27:53 +0100 Subject: [PATCH 27/30] Sorting the field list to make sure we have the important fields at the top (if known). --- packages/backend-core/src/sql/sql.ts | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index ff32026814..55f71d76b0 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -55,6 +55,20 @@ function getRelationshipLimit() { return envLimit || 500 } +function prioritisedArraySort(toSort: string[], priorities: string[]) { + return toSort.sort((a, b) => { + const aPriority = priorities.find(field => field && a.endsWith(field)) + const bPriority = priorities.find(field => field && b.endsWith(field)) + if (aPriority && !bPriority) { + return -1 + } + if (!aPriority && bPriority) { + return 1 + } + return a.localeCompare(b) + }) +} + function getTableName(table?: Table): string | undefined { // SQS uses the table ID rather than the table name if ( @@ -924,11 +938,12 @@ class InternalBuilder { const requiredFields = [ ...(relatedTable?.primary || []), relatedTable?.primaryDisplay, - ] - let relationshipFields = fields - .filter(field => field.split(".")[0] === toAlias) - // sort the required fields to first in the list - .sort(field => (requiredFields.includes(field) ? 1 : -1)) + ].filter(field => field) as string[] + // sort the required fields to first in the list, so they don't get sliced out + let relationshipFields = prioritisedArraySort( + fields.filter(field => field.split(".")[0] === toAlias), + requiredFields + ) relationshipFields = relationshipFields.slice( 0, From ec400dee6f73b510d5b96378abb66f7877cda5cb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 16 Sep 2024 19:20:58 +0100 Subject: [PATCH 28/30] Fixing test cases. --- packages/server/src/integrations/tests/sql.spec.ts | 4 ++-- packages/server/src/integrations/tests/sqlAlias.spec.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index cf5b88b0fd..5a505fbb40 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -162,7 +162,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "production"."brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "production"."products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) @@ -170,7 +170,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ bindings: [limit, relationshipLimit], - sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('product_id',"products"."product_id",'product_name',"products"."product_name",'brand_id',"products"."brand_id")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, + sql: `with "paginated" as (select "brands".* from "brands" order by "test"."id" asc limit $1) select "brands".*, (select json_agg(json_build_object('brand_id',"products"."brand_id",'product_id',"products"."product_id",'product_name',"products"."product_name")) from (select "products".* from "products" as "products" where "products"."brand_id" = "brands"."brand_id" order by "products"."brand_id" asc limit $2) as "products") as "products" from "paginated" as "brands" order by "test"."id" asc`, }) }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 9548499c65..fc5af4238c 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -63,7 +63,7 @@ describe("Captures of real examples", () => { bindings: [primaryLimit, relationshipLimit, relationshipLimit], sql: expect.stringContaining( multiline( - `select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid",'executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")` + `select json_agg(json_build_object('completed',"b"."completed",'completed',"b"."completed",'executorid',"b"."executorid",'executorid',"b"."executorid",'qaid',"b"."qaid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskid',"b"."taskid",'taskname',"b"."taskname",'taskname',"b"."taskname")` ) ), }) @@ -95,7 +95,7 @@ describe("Captures of real examples", () => { sql: expect.stringContaining( multiline( `with "paginated" as (select "a".* from "products" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc limit $1) - select "a".*, (select json_agg(json_build_object('executorid',"b"."executorid",'taskname',"b"."taskname",'taskid',"b"."taskid",'completed',"b"."completed",'qaid',"b"."qaid")) + select "a".*, (select json_agg(json_build_object('completed',"b"."completed",'executorid',"b"."executorid",'qaid',"b"."qaid",'taskid',"b"."taskid",'taskname',"b"."taskname")) from (select "b".* from "tasks" as "b" inner join "products_tasks" as "c" on "b"."taskid" = "c"."taskid" where "c"."productid" = "a"."productid" order by "b"."taskid" asc limit $2) as "b") as "tasks" from "paginated" as "a" order by "a"."productname" asc nulls first, "a"."productid" asc` ) @@ -113,7 +113,7 @@ describe("Captures of real examples", () => { bindings: [...filters, relationshipLimit, relationshipLimit], sql: multiline( `with "paginated" as (select "a".* from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) - select "a".*, (select json_agg(json_build_object('productname',"b"."productname",'productid',"b"."productid")) + select "a".*, (select json_agg(json_build_object('productid',"b"."productid",'productname',"b"."productname")) from (select "b".* from "products" as "b" inner join "products_tasks" as "c" on "b"."productid" = "c"."productid" where "c"."taskid" = "a"."taskid" order by "b"."productid" asc limit $4) as "b") as "products" from "paginated" as "a" order by "a"."taskid" asc` ), From 614132eb0462a071cbbcc6f0c867e589e1502108 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Mon, 16 Sep 2024 20:40:02 +0000 Subject: [PATCH 29/30] Bump version to 2.32.5 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index a9c50ea4b5..c710d888c7 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "2.32.4", + "version": "2.32.5", "npmClient": "yarn", "packages": [ "packages/*", From 5e245d11e5963774b8c16f478624c41e031abcb7 Mon Sep 17 00:00:00 2001 From: Dean Date: Tue, 17 Sep 2024 09:20:23 +0100 Subject: [PATCH 30/30] Bump account portal to latest --- packages/account-portal | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/account-portal b/packages/account-portal index 25c6ed45c8..905773d708 160000 --- a/packages/account-portal +++ b/packages/account-portal @@ -1 +1 @@ -Subproject commit 25c6ed45c826f27b1ee7a7d1460dc1a386c6c471 +Subproject commit 905773d70854a43c6ef2461c7a49671bff56fedc