From 2e636475cf40f029c413bfb68d9608b3776d1c2a Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 9 May 2024 19:47:48 +0200 Subject: [PATCH 01/10] Update packages --- packages/server/package.json | 3 +- yarn.lock | 78 +++--------------------------------- 2 files changed, 7 insertions(+), 74 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 63df6cd15f..ce8e476203 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,8 +79,7 @@ "dotenv": "8.2.0", "form-data": "4.0.0", "global-agent": "3.0.0", - "google-auth-library": "7.12.0", - "google-spreadsheet": "3.2.0", + "google-spreadsheet": "4.1.1", "ioredis": "5.3.2", "isolated-vm": "^4.7.2", "jimp": "0.22.10", diff --git a/yarn.lock b/yarn.lock index 194fec23dc..3f80d0ac1d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6983,7 +6983,7 @@ axios-retry@^3.1.9: "@babel/runtime" "^7.15.4" is-retry-allowed "^2.2.0" -axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.21.4, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.5.0: +axios@0.24.0, axios@1.1.3, axios@1.6.3, axios@^0.21.1, axios@^0.26.0, axios@^1.0.0, axios@^1.1.3, axios@^1.4.0, axios@^1.5.0: version "1.6.3" resolved "https://registry.yarnpkg.com/axios/-/axios-1.6.3.tgz#7f50f23b3aa246eff43c54834272346c396613f4" integrity sha512-fWyNdeawGam70jXSVlKl+SUNVcL6j6W79CuSIPfi6HnDUmSCH6gyUys/HrqHeA/wU0Az41rRgean494d0Jb+ww== @@ -11071,17 +11071,6 @@ gauge@^4.0.3: strip-ansi "^6.0.1" wide-align "^1.1.5" -gaxios@^4.0.0: - version "4.3.3" - resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-4.3.3.tgz#d44bdefe52d34b6435cc41214fdb160b64abfc22" - integrity sha512-gSaYYIO1Y3wUtdfHmjDUZ8LWaxJQpiavzbF5Kq53akSzvmVg0RfyOcFDbO1KJ/KCGRFz2qG+lS81F0nkr7cRJA== - dependencies: - abort-controller "^3.0.0" - extend "^3.0.2" - https-proxy-agent "^5.0.0" - is-stream "^2.0.0" - node-fetch "^2.6.7" - gaxios@^5.0.0, gaxios@^5.0.1: version "5.1.3" resolved "https://registry.yarnpkg.com/gaxios/-/gaxios-5.1.3.tgz#f7fa92da0fe197c846441e5ead2573d4979e9013" @@ -11092,14 +11081,6 @@ gaxios@^5.0.0, gaxios@^5.0.1: is-stream "^2.0.0" node-fetch "^2.6.9" -gcp-metadata@^4.2.0: - version "4.3.1" - resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-4.3.1.tgz#fb205fe6a90fef2fd9c85e6ba06e5559ee1eefa9" - integrity sha512-x850LS5N7V1F3UcV7PoupzGsyD6iVwTVvsh3tbXfkctZnBnjW5yu5z1/3k3SehF7TyoTIe78rJs02GMMy+LF+A== - dependencies: - gaxios "^4.0.0" - json-bigint "^1.0.0" - gcp-metadata@^5.3.0: version "5.3.0" resolved "https://registry.yarnpkg.com/gcp-metadata/-/gcp-metadata-5.3.0.tgz#6f45eb473d0cb47d15001476b48b663744d25408" @@ -11506,36 +11487,6 @@ gonzales-pe@^4.2.3, gonzales-pe@^4.3.0: dependencies: minimist "^1.2.5" -google-auth-library@7.12.0: - version "7.12.0" - resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-7.12.0.tgz#7965db6bc20cb31f2df05a08a296bbed6af69426" - integrity sha512-RS/whvFPMoF1hQNxnoVET3DWKPBt1Xgqe2rY0k+Jn7TNhoHlwdnSe7Rlcbo2Nub3Mt2lUVz26X65aDQrWp6x8w== - dependencies: - arrify "^2.0.0" - base64-js "^1.3.0" - ecdsa-sig-formatter "^1.0.11" - fast-text-encoding "^1.0.0" - gaxios "^4.0.0" - gcp-metadata "^4.2.0" - gtoken "^5.0.4" - jws "^4.0.0" - lru-cache "^6.0.0" - -google-auth-library@^6.1.3: - version "6.1.6" - resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-6.1.6.tgz#deacdcdb883d9ed6bac78bb5d79a078877fdf572" - integrity sha512-Q+ZjUEvLQj/lrVHF/IQwRo6p3s8Nc44Zk/DALsN+ac3T4HY/g/3rrufkgtl+nZ1TW7DNAw5cTChdVp4apUXVgQ== - dependencies: - arrify "^2.0.0" - base64-js "^1.3.0" - ecdsa-sig-formatter "^1.0.11" - fast-text-encoding "^1.0.0" - gaxios "^4.0.0" - gcp-metadata "^4.2.0" - gtoken "^5.0.4" - jws "^4.0.0" - lru-cache "^6.0.0" - google-auth-library@^8.0.1, google-auth-library@^8.0.2: version "8.9.0" resolved "https://registry.yarnpkg.com/google-auth-library/-/google-auth-library-8.9.0.tgz#15a271eb2ec35d43b81deb72211bd61b1ef14dd0" @@ -11572,13 +11523,6 @@ google-gax@^3.5.7: protobufjs-cli "1.1.1" retry-request "^5.0.0" -google-p12-pem@^3.1.3: - version "3.1.4" - resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-3.1.4.tgz#123f7b40da204de4ed1fbf2fd5be12c047fc8b3b" - integrity sha512-HHuHmkLgwjdmVRngf5+gSmpkyaRI6QmOg77J8tkNBHhNEI62sGHyw4/+UkgyZEI7h84NbWprXDJ+sa3xOYFvTg== - dependencies: - node-forge "^1.3.1" - google-p12-pem@^4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/google-p12-pem/-/google-p12-pem-4.0.1.tgz#82841798253c65b7dc2a4e5fe9df141db670172a" @@ -11586,13 +11530,12 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -google-spreadsheet@3.2.0: - version "3.2.0" - resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-3.2.0.tgz#ce8aa75c15705aa950ad52b091a6fc4d33dcb329" - integrity sha512-z7XMaqb+26rdo8p51r5O03u8aPLAPzn5YhOXYJPcf2hdMVr0dUbIARgdkRdmGiBeoV/QoU/7VNhq1MMCLZv3kQ== +google-spreadsheet@4.1.1: + version "4.1.1" + resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.1.tgz#56c422c86b51a2ea9dad21c40cd69bccbd58591f" + integrity sha512-Npk/xAMTgxEt/m/X9EXIqdY6CEYGiqUHrSuiLnNSKli5H+wiOQLSLsnfMxcdNPH6aSh6GttZm6QJhrnsxjwpZQ== dependencies: - axios "^0.21.4" - google-auth-library "^6.1.3" + axios "^1.4.0" lodash "^4.17.21" gopd@^1.0.1: @@ -11678,15 +11621,6 @@ graphemer@^1.4.0: resolved "https://registry.yarnpkg.com/graphemer/-/graphemer-1.4.0.tgz#fb2f1d55e0e3a1849aeffc90c4fa0dd53a0e66c6" integrity sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag== -gtoken@^5.0.4: - version "5.3.2" - resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-5.3.2.tgz#deb7dc876abe002178e0515e383382ea9446d58f" - integrity sha512-gkvEKREW7dXWF8NV8pVrKfW7WqReAmjjkMBh6lNCCGOM4ucS0r0YyXXl0r/9Yj8wcW/32ISkfc8h5mPTDbtifQ== - dependencies: - gaxios "^4.0.0" - google-p12-pem "^3.1.3" - jws "^4.0.0" - gtoken@^6.1.0: version "6.1.2" resolved "https://registry.yarnpkg.com/gtoken/-/gtoken-6.1.2.tgz#aeb7bdb019ff4c3ba3ac100bbe7b6e74dce0e8bc" From 9414c34c218b2ec58c9a86035f246b9986009669 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 9 May 2024 20:11:27 +0200 Subject: [PATCH 02/10] Update types --- packages/server/package.json | 2 +- yarn.lock | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index ce8e476203..7d5dd6e9a1 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -124,7 +124,7 @@ "@swc/jest": "0.2.27", "@types/archiver": "6.0.2", "@types/global-agent": "2.1.1", - "@types/google-spreadsheet": "3.1.5", + "@types/google-spreadsheet": "4.0.0", "@types/jest": "29.5.5", "@types/koa": "2.13.4", "@types/koa-send": "^4.1.6", diff --git a/yarn.lock b/yarn.lock index 3f80d0ac1d..1351f34b1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5272,10 +5272,12 @@ resolved "https://registry.yarnpkg.com/@types/global-agent/-/global-agent-2.1.1.tgz#3f93185e48a3a36e377a52a8301320cd162a831b" integrity sha512-sVox8Phk1UKgP6LQPAdeRxfww6vHKt7Bf59dXzYLsQBUEMEn8S10a+ESp/yO0i4fJ3WS4+CIuz42hgJcuA+3mA== -"@types/google-spreadsheet@3.1.5": - version "3.1.5" - resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-3.1.5.tgz#2bdc6f9f5372551e0506cb6ef3f562adcf44fc2e" - integrity sha512-7N+mDtZ1pmya2RRFPPl4KYc2TRgiqCNBLUZfyrKfER+u751JgCO+C24/LzF70UmUm/zhHUbzRZ5mtfaxekQ1ZQ== +"@types/google-spreadsheet@4.0.0": + version "4.0.0" + resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-4.0.0.tgz#a5828b3e5461eec0e138b8e4a7ab15fd69c3af30" + integrity sha512-GdfhBRQcN1KhpZGbhOEu/Cj2LimDLUW7hVB/GD552F4IY4FuYFwYwIz1TIB7rRqMVnMy99RdwpgXPX0G5Uu+7g== + dependencies: + google-spreadsheet "*" "@types/graceful-fs@^4.1.3": version "4.1.6" @@ -11530,7 +11532,7 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -google-spreadsheet@4.1.1: +google-spreadsheet@*, google-spreadsheet@4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.1.tgz#56c422c86b51a2ea9dad21c40cd69bccbd58591f" integrity sha512-Npk/xAMTgxEt/m/X9EXIqdY6CEYGiqUHrSuiLnNSKli5H+wiOQLSLsnfMxcdNPH6aSh6GttZm6QJhrnsxjwpZQ== From f8c0a3b5c0a041e2345ef0d40d22c6eb7cc051b0 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 9 May 2024 20:13:26 +0200 Subject: [PATCH 03/10] Remove types --- packages/server/package.json | 1 - yarn.lock | 9 +-------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 7d5dd6e9a1..0e12d387a6 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -124,7 +124,6 @@ "@swc/jest": "0.2.27", "@types/archiver": "6.0.2", "@types/global-agent": "2.1.1", - "@types/google-spreadsheet": "4.0.0", "@types/jest": "29.5.5", "@types/koa": "2.13.4", "@types/koa-send": "^4.1.6", diff --git a/yarn.lock b/yarn.lock index 1351f34b1f..3120531556 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5272,13 +5272,6 @@ resolved "https://registry.yarnpkg.com/@types/global-agent/-/global-agent-2.1.1.tgz#3f93185e48a3a36e377a52a8301320cd162a831b" integrity sha512-sVox8Phk1UKgP6LQPAdeRxfww6vHKt7Bf59dXzYLsQBUEMEn8S10a+ESp/yO0i4fJ3WS4+CIuz42hgJcuA+3mA== -"@types/google-spreadsheet@4.0.0": - version "4.0.0" - resolved "https://registry.yarnpkg.com/@types/google-spreadsheet/-/google-spreadsheet-4.0.0.tgz#a5828b3e5461eec0e138b8e4a7ab15fd69c3af30" - integrity sha512-GdfhBRQcN1KhpZGbhOEu/Cj2LimDLUW7hVB/GD552F4IY4FuYFwYwIz1TIB7rRqMVnMy99RdwpgXPX0G5Uu+7g== - dependencies: - google-spreadsheet "*" - "@types/graceful-fs@^4.1.3": version "4.1.6" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.6.tgz#e14b2576a1c25026b7f02ede1de3b84c3a1efeae" @@ -11532,7 +11525,7 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -google-spreadsheet@*, google-spreadsheet@4.1.1: +google-spreadsheet@4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.1.tgz#56c422c86b51a2ea9dad21c40cd69bccbd58591f" integrity sha512-Npk/xAMTgxEt/m/X9EXIqdY6CEYGiqUHrSuiLnNSKli5H+wiOQLSLsnfMxcdNPH6aSh6GttZm6QJhrnsxjwpZQ== From 004e13a349a239b8cb21839d2743292a9a8fbf21 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Thu, 9 May 2024 20:13:33 +0200 Subject: [PATCH 04/10] Fix --- .../server/src/integrations/googlesheets.ts | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index da98ac4875..b2dd7b73e5 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -158,12 +158,12 @@ const SCHEMA: Integration = { class GoogleSheetsIntegration implements DatasourcePlus { private readonly config: GoogleSheetsConfig - private client: GoogleSpreadsheet + private readonly spreadsheetId: string + private client: GoogleSpreadsheet = undefined! constructor(config: GoogleSheetsConfig) { this.config = config - const spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId) - this.client = new GoogleSpreadsheet(spreadsheetId) + this.spreadsheetId = this.cleanSpreadsheetUrl(this.config.spreadsheetId) } async testConnection(): Promise { @@ -252,7 +252,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { access_token: tokenResponse.access_token, }) - this.client.useOAuth2Client(oauthClient) + this.client = new GoogleSpreadsheet(this.spreadsheetId, oauthClient) await this.client.loadInfo() } catch (err: any) { // this happens for xlsx imports @@ -385,13 +385,17 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - buildRowObject(headers: string[], values: string[], rowNumber: number) { + buildRowObject( + headers: string[], + values: Record, + rowNumber: number + ) { const rowObject: { rowNumber: number } & Row = { rowNumber, _id: rowNumber.toString(), } for (let i = 0; i < headers.length; i++) { - rowObject[headers[i]] = values[i] + rowObject[headers[i]] = values[headers[i]] } return rowObject } @@ -475,7 +479,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { typeof query.row === "string" ? JSON.parse(query.row) : query.row const row = await sheet.addRow(rowToInsert) return [ - this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber), + this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber), ] } catch (err) { console.error("Error writing to google sheets", err) @@ -493,7 +497,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } const rows = await sheet.addRows(rowsToInsert) return rows.map(row => - this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber) + this.buildRowObject(sheet.headerValues, row.toObject(), row.rowNumber) ) } catch (err) { console.error("Error bulk writing to google sheets", err) @@ -548,7 +552,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { let response = [] for (let row of filtered) { response.push( - this.buildRowObject(headerValues, row._rawData, row._rowNumber) + this.buildRowObject(headerValues, row.toObject(), row._rowNumber) ) } @@ -598,10 +602,10 @@ class GoogleSheetsIntegration implements DatasourcePlus { const updateValues = typeof query.row === "string" ? JSON.parse(query.row) : query.row for (let key in updateValues) { - row[key] = updateValues[key] + row.set(key, updateValues[key]) - if (row[key] === null) { - row[key] = "" + if (row.get(key) === null) { + row.set(key, "") } const { type, subtype, constraints } = query.table.schema[key] @@ -615,7 +619,11 @@ class GoogleSheetsIntegration implements DatasourcePlus { } await row.save() return [ - this.buildRowObject(sheet.headerValues, row._rawData, row._rowNumber), + this.buildRowObject( + sheet.headerValues, + row.toObject(), + row.rowNumber + ), ] } else { throw new Error("Row does not exist.") From 63160a394b94d61f7f9889db995e0ffb93e0ef17 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Mon, 13 May 2024 15:22:55 +0200 Subject: [PATCH 05/10] Fix --- packages/server/src/integrations/googlesheets.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index b2dd7b73e5..6d20987ae7 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -613,8 +613,8 @@ class GoogleSheetsIntegration implements DatasourcePlus { type === FieldType.BB_REFERENCE && subtype === BBReferenceFieldSubType.USER && constraints?.type !== "array" - if (isDeprecatedSingleUser && Array.isArray(row[key])) { - row[key] = row[key][0] + if (isDeprecatedSingleUser && Array.isArray(row.get(key))) { + row.set(key, row.get(key)[0]) } } await row.save() From 5b5e7e47a2ceab5a37fb3e491d2b3dce26deb5a3 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 13 May 2024 18:11:40 +0100 Subject: [PATCH 06/10] Fixing some issues with updating rows with attachments, there were some UI bugs that had been raised (undefined being displayed) and the uploading from a URL was not working as expected, due to the stream not matching the expected format of the S3 SDK - this has been fixed by converting the stream to a true readable stream with . --- .../backend-core/src/objectStore/utils.ts | 9 ++--- .../SetupPanel/AutomationBlockSetup.svelte | 19 +++++----- .../SetupPanel/RowSelectorTypes.svelte | 12 +++---- .../server/src/automations/automationUtils.ts | 4 +-- .../server/src/automations/steps/updateRow.ts | 23 ++++++------ .../src/utilities/rowProcessor/attachments.ts | 21 ++++++++--- .../src/utilities/rowProcessor/index.ts | 36 ++++++++++--------- 7 files changed, 70 insertions(+), 54 deletions(-) diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index 5b9c2e3646..2edb075b92 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -9,6 +9,9 @@ import { AutomationAttachmentContent, BucketedContent, } from "@budibase/types" +import stream from "stream" +import streamWeb from "node:stream/web" + /**************************************************** * NOTE: When adding a new bucket - name * * sure that S3 usages (like budibase-infra) * @@ -53,12 +56,10 @@ export const bucketTTLConfig = ( Rules: [lifecycleRule], } - const params = { + return { Bucket: bucketName, LifecycleConfiguration: lifecycleConfiguration, } - - return params } async function processUrlAttachment( @@ -71,7 +72,7 @@ async function processUrlAttachment( const fallbackFilename = path.basename(new URL(attachment.url).pathname) return { filename: attachment.filename || fallbackFilename, - content: response.body, + content: stream.Readable.fromWeb(response.body as streamWeb.ReadableStream), } } diff --git a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte index 85af5bbafd..879927343f 100644 --- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte @@ -374,6 +374,16 @@ return `${value.title || (key === "row" ? "Table" : key)} ${requiredSuffix}` } + function handleAttachmentParams(keyValueObj) { + let params = {} + if (keyValueObj?.length) { + for (let param of keyValueObj) { + params[param.url] = param.filename + } + } + return params + } + onMount(async () => { try { await environment.loadVariables() @@ -381,15 +391,6 @@ console.error(error) } }) - const handleAttachmentParams = keyValuObj => { - let params = {} - if (keyValuObj?.length) { - for (let param of keyValuObj) { - params[param.url] = param.filename - } - } - return params - }
diff --git a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte index 9b4e5e36f6..1b52e35314 100644 --- a/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte +++ b/packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte @@ -25,21 +25,21 @@ return !!schema.constraints?.inclusion?.length } - const handleAttachmentParams = keyValuObj => { + function handleAttachmentParams(keyValueObj) { let params = {} if ( schema.type === FieldType.ATTACHMENT_SINGLE && - Object.keys(keyValuObj).length === 0 + Object.keys(keyValueObj).length === 0 ) { return [] } - if (!Array.isArray(keyValuObj)) { - keyValuObj = [keyValuObj] + if (!Array.isArray(keyValueObj) && keyValueObj) { + keyValueObj = [keyValueObj] } - if (keyValuObj.length) { - for (let param of keyValuObj) { + if (keyValueObj.length) { + for (let param of keyValueObj) { params[param.url] = param.filename } } diff --git a/packages/server/src/automations/automationUtils.ts b/packages/server/src/automations/automationUtils.ts index c94c166be1..88b279b771 100644 --- a/packages/server/src/automations/automationUtils.ts +++ b/packages/server/src/automations/automationUtils.ts @@ -163,7 +163,7 @@ async function generateAttachmentRow(attachment: AutomationAttachment) { try { const { filename } = attachment - const extension = path.extname(filename) + const extension = path.extname(filename).replaceAll(".", "") const attachmentResult = await objectStore.processAutomationAttachment( attachment ) @@ -183,7 +183,7 @@ async function generateAttachmentRow(attachment: AutomationAttachment) { return { size, name: filename, - extension, + extension: extension, key: s3Key, } } catch (error) { diff --git a/packages/server/src/automations/steps/updateRow.ts b/packages/server/src/automations/steps/updateRow.ts index 348c5e8373..32c8addd7a 100644 --- a/packages/server/src/automations/steps/updateRow.ts +++ b/packages/server/src/automations/steps/updateRow.ts @@ -94,18 +94,6 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { } } - // have to clean up the row, remove the table from it - const ctx: any = buildCtx(appId, emitter, { - body: { - ...inputs.row, - _id: inputs.rowId, - }, - params: { - rowId: inputs.rowId, - tableId: tableId, - }, - }) - try { if (tableId) { inputs.row = await automationUtils.cleanUpRow( @@ -118,6 +106,17 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) { inputs.row ) } + // have to clean up the row, remove the table from it + const ctx: any = buildCtx(appId, emitter, { + body: { + ...inputs.row, + _id: inputs.rowId, + }, + params: { + rowId: inputs.rowId, + tableId: tableId, + }, + }) await rowController.patch(ctx) return { row: ctx.body, diff --git a/packages/server/src/utilities/rowProcessor/attachments.ts b/packages/server/src/utilities/rowProcessor/attachments.ts index da52d6a631..bfa216c25b 100644 --- a/packages/server/src/utilities/rowProcessor/attachments.ts +++ b/packages/server/src/utilities/rowProcessor/attachments.ts @@ -1,6 +1,12 @@ import { ObjectStoreBuckets } from "../../constants" import { context, db as dbCore, objectStore } from "@budibase/backend-core" -import { FieldType, RenameColumn, Row, Table } from "@budibase/types" +import { + FieldType, + RenameColumn, + Row, + RowAttachment, + Table, +} from "@budibase/types" export class AttachmentCleanup { static async coreCleanup(fileListFn: () => string[]): Promise { @@ -21,7 +27,7 @@ export class AttachmentCleanup { private static extractAttachmentKeys( type: FieldType, - rowData: any + rowData: RowAttachment[] | RowAttachment ): string[] { if ( type !== FieldType.ATTACHMENTS && @@ -34,10 +40,15 @@ export class AttachmentCleanup { return [] } - if (type === FieldType.ATTACHMENTS) { - return rowData.map((attachment: any) => attachment.key) + if (type === FieldType.ATTACHMENTS && Array.isArray(rowData)) { + return rowData + .filter(attachment => attachment.key) + .map(attachment => attachment.key) + } else if ("key" in rowData) { + return [rowData.key] } - return [rowData.key] + + return [] } private static async tableChange( diff --git a/packages/server/src/utilities/rowProcessor/index.ts b/packages/server/src/utilities/rowProcessor/index.ts index b71f40e870..e7bc725285 100644 --- a/packages/server/src/utilities/rowProcessor/index.ts +++ b/packages/server/src/utilities/rowProcessor/index.ts @@ -1,11 +1,11 @@ import * as linkRows from "../../db/linkedRows" -import { processFormulas, fixAutoColumnSubType } from "./utils" +import { fixAutoColumnSubType, processFormulas } from "./utils" import { objectStore, utils } from "@budibase/backend-core" import { InternalTables } from "../../db/utils" import { TYPE_TRANSFORM_MAP } from "./map" import { - FieldType, AutoFieldSubType, + FieldType, Row, RowAttachment, Table, @@ -221,27 +221,31 @@ export async function outputProcessing( opts.squash = true } - // process complex types: attachements, bb references... + // process complex types: attachments, bb references... for (let [property, column] of Object.entries(table.schema)) { - if (column.type === FieldType.ATTACHMENTS) { + if ( + column.type === FieldType.ATTACHMENTS || + column.type === FieldType.ATTACHMENT_SINGLE + ) { for (let row of enriched) { - if (row[property] == null || !Array.isArray(row[property])) { + if (row[property] == null) { continue } - row[property].forEach((attachment: RowAttachment) => { - if (!attachment.url) { + const process = (attachment: RowAttachment) => { + if (!attachment.url && attachment.key) { attachment.url = objectStore.getAppFileUrl(attachment.key) } - }) - } - } else if (column.type === FieldType.ATTACHMENT_SINGLE) { - for (let row of enriched) { - if (!row[property] || Object.keys(row[property]).length === 0) { - continue + return attachment } - - if (!row[property].url) { - row[property].url = objectStore.getAppFileUrl(row[property].key) + if (typeof row[property] === "string") { + row[property] = JSON.parse(row[property]) + } + if (Array.isArray(row[property])) { + row[property].forEach((attachment: RowAttachment) => { + process(attachment) + }) + } else { + process(row[property]) } } } else if ( From d2e72889d63e63a3edf1746199aa8062d2a134c8 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 09:39:55 +0200 Subject: [PATCH 07/10] Update --- packages/server/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/package.json b/packages/server/package.json index 0e12d387a6..ab6738635d 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -79,7 +79,7 @@ "dotenv": "8.2.0", "form-data": "4.0.0", "global-agent": "3.0.0", - "google-spreadsheet": "4.1.1", + "google-spreadsheet": "4.1.2", "ioredis": "5.3.2", "isolated-vm": "^4.7.2", "jimp": "0.22.10", diff --git a/yarn.lock b/yarn.lock index 3120531556..cb97cfe0af 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11525,10 +11525,10 @@ google-p12-pem@^4.0.0: dependencies: node-forge "^1.3.1" -google-spreadsheet@4.1.1: - version "4.1.1" - resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.1.tgz#56c422c86b51a2ea9dad21c40cd69bccbd58591f" - integrity sha512-Npk/xAMTgxEt/m/X9EXIqdY6CEYGiqUHrSuiLnNSKli5H+wiOQLSLsnfMxcdNPH6aSh6GttZm6QJhrnsxjwpZQ== +google-spreadsheet@4.1.2: + version "4.1.2" + resolved "https://registry.yarnpkg.com/google-spreadsheet/-/google-spreadsheet-4.1.2.tgz#92e30fdba7e0d78c55d50731528df7835d58bfee" + integrity sha512-HFBweDAkOcyC2qO9kmWESKbNuOcn+R7UzZN/tj5LLNxVv8FHmg113u0Ow+yaKwwIOt/NnDtPLuptAhaxTs0FYw== dependencies: axios "^1.4.0" lodash "^4.17.21" From c9d5112fcc6522e1035e2a4b8ae559b466329e53 Mon Sep 17 00:00:00 2001 From: Adria Navarro Date: Tue, 14 May 2024 11:35:01 +0200 Subject: [PATCH 08/10] Change methods to private --- .../server/src/integrations/googlesheets.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/googlesheets.ts b/packages/server/src/integrations/googlesheets.ts index 6d20987ae7..dc945b454a 100644 --- a/packages/server/src/integrations/googlesheets.ts +++ b/packages/server/src/integrations/googlesheets.ts @@ -191,7 +191,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { * @param spreadsheetId - the URL or standard spreadsheetId of the google sheet * @returns spreadsheet Id of the google sheet */ - cleanSpreadsheetUrl(spreadsheetId: string) { + private cleanSpreadsheetUrl(spreadsheetId: string) { if (!spreadsheetId) { throw new Error( "You must set a spreadsheet ID in your configuration to fetch tables." @@ -201,7 +201,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { return parts.length > 5 ? parts[5] : spreadsheetId } - async fetchAccessToken( + private async fetchAccessToken( payload: AuthTokenRequest ): Promise { const response = await fetch("https://www.googleapis.com/oauth2/v4/token", { @@ -226,7 +226,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { return json } - async connect() { + private async connect() { try { await setupCreationAuth(this.config) @@ -271,7 +271,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { return sheets.map(s => s.title) } - getTableSchema( + private getTableSchema( title: string, headerValues: string[], datasourceId: string, @@ -385,7 +385,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - buildRowObject( + private buildRowObject( headers: string[], values: Record, rowNumber: number @@ -400,7 +400,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { return rowObject } - async createTable(name?: string) { + private async createTable(name?: string) { if (!name) { throw new Error("Must provide name for new sheet.") } @@ -413,7 +413,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - async updateTable(table: TableRequest) { + private async updateTable(table: TableRequest) { await this.connect() const sheet = this.client.sheetsByTitle[table.name] await sheet.loadHeaderRow() @@ -460,7 +460,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - async deleteTable(sheet: any) { + private async deleteTable(sheet: any) { try { await this.connect() const sheetToDelete = this.client.sheetsByTitle[sheet] @@ -487,7 +487,7 @@ class GoogleSheetsIntegration implements DatasourcePlus { } } - async createBulk(query: { sheet: string; rows: Row[] }) { + private async createBulk(query: { sheet: string; rows: Row[] }) { try { await this.connect() const sheet = this.client.sheetsByTitle[query.sheet] From 39147b80b627e0d26359f7aa176ed5fe3354e6ab Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 14 May 2024 11:23:50 +0100 Subject: [PATCH 09/10] PR comments. --- packages/backend-core/src/objectStore/utils.ts | 3 +++ packages/server/src/automations/automationUtils.ts | 7 +++++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/objectStore/utils.ts b/packages/backend-core/src/objectStore/utils.ts index 2edb075b92..30c2fefbf1 100644 --- a/packages/backend-core/src/objectStore/utils.ts +++ b/packages/backend-core/src/objectStore/utils.ts @@ -70,6 +70,9 @@ async function processUrlAttachment( throw new Error(`Unexpected response ${response.statusText}`) } const fallbackFilename = path.basename(new URL(attachment.url).pathname) + if (!response.body) { + throw new Error("No response received for attachment") + } return { filename: attachment.filename || fallbackFilename, content: stream.Readable.fromWeb(response.body as streamWeb.ReadableStream), diff --git a/packages/server/src/automations/automationUtils.ts b/packages/server/src/automations/automationUtils.ts index 88b279b771..cb09f860da 100644 --- a/packages/server/src/automations/automationUtils.ts +++ b/packages/server/src/automations/automationUtils.ts @@ -163,7 +163,10 @@ async function generateAttachmentRow(attachment: AutomationAttachment) { try { const { filename } = attachment - const extension = path.extname(filename).replaceAll(".", "") + let extension = path.extname(filename) + if (extension.startsWith(".")) { + extension = extension.substring(1, extension.length) + } const attachmentResult = await objectStore.processAutomationAttachment( attachment ) @@ -182,8 +185,8 @@ async function generateAttachmentRow(attachment: AutomationAttachment) { return { size, + extension, name: filename, - extension: extension, key: s3Key, } } catch (error) { From 7eb03623ef45c6ee8b410aaa9866163c3907dfcc Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Tue, 14 May 2024 11:13:07 +0000 Subject: [PATCH 10/10] Bump version to 2.26.2 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index b29f82c71a..1e49def198 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.26.1", + "version": "2.26.2", "npmClient": "yarn", "packages": [ "packages/*",