diff --git a/lerna.json b/lerna.json index f9217ff0a5..c6c7f9e17c 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.2.11-alpha.2", + "version": "2.2.11", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 8bf00e14f9..3f2cbb5510 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -23,7 +23,7 @@ }, "dependencies": { "@budibase/nano": "10.1.1", - "@budibase/types": "2.2.11-alpha.2", + "@budibase/types": "^2.2.11", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index 77cd6fe7ed..d268de4061 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/string-templates": "2.2.11-alpha.2", + "@budibase/string-templates": "^2.2.11", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", "@spectrum-css/avatar": "3.0.2", diff --git a/packages/builder/package.json b/packages/builder/package.json index 453b5446eb..9c532dcb2f 100644 --- a/packages/builder/package.json +++ b/packages/builder/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/builder", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "license": "GPL-3.0", "private": true, "scripts": { @@ -71,10 +71,10 @@ } }, "dependencies": { - "@budibase/bbui": "2.2.11-alpha.2", - "@budibase/client": "2.2.11-alpha.2", - "@budibase/frontend-core": "2.2.11-alpha.2", - "@budibase/string-templates": "2.2.11-alpha.2", + "@budibase/bbui": "^2.2.11", + "@budibase/client": "^2.2.11", + "@budibase/frontend-core": "^2.2.11", + "@budibase/string-templates": "^2.2.11", "@sentry/browser": "5.19.1", "@spectrum-css/page": "^3.0.1", "@spectrum-css/vars": "^3.0.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 0dfb466565..2bfb020219 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/cli", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase CLI, for developers, self hosting and migrations.", "main": "src/index.js", "bin": { @@ -26,9 +26,9 @@ "outputPath": "build" }, "dependencies": { - "@budibase/backend-core": "2.2.11-alpha.2", - "@budibase/string-templates": "2.2.11-alpha.2", - "@budibase/types": "2.2.11-alpha.2", + "@budibase/backend-core": "^2.2.11", + "@budibase/string-templates": "^2.2.11", + "@budibase/types": "^2.2.11", "axios": "0.21.2", "chalk": "4.1.0", "cli-progress": "3.11.2", diff --git a/packages/client/package.json b/packages/client/package.json index ea1fa187cd..40f75ba9ee 100644 --- a/packages/client/package.json +++ b/packages/client/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/client", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "license": "MPL-2.0", "module": "dist/budibase-client.js", "main": "dist/budibase-client.js", @@ -19,9 +19,9 @@ "dev:builder": "rollup -cw" }, "dependencies": { - "@budibase/bbui": "2.2.11-alpha.2", - "@budibase/frontend-core": "2.2.11-alpha.2", - "@budibase/string-templates": "2.2.11-alpha.2", + "@budibase/bbui": "^2.2.11", + "@budibase/frontend-core": "^2.2.11", + "@budibase/string-templates": "^2.2.11", "@spectrum-css/button": "^3.0.3", "@spectrum-css/card": "^3.0.3", "@spectrum-css/divider": "^1.0.3", diff --git a/packages/frontend-core/package.json b/packages/frontend-core/package.json index 241e860102..34f332fc96 100644 --- a/packages/frontend-core/package.json +++ b/packages/frontend-core/package.json @@ -1,12 +1,12 @@ { "name": "@budibase/frontend-core", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase frontend core libraries used in builder and client", "author": "Budibase", "license": "MPL-2.0", "svelte": "src/index.js", "dependencies": { - "@budibase/bbui": "2.2.11-alpha.2", + "@budibase/bbui": "^2.2.11", "lodash": "^4.17.21", "svelte": "^3.46.2" } diff --git a/packages/sdk/package.json b/packages/sdk/package.json index 58bacc8e00..db65bf176d 100644 --- a/packages/sdk/package.json +++ b/packages/sdk/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/sdk", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase Public API SDK", "author": "Budibase", "license": "MPL-2.0", diff --git a/packages/server/package.json b/packages/server/package.json index e9613265f6..61589dc7e3 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/server", "email": "hi@budibase.com", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase Web Server", "main": "src/index.ts", "repository": { @@ -43,11 +43,11 @@ "license": "GPL-3.0", "dependencies": { "@apidevtools/swagger-parser": "10.0.3", - "@budibase/backend-core": "2.2.11-alpha.2", - "@budibase/client": "2.2.11-alpha.2", - "@budibase/pro": "2.2.11-alpha.2", - "@budibase/string-templates": "2.2.11-alpha.2", - "@budibase/types": "2.2.11-alpha.2", + "@budibase/backend-core": "^2.2.11", + "@budibase/client": "^2.2.11", + "@budibase/pro": "2.2.11", + "@budibase/string-templates": "^2.2.11", + "@budibase/types": "^2.2.11", "@bull-board/api": "3.7.0", "@bull-board/koa": "3.9.4", "@elastic/elasticsearch": "7.10.0", diff --git a/packages/server/src/api/routes/tests/analytics.spec.js b/packages/server/src/api/routes/tests/analytics.spec.js index 51196943e1..f8b3a6763f 100644 --- a/packages/server/src/api/routes/tests/analytics.spec.js +++ b/packages/server/src/api/routes/tests/analytics.spec.js @@ -1,5 +1,5 @@ const setup = require("./utilities") -const { events, constants, db } = require("@budibase/backend-core") +const { events, constants } = require("@budibase/backend-core") describe("/static", () => { let request = setup.getRequest() diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts index a4eec35361..b63d4c5a40 100644 --- a/packages/server/src/sdk/app/backups/imports.ts +++ b/packages/server/src/sdk/app/backups/imports.ts @@ -1,17 +1,16 @@ import { db as dbCore, objectStore } from "@budibase/backend-core" -import { Database } from "@budibase/types" +import { Database, Row } from "@budibase/types" import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils" import { budibaseTempDir } from "../../../utilities/budibaseDir" import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants" import { downloadTemplate } from "../../../utilities/fileSystem" -import { FieldTypes, ObjectStoreBuckets } from "../../../constants" +import { ObjectStoreBuckets } from "../../../constants" import { join } from "path" import fs from "fs" import sdk from "../../" import { Automation, AutomationTriggerStepId, - CouchFindOptions, RowAttachment, } from "@budibase/types" const uuid = require("uuid/v4") @@ -25,54 +24,45 @@ type TemplateType = { key?: string } -async function updateAttachmentColumns(prodAppId: string, db: Database) { +function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) { + // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv + const urlParts = attachment.key.split("/") + // remove the app ID + urlParts.shift() + // add new app ID + urlParts.unshift(appId) + const key = urlParts.join("/") + return { + ...attachment, + key, + url: "", // calculated on retrieval using key + } +} + +export async function updateAttachmentColumns(prodAppId: string, db: Database) { // iterate through attachment documents and update them const tables = await sdk.tables.getAllInternalTables(db) + let updatedRows: Row[] = [] for (let table of tables) { - const attachmentCols: string[] = [] - for (let [key, column] of Object.entries(table.schema)) { - if (column.type === FieldTypes.ATTACHMENT) { - attachmentCols.push(key) - } - } - // no attachment columns, nothing to do - if (attachmentCols.length === 0) { - continue - } - // use the CouchDB Mango query API to lookup rows that have attachments - const params: CouchFindOptions = { - selector: { - _id: { - $regex: `^${TABLE_ROW_PREFIX}`, - }, - }, - } - attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) - const { rows } = await dbCore.directCouchFind(db.name, params) - for (let row of rows) { - for (let column of attachmentCols) { - if (!Array.isArray(row[column])) { - continue - } - row[column] = row[column].map((attachment: RowAttachment) => { - // Key looks like: appId/attachments/file.csv - const urlParts = attachment.key.split("/") - // remove the app ID - urlParts.shift() - // add new app ID - urlParts.unshift(prodAppId) - const key = urlParts.join("/") - return { - ...attachment, - key, - url: "", // calculated on retrieval using key + const { rows, columns } = await sdk.rows.getRowsWithAttachments( + db.name, + table + ) + updatedRows = updatedRows.concat( + rows.map(row => { + for (let column of columns) { + if (Array.isArray(row[column])) { + row[column] = row[column].map((attachment: RowAttachment) => + rewriteAttachmentUrl(prodAppId, attachment) + ) } - }) - } - } - // write back the updated attachments - await db.bulkDocs(rows) + } + return row + }) + ) } + // write back the updated attachments + await db.bulkDocs(updatedRows) } async function updateAutomations(prodAppId: string, db: Database) { diff --git a/packages/server/src/sdk/app/rows/attachments.ts b/packages/server/src/sdk/app/rows/attachments.ts new file mode 100644 index 0000000000..67f58f8f2c --- /dev/null +++ b/packages/server/src/sdk/app/rows/attachments.ts @@ -0,0 +1,60 @@ +import { CouchFindOptions, Table, Row } from "@budibase/types" +import { db as dbCore } from "@budibase/backend-core" +import { DocumentType, SEPARATOR } from "../../../db/utils" +import { FieldTypes } from "../../../constants" + +// default limit - seems to work well for performance +export const FIND_LIMIT = 25 + +function generateAttachmentFindParams( + tableId: string, + attachmentCols: string[], + bookmark: null | string +) { + const params: CouchFindOptions = { + selector: { + _id: { + $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`, + }, + }, + limit: FIND_LIMIT, + } + attachmentCols.forEach(col => (params.selector[col] = { $exists: true })) + if (bookmark) { + params.bookmark = bookmark + } + return params +} + +export async function getRowsWithAttachments(appId: string, table: Table) { + // iterate through attachment documents and update them + const db = dbCore.getDB(appId) + const attachmentCols: string[] = [] + for (let [key, column] of Object.entries(table.schema)) { + if (column.type === FieldTypes.ATTACHMENT) { + attachmentCols.push(key) + } + } + // no attachment columns, nothing to do + if (attachmentCols.length === 0) { + return { rows: [], columns: [] } + } + let bookmark: null | string = null, + rowsLength = 0, + rowList: Row[] = [] + do { + const params = generateAttachmentFindParams( + table._id!, + attachmentCols, + bookmark + ) + // use the CouchDB Mango query API to lookup rows that have attachments + const resp = await dbCore.directCouchFind(db.name, params) + bookmark = resp.bookmark + rowsLength = resp.rows.length + const rows = resp.rows + rowList = rowList.concat(rows) + } while (rowsLength === FIND_LIMIT) + // write back the updated attachments + return { rows: rowList, columns: attachmentCols } +} diff --git a/packages/server/src/sdk/app/rows/index.ts b/packages/server/src/sdk/app/rows/index.ts new file mode 100644 index 0000000000..12a44ded67 --- /dev/null +++ b/packages/server/src/sdk/app/rows/index.ts @@ -0,0 +1,7 @@ +import * as attachments from "./attachments" +import * as rows from "./rows" + +export default { + ...attachments, + ...rows, +} diff --git a/packages/server/src/sdk/app/rows/rows.ts b/packages/server/src/sdk/app/rows/rows.ts new file mode 100644 index 0000000000..8709180f0b --- /dev/null +++ b/packages/server/src/sdk/app/rows/rows.ts @@ -0,0 +1,18 @@ +import { db as dbCore, context } from "@budibase/backend-core" +import { Database, Row } from "@budibase/types" +import { getRowParams } from "../../../db/utils" + +export async function getAllInternalRows(appId?: string) { + let db: Database + if (appId) { + db = dbCore.getDB(appId) + } else { + db = context.getAppDB() + } + const response = await db.allDocs( + getRowParams(null, null, { + include_docs: true, + }) + ) + return response.rows.map(row => row.doc) as Row[] +} diff --git a/packages/server/src/sdk/index.ts b/packages/server/src/sdk/index.ts index 19df8b4388..2ee40992c8 100644 --- a/packages/server/src/sdk/index.ts +++ b/packages/server/src/sdk/index.ts @@ -2,6 +2,7 @@ import { default as backups } from "./app/backups" import { default as tables } from "./app/tables" import { default as automations } from "./app/automations" import { default as applications } from "./app/applications" +import { default as rows } from "./app/rows" import { default as users } from "./users" const sdk = { @@ -9,6 +10,7 @@ const sdk = { tables, automations, applications, + rows, users, } diff --git a/packages/server/src/sdk/tests/attachments.spec.ts b/packages/server/src/sdk/tests/attachments.spec.ts new file mode 100644 index 0000000000..f1abb246f8 --- /dev/null +++ b/packages/server/src/sdk/tests/attachments.spec.ts @@ -0,0 +1,76 @@ +import newid from "../../db/newid" + +const attachment = { + size: 73479, + name: "2022-12-14 11_11_44-.png", + url: "/prod-budi-app-assets/app_bbb/attachments/a.png", + extension: "png", + key: "app_bbb/attachments/a.png", +} + +const row = { + _id: "ro_ta_aaa", + photo: [attachment], + otherCol: "string", +} + +const table = { + _id: "ta_aaa", + name: "photos", + schema: { + photo: { + type: "attachment", + name: "photo", + }, + otherCol: { + type: "string", + name: "otherCol", + }, + }, +} + +jest.mock("@budibase/backend-core", () => { + const core = jest.requireActual("@budibase/backend-core") + return { + ...core, + db: { + ...core.db, + directCouchFind: jest.fn(), + }, + } +}) + +import { db as dbCore } from "@budibase/backend-core" +import sdk from "../index" + +describe("should be able to re-write attachment URLs", () => { + it("it should update URLs on a number of rows over the limit", async () => { + const db = dbCore.getDB("app_aaa") + await db.put(table) + const limit = 30 + let rows = [] + for (let i = 0; i < limit; i++) { + const rowToWrite = { + ...row, + _id: `${row._id}_${newid()}`, + } + const { rev } = await db.put(rowToWrite) + rows.push({ + ...rowToWrite, + _rev: rev, + }) + } + + dbCore.directCouchFind + // @ts-ignore + .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" }) + .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" }) + await sdk.backups.updateAttachmentColumns(db.name, db) + const finalRows = await sdk.rows.getAllInternalRows(db.name) + for (let rowToCheck of finalRows) { + expect(rowToCheck.otherCol).toBe(row.otherCol) + expect(rowToCheck.photo[0].url).toBe("") + expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`) + } + }) +}) diff --git a/packages/string-templates/package.json b/packages/string-templates/package.json index f52d6a8e30..4aa04d25f7 100644 --- a/packages/string-templates/package.json +++ b/packages/string-templates/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/string-templates", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Handlebars wrapper for Budibase templating.", "main": "src/index.cjs", "module": "dist/bundle.mjs", diff --git a/packages/types/package.json b/packages/types/package.json index 05820ae203..d7a3055ba8 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/types", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase types", "main": "dist/index.js", "types": "dist/index.d.ts", diff --git a/packages/worker/package.json b/packages/worker/package.json index a6f6a105ea..21d65e52ae 100644 --- a/packages/worker/package.json +++ b/packages/worker/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/worker", "email": "hi@budibase.com", - "version": "2.2.11-alpha.2", + "version": "2.2.11", "description": "Budibase background service", "main": "src/index.ts", "repository": { @@ -36,10 +36,10 @@ "author": "Budibase", "license": "GPL-3.0", "dependencies": { - "@budibase/backend-core": "2.2.11-alpha.2", - "@budibase/pro": "2.2.11-alpha.2", - "@budibase/string-templates": "2.2.11-alpha.2", - "@budibase/types": "2.2.11-alpha.2", + "@budibase/backend-core": "^2.2.11", + "@budibase/pro": "2.2.11", + "@budibase/string-templates": "^2.2.11", + "@budibase/types": "^2.2.11", "@koa/router": "8.0.8", "@sentry/node": "6.17.7", "@techpass/passport-openidconnect": "0.3.2",