Merge branch 'master' of github.com:Budibase/budibase into develop

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.11-alpha.2",
+    "@budibase/types": "^2.2.11",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.11-alpha.2",
+    "@budibase/string-templates": "^2.2.11",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",
     "@spectrum-css/avatar": "3.0.2",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.11-alpha.2",
-    "@budibase/client": "2.2.11-alpha.2",
-    "@budibase/frontend-core": "2.2.11-alpha.2",
-    "@budibase/string-templates": "2.2.11-alpha.2",
+    "@budibase/bbui": "^2.2.11",
+    "@budibase/client": "^2.2.11",
+    "@budibase/frontend-core": "^2.2.11",
+    "@budibase/string-templates": "^2.2.11",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.2.11-alpha.2",
-    "@budibase/string-templates": "2.2.11-alpha.2",
-    "@budibase/types": "2.2.11-alpha.2",
+    "@budibase/backend-core": "^2.2.11",
+    "@budibase/string-templates": "^2.2.11",
+    "@budibase/types": "^2.2.11",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.11-alpha.2",
-    "@budibase/frontend-core": "2.2.11-alpha.2",
-    "@budibase/string-templates": "2.2.11-alpha.2",
+    "@budibase/bbui": "^2.2.11",
+    "@budibase/frontend-core": "^2.2.11",
+    "@budibase/string-templates": "^2.2.11",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.2.11-alpha.2",
+    "@budibase/bbui": "^2.2.11",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.11-alpha.2",
-    "@budibase/client": "2.2.11-alpha.2",
-    "@budibase/pro": "2.2.11-alpha.2",
-    "@budibase/string-templates": "2.2.11-alpha.2",
-    "@budibase/types": "2.2.11-alpha.2",
+    "@budibase/backend-core": "^2.2.11",
+    "@budibase/client": "^2.2.11",
+    "@budibase/pro": "2.2.11",
+    "@budibase/string-templates": "^2.2.11",
+    "@budibase/types": "^2.2.11",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/api/routes/tests/analytics.spec.js

@@ -1,5 +1,5 @@
 const setup = require("./utilities")
-const { events, constants, db } = require("@budibase/backend-core")
+const { events, constants } = require("@budibase/backend-core")
 
 describe("/static", () => {
   let request = setup.getRequest()

packages/server/src/sdk/app/backups/imports.ts

@@ -1,17 +1,16 @@
 import { db as dbCore, objectStore } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
+import { Database, Row } from "@budibase/types"
 import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
 import { budibaseTempDir } from "../../../utilities/budibaseDir"
 import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
 import { downloadTemplate } from "../../../utilities/fileSystem"
-import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
+import { ObjectStoreBuckets } from "../../../constants"
 import { join } from "path"
 import fs from "fs"
 import sdk from "../../"
 import {
   Automation,
   AutomationTriggerStepId,
-  CouchFindOptions,
   RowAttachment,
 } from "@budibase/types"
 const uuid = require("uuid/v4")
@@ -25,54 +24,45 @@ type TemplateType = {
   key?: string
 }
 
-async function updateAttachmentColumns(prodAppId: string, db: Database) {
+function rewriteAttachmentUrl(appId: string, attachment: RowAttachment) {
+  // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+  const urlParts = attachment.key.split("/")
+  // remove the app ID
+  urlParts.shift()
+  // add new app ID
+  urlParts.unshift(appId)
+  const key = urlParts.join("/")
+  return {
+    ...attachment,
+    key,
+    url: "", // calculated on retrieval using key
+  }
+}
+
+export async function updateAttachmentColumns(prodAppId: string, db: Database) {
   // iterate through attachment documents and update them
   const tables = await sdk.tables.getAllInternalTables(db)
+  let updatedRows: Row[] = []
   for (let table of tables) {
-    const attachmentCols: string[] = []
-    for (let [key, column] of Object.entries(table.schema)) {
-      if (column.type === FieldTypes.ATTACHMENT) {
-        attachmentCols.push(key)
-      }
-    }
-    // no attachment columns, nothing to do
-    if (attachmentCols.length === 0) {
-      continue
-    }
-    // use the CouchDB Mango query API to lookup rows that have attachments
-    const params: CouchFindOptions = {
-      selector: {
-        _id: {
-          $regex: `^${TABLE_ROW_PREFIX}`,
-        },
-      },
-    }
-    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
-    const { rows } = await dbCore.directCouchFind(db.name, params)
-    for (let row of rows) {
-      for (let column of attachmentCols) {
-        if (!Array.isArray(row[column])) {
-          continue
-        }
-        row[column] = row[column].map((attachment: RowAttachment) => {
-          // Key looks like: appId/attachments/file.csv
-          const urlParts = attachment.key.split("/")
-          // remove the app ID
-          urlParts.shift()
-          // add new app ID
-          urlParts.unshift(prodAppId)
-          const key = urlParts.join("/")
-          return {
-            ...attachment,
-            key,
-            url: "", // calculated on retrieval using key
+    const { rows, columns } = await sdk.rows.getRowsWithAttachments(
+      db.name,
+      table
+    )
+    updatedRows = updatedRows.concat(
+      rows.map(row => {
+        for (let column of columns) {
+          if (Array.isArray(row[column])) {
+            row[column] = row[column].map((attachment: RowAttachment) =>
+              rewriteAttachmentUrl(prodAppId, attachment)
+            )
           }
-        })
-      }
-    }
-    // write back the updated attachments
-    await db.bulkDocs(rows)
+        }
+        return row
+      })
+    )
   }
+  // write back the updated attachments
+  await db.bulkDocs(updatedRows)
 }
 
 async function updateAutomations(prodAppId: string, db: Database) {

packages/server/src/sdk/app/rows/attachments.ts

@@ -0,0 +1,60 @@
+import { CouchFindOptions, Table, Row } from "@budibase/types"
+import { db as dbCore } from "@budibase/backend-core"
+import { DocumentType, SEPARATOR } from "../../../db/utils"
+import { FieldTypes } from "../../../constants"
+
+// default limit - seems to work well for performance
+export const FIND_LIMIT = 25
+
+function generateAttachmentFindParams(
+  tableId: string,
+  attachmentCols: string[],
+  bookmark: null | string
+) {
+  const params: CouchFindOptions = {
+    selector: {
+      _id: {
+        $regex: `^${DocumentType.ROW}${SEPARATOR}${tableId}`,
+      },
+    },
+    limit: FIND_LIMIT,
+  }
+  attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+  if (bookmark) {
+    params.bookmark = bookmark
+  }
+  return params
+}
+
+export async function getRowsWithAttachments(appId: string, table: Table) {
+  // iterate through attachment documents and update them
+  const db = dbCore.getDB(appId)
+  const attachmentCols: string[] = []
+  for (let [key, column] of Object.entries(table.schema)) {
+    if (column.type === FieldTypes.ATTACHMENT) {
+      attachmentCols.push(key)
+    }
+  }
+  // no attachment columns, nothing to do
+  if (attachmentCols.length === 0) {
+    return { rows: [], columns: [] }
+  }
+  let bookmark: null | string = null,
+    rowsLength = 0,
+    rowList: Row[] = []
+  do {
+    const params = generateAttachmentFindParams(
+      table._id!,
+      attachmentCols,
+      bookmark
+    )
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const resp = await dbCore.directCouchFind(db.name, params)
+    bookmark = resp.bookmark
+    rowsLength = resp.rows.length
+    const rows = resp.rows
+    rowList = rowList.concat(rows)
+  } while (rowsLength === FIND_LIMIT)
+  // write back the updated attachments
+  return { rows: rowList, columns: attachmentCols }
+}

packages/server/src/sdk/app/rows/index.ts

@@ -0,0 +1,7 @@
+import * as attachments from "./attachments"
+import * as rows from "./rows"
+
+export default {
+  ...attachments,
+  ...rows,
+}

packages/server/src/sdk/app/rows/rows.ts

@@ -0,0 +1,18 @@
+import { db as dbCore, context } from "@budibase/backend-core"
+import { Database, Row } from "@budibase/types"
+import { getRowParams } from "../../../db/utils"
+
+export async function getAllInternalRows(appId?: string) {
+  let db: Database
+  if (appId) {
+    db = dbCore.getDB(appId)
+  } else {
+    db = context.getAppDB()
+  }
+  const response = await db.allDocs(
+    getRowParams(null, null, {
+      include_docs: true,
+    })
+  )
+  return response.rows.map(row => row.doc) as Row[]
+}

packages/server/src/sdk/index.ts

@@ -2,6 +2,7 @@ import { default as backups } from "./app/backups"
 import { default as tables } from "./app/tables"
 import { default as automations } from "./app/automations"
 import { default as applications } from "./app/applications"
+import { default as rows } from "./app/rows"
 import { default as users } from "./users"
 
 const sdk = {
@@ -9,6 +10,7 @@ const sdk = {
   tables,
   automations,
   applications,
+  rows,
   users,
 }
 

packages/server/src/sdk/tests/attachments.spec.ts

@@ -0,0 +1,76 @@
+import newid from "../../db/newid"
+
+const attachment = {
+  size: 73479,
+  name: "2022-12-14 11_11_44-.png",
+  url: "/prod-budi-app-assets/app_bbb/attachments/a.png",
+  extension: "png",
+  key: "app_bbb/attachments/a.png",
+}
+
+const row = {
+  _id: "ro_ta_aaa",
+  photo: [attachment],
+  otherCol: "string",
+}
+
+const table = {
+  _id: "ta_aaa",
+  name: "photos",
+  schema: {
+    photo: {
+      type: "attachment",
+      name: "photo",
+    },
+    otherCol: {
+      type: "string",
+      name: "otherCol",
+    },
+  },
+}
+
+jest.mock("@budibase/backend-core", () => {
+  const core = jest.requireActual("@budibase/backend-core")
+  return {
+    ...core,
+    db: {
+      ...core.db,
+      directCouchFind: jest.fn(),
+    },
+  }
+})
+
+import { db as dbCore } from "@budibase/backend-core"
+import sdk from "../index"
+
+describe("should be able to re-write attachment URLs", () => {
+  it("it should update URLs on a number of rows over the limit", async () => {
+    const db = dbCore.getDB("app_aaa")
+    await db.put(table)
+    const limit = 30
+    let rows = []
+    for (let i = 0; i < limit; i++) {
+      const rowToWrite = {
+        ...row,
+        _id: `${row._id}_${newid()}`,
+      }
+      const { rev } = await db.put(rowToWrite)
+      rows.push({
+        ...rowToWrite,
+        _rev: rev,
+      })
+    }
+
+    dbCore.directCouchFind
+      // @ts-ignore
+      .mockReturnValueOnce({ rows: rows.slice(0, 25), bookmark: "aaa" })
+      .mockReturnValueOnce({ rows: rows.slice(25, limit), bookmark: "bbb" })
+    await sdk.backups.updateAttachmentColumns(db.name, db)
+    const finalRows = await sdk.rows.getAllInternalRows(db.name)
+    for (let rowToCheck of finalRows) {
+      expect(rowToCheck.otherCol).toBe(row.otherCol)
+      expect(rowToCheck.photo[0].url).toBe("")
+      expect(rowToCheck.photo[0].key).toBe(`${db.name}/attachments/a.png`)
+    }
+  })
+})

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.2.11-alpha.2",
+  "version": "2.2.11",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.2.11-alpha.2",
-    "@budibase/pro": "2.2.11-alpha.2",
-    "@budibase/string-templates": "2.2.11-alpha.2",
-    "@budibase/types": "2.2.11-alpha.2",
+    "@budibase/backend-core": "^2.2.11",
+    "@budibase/pro": "2.2.11",
+    "@budibase/string-templates": "^2.2.11",
+    "@budibase/types": "^2.2.11",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",