Merge branch 'master' into BUDI-8046/allow-syncing-existing-users

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.20.10",
+  "version": "2.20.11",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit ab324e35d855012bd0f49caa53c6dd765223c6fa
+Subproject commit de6d44c372a7f48ca0ce8c6c0c19311d4bc21646

packages/backend-core/src/objectStore/buckets/plugins.ts

@@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types"
 
 // URLS
 
-export function enrichPluginURLs(plugins: Plugin[]) {
+export function enrichPluginURLs(plugins?: Plugin[]): Plugin[] {
   if (!plugins || !plugins.length) {
     return []
   }

packages/builder/src/components/design/settings/controls/ButtonActionEditor/actions/ExportData.svelte

@@ -1,9 +1,9 @@
 <script>
-  import { Label, Select, Body, Multiselect } from "@budibase/bbui"
+  import { Label, Select, Body } from "@budibase/bbui"
-  import { findAllMatchingComponents, findComponent } from "helpers/components"
-  import { selectedScreen } from "stores/builder"
   import { onMount } from "svelte"
-  import { getDatasourceForProvider, getSchemaForDatasource } from "dataBinding"
+  import ColumnEditor from "../../ColumnEditor/ColumnEditor.svelte"
+  import { findAllMatchingComponents } from "helpers/components"
+  import { selectedScreen } from "stores/builder"
 
   export let parameters
 
@@ -18,37 +18,65 @@
     },
   ]
 
+  const DELIMITERS = [
+    {
+      label: ",",
+      value: ",",
+    },
+    {
+      label: ";",
+      value: ";",
+    },
+    {
+      label: ":",
+      value: ":",
+    },
+    {
+      label: "|",
+      value: "|",
+    },
+    {
+      label: "~",
+      value: "~",
+    },
+    {
+      label: "[tab]",
+      value: "\t",
+    },
+    {
+      label: "[space]",
+      value: " ",
+    },
+  ]
+
   $: tables = findAllMatchingComponents($selectedScreen?.props, component =>
     component._component.endsWith("table")
-  ).map(table => ({
+  )
-    label: table._instanceName,
-    value: table._id,
-  }))
   $: tableBlocks = findAllMatchingComponents(
     $selectedScreen?.props,
     component => component._component.endsWith("tableblock")
-  ).map(block => ({
+  )
-    label: block._instanceName,
+  $: components = tables.concat(tableBlocks)
-    value: `${block._id}-table`,
+  $: componentOptions = components.map(table => ({
+    label: table._instanceName,
+    value: table._component.includes("tableblock")
+      ? `${table._id}-table`
+      : table._id,
   }))
-  $: componentOptions = tables.concat(tableBlocks)
+  $: selectedTableId = parameters.tableComponentId?.includes("-")
-  $: columnOptions = getColumnOptions(parameters.tableComponentId)
+    ? parameters.tableComponentId.split("-")[0]
-
+    : parameters.tableComponentId
-  const getColumnOptions = tableId => {
+  $: selectedTable = components.find(
-    // Strip block suffix if block component
+    component => component._id === selectedTableId
-    if (tableId?.includes("-")) {
+  )
-      tableId = tableId.split("-")[0]
-    }
-    const selectedTable = findComponent($selectedScreen?.props, tableId)
-    const datasource = getDatasourceForProvider($selectedScreen, selectedTable)
-    const { schema } = getSchemaForDatasource($selectedScreen, datasource)
-    return Object.keys(schema || {})
-  }
 
   onMount(() => {
     if (!parameters.type) {
       parameters.type = "csv"
     }
+    if (!parameters.delimiter) {
+      parameters.delimiter = ","
+    }
   })
 </script>
 
@@ -67,13 +95,29 @@
       options={componentOptions}
       on:change={() => (parameters.columns = [])}
     />
+    <span />
     <Label small>Export as</Label>
     <Select bind:value={parameters.type} options={FORMATS} />
+    <Select
+      bind:value={parameters.delimiter}
+      placeholder={null}
+      options={DELIMITERS}
+      disabled={parameters.type !== "csv"}
+    />
     <Label small>Export columns</Label>
-    <Multiselect
+    <ColumnEditor
-      placeholder="All columns"
       bind:value={parameters.columns}
-      options={columnOptions}
+      allowCellEditing={false}
+      componentInstance={selectedTable}
+      on:change={e => {
+        const columns = e.detail
+        parameters.customHeaders = columns.reduce((headerMap, column) => {
+          return {
+            [column.name]: column.displayName,
+            ...headerMap,
+          }
+        }, {})
+      }}
     />
   </div>
 </div>
@@ -97,8 +141,8 @@
   .params {
     display: grid;
     column-gap: var(--spacing-xs);
-    row-gap: var(--spacing-s);
+    row-gap: var(--spacing-m);
-    grid-template-columns: 90px 1fr;
+    grid-template-columns: 90px 1fr 90px;
     align-items: center;
   }
 </style>

packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte

@@ -29,6 +29,12 @@
     allowLinks: true,
   })
 
+  $: {
+    value = (value || []).filter(
+      column => (schema || {})[column.name || column] !== undefined
+    )
+  }
+
   const getText = value => {
     if (!value?.length) {
       return "All columns"

packages/client/src/utils/buttonActions.js

@@ -341,7 +341,11 @@ const exportDataHandler = async action => {
         tableId: selection.tableId,
         rows: selection.selectedRows,
         format: action.parameters.type,
-        columns: action.parameters.columns,
+        columns: action.parameters.columns?.map(
+          column => column.name || column
+        ),
+        delimiter: action.parameters.delimiter,
+        customHeaders: action.parameters.customHeaders,
       })
       download(
         new Blob([data], { type: "text/plain" }),

packages/frontend-core/src/api/rows.js

@@ -89,13 +89,24 @@ export const buildRowEndpoints = API => ({
    * @param rows the array of rows to export
    * @param format the format to export (csv or json)
    * @param columns which columns to export (all if undefined)
+   * @param delimiter how values should be separated in a CSV (default is comma)
    */
-  exportRows: async ({ tableId, rows, format, columns, search }) => {
+  exportRows: async ({
+    tableId,
+    rows,
+    format,
+    columns,
+    search,
+    delimiter,
+    customHeaders,
+  }) => {
     return await API.post({
       url: `/api/${tableId}/rows/exportRows?format=${format}`,
       body: {
         rows,
         columns,
+        delimiter,
+        customHeaders,
         ...search,
       },
       parseResponse: async response => {

packages/server/scripts/test.sh

@@ -3,12 +3,12 @@ set -e
 
 if [[ -n $CI ]]
 then
-  export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot"
+  export NODE_OPTIONS="--max-old-space-size=4096 --no-node-snapshot $NODE_OPTIONS"
   echo "jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@"
   jest --coverage --maxWorkers=2 --forceExit --workerIdleMemoryLimit=2000MB --bail $@
 else
   # --maxWorkers performs better in development
-  export NODE_OPTIONS="--no-node-snapshot"
+  export NODE_OPTIONS="--no-node-snapshot $NODE_OPTIONS"
   echo "jest --coverage --maxWorkers=2 --forceExit $@"
   jest --coverage --maxWorkers=2 --forceExit $@
 fi

packages/server/src/api/controllers/application.ts

@@ -47,6 +47,9 @@ import {
   PlanType,
   Screen,
   UserCtx,
+  CreateAppRequest,
+  FetchAppDefinitionResponse,
+  FetchAppPackageResponse,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
 import sdk from "../../sdk"
@@ -58,23 +61,23 @@ import * as appMigrations from "../../appMigrations"
 async function getLayouts() {
   const db = context.getAppDB()
   return (
-    await db.allDocs(
+    await db.allDocs<Layout>(
       getLayoutParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map((row: any) => row.doc)
+  ).rows.map(row => row.doc!)
 }
 
 async function getScreens() {
   const db = context.getAppDB()
   return (
-    await db.allDocs(
+    await db.allDocs<Screen>(
       getScreenParams(null, {
         include_docs: true,
       })
     )
-  ).rows.map((row: any) => row.doc)
+  ).rows.map(row => row.doc!)
 }
 
 function getUserRoleId(ctx: UserCtx) {
@@ -116,8 +119,8 @@ function checkAppName(
 }
 
 interface AppTemplate {
-  templateString: string
+  templateString?: string
-  useTemplate: string
+  useTemplate?: string
   file?: {
     type: string
     path: string
@@ -174,14 +177,16 @@ export const addSampleData = async (ctx: UserCtx) => {
   ctx.status = 200
 }
 
-export async function fetch(ctx: UserCtx) {
+export async function fetch(ctx: UserCtx<void, App[]>) {
   ctx.body = await sdk.applications.fetch(
     ctx.query.status as AppStatus,
     ctx.user
   )
 }
 
-export async function fetchAppDefinition(ctx: UserCtx) {
+export async function fetchAppDefinition(
+  ctx: UserCtx<void, FetchAppDefinitionResponse>
+) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
   const accessController = new roles.AccessController()
@@ -196,10 +201,12 @@ export async function fetchAppDefinition(ctx: UserCtx) {
   }
 }
 
-export async function fetchAppPackage(ctx: UserCtx) {
+export async function fetchAppPackage(
+  ctx: UserCtx<void, FetchAppPackageResponse>
+) {
   const db = context.getAppDB()
   const appId = context.getAppId()
-  let application = await db.get<any>(DocumentType.APP_METADATA)
+  let application = await db.get<App>(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
   let screens = await getScreens()
   const license = await licensing.cache.getCachedLicense()
@@ -231,17 +238,21 @@ export async function fetchAppPackage(ctx: UserCtx) {
   }
 }
 
-async function performAppCreate(ctx: UserCtx) {
+async function performAppCreate(ctx: UserCtx<CreateAppRequest, App>) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
-  const name = ctx.request.body.name,
+  const {
-    possibleUrl = ctx.request.body.url,
+    name,
-    encryptionPassword = ctx.request.body.encryptionPassword
+    url,
+    encryptionPassword,
+    useTemplate,
+    templateKey,
+    templateString,
+  } = ctx.request.body
 
   checkAppName(ctx, apps, name)
-  const url = sdk.applications.getAppUrl({ name, url: possibleUrl })
+  const appUrl = sdk.applications.getAppUrl({ name, url })
-  checkAppUrl(ctx, apps, url)
+  checkAppUrl(ctx, apps, appUrl)
 
-  const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig: AppTemplate = {
     useTemplate,
     key: templateKey,
@@ -268,7 +279,7 @@ async function performAppCreate(ctx: UserCtx) {
       version: envCore.VERSION,
       componentLibraries: ["@budibase/standard-components"],
       name: name,
-      url: url,
+      url: appUrl,
       template: templateKey,
       instance,
       tenantId: tenancy.getTenantId(),
@@ -420,7 +431,9 @@ export async function create(ctx: UserCtx) {
 
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
-export async function update(ctx: UserCtx) {
+export async function update(
+  ctx: UserCtx<{ name?: string; url?: string }, App>
+) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation
   const name = ctx.request.body.name,
@@ -493,7 +506,7 @@ export async function revertClient(ctx: UserCtx) {
   const revertedToVersion = application.revertableVersion
   const appPackageUpdates = {
     version: revertedToVersion,
-    revertableVersion: null,
+    revertableVersion: undefined,
   }
   const app = await updateAppPackage(appPackageUpdates, ctx.params.appId)
   await events.app.versionReverted(app, currentVersion, revertedToVersion)
@@ -613,12 +626,15 @@ export async function importToApp(ctx: UserCtx) {
   ctx.body = { message: "app updated" }
 }
 
-export async function updateAppPackage(appPackage: any, appId: any) {
+export async function updateAppPackage(
+  appPackage: Partial<App>,
+  appId: string
+) {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get<App>(DocumentType.APP_METADATA)
 
-    const newAppPackage = { ...application, ...appPackage }
+    const newAppPackage: App = { ...application, ...appPackage }
     if (appPackage._rev !== application._rev) {
       newAppPackage._rev = application._rev
     }

packages/server/src/api/controllers/row/index.ts

@@ -223,7 +223,8 @@ export const exportRows = async (
 
   const format = ctx.query.format
 
-  const { rows, columns, query, sort, sortOrder } = ctx.request.body
+  const { rows, columns, query, sort, sortOrder, delimiter, customHeaders } =
+    ctx.request.body
   if (typeof format !== "string" || !exporters.isFormat(format)) {
     ctx.throw(
       400,
@@ -241,6 +242,8 @@ export const exportRows = async (
     query,
     sort,
     sortOrder,
+    delimiter,
+    customHeaders,
   })
   ctx.attachment(fileName)
   ctx.body = apiFileReturn(content)

packages/server/src/api/controllers/view/exporters.ts

@@ -1,7 +1,19 @@
 import { Row, TableSchema } from "@budibase/types"
 
-export function csv(headers: string[], rows: Row[]) {
+function getHeaders(
-  let csv = headers.map(key => `"${key}"`).join(",")
+  headers: string[],
+  customHeaders: { [key: string]: string }
+) {
+  return headers.map(header => `"${customHeaders[header] || header}"`)
+}
+
+export function csv(
+  headers: string[],
+  rows: Row[],
+  delimiter: string = ",",
+  customHeaders: { [key: string]: string } = {}
+) {
+  let csv = getHeaders(headers, customHeaders).join(delimiter)
 
   for (let row of rows) {
     csv = `${csv}\n${headers
@@ -15,7 +27,7 @@ export function csv(headers: string[], rows: Row[]) {
             : ""
         return val.trim()
       })
-      .join(",")}`
+      .join(delimiter)}`
   }
   return csv
 }

packages/server/src/api/routes/application.ts

@@ -4,7 +4,6 @@ import * as deploymentController from "../controllers/deploy"
 import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import { applicationValidator } from "./utils/validators"
-import { importToApp } from "../controllers/application"
 
 const router: Router = new Router()
 

packages/server/src/api/routes/tests/application.spec.ts

@@ -11,65 +11,54 @@ jest.mock("../../../utilities/redis", () => ({
   checkDebounce: jest.fn(),
   shutdown: jest.fn(),
 }))
-import { clearAllApps, checkBuilderEndpoint } from "./utilities/TestFunctions"
+import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 import { AppStatus } from "../../../db/utils"
 import { events, utils, context } from "@budibase/backend-core"
 import env from "../../../environment"
-
+import type { App } from "@budibase/types"
-jest.setTimeout(15000)
+import tk from "timekeeper"
 
 describe("/applications", () => {
-  let request = setup.getRequest()
   let config = setup.getConfig()
+  let app: App
 
   afterAll(setup.afterAll)
-
+  beforeAll(async () => await config.init())
-  beforeAll(async () => {
-    await config.init()
-  })
 
   beforeEach(async () => {
+    app = await config.api.application.create({ name: utils.newid() })
+    const deployment = await config.api.application.publish(app.appId)
+    expect(deployment.status).toBe("SUCCESS")
     jest.clearAllMocks()
   })
 
   describe("create", () => {
     it("creates empty app", async () => {
-      const res = await request
+      const app = await config.api.application.create({ name: utils.newid() })
-        .post("/api/applications")
+      expect(app._id).toBeDefined()
-        .field("name", utils.newid())
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
       expect(events.app.created).toBeCalledTimes(1)
     })
 
     it("creates app from template", async () => {
-      const res = await request
+      const app = await config.api.application.create({
-        .post("/api/applications")
+        name: utils.newid(),
-        .field("name", utils.newid())
+        useTemplate: "true",
-        .field("useTemplate", "true")
+        templateKey: "test",
-        .field("templateKey", "test")
+        templateString: "{}",
-        .field("templateString", "{}") // override the file download
+      })
-        .set(config.defaultHeaders())
+      expect(app._id).toBeDefined()
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
       expect(events.app.created).toBeCalledTimes(1)
       expect(events.app.templateImported).toBeCalledTimes(1)
     })
 
     it("creates app from file", async () => {
-      const res = await request
+      const app = await config.api.application.create({
-        .post("/api/applications")
+        name: utils.newid(),
-        .field("name", utils.newid())
+        useTemplate: "true",
-        .field("useTemplate", "true")
+        templateFile: "src/api/routes/tests/data/export.txt",
-        .set(config.defaultHeaders())
+      })
-        .attach("templateFile", "src/api/routes/tests/data/export.txt")
+      expect(app._id).toBeDefined()
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._id).toBeDefined()
       expect(events.app.created).toBeCalledTimes(1)
       expect(events.app.fileImported).toBeCalledTimes(1)
     })
@@ -84,24 +73,21 @@ describe("/applications", () => {
     })
 
     it("migrates navigation settings from old apps", async () => {
-      const res = await request
+      const app = await config.api.application.create({
-        .post("/api/applications")
+        name: utils.newid(),
-        .field("name", "Old App")
+        useTemplate: "true",
-        .field("useTemplate", "true")
+        templateFile: "src/api/routes/tests/data/old-app.txt",
-        .set(config.defaultHeaders())
+      })
-        .attach("templateFile", "src/api/routes/tests/data/old-app.txt")
+      expect(app._id).toBeDefined()
-        .expect("Content-Type", /json/)
+      expect(app.navigation).toBeDefined()
-        .expect(200)
+      expect(app.navigation!.hideLogo).toBe(true)
-      expect(res.body._id).toBeDefined()
+      expect(app.navigation!.title).toBe("Custom Title")
-      expect(res.body.navigation).toBeDefined()
+      expect(app.navigation!.hideLogo).toBe(true)
-      expect(res.body.navigation.hideLogo).toBe(true)
+      expect(app.navigation!.navigation).toBe("Left")
-      expect(res.body.navigation.title).toBe("Custom Title")
+      expect(app.navigation!.navBackground).toBe(
-      expect(res.body.navigation.hideLogo).toBe(true)
-      expect(res.body.navigation.navigation).toBe("Left")
-      expect(res.body.navigation.navBackground).toBe(
         "var(--spectrum-global-color-blue-600)"
       )
-      expect(res.body.navigation.navTextColor).toBe(
+      expect(app.navigation!.navTextColor).toBe(
         "var(--spectrum-global-color-gray-50)"
       )
       expect(events.app.created).toBeCalledTimes(1)
@@ -110,164 +96,106 @@ describe("/applications", () => {
   })
 
   describe("fetch", () => {
-    beforeEach(async () => {
-      // Clean all apps but the onde from config
-      await clearAllApps(config.getTenantId(), [config.getAppId()!])
-    })
-
     it("lists all applications", async () => {
-      await config.createApp("app1")
+      const apps = await config.api.application.fetch({ status: AppStatus.DEV })
-      await config.createApp("app2")
+      expect(apps.length).toBeGreaterThan(0)
-
-      const res = await request
-        .get(`/api/applications?status=${AppStatus.DEV}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-
-      // two created apps + the inited app
-      expect(res.body.length).toBe(3)
     })
   })
 
   describe("fetchAppDefinition", () => {
     it("should be able to get an apps definition", async () => {
-      const res = await request
+      const res = await config.api.application.getDefinition(app.appId)
-        .get(`/api/applications/${config.getAppId()}/definition`)
+      expect(res.libraries.length).toEqual(1)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.libraries.length).toEqual(1)
     })
   })
 
   describe("fetchAppPackage", () => {
     it("should be able to fetch the app package", async () => {
-      const res = await request
+      const res = await config.api.application.getAppPackage(app.appId)
-        .get(`/api/applications/${config.getAppId()}/appPackage`)
+      expect(res.application).toBeDefined()
-        .set(config.defaultHeaders())
+      expect(res.application.appId).toEqual(config.getAppId())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.application).toBeDefined()
-      expect(res.body.application.appId).toEqual(config.getAppId())
     })
   })
 
   describe("update", () => {
     it("should be able to update the app package", async () => {
-      const res = await request
+      const updatedApp = await config.api.application.update(app.appId, {
-        .put(`/api/applications/${config.getAppId()}`)
+        name: "TEST_APP",
-        .send({
+      })
-          name: "TEST_APP",
+      expect(updatedApp._rev).toBeDefined()
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body._rev).toBeDefined()
       expect(events.app.updated).toBeCalledTimes(1)
     })
   })
 
   describe("publish", () => {
     it("should publish app with dev app ID", async () => {
-      const appId = config.getAppId()
+      await config.api.application.publish(app.appId)
-      await request
-        .post(`/api/applications/${appId}/publish`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       expect(events.app.published).toBeCalledTimes(1)
     })
 
     it("should publish app with prod app ID", async () => {
-      const appId = config.getProdAppId()
+      await config.api.application.publish(app.appId.replace("_dev", ""))
-      await request
-        .post(`/api/applications/${appId}/publish`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       expect(events.app.published).toBeCalledTimes(1)
     })
   })
 
   describe("manage client library version", () => {
     it("should be able to update the app client library version", async () => {
-      await request
+      await config.api.application.updateClient(app.appId)
-        .post(`/api/applications/${config.getAppId()}/client/update`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       expect(events.app.versionUpdated).toBeCalledTimes(1)
     })
 
     it("should be able to revert the app client library version", async () => {
-      // We need to first update the version so that we can then revert
+      await config.api.application.updateClient(app.appId)
-      await request
+      await config.api.application.revertClient(app.appId)
-        .post(`/api/applications/${config.getAppId()}/client/update`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      await request
-        .post(`/api/applications/${config.getAppId()}/client/revert`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       expect(events.app.versionReverted).toBeCalledTimes(1)
     })
   })
 
   describe("edited at", () => {
-    it("middleware should set edited at", async () => {
+    it("middleware should set updatedAt", async () => {
-      const headers = config.defaultHeaders()
+      const app = await tk.withFreeze(
-      headers["referer"] = `/${config.getAppId()}/test`
+        "2021-01-01",
-      const res = await request
+        async () => await config.api.application.create({ name: utils.newid() })
-        .put(`/api/applications/${config.getAppId()}`)
+      )
-        .send({
+      expect(app.updatedAt).toEqual("2021-01-01T00:00:00.000Z")
-          name: "UPDATED_NAME",
+
-        })
+      const updatedApp = await tk.withFreeze(
-        .set(headers)
+        "2021-02-01",
-        .expect("Content-Type", /json/)
+        async () =>
-        .expect(200)
+          await config.api.application.update(app.appId, {
-      expect(res.body._rev).toBeDefined()
+            name: "UPDATED_NAME",
-      // retrieve the app to check it
+          })
-      const getRes = await request
+      )
-        .get(`/api/applications/${config.getAppId()}/appPackage`)
+      expect(updatedApp._rev).toBeDefined()
-        .set(headers)
+      expect(updatedApp.updatedAt).toEqual("2021-02-01T00:00:00.000Z")
-        .expect("Content-Type", /json/)
+
-        .expect(200)
+      const fetchedApp = await config.api.application.get(app.appId)
-      expect(getRes.body.application.updatedAt).toBeDefined()
+      expect(fetchedApp.updatedAt).toEqual("2021-02-01T00:00:00.000Z")
     })
   })
 
   describe("sync", () => {
     it("app should sync correctly", async () => {
-      const res = await request
+      const { message } = await config.api.application.sync(app.appId)
-        .post(`/api/applications/${config.getAppId()}/sync`)
+      expect(message).toEqual("App sync completed successfully.")
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual("App sync completed successfully.")
     })
 
     it("app should not sync if production", async () => {
-      const res = await request
+      const { message } = await config.api.application.sync(
-        .post(`/api/applications/app_123456/sync`)
+        app.appId.replace("_dev", ""),
-        .set(config.defaultHeaders())
+        { statusCode: 400 }
-        .expect("Content-Type", /json/)
+      )
-        .expect(400)
+
-      expect(res.body.message).toEqual(
+      expect(message).toEqual(
         "This action cannot be performed for production apps"
       )
     })
 
     it("app should not sync if sync is disabled", async () => {
       env._set("DISABLE_AUTO_PROD_APP_SYNC", true)
-      const res = await request
+      const { message } = await config.api.application.sync(app.appId)
-        .post(`/api/applications/${config.getAppId()}/sync`)
+      expect(message).toEqual(
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
-      expect(res.body.message).toEqual(
         "App sync disabled. You can reenable with the DISABLE_AUTO_PROD_APP_SYNC environment variable."
       )
       env._set("DISABLE_AUTO_PROD_APP_SYNC", false)
@@ -275,51 +203,26 @@ describe("/applications", () => {
   })
 
   describe("unpublish", () => {
-    beforeEach(async () => {
-      // We want to republish as the unpublish will delete the prod app
-      await config.publish()
-    })
-
     it("should unpublish app with dev app ID", async () => {
-      const appId = config.getAppId()
+      await config.api.application.unpublish(app.appId)
-      await request
-        .post(`/api/applications/${appId}/unpublish`)
-        .set(config.defaultHeaders())
-        .expect(204)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
 
     it("should unpublish app with prod app ID", async () => {
-      const appId = config.getProdAppId()
+      await config.api.application.unpublish(app.appId.replace("_dev", ""))
-      await request
-        .post(`/api/applications/${appId}/unpublish`)
-        .set(config.defaultHeaders())
-        .expect(204)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
   })
 
   describe("delete", () => {
     it("should delete published app and dev apps with dev app ID", async () => {
-      await config.createApp("to-delete")
+      await config.api.application.delete(app.appId)
-      const appId = config.getAppId()
-      await request
-        .delete(`/api/applications/${appId}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       expect(events.app.deleted).toBeCalledTimes(1)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
 
     it("should delete published app and dev app with prod app ID", async () => {
-      await config.createApp("to-delete")
+      await config.api.application.delete(app.appId.replace("_dev", ""))
-      const appId = config.getProdAppId()
-      await request
-        .delete(`/api/applications/${appId}`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
       expect(events.app.deleted).toBeCalledTimes(1)
       expect(events.app.unpublished).toBeCalledTimes(1)
     })
@@ -327,28 +230,18 @@ describe("/applications", () => {
 
   describe("POST /api/applications/:appId/sync", () => {
     it("should not sync automation logs", async () => {
-      // setup the apps
-      await config.createApp("testing-auto-logs")
       const automation = await config.createAutomation()
-      await config.publish()
+      await context.doInAppContext(app.appId, () =>
-      await context.doInAppContext(config.getProdAppId(), () => {
+        config.createAutomationLog(automation)
-        return config.createAutomationLog(automation)
+      )
-      })
 
-      // do the sync
+      await config.api.application.sync(app.appId)
-      const appId = config.getAppId()
-      await request
-        .post(`/api/applications/${appId}/sync`)
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
 
       // does exist in prod
       const prodLogs = await config.getAutomationLogs()
       expect(prodLogs.data.length).toBe(1)
 
-      // delete prod app so we revert to dev log search
+      await config.api.application.unpublish(app.appId)
-      await config.unpublish()
 
       // doesn't exist in dev
       const devLogs = await config.getAutomationLogs()

packages/server/src/sdk/app/rows/search.ts

@@ -36,11 +36,13 @@ export async function search(options: SearchParams): Promise<{
 export interface ExportRowsParams {
   tableId: string
   format: Format
+  delimiter?: string
   rowIds?: string[]
   columns?: string[]
   query?: SearchFilters
   sort?: string
   sortOrder?: SortOrder
+  customHeaders?: { [key: string]: string }
 }
 
 export interface ExportRowsResult {

packages/server/src/sdk/app/rows/search/external.ts

@@ -101,7 +101,17 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, columns, rowIds, query, sort, sortOrder } = options
+  const {
+    tableId,
+    format,
+    columns,
+    rowIds,
+    query,
+    sort,
+    sortOrder,
+    delimiter,
+    customHeaders,
+  } = options
   const { datasourceId, tableName } = breakExternalTableId(tableId)
 
   let requestQuery: SearchFilters = {}
@@ -153,12 +163,17 @@ export async function exportRows(
     rows = result.rows
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns)
+  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
 
   let content: string
   switch (format) {
     case exporters.Format.CSV:
-      content = exporters.csv(headers ?? Object.keys(schema), exportRows)
+      content = exporters.csv(
+        headers ?? Object.keys(schema),
+        exportRows,
+        delimiter,
+        customHeaders
+      )
       break
     case exporters.Format.JSON:
       content = exporters.json(exportRows)

packages/server/src/sdk/app/rows/search/internal.ts

@@ -84,7 +84,17 @@ export async function search(options: SearchParams) {
 export async function exportRows(
   options: ExportRowsParams
 ): Promise<ExportRowsResult> {
-  const { tableId, format, rowIds, columns, query, sort, sortOrder } = options
+  const {
+    tableId,
+    format,
+    rowIds,
+    columns,
+    query,
+    sort,
+    sortOrder,
+    delimiter,
+    customHeaders,
+  } = options
   const db = context.getAppDB()
   const table = await sdk.tables.getTable(tableId)
 
@@ -124,11 +134,16 @@ export async function exportRows(
     rows = result
   }
 
-  let exportRows = cleanExportRows(rows, schema, format, columns)
+  let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
   if (format === Format.CSV) {
     return {
       fileName: "export.csv",
-      content: csv(headers ?? Object.keys(rows[0]), exportRows),
+      content: csv(
+        headers ?? Object.keys(rows[0]),
+        exportRows,
+        delimiter,
+        customHeaders
+      ),
     }
   } else if (format === Format.JSON) {
     return {

packages/server/src/sdk/app/rows/utils.ts

@@ -16,7 +16,8 @@ export function cleanExportRows(
   rows: any[],
   schema: TableSchema,
   format: string,
-  columns?: string[]
+  columns?: string[],
+  customHeaders: { [key: string]: string } = {}
 ) {
   let cleanRows = [...rows]
 
@@ -44,11 +45,27 @@ export function cleanExportRows(
         }
       }
     }
+  } else if (format === Format.JSON) {
+    // Replace row keys with custom headers
+    for (let row of cleanRows) {
+      renameKeys(customHeaders, row)
+    }
   }
 
   return cleanRows
 }
 
+function renameKeys(keysMap: { [key: string]: any }, row: any) {
+  for (const key in keysMap) {
+    Object.defineProperty(
+      row,
+      keysMap[key],
+      Object.getOwnPropertyDescriptor(row, key) || {}
+    )
+    delete row[key]
+  }
+}
+
 function isForeignKey(key: string, table: Table) {
   const relationships = Object.values(table.schema).filter(isRelationshipColumn)
   return relationships.some(

packages/server/src/tests/utilities/api/application.ts

@@ -1,17 +1,96 @@
 import { Response } from "supertest"
-import { App } from "@budibase/types"
+import {
+  App,
+  type CreateAppRequest,
+  type FetchAppDefinitionResponse,
+  type FetchAppPackageResponse,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
+import { AppStatus } from "../../../db/utils"
+import { constants } from "@budibase/backend-core"
 
 export class ApplicationAPI extends TestAPI {
   constructor(config: TestConfiguration) {
     super(config)
   }
 
+  create = async (app: CreateAppRequest): Promise<App> => {
+    const request = this.request
+      .post("/api/applications")
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    for (const key of Object.keys(app)) {
+      request.field(key, (app as any)[key])
+    }
+
+    if (app.templateFile) {
+      request.attach("templateFile", app.templateFile)
+    }
+
+    const result = await request
+
+    if (result.statusCode !== 200) {
+      throw new Error(JSON.stringify(result.body))
+    }
+
+    return result.body as App
+  }
+
+  delete = async (appId: string): Promise<void> => {
+    await this.request
+      .delete(`/api/applications/${appId}`)
+      .set(this.config.defaultHeaders())
+      .expect(200)
+  }
+
+  publish = async (
+    appId: string
+  ): Promise<{ _id: string; status: string; appUrl: string }> => {
+    // While the publish endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
+    const result = await this.request
+      .post(`/api/applications/${appId}/publish`)
+      .set(headers)
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as { _id: string; status: string; appUrl: string }
+  }
+
+  unpublish = async (appId: string): Promise<void> => {
+    await this.request
+      .post(`/api/applications/${appId}/unpublish`)
+      .set(this.config.defaultHeaders())
+      .expect(204)
+  }
+
+  sync = async (
+    appId: string,
+    { statusCode }: { statusCode: number } = { statusCode: 200 }
+  ): Promise<{ message: string }> => {
+    const result = await this.request
+      .post(`/api/applications/${appId}/sync`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(statusCode)
+    return result.body
+  }
+
   getRaw = async (appId: string): Promise<Response> => {
+    // While the appPackage endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
     const result = await this.request
       .get(`/api/applications/${appId}/appPackage`)
-      .set(this.config.defaultHeaders())
+      .set(headers)
       .expect("Content-Type", /json/)
       .expect(200)
     return result
@@ -21,4 +100,94 @@ export class ApplicationAPI extends TestAPI {
     const result = await this.getRaw(appId)
     return result.body.application as App
   }
+
+  getDefinition = async (
+    appId: string
+  ): Promise<FetchAppDefinitionResponse> => {
+    const result = await this.request
+      .get(`/api/applications/${appId}/definition`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as FetchAppDefinitionResponse
+  }
+
+  getAppPackage = async (appId: string): Promise<FetchAppPackageResponse> => {
+    const result = await this.request
+      .get(`/api/applications/${appId}/appPackage`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body
+  }
+
+  update = async (
+    appId: string,
+    app: { name?: string; url?: string }
+  ): Promise<App> => {
+    const request = this.request
+      .put(`/api/applications/${appId}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+
+    for (const key of Object.keys(app)) {
+      request.field(key, (app as any)[key])
+    }
+
+    const result = await request
+
+    if (result.statusCode !== 200) {
+      throw new Error(JSON.stringify(result.body))
+    }
+
+    return result.body as App
+  }
+
+  updateClient = async (appId: string): Promise<void> => {
+    // While the updateClient endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
+    const response = await this.request
+      .post(`/api/applications/${appId}/client/update`)
+      .set(headers)
+      .expect("Content-Type", /json/)
+
+    if (response.statusCode !== 200) {
+      throw new Error(JSON.stringify(response.body))
+    }
+  }
+
+  revertClient = async (appId: string): Promise<void> => {
+    // While the revertClient endpoint does take an :appId parameter, it doesn't
+    // use it. It uses the appId from the context.
+    let headers = {
+      ...this.config.defaultHeaders(),
+      [constants.Header.APP_ID]: appId,
+    }
+    const response = await this.request
+      .post(`/api/applications/${appId}/client/revert`)
+      .set(headers)
+      .expect("Content-Type", /json/)
+
+    if (response.statusCode !== 200) {
+      throw new Error(JSON.stringify(response.body))
+    }
+  }
+
+  fetch = async ({ status }: { status?: AppStatus } = {}): Promise<App[]> => {
+    let query = []
+    if (status) {
+      query.push(`status=${status}`)
+    }
+
+    const result = await this.request
+      .get(`/api/applications${query.length ? `?${query.join("&")}` : ""}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+    return result.body as App[]
+  }
 }

packages/types/src/api/web/app/rows.ts

@@ -37,6 +37,8 @@ export interface ExportRowsRequest {
   query?: SearchFilters
   sort?: string
   sortOrder?: SortOrder
+  delimiter?: string
+  customHeaders?: { [key: string]: string }
 }
 
 export type ExportRowsResponse = ReadStream

packages/types/src/api/web/application.ts

@@ -0,0 +1,29 @@
+import type { PlanType } from "../../sdk"
+import type { Layout, App, Screen } from "../../documents"
+
+export interface CreateAppRequest {
+  name: string
+  url?: string
+  useTemplate?: string
+  templateName?: string
+  templateKey?: string
+  templateFile?: string
+  includeSampleData?: boolean
+  encryptionPassword?: string
+  templateString?: string
+}
+
+export interface FetchAppDefinitionResponse {
+  layouts: Layout[]
+  screens: Screen[]
+  libraries: string[]
+}
+
+export interface FetchAppPackageResponse {
+  application: App
+  licenseType: PlanType
+  screens: Screen[]
+  layouts: Layout[]
+  clientLibPath: string
+  hasLock: boolean
+}

packages/types/src/api/web/index.ts

@@ -1,3 +1,4 @@
+export * from "./application"
 export * from "./analytics"
 export * from "./auth"
 export * from "./user"

packages/types/src/documents/app/app.ts

@@ -1,4 +1,4 @@
-import { User, Document } from "../"
+import { User, Document, Plugin } from "../"
 import { SocketSession } from "../../sdk"
 
 export type AppMetadataErrors = { [key: string]: string[] }
@@ -24,6 +24,8 @@ export interface App extends Document {
   icon?: AppIcon
   features?: AppFeatures
   automations?: AutomationSettings
+  usedPlugins?: Plugin[]
+  upgradableVersion?: string
 }
 
 export interface AppInstance {

packages/worker/scripts/test.sh

@@ -4,10 +4,10 @@ set -e
 if [[ -n $CI ]]
 then
   # Running in ci, where resources are limited
-  echo "jest --coverage --maxWorkers=2  --forceExit --bail"
+  echo "jest --coverage --maxWorkers=2  --forceExit --bail $@"
-  jest --coverage --maxWorkers=2  --forceExit --bail
+  jest --coverage --maxWorkers=2  --forceExit --bail $@
 else
   # --maxWorkers performs better in development
-  echo "jest --coverage --maxWorkers=2 --forceExit"
+  echo "jest --coverage --maxWorkers=2 --forceExit $@"
-  jest --coverage --maxWorkers=2 --forceExit
+  jest --coverage --maxWorkers=2 --forceExit $@
 fi

qa-core/src/internal-api/api/apis/AppAPI.ts

@@ -1,11 +1,10 @@
-import { App } from "@budibase/types"
+import { App, CreateAppRequest } from "@budibase/types"
 import { Response } from "node-fetch"
 import {
   RouteConfig,
   AppPackageResponse,
   DeployConfig,
   MessageResponse,
-  CreateAppRequest,
 } from "../../../types"
 import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient"
 import BaseAPI from "./BaseAPI"

qa-core/src/internal-api/fixtures/applications.ts

@@ -1,5 +1,5 @@
 import { generator } from "../../shared"
-import { CreateAppRequest } from "../../types"
+import { CreateAppRequest } from "@budibase/types"
 
 function uniqueWord() {
   return generator.word() + generator.hash()

qa-core/src/internal-api/tests/tables/tables.spec.ts

@@ -13,17 +13,6 @@ describe("Internal API - Table Operations", () => {
     await config.afterAll()
   })
 
-  async function createAppFromTemplate() {
-    return config.api.apps.create({
-      name: generator.word(),
-      url: `/${generator.word()}`,
-      useTemplate: "true",
-      templateName: "Near Miss Register",
-      templateKey: "app/near-miss-register",
-      templateFile: undefined,
-    })
-  }
-
   it("Create and delete table, columns and rows", async () => {
     // create the app
     await config.createApp(fixtures.apps.appFromTemplate())

qa-core/src/shared/BudibaseTestConfiguration.ts

@@ -1,8 +1,8 @@
 import { BudibaseInternalAPI } from "../internal-api"
 import { AccountInternalAPI } from "../account-api"
-import { APIRequestOpts, CreateAppRequest, State } from "../types"
+import { APIRequestOpts, State } from "../types"
 import * as fixtures from "../internal-api/fixtures"
-import { CreateAccountRequest } from "@budibase/types"
+import { CreateAccountRequest, CreateAppRequest } from "@budibase/types"
 
 export default class BudibaseTestConfiguration {
   // apis

qa-core/src/types/app.ts

@@ -1,10 +0,0 @@
-// TODO: Integrate with budibase
-export interface CreateAppRequest {
-  name: string
-  url: string
-  useTemplate?: string
-  templateName?: string
-  templateKey?: string
-  templateFile?: string
-  includeSampleData?: boolean
-}

qa-core/src/types/index.ts

@@ -1,6 +1,5 @@
 export * from "./api"
 export * from "./apiKeyResponse"
-export * from "./app"
 export * from "./appPackage"
 export * from "./deploy"
 export * from "./newAccount"