Merge branch 'develop' into user-table-changes

.github/workflows/budibase_ci.yml

@@ -25,6 +25,13 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
+      - name: Maximize build space
+        uses: easimon/maximize-build-space@master
+        with:
+          root-reserve-mb: 35000
+          swap-size-mb: 1024
+          remove-android: 'true'
+          remove-dotnet: 'true'
       - name: Checkout repo and submodules
         uses: actions/checkout@v3
         if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'

.github/workflows/stale_bot.yml

@@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale
 on:
   workflow_dispatch:
   schedule:
-    - cron: '30 1 * * *' # 1:30 every morning
+    - cron: '*/30 * * * *' # Every 30 mins
 
 jobs:
   stale:

.prettierignore

@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify
-packages/sdk/sdk
+packages/sdk/sdk
+packages/pro/coverage

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.9.40-alpha.7",
+  "version": "2.10.9-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/scripts/build.js

@@ -1,4 +1,5 @@
 #!/usr/bin/node
 const coreBuild = require("../../../scripts/build")
 
+coreBuild("./src/plugin/index.ts", "./dist/plugins.js")
 coreBuild("./src/index.ts", "./dist/index.js")

packages/bbui/src/Banner/Banner.svelte

@@ -66,6 +66,10 @@
     pointer-events: all;
     width: 100%;
   }
+
+  .spectrum-Toast--neutral {
+    background-color: var(--grey-2);
+  }
   .spectrum-Button {
     border: 1px solid rgba(255, 255, 255, 0.2);
   }

packages/bbui/src/Notification/Notification.svelte

@@ -27,7 +27,11 @@
   <div class="spectrum-Toast-body" class:actionBody={!!action}>
     <div class="wrap spectrum-Toast-content">{message || ""}</div>
     {#if action}
-      <ActionButton quiet emphasized on:click={action}>
+      <ActionButton
+        quiet
+        emphasized
+        on:click={() => action(() => dispatch("dismiss"))}
+      >
         <div style="color: white; font-weight: 600;">{actionMessage}</div>
       </ActionButton>
     {/if}

packages/bbui/src/Notification/NotificationDisplay.svelte

@@ -8,7 +8,7 @@
 
 <Portal target=".modal-container">
   <div class="notifications">
-    {#each $notifications as { type, icon, message, id, dismissable, action, wide } (id)}
+    {#each $notifications as { type, icon, message, id, dismissable, action, actionMessage, wide } (id)}
       <div transition:fly={{ y: 30 }}>
         <Notification
           {type}
@@ -16,6 +16,7 @@
           {message}
           {dismissable}
           {action}
+          {actionMessage}
           {wide}
           on:dismiss={() => notifications.dismiss(id)}
         />

packages/bbui/src/Stores/banner.js

@@ -1,6 +1,7 @@
 import { writable } from "svelte/store"
 
 export const BANNER_TYPES = {
+  NEUTRAL: "neutral",
   INFO: "info",
   NEGATIVE: "negative",
   WARNING: "warning",

packages/bbui/src/Stores/notifications.js

@@ -27,7 +27,9 @@ export const createNotificationStore = () => {
       icon = "",
       autoDismiss = true,
       action = null,
+      actionMessage = null,
       wide = false,
+      dismissTimeout = NOTIFICATION_TIMEOUT,
     }
   ) => {
     if (block) {
@@ -44,14 +46,16 @@ export const createNotificationStore = () => {
           icon,
           dismissable: !autoDismiss,
           action,
+          actionMessage,
           wide,
+          dismissTimeout,
         },
       ]
     })
     if (autoDismiss) {
       const timeoutId = setTimeout(() => {
         dismissNotification(_id)
-      }, NOTIFICATION_TIMEOUT)
+      }, dismissTimeout)
       timeoutIds.add(timeoutId)
     }
   }

packages/builder/src/builderStore/dataBinding.js

@@ -504,22 +504,33 @@ const getDeviceBindings = () => {
   let bindings = []
   if (get(store).clientFeatures?.deviceAwareness) {
     const safeDevice = makePropSafe("device")
-    bindings.push({
-      type: "context",
-      runtimeBinding: `${safeDevice}.${makePropSafe("mobile")}`,
-      readableBinding: `Device.Mobile`,
-      category: "Device",
-      icon: "DevicePhone",
-      display: { type: "boolean", name: "mobile" },
-    })
-    bindings.push({
-      type: "context",
-      runtimeBinding: `${safeDevice}.${makePropSafe("tablet")}`,
-      readableBinding: `Device.Tablet`,
-      category: "Device",
-      icon: "DevicePhone",
-      display: { type: "boolean", name: "tablet" },
-    })
+
+    bindings = [
+      {
+        type: "context",
+        runtimeBinding: `${safeDevice}.${makePropSafe("mobile")}`,
+        readableBinding: `Device.Mobile`,
+        category: "Device",
+        icon: "DevicePhone",
+        display: { type: "boolean", name: "mobile" },
+      },
+      {
+        type: "context",
+        runtimeBinding: `${safeDevice}.${makePropSafe("tablet")}`,
+        readableBinding: `Device.Tablet`,
+        category: "Device",
+        icon: "DevicePhone",
+        display: { type: "boolean", name: "tablet" },
+      },
+      {
+        type: "context",
+        runtimeBinding: `${safeDevice}.${makePropSafe("theme")}`,
+        readableBinding: `App.Theme`,
+        category: "Device",
+        icon: "DevicePhone",
+        display: { type: "string", name: "App Theme" },
+      },
+    ]
   }
   return bindings
 }

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -55,7 +55,7 @@
   let linkEditDisabled
   let primaryDisplay
   let indexes = [...($tables.selected.indexes || [])]
-  let isCreating
+  let isCreating = undefined
 
   let table = $tables.selected
   let confirmDeleteDialog
@@ -75,11 +75,11 @@
   }
 
   const initialiseField = (field, savingColumn) => {
+    isCreating = !field
     if (field && !savingColumn) {
       editableColumn = cloneDeep(field)
       originalName = editableColumn.name ? editableColumn.name + "" : null
       linkEditDisabled = originalName != null
-      isCreating = originalName == null
       primaryDisplay =
         $tables.selected.primaryDisplay == null ||
         $tables.selected.primaryDisplay === editableColumn.name
@@ -523,7 +523,7 @@
   {:else if editableColumn.type === "number" && !editableColumn.autocolumn}
     <div class="split-label">
       <div class="label-length">
-        <Label size="M">Max Value</Label>
+        <Label size="M">Min Value</Label>
       </div>
       <div class="input-length">
         <Input
@@ -584,6 +584,7 @@
               { label: "Dynamic", value: "dynamic" },
               { label: "Static", value: "static" },
             ]}
+            disabled={!isCreating}
             getOptionLabel={option => option.label}
             getOptionValue={option => option.value}
             tooltip="Dynamic formula are calculated when retrieved, but cannot be filtered or sorted by,

packages/builder/src/components/design/settings/componentSettings.js

@@ -1,4 +1,4 @@
-import { Checkbox, Select, RadioGroup, Stepper } from "@budibase/bbui"
+import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui"
 import DataSourceSelect from "./controls/DataSourceSelect.svelte"
 import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte"
 import DataProviderSelect from "./controls/DataProviderSelect.svelte"
@@ -60,6 +60,7 @@ const componentMap = {
   "field/longform": FormFieldSelect,
   "field/datetime": FormFieldSelect,
   "field/attachment": FormFieldSelect,
+  "field/s3": Input,
   "field/link": FormFieldSelect,
   "field/array": FormFieldSelect,
   "field/json": FormFieldSelect,

packages/builder/src/pages/builder/_layout.svelte

@@ -3,6 +3,7 @@
   import { admin, auth, licensing } from "stores/portal"
   import { onMount } from "svelte"
   import { CookieUtils, Constants } from "@budibase/frontend-core"
+  import { banner, BANNER_TYPES } from "@budibase/bbui"
   import { API } from "api"
   import Branding from "./Branding.svelte"
 
@@ -16,6 +17,32 @@
   $: user = $auth.user
 
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
+  let showVerificationPrompt = false
+
+  const checkVerification = user => {
+    if (!showVerificationPrompt && user?.account?.verified === false) {
+      showVerificationPrompt = true
+      banner.queue([
+        {
+          message: `Please verify your account. We've sent the verification link to ${user.email}`,
+          type: BANNER_TYPES.NEUTRAL,
+          showCloseButton: false,
+          extraButtonAction: () => {
+            fetch(`${$admin.accountPortalUrl}/api/auth/reset`, {
+              method: "POST",
+              headers: {
+                "Content-Type": "application/json",
+              },
+              body: JSON.stringify({ email: user.email }),
+            })
+          },
+          extraButtonText: "Resend email",
+        },
+      ])
+    }
+  }
+
+  $: checkVerification(user)
 
   const validateTenantId = async () => {
     const host = window.location.host

packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte

@@ -126,7 +126,7 @@
           user,
           prodAppId
         )
-        const isAppBuilder = sdk.users.hasAppBuilderPermissions(user, prodAppId)
+        const isAppBuilder = user.builder?.apps?.includes(prodAppId)
         let role
         if (isAdminOrGlobalBuilder) {
           role = Constants.Roles.ADMIN

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte

@@ -27,12 +27,14 @@
     if (datasource.source === IntegrationTypes.COUCHDB) {
       return datasource.config.database
     }
-    if (
-      datasource.source === IntegrationTypes.DYNAMODB ||
-      datasource.source === IntegrationTypes.S3
-    ) {
+    if (datasource.source === IntegrationTypes.DYNAMODB) {
       return `${datasource.config.endpoint}:${datasource.config.region}`
     }
+    if (datasource.source === IntegrationTypes.S3) {
+      return datasource.config.endpoint
+        ? `${datasource.config.endpoint}:${datasource.config.region}`
+        : `s3.${datasource.config.region}.amazonaws.com`
+    }
     if (datasource.source === IntegrationTypes.ELASTICSEARCH) {
       return datasource.config.url
     }

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -21,6 +21,7 @@
     $selectedScreen,
     $store.selectedComponentId
   )
+
   $: componentBindings = getComponentBindableProperties(
     $selectedScreen,
     $store.selectedComponentId

packages/builder/src/pages/builder/portal/settings/version.svelte

@@ -63,8 +63,7 @@
       )
       const githubResponse = await githubCheck.json()
 
-      //Get tag and remove the v infront of the tage name e.g. v1.0.0 is 1.0.0
-      githubVersion = githubResponse.tag_name.slice(1)
+      githubVersion = githubResponse.tag_name
 
       //Get the release date and output it in the local time format
       githubPublishedDate = new Date(githubResponse.published_at)

packages/builder/src/pages/builder/portal/users/users/[userId].svelte

@@ -111,7 +111,7 @@
       })
     }
     return availableApps.map(app => {
-      const prodAppId = apps.getProdAppID(app.appId)
+      const prodAppId = apps.getProdAppID(app.devId)
       return {
         name: app.name,
         devId: app.devId,

packages/builder/src/stores/portal/licensing.js

@@ -107,7 +107,7 @@ export const createLicensingStore = () => {
         Constants.Features.USER_GROUPS
       )
       const backupsEnabled = license.features.includes(
-        Constants.Features.BACKUPS
+        Constants.Features.APP_BACKUPS
       )
       const scimEnabled = license.features.includes(Constants.Features.SCIM)
       const environmentVariablesEnabled = license.features.includes(

packages/client/manifest.json

@@ -3721,7 +3721,7 @@
     },
     "settings": [
       {
-        "type": "field/attachment",
+        "type": "field/s3",
         "label": "Field",
         "key": "field",
         "required": true

packages/client/src/components/app/forms/S3Upload.svelte

@@ -2,6 +2,7 @@
   import Field from "./Field.svelte"
   import { CoreDropzone, ProgressCircle } from "@budibase/bbui"
   import { getContext, onMount, onDestroy } from "svelte"
+  import { cloneDeep } from "../../../../../bbui/src/helpers"
 
   export let datasourceId
   export let bucket
@@ -14,6 +15,7 @@
 
   let fieldState
   let fieldApi
+  let localFiles = []
 
   const { API, notificationStore, uploadStore } = getContext("sdk")
   const component = getContext("component")
@@ -90,9 +92,17 @@
   }
 
   const handleChange = e => {
-    const changed = fieldApi.setValue(e.detail)
+    localFiles = e.detail
+    let files = cloneDeep(e.detail) || []
+    // remove URL as it contains the full base64 image data
+    files.forEach(file => {
+      if (file.type?.startsWith("image")) {
+        delete file.url
+      }
+    })
+    const changed = fieldApi.setValue(files)
     if (onChange && changed) {
-      onChange({ value: e.detail })
+      onChange({ value: files })
     }
   }
 
@@ -118,7 +128,7 @@
   <div class="content">
     {#if fieldState}
       <CoreDropzone
-        value={fieldState.value}
+        value={localFiles}
         disabled={loading || fieldState.disabled}
         error={fieldState.error}
         on:change={handleChange}

packages/client/src/components/context/DeviceBindingsProvider.svelte

@@ -1,6 +1,7 @@
 <script>
   import Provider from "./Provider.svelte"
   import { onMount, onDestroy } from "svelte"
+  import { themeStore } from "stores"
 
   let width = window.innerWidth
   let height = window.innerHeight
@@ -13,11 +14,14 @@
     }
   })
 
+  $: theme = $themeStore.theme
+
   $: data = {
     mobile: width && width < tabletBreakpoint,
     tablet: width && width >= tabletBreakpoint && width < desktopBreakpoint,
     width,
     height,
+    theme,
   }
 
   onMount(() => {

packages/frontend-core/src/components/grid/cells/DateCell.svelte

@@ -1,5 +1,5 @@
 <script>
-  import dayjs from "dayjs"
+  import { dayjs } from "dayjs"
   import { CoreDatePicker, Icon } from "@budibase/bbui"
   import { onMount } from "svelte"
 

packages/frontend-core/src/components/grid/cells/HeaderCell.svelte

@@ -20,6 +20,8 @@
     config,
     ui,
     columns,
+    definition,
+    datasource,
   } = getContext("grid")
 
   const bannedDisplayColumnTypes = [
@@ -118,6 +120,33 @@
     open = false
   }
 
+  const duplicateColumn = async () => {
+    open = false
+
+    // Generate new name
+    let newName = `${column.name} copy`
+    let attempts = 2
+    while ($definition.schema[newName]) {
+      newName = `${column.name} copy ${attempts++}`
+    }
+
+    // Save schema with new column
+    const existingColumnDefinition = $definition.schema[column.name]
+    await datasource.actions.saveDefinition({
+      ...$definition,
+      schema: {
+        ...$definition.schema,
+        [newName]: {
+          ...existingColumnDefinition,
+          name: newName,
+          schema: {
+            ...existingColumnDefinition.schema,
+          },
+        },
+      },
+    })
+  }
+
   onMount(() => subscribe("close-edit-column", cancelEdit))
 </script>
 
@@ -192,6 +221,13 @@
       >
         Edit column
       </MenuItem>
+      <MenuItem
+        icon="Duplicate"
+        on:click={duplicateColumn}
+        disabled={!$config.canEditColumns}
+      >
+        Duplicate column
+      </MenuItem>
       <MenuItem
         icon="Label"
         on:click={makeDisplayColumn}

packages/frontend-core/src/constants.js

@@ -52,11 +52,6 @@ export const BuilderRoleDescriptions = [
     icon: "User",
     label: "App user - Only has access to published apps",
   },
-  {
-    value: BudibaseRoles.Developer,
-    icon: "Hammer",
-    label: "Developer - Access to the app builder",
-  },
   {
     value: BudibaseRoles.Admin,
     icon: "Draw",

packages/server/scripts/integrations/postgres/init.sql

@@ -1,7 +1,7 @@
 SELECT 'CREATE DATABASE main'
 WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
 CREATE SCHEMA "test-1";
-CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer');
+CREATE TYPE person_job AS ENUM ('qa', 'programmer', 'designer', 'support');
 CREATE TABLE Persons (
     PersonID SERIAL PRIMARY KEY,
     LastName varchar(255),
@@ -51,6 +51,7 @@ CREATE TABLE CompositeTable (
 );
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Mike', 'Hughes', '123 Fake Street', 'Belfast', 'qa');
 INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer');
+INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE);
 INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE);
 INSERT INTO Products (ProductName) VALUES ('Computers');

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,4 +1,5 @@
 import {
+  AutoReason,
   Datasource,
   FieldSchema,
   FieldType,
@@ -24,7 +25,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
-import { FieldTypes } from "../../../constants"
+import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
@@ -259,6 +260,15 @@ function isOneSide(field: FieldSchema) {
   )
 }
 
+function isEditableColumn(column: FieldSchema) {
+  const isExternalAutoColumn =
+    column.autocolumn &&
+    column.autoReason !== AutoReason.FOREIGN_KEY &&
+    column.subtype !== AutoFieldSubTypes.AUTO_ID
+  const isFormula = column.type === FieldTypes.FORMULA
+  return !(isExternalAutoColumn || isFormula)
+}
+
 export class ExternalRequest {
   private operation: Operation
   private tableId: string
@@ -295,11 +305,7 @@ export class ExternalRequest {
       manyRelationships: ManyRelationship[] = []
     for (let [key, field] of Object.entries(table.schema)) {
       // if set already, or not set just skip it
-      if (
-        row[key] == null ||
-        newRow[key] ||
-        !sdk.tables.isEditableColumn(field)
-      ) {
+      if (row[key] == null || newRow[key] || !isEditableColumn(field)) {
         continue
       }
       // if its an empty string then it means return the column to null (if possible)

packages/server/src/api/controllers/row/external.ts

@@ -18,6 +18,8 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
+import { inputProcessing } from "../../../utilities/rowProcessor"
+import { cloneDeep, isEqual } from "lodash"
 
 export async function handleRequest(
   operation: Operation,
@@ -26,20 +28,8 @@ export async function handleRequest(
 ) {
   // make sure the filters are cleaned up, no empty strings for equals, fuzzy or string
   if (opts && opts.filters) {
-    for (let filterField of NoEmptyFilterStrings) {
-      if (!opts.filters[filterField]) {
-        continue
-      }
-      // @ts-ignore
-      for (let [key, value] of Object.entries(opts.filters[filterField])) {
-        if (!value || value === "") {
-          // @ts-ignore
-          delete opts.filters[filterField][key]
-        }
-      }
-    }
+    opts.filters = utils.removeEmptyFilters(opts.filters)
   }
-
   if (
     !dataFilters.hasFilters(opts?.filters) &&
     opts?.filters?.onEmptyFilter === EmptyFilterOption.RETURN_NONE
@@ -88,10 +78,24 @@ export async function save(ctx: UserCtx) {
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  const { table: updatedTable, row } = inputProcessing(
+    ctx.user,
+    cloneDeep(table),
+    inputs
+  )
+
   const response = await handleRequest(Operation.CREATE, tableId, {
-    row: inputs,
+    row,
   })
+
   const responseRow = response as { row: Row }
+
+  if (!isEqual(table, updatedTable)) {
+    await sdk.tables.saveTable(updatedTable)
+  }
+
   const rowId = responseRow.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
@@ -106,10 +110,18 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
-  return sdk.rows.external.getRow(tableId, id)
+  const row = await sdk.rows.external.getRow(tableId, id, {
+    relationships: true,
+  })
+
+  if (!row) {
+    ctx.throw(404)
+  }
+
+  return row
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -119,7 +131,7 @@ export async function destroy(ctx: UserCtx) {
     id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
-  return { response: { ok: true }, row }
+  return { response: { ok: true, id: _id }, row }
 }
 
 export async function bulkDestroy(ctx: UserCtx) {

packages/server/src/api/controllers/row/index.ts

@@ -14,6 +14,10 @@ import {
   SearchRowResponse,
   SearchRowRequest,
   SearchParams,
+  GetRowResponse,
+  ValidateResponse,
+  ExportRowsRequest,
+  ExportRowsResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -111,7 +115,7 @@ export async function fetch(ctx: any) {
   })
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx<void, GetRowResponse>) {
   const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
     datasourceId: tableId,
@@ -214,11 +218,11 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   })
 }
 
-export async function validate(ctx: Ctx) {
+export async function validate(ctx: Ctx<Row, ValidateResponse>) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
   if (isExternalTable(tableId)) {
-    ctx.body = { valid: true }
+    ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({
       row: ctx.request.body,
@@ -237,7 +241,9 @@ export async function fetchEnrichedRow(ctx: any) {
   )
 }
 
-export const exportRows = async (ctx: any) => {
+export const exportRows = async (
+  ctx: Ctx<ExportRowsRequest, ExportRowsResponse>
+) => {
   const tableId = utils.getTableId(ctx)
 
   const format = ctx.query.format

packages/server/src/api/controllers/row/internal.ts

@@ -131,7 +131,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const tableId = utils.getTableId(ctx),
     rowId = ctx.params.rowId
   const table = await sdk.tables.getTable(tableId)

packages/server/src/api/controllers/row/tests/utils.spec.ts

@@ -0,0 +1,21 @@
+import * as utils from "../utils"
+
+describe("removeEmptyFilters", () => {
+  it("0 should not be removed", () => {
+    const filters = utils.removeEmptyFilters({
+      equal: {
+        column: 0,
+      },
+    })
+    expect((filters.equal as any).column).toBe(0)
+  })
+
+  it("empty string should be removed", () => {
+    const filters = utils.removeEmptyFilters({
+      equal: {
+        column: "",
+      },
+    })
+    expect(Object.values(filters.equal as any).length).toBe(0)
+  })
+})

packages/server/src/api/controllers/row/utils.ts

@@ -1,8 +1,15 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
-import { FieldTypes } from "../../../constants"
+import {
+  Ctx,
+  FieldType,
+  Row,
+  SearchFilters,
+  Table,
+  UserCtx,
+} from "@budibase/types"
+import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
 import sdk from "../../../sdk"
 
 import validateJs from "validate.js"
@@ -27,7 +34,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
-  let row
+  let row: Row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
     ctx.params = {
@@ -139,3 +146,32 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+// don't do a pure falsy check, as 0 is included
+// https://github.com/Budibase/budibase/issues/10118
+export function removeEmptyFilters(filters: SearchFilters) {
+  for (let filterField of NoEmptyFilterStrings) {
+    if (!filters[filterField]) {
+      continue
+    }
+
+    for (let filterType of Object.keys(filters)) {
+      if (filterType !== filterField) {
+        continue
+      }
+      // don't know which one we're checking, type could be anything
+      const value = filters[filterType] as unknown
+      if (typeof value === "object") {
+        for (let [key, value] of Object.entries(
+          filters[filterType] as object
+        )) {
+          if (value == null || value === "") {
+            // @ts-ignore
+            delete filters[filterField][key]
+          }
+        }
+      }
+    }
+  }
+  return filters
+}

packages/server/src/api/controllers/table/index.ts

@@ -78,9 +78,9 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
   ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
+    ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
   ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, savedTable)
+  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/api/routes/tests/table.spec.ts

@@ -1,6 +1,6 @@
 import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
-import { FieldType, Table } from "@budibase/types"
+import { FieldType, Table, ViewCalculation } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 const { basicTable } = setup.structures
@@ -90,8 +90,10 @@ describe("/tables", () => {
       await config.createLegacyView({
         name: "TestView",
         field: "Price",
-        calculation: "stats",
-        tableId: testTable._id,
+        calculation: ViewCalculation.STATISTICS,
+        tableId: testTable._id!,
+        schema: {},
+        filters: [],
       })
 
       const testRow = await request
@@ -254,7 +256,7 @@ describe("/tables", () => {
       }))
 
       await config.api.viewV2.create({ tableId })
-      await config.createLegacyView({ tableId, name: generator.guid() })
+      await config.createLegacyView()
 
       const res = await config.api.table.fetch()
 
@@ -366,7 +368,7 @@ describe("/tables", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.message).toEqual(`Table ${testTable._id} deleted.`)
-      const dependentTable = await config.getTable(linkedTable._id)
+      const dependentTable = await config.api.table.get(linkedTable._id!)
       expect(dependentTable.schema.TestTable).not.toBeDefined()
     })
 

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -6,6 +6,7 @@ import {
   SortOrder,
   SortType,
   Table,
+  UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
 } from "@budibase/types"
@@ -418,9 +419,12 @@ describe.each([
       const res = await config.api.viewV2.create(newView)
       const view = await config.api.viewV2.get(res.id)
       expect(view!.schema?.Price).toBeUndefined()
-      const updatedTable = await config.getTable(table._id!)
-      const viewSchema = updatedTable.views[view!.name!].schema
-      expect(viewSchema.Price.visible).toEqual(false)
+      const updatedTable = await config.api.table.get(table._id!)
+      const viewSchema = updatedTable.views![view!.name!].schema as Record<
+        string,
+        UIFieldMetadata
+      >
+      expect(viewSchema.Price?.visible).toEqual(false)
     })
   })
 })

packages/server/src/db/tests/linkController.spec.js

@@ -6,7 +6,7 @@ const { RelationshipType } = require("../../constants")
 const { cloneDeep } = require("lodash/fp")
 
 describe("test the link controller", () => {
-  let config = new TestConfig(false)
+  let config = new TestConfig()
   let table1, table2, appId
 
   beforeAll(async () => {

packages/server/src/db/tests/linkTests.spec.js

@@ -4,7 +4,7 @@ const linkUtils = require("../linkedRows/linkUtils")
 const { context } = require("@budibase/backend-core")
 
 describe("test link functionality", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
   let appId
 
   describe("getLinkedTable", () => {

packages/server/src/integration-test/postgres.spec.ts

@@ -12,18 +12,15 @@ import {
   FieldType,
   RelationshipType,
   Row,
-  SourceName,
   Table,
 } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { databaseTestProviders } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
-jest.setTimeout(30000)
-
 jest.unmock("pg")
 jest.mock("../websockets")
 
@@ -35,62 +32,18 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let host: string
-  let port: number
-  const containers: StartedTestContainer[] = []
-
   beforeAll(async () => {
-    const containerPostgres = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
-        )
-      )
-      .start()
-
-    host = containerPostgres.getContainerIpAddress()
-    port = containerPostgres.getMappedPort(5432)
-
     await config.init()
     const apiKey = await config.generateApiKey()
 
-    containers.push(containerPostgres)
-
     makeRequest = generateMakeRequest(apiKey, true)
-  })
 
-  afterAll(async () => {
-    for (let container of containers) {
-      await container.stop()
-    }
+    postgresDatasource = await config.api.datasource.create(
+      await databaseTestProviders.postgres.getDsConfig()
+    )
   })
 
-  function pgDatasourceConfig() {
-    return {
-      datasource: {
-        type: "datasource",
-        source: SourceName.POSTGRES,
-        plus: true,
-        config: {
-          host,
-          port,
-          database: "postgres",
-          user: "postgres",
-          password: "password",
-          schema: "public",
-          ssl: false,
-          rejectUnauthorized: false,
-          ca: false,
-        },
-      },
-    }
-  }
-
   beforeEach(async () => {
-    postgresDatasource = await config.createDatasource(pgDatasourceConfig())
-
     async function createAuxTable(prefix: string) {
       return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
@@ -226,25 +179,6 @@ describe("postgres integrations", () => {
     let { rowData } = opts as any
     let foreignRows: ForeignRowsInfo[] = []
 
-    async function createForeignRow(tableInfo: ForeignTableInfo) {
-      const foreignKey = `fk_${tableInfo.table.name}_${tableInfo.fieldName}`
-
-      const foreignRow = await config.createRow({
-        tableId: tableInfo.table._id,
-        title: generator.name(),
-      })
-
-      rowData = {
-        ...rowData,
-        [foreignKey]: foreignRow.id,
-      }
-      foreignRows.push({
-        row: foreignRow,
-
-        relationshipType: tableInfo.relationshipType,
-      })
-    }
-
     if (opts?.createForeignRows?.createOneToMany) {
       const foreignKey = `fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`
 
@@ -322,6 +256,14 @@ describe("postgres integrations", () => {
     })
   }
 
+  const createRandomTableWithRows = async () => {
+    const tableId = (await createDefaultPgTable())._id!
+    return await config.api.row.save(tableId, {
+      tableId,
+      title: generator.name(),
+    })
+  }
+
   async function populatePrimaryRows(
     count: number,
     opts?: {
@@ -357,9 +299,9 @@ describe("postgres integrations", () => {
       config: {
         ca: false,
         database: "postgres",
-        host,
+        host: postgresDatasource.config!.host,
         password: "--secret-value--",
-        port,
+        port: postgresDatasource.config!.port,
         rejectUnauthorized: false,
         schema: "public",
         ssl: false,
@@ -401,12 +343,16 @@ describe("postgres integrations", () => {
 
       it("multiple rows can be persisted", async () => {
         const numberOfRows = 10
-        const newRows = Array(numberOfRows).fill(generateRandomPrimaryRowData())
+        const newRows: Row[] = Array(numberOfRows).fill(
+          generateRandomPrimaryRowData()
+        )
 
-        for (const newRow of newRows) {
-          const res = await createRow(primaryPostgresTable._id, newRow)
-          expect(res.status).toBe(200)
-        }
+        await Promise.all(
+          newRows.map(async newRow => {
+            const res = await createRow(primaryPostgresTable._id, newRow)
+            expect(res.status).toBe(200)
+          })
+        )
 
         const persistedRows = await config.getRows(primaryPostgresTable._id!)
         expect(persistedRows).toHaveLength(numberOfRows)
@@ -567,7 +513,7 @@ describe("postgres integrations", () => {
           foreignRows = createdRow.foreignRows
         })
 
-        it("only one to many foreign keys are retrieved", async () => {
+        it("only one to primary keys are retrieved", async () => {
           const res = await getRow(primaryPostgresTable._id, row.id)
 
           expect(res.status).toBe(200)
@@ -575,6 +521,12 @@ describe("postgres integrations", () => {
           const one2ManyForeignRows = foreignRows.filter(
             x => x.relationshipType === RelationshipType.ONE_TO_MANY
           )
+          const many2OneForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_ONE
+          )
+          const many2ManyForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_MANY
+          )
           expect(one2ManyForeignRows).toHaveLength(1)
 
           expect(res.body).toEqual({
@@ -585,9 +537,25 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               one2ManyForeignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              one2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2OneForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -616,9 +584,13 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               foreignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -645,9 +617,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -674,9 +650,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
     })
@@ -730,12 +710,6 @@ describe("postgres integrations", () => {
       describe("given than multiple tables have multiple rows", () => {
         const rowsCount = 6
         beforeEach(async () => {
-          const createRandomTableWithRows = async () =>
-            await config.createRow({
-              tableId: (await createDefaultPgTable())._id,
-              title: generator.name(),
-            })
-
           await createRandomTableWithRows()
           await createRandomTableWithRows()
 
@@ -1023,12 +997,6 @@ describe("postgres integrations", () => {
       const rowsCount = 6
 
       beforeEach(async () => {
-        const createRandomTableWithRows = async () =>
-          await config.createRow({
-            tableId: (await createDefaultPgTable())._id,
-            title: generator.name(),
-          })
-
         await createRandomTableWithRows()
         await populatePrimaryRows(rowsCount)
         await createRandomTableWithRows()
@@ -1046,24 +1014,25 @@ describe("postgres integrations", () => {
 
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
-      const config = pgDatasourceConfig()
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const response = await config.api.datasource.verify({
+        datasource: await databaseTestProviders.postgres.getDsConfig(),
+      })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const config = pgDatasourceConfig()
-      config.datasource.config.password = "wrongpassword"
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
+      const response = await config.api.datasource.verify({
+        datasource: {
+          ...dbConfig,
+          config: {
+            ...dbConfig.config,
+            password: "wrongpassword",
+          },
+        },
+      })
+
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(false)
       expect(response.body.error).toBeDefined()

packages/server/src/integrations/base/sql.ts

@@ -58,7 +58,7 @@ function parse(input: any) {
     return null
   }
   if (isIsoDateString(input)) {
-    return new Date(input)
+    return new Date(input.trim())
   }
   return input
 }

packages/server/src/integrations/tests/sql.spec.ts

@@ -657,4 +657,29 @@ describe("SQL query builder", () => {
       sql: `select * from (select top (@p0) * from [test] order by [test].[id] asc) as [test]`,
     })
   })
+
+  it("should not parse JSON string as Date", () => {
+    let query = new Sql(SqlClient.POSTGRES, limit)._query(
+      generateCreateJson(TABLE_NAME, {
+        name: '{ "created_at":"2023-09-09T03:21:06.024Z" }',
+      })
+    )
+    expect(query).toEqual({
+      bindings: ['{ "created_at":"2023-09-09T03:21:06.024Z" }'],
+      sql: `insert into \"test\" (\"name\") values ($1) returning *`,
+    })
+  })
+
+  it("should parse and trim valid string as Date", () => {
+    const dateObj = new Date("2023-09-09T03:21:06.024Z")
+    let query = new Sql(SqlClient.POSTGRES, limit)._query(
+      generateCreateJson(TABLE_NAME, {
+        name: " 2023-09-09T03:21:06.024Z ",
+      })
+    )
+    expect(query).toEqual({
+      bindings: [dateObj],
+      sql: `insert into \"test\" (\"name\") values ($1) returning *`,
+    })
+  })
 })

packages/server/src/integrations/tests/utils/index.ts

@@ -0,0 +1,14 @@
+jest.unmock("pg")
+
+import { Datasource } from "@budibase/types"
+import * as pg from "./postgres"
+
+jest.setTimeout(30000)
+
+export interface DatabasePlusTestProvider {
+  getDsConfig(): Promise<Datasource>
+}
+
+export const databaseTestProviders = {
+  postgres: pg,
+}

packages/server/src/integrations/tests/utils/postgres.ts

@@ -0,0 +1,38 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function getDsConfig(): Promise<Datasource> {
+  if (!container) {
+    container = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .withWaitStrategy(
+        Wait.forLogMessage(
+          "PostgreSQL init process complete; ready for start up."
+        )
+      )
+      .start()
+  }
+
+  const host = container.getContainerIpAddress()
+  const port = container.getMappedPort(5432)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.POSTGRES,
+    plus: true,
+    config: {
+      host,
+      port,
+      database: "postgres",
+      user: "postgres",
+      password: "password",
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
+  }
+}

packages/server/src/integrations/utils.ts

@@ -182,11 +182,12 @@ export function getSqlQuery(query: SqlQuery | string): SqlQuery {
 export const isSQL = helpers.isSQL
 
 export function isIsoDateString(str: string) {
-  if (!/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z/.test(str)) {
+  const trimmedValue = str.trim()
+  if (!/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z$/.test(trimmedValue)) {
     return false
   }
-  let d = new Date(str)
-  return d.toISOString() === str
+  let d = new Date(trimmedValue)
+  return d.toISOString() === trimmedValue
 }
 
 /**

packages/server/src/migrations/tests/index.spec.ts

@@ -11,6 +11,7 @@ import { MIGRATIONS } from "../"
 import * as helpers from "./helpers"
 
 import tk from "timekeeper"
+import { View } from "@budibase/types"
 const timestamp = new Date().toISOString()
 tk.freeze(timestamp)
 
@@ -52,7 +53,9 @@ describe("migrations", () => {
         await config.createTable()
         await config.createLegacyView()
         await config.createTable()
-        await config.createLegacyView(structures.view(config.table!._id!))
+        await config.createLegacyView(
+          structures.view(config.table!._id!) as View
+        )
         await config.createScreen()
         await config.createScreen()
 

packages/server/src/sdk/app/permissions/index.ts

@@ -61,11 +61,7 @@ export async function getInheritablePermissions(
 export async function allowsExplicitPermissions(resourceId: string) {
   if (isViewID(resourceId)) {
     const allowed = await features.isViewPermissionEnabled()
-    const minPlan = !allowed
-      ? env.SELF_HOSTED
-        ? PlanType.BUSINESS
-        : PlanType.PREMIUM
-      : undefined
+    const minPlan = !allowed ? PlanType.BUSINESS : undefined
 
     return {
       allowed,
@@ -92,7 +88,7 @@ export async function getResourcePerms(
     // update the various roleIds in the resource permissions
     for (let role of rolesList) {
       const rolePerms = allowsExplicitPerm
-        ? roles.checkForRoleResourceArray(role.permissions, resourceId)
+        ? roles.checkForRoleResourceArray(role.permissions || {}, resourceId)
         : {}
       if (rolePerms[resourceId]?.indexOf(level) > -1) {
         permissions[level] = {

packages/server/src/sdk/app/rows/search.ts

@@ -30,7 +30,7 @@ export interface ExportRowsParams {
   format: Format
   rowIds?: string[]
   columns?: string[]
-  query: SearchFilters
+  query?: SearchFilters
 }
 
 export interface ExportRowsResult {

packages/server/src/sdk/app/tables/index.ts

@@ -12,7 +12,7 @@ import {
   TableViewsResponse,
 } from "@budibase/types"
 import datasources from "../datasources"
-import { isEditableColumn, populateExternalTableSchemas } from "./validation"
+import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
@@ -73,12 +73,23 @@ function enrichViewSchemas(table: Table): TableResponse {
   }
 }
 
+async function saveTable(table: Table) {
+  const db = context.getAppDB()
+  if (isExternalTable(table._id!)) {
+    const datasource = await sdk.datasources.get(table.sourceId!)
+    datasource.entities![table.name] = table
+    await db.put(datasource)
+  } else {
+    await db.put(table)
+  }
+}
+
 export default {
   getAllInternalTables,
   getAllExternalTables,
   getExternalTable,
   getTable,
   populateExternalTableSchemas,
-  isEditableColumn,
   enrichViewSchemas,
+  saveTable,
 }

packages/server/src/sdk/app/tables/validation.ts

@@ -55,13 +55,6 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   return datasource
 }
 
-export function isEditableColumn(column: FieldSchema) {
-  const isAutoColumn =
-    column.autocolumn && column.autoReason !== AutoReason.FOREIGN_KEY
-  const isFormula = column.type === FieldTypes.FORMULA
-  return !(isAutoColumn || isFormula)
-}
-
 export function populateExternalTableSchemas(datasource: Datasource) {
   return checkForeignKeysAreAutoColumns(datasource)
 }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -50,6 +50,11 @@ import {
   SearchFilters,
   UserRoles,
   Automation,
+  View,
+  FieldType,
+  RelationshipType,
+  ViewV2,
+  CreateViewRequest,
 } from "@budibase/types"
 
 import API from "./api"
@@ -75,9 +80,8 @@ class TestConfiguration {
   globalUserId: any
   userMetadataId: any
   table?: Table
-  linkedTable: any
   automation: any
-  datasource: any
+  datasource?: Datasource
   tenantId?: string
   defaultUserValues: DefaultUserValues
   api: API
@@ -527,7 +531,7 @@ class TestConfiguration {
   // TABLE
 
   async updateTable(
-    config?: any,
+    config?: Table,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
@@ -542,33 +546,50 @@ class TestConfiguration {
     if (config != null && config._id) {
       delete config._id
     }
+    config = config || basicTable()
+    if (this.datasource && !config.sourceId) {
+      config.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        config.type = "external"
+      }
+    }
+
     return this.updateTable(config, options)
   }
 
   async getTable(tableId?: string) {
-    tableId = tableId || this.table?._id
+    tableId = tableId || this.table!._id!
     return this._req(null, { tableId }, controllers.table.find)
   }
 
-  async createLinkedTable(relationshipType?: string, links: any = ["link"]) {
+  async createLinkedTable(
+    relationshipType = RelationshipType.ONE_TO_MANY,
+    links: any = ["link"],
+    config?: Table
+  ) {
     if (!this.table) {
       throw "Must have created a table first."
     }
-    const tableConfig: any = basicTable()
+    const tableConfig = config || basicTable()
     tableConfig.primaryDisplay = "name"
     for (let link of links) {
       tableConfig.schema[link] = {
-        type: "link",
+        type: FieldType.LINK,
         fieldName: link,
         tableId: this.table._id,
         name: link,
-      }
-      if (relationshipType) {
-        tableConfig.schema[link].relationshipType = relationshipType
+        relationshipType,
       }
     }
+
+    if (this.datasource && !tableConfig.sourceId) {
+      tableConfig.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        tableConfig.type = "external"
+      }
+    }
+
     const linkedTable = await this.createTable(tableConfig)
-    this.linkedTable = linkedTable
     return linkedTable
   }
 
@@ -621,17 +642,36 @@ class TestConfiguration {
 
   // VIEW
 
-  async createLegacyView(config?: any) {
-    if (!this.table) {
+  async createLegacyView(config?: View) {
+    if (!this.table && !config) {
       throw "Test requires table to be configured."
     }
     const view = config || {
-      tableId: this.table._id,
-      name: "ViewTest",
+      tableId: this.table!._id,
+      name: generator.guid(),
     }
     return this._req(view, null, controllers.view.v1.save)
   }
 
+  async createView(
+    config?: Omit<CreateViewRequest, "tableId" | "name"> & {
+      name?: string
+      tableId?: string
+    }
+  ) {
+    if (!this.table && !config?.tableId) {
+      throw "Test requires table to be configured."
+    }
+
+    const view: CreateViewRequest = {
+      ...config,
+      tableId: config?.tableId || this.table!._id!,
+      name: config?.name || generator.word(),
+    }
+
+    return await this.api.viewV2.create(view)
+  }
+
   // AUTOMATION
 
   async createAutomation(config?: any) {
@@ -677,17 +717,17 @@ class TestConfiguration {
     config = config || basicDatasource()
     const response = await this._req(config, null, controllers.datasource.save)
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
-  async updateDatasource(datasource: any) {
+  async updateDatasource(datasource: Datasource): Promise<Datasource> {
     const response = await this._req(
       datasource,
       { datasourceId: datasource._id },
       controllers.datasource.update
     )
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
   async restDatasource(cfg?: any) {
@@ -771,7 +811,7 @@ class TestConfiguration {
     if (!this.datasource && !config) {
       throw "No datasource created for query."
     }
-    config = config || basicQuery(this.datasource._id)
+    config = config || basicQuery(this.datasource!._id!)
     return this._req(config, null, controllers.query.save)
   }
 

packages/server/src/tests/utilities/api/datasource.ts

@@ -0,0 +1,57 @@
+import {
+  CreateDatasourceRequest,
+  Datasource,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class DatasourceAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (
+    config: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const body: CreateDatasourceRequest = {
+      datasource: config,
+      tablesFilter: [],
+    }
+    const result = await this.request
+      .post(`/api/datasources`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+
+  update = async (
+    datasource: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const result = await this.request
+      .put(`/api/datasources/${datasource._id}`)
+      .send(datasource)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+
+  verify = async (
+    data: VerifyDatasourceRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const result = await this.request
+      .post(`/api/datasources/verify`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result
+  }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -3,17 +3,23 @@ import { PermissionAPI } from "./permission"
 import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
+import { DatasourceAPI } from "./datasource"
+import { LegacyViewAPI } from "./legacyView"
 
 export default class API {
   table: TableAPI
+  legacyView: LegacyViewAPI
   viewV2: ViewV2API
   row: RowAPI
   permission: PermissionAPI
+  datasource: DatasourceAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
+    this.legacyView = new LegacyViewAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.row = new RowAPI(config)
     this.permission = new PermissionAPI(config)
+    this.datasource = new DatasourceAPI(config)
   }
 }

packages/server/src/tests/utilities/api/legacyView.ts

@@ -0,0 +1,16 @@
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class LegacyViewAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  get = async (id: string, { expectStatus } = { expectStatus: 200 }) => {
+    return await this.request
+      .get(`/api/views/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
+}

packages/server/src/tests/utilities/api/row.ts

@@ -1,4 +1,10 @@
-import { PatchRowRequest, SaveRowRequest, Row } from "@budibase/types"
+import {
+  PatchRowRequest,
+  SaveRowRequest,
+  Row,
+  ValidateResponse,
+  ExportRowsRequest,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -22,6 +28,21 @@ export class RowAPI extends TestAPI {
     return request
   }
 
+  getEnriched = async (
+    sourceId: string,
+    rowId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .get(`/api/${sourceId}/${rowId}/enrich`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+    if (expectStatus !== 404) {
+      request.expect("Content-Type", /json/)
+    }
+    return request
+  }
+
   save = async (
     sourceId: string,
     row: SaveRowRequest,
@@ -36,6 +57,20 @@ export class RowAPI extends TestAPI {
     return resp.body as Row
   }
 
+  validate = async (
+    sourceId: string,
+    row: SaveRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ValidateResponse> => {
+    const resp = await this.request
+      .post(`/api/${sourceId}/rows/validate`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return resp.body as ValidateResponse
+  }
+
   patch = async (
     sourceId: string,
     row: PatchRowRequest,
@@ -51,14 +86,40 @@ export class RowAPI extends TestAPI {
 
   delete = async (
     sourceId: string,
-    rows: Row[],
+    rows: Row | string | (Row | string)[],
     { expectStatus } = { expectStatus: 200 }
   ) => {
     return this.request
       .delete(`/api/${sourceId}/rows`)
-      .send({ rows })
+      .send(Array.isArray(rows) ? { rows } : rows)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
+
+  fetch = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .get(`/api/${sourceId}/rows`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
+
+  exportRows = async (
+    tableId: string,
+    body: ExportRowsRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .post(`/api/${tableId}/rows/exportRows?format=json`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return request
+  }
 }

packages/server/src/tests/utilities/api/table.ts

@@ -1,4 +1,4 @@
-import { Table } from "@budibase/types"
+import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -7,6 +7,19 @@ export class TableAPI extends TestAPI {
     super(config)
   }
 
+  create = async (
+    data: SaveTableRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<SaveTableResponse> => {
+    const res = await this.request
+      .post(`/api/tables`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return res.body
+  }
+
   fetch = async (
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Table[]> => {

packages/server/src/tests/utilities/api/viewV2.ts

@@ -23,8 +23,8 @@ export class ViewV2API extends TestAPI {
     if (!tableId && !this.config.table) {
       throw "Test requires table to be configured."
     }
-    const table = this.config.table
-    tableId = table!._id!
+
+    tableId = tableId || this.config.table!._id!
     const view = {
       tableId,
       name: generator.guid(),

packages/types/src/api/web/app/row.ts

@@ -1,5 +1,7 @@
 import { Row } from "../../../documents/app/row"
 
+export interface GetRowResponse extends Row {}
+
 export interface DeleteRows {
   rows: (Row | string)[]
 }
@@ -9,3 +11,8 @@ export interface DeleteRow {
 }
 
 export type DeleteRowRequest = DeleteRows | DeleteRow
+
+export interface ValidateResponse {
+  valid: boolean
+  errors: Record<string, any>
+}

packages/types/src/api/web/app/rows.ts

@@ -1,5 +1,6 @@
-import { SearchParams } from "../../../sdk"
+import { SearchFilters, SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
+import { ReadStream } from "fs"
 
 export interface SaveRowRequest extends Row {}
 
@@ -28,3 +29,11 @@ export interface SearchViewRowRequest
 export interface SearchRowResponse {
   rows: any[]
 }
+
+export interface ExportRowsRequest {
+  rows: string[]
+  columns?: string[]
+  query?: SearchFilters
+}
+
+export type ExportRowsResponse = ReadStream

packages/worker/src/api/routes/global/tests/scim.spec.ts

@@ -10,6 +10,8 @@ import {
 import { TestConfiguration } from "../../../../tests"
 import { events } from "@budibase/backend-core"
 
+// this test can 409 - retries reduce issues with this
+jest.retryTimes(2)
 jest.setTimeout(30000)
 
 mocks.licenses.useScimIntegration()

qa-core/src/integrations/external-schema/postgres.integration.spec.ts

@@ -17,8 +17,7 @@ describe("getExternalSchema", () => {
     }
 
     beforeAll(async () => {
-      // This is left on propose without a tag, so if a new version introduces a breaking change we will be notified
-      const container = await new GenericContainer("postgres")
+      const container = await new GenericContainer("postgres:13.12")
         .withExposedPorts(5432)
         .withEnv("POSTGRES_PASSWORD", "password")
         .start()

yarn.lock

@@ -6269,6 +6269,14 @@
     "@types/tedious" "*"
     tarn "^3.0.1"
 
+"@types/node-fetch@2.6.1":
+  version "2.6.1"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.1.tgz#8f127c50481db65886800ef496f20bbf15518975"
+  integrity sha512-oMqjURCaxoSIsHSr1E47QHzbmzNR5rK8McHuNb11BOM9cHcIK3Avy0s/b2JlXHoQGTYS3NsvWzV1M0iK7l0wbA==
+  dependencies:
+    "@types/node" "*"
+    form-data "^3.0.0"
+
 "@types/node-fetch@2.6.4":
   version "2.6.4"
   resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.4.tgz#1bc3a26de814f6bf466b25aeb1473fa1afe6a660"
@@ -6290,6 +6298,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.18.tgz#8dfb97f0da23c2293e554c5a50d61ef134d7697f"
   integrity sha512-DHQpWGjyQKSHj3ebjFI/wRKcqQcdR+MoFBygntYOZytCqNfkd2ZC4ARDJ2DQqhjH5p85Nnd3jhUJIXrszFX/JA==
 
+"@types/node@14.18.20":
+  version "14.18.20"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-14.18.20.tgz#268f028b36eaf51181c3300252f605488c4f0650"
+  integrity sha512-Q8KKwm9YqEmUBRsqJ2GWJDtXltBDxTdC4m5vTdXBolu2PeQh8LX+f6BTwU+OuXPu37fLxoN6gidqBmnky36FXA==
+
 "@types/node@16.9.1":
   version "16.9.1"
   resolved "https://registry.yarnpkg.com/@types/node/-/node-16.9.1.tgz#0611b37db4246c937feef529ddcc018cf8e35708"