Merge remote-tracking branch 'origin/develop' into feature/auto-screen-ui

hosting/docker-compose.yaml

@@ -45,6 +45,7 @@ services:
       MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
       MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
       MINIO_URL: http://minio-service:9000
+      APPS_URL: http://app-service:4002
       COUCH_DB_USERNAME: ${COUCH_DB_USER}
       COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
       COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -113,6 +113,8 @@ spec:
           value: {{ .Values.globals.smtp.port | quote }}
         - name: SMTP_FROM_ADDRESS
           value: {{ .Values.globals.smtp.from | quote }}
+        - name: APPS_URL
+          value: http://app-service:{{ .Values.services.apps.port }}
         image: budibase/worker
         imagePullPolicy: Always
         name: bbworker

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/constants.js

@@ -6,6 +6,7 @@ exports.UserStatus = {
 exports.Cookies = {
   CurrentApp: "budibase:currentapp",
   Auth: "budibase:auth",
+  Init: "budibase:init",
   OIDC_CONFIG: "budibase:oidc:config",
 }
 

packages/auth/src/db/utils.js

@@ -152,6 +152,17 @@ exports.getDeployedAppID = appId => {
   return appId
 }
 
+/**
+ * Convert a deployed app ID to a development app ID.
+ */
+exports.getDevelopmentAppID = appId => {
+  if (!appId.startsWith(exports.APP_DEV_PREFIX)) {
+    const id = appId.split(exports.APP_PREFIX)[1]
+    return `${exports.APP_DEV_PREFIX}${id}`
+  }
+  return appId
+}
+
 exports.getCouchUrl = () => {
   if (!env.COUCH_DB_URL) return
 
@@ -248,6 +259,24 @@ exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => {
   }
 }
 
+/**
+ * Utility function for getAllApps but filters to production apps only.
+ */
+exports.getDeployedAppIDs = async CouchDB => {
+  return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(
+    id => !exports.isDevAppID(id)
+  )
+}
+
+/**
+ * Utility function for the inverse of above.
+ */
+exports.getDevAppIDs = async CouchDB => {
+  return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(id =>
+    exports.isDevAppID(id)
+  )
+}
+
 exports.dbExists = async (CouchDB, dbName) => {
   let exists = false
   try {

packages/auth/src/middleware/passport/local.js

@@ -9,6 +9,8 @@ const { createASession } = require("../../security/sessions")
 const { getTenantId } = require("../../tenancy")
 
 const INVALID_ERR = "Invalid Credentials"
+const SSO_NO_PASSWORD = "SSO user does not have a password set"
+const EXPIRED = "This account has expired. Please reset your password"
 
 exports.options = {
   passReqToCallback: true,
@@ -36,6 +38,19 @@ exports.authenticate = async function (ctx, email, password, done) {
     return authError(done, INVALID_ERR)
   }
 
+  // check that the user has a stored password before proceeding
+  if (!dbUser.password) {
+    if (
+      (dbUser.account && dbUser.account.authType === "sso") || // root account sso
+      dbUser.thirdPartyProfile // internal sso
+    ) {
+      return authError(done, SSO_NO_PASSWORD)
+    }
+
+    console.error("Non SSO usser has no password set", dbUser)
+    return authError(done, EXPIRED)
+  }
+
   // authenticate
   if (await compare(password, dbUser.password)) {
     const sessionId = newid()

packages/auth/src/utils.js

@@ -181,8 +181,8 @@ exports.saveUser = async (
 
     // check budibase users in other tenants
     if (env.MULTI_TENANCY) {
-      dbUser = await getTenantUser(email)
+      const tenantUser = await getTenantUser(email)
-      if (dbUser != null && dbUser.tenantId !== tenantId) {
+      if (tenantUser != null && tenantUser.tenantId !== tenantId) {
         throw `Email address ${email} already in use.`
       }
     }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Form/Combobox.svelte

@@ -36,5 +36,7 @@
     {getOptionLabel}
     {getOptionValue}
     on:change={onChange}
+    on:pick
+    on:type
   />
 </Field>

packages/bbui/src/Form/Core/Checkbox.svelte

@@ -21,6 +21,7 @@
 <label
   class="spectrum-Checkbox spectrum-Checkbox--emphasized {sizeClass}"
   class:is-invalid={!!error}
+  class:checked={value}
 >
   <input
     checked={value}
@@ -50,6 +51,16 @@
 </label>
 
 <style>
+  .spectrum-Checkbox--sizeL .spectrum-Checkbox-checkmark {
+    transform: scale(1.1);
+    left: 55%;
+    top: 55%;
+  }
+  .spectrum-Checkbox--sizeXL .spectrum-Checkbox-checkmark {
+    transform: scale(1.2);
+    left: 60%;
+    top: 60%;
+  }
   .spectrum-Checkbox-input {
     opacity: 0;
   }

packages/bbui/src/Form/Core/Combobox.svelte

@@ -40,8 +40,15 @@
     open = false
   }
 
-  const onChange = e => {
+  const onType = e => {
-    selectOption(e.target.value)
+    const value = e.target.value
+    dispatch("type", value)
+    selectOption(value)
+  }
+
+  const onPick = value => {
+    dispatch("pick", value)
+    selectOption(value)
   }
 </script>
 
@@ -62,7 +69,7 @@
       type="text"
       on:focus={() => (focus = true)}
       on:blur={() => (focus = false)}
-      on:change={onChange}
+      on:change={onType}
       value={value || ""}
       placeholder={placeholder || ""}
       {disabled}
@@ -99,7 +106,7 @@
               role="option"
               aria-selected="true"
               tabindex="0"
-              on:click={() => selectOption(getOptionValue(option))}
+              on:click={() => onPick(getOptionValue(option))}
             >
               <span class="spectrum-Menu-itemLabel"
                 >{getOptionLabel(option)}</span

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.173-alpha.6",
+    "@budibase/bbui": "^0.9.180-alpha.6",
-    "@budibase/client": "^0.9.173-alpha.6",
+    "@budibase/client": "^0.9.180-alpha.6",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.173-alpha.6",
+    "@budibase/string-templates": "^0.9.180-alpha.6",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -4,6 +4,7 @@ import {
   findAllMatchingComponents,
   findComponent,
   findComponentPath,
+  getComponentSettings,
 } from "./storeUtils"
 import { store } from "builderStore"
 import { queries as queriesStores, tables as tablesStore } from "stores/backend"
@@ -38,6 +39,25 @@ export const getBindableProperties = (asset, componentId) => {
   ]
 }
 
+/**
+ * Gets the bindable properties exposed by a certain component.
+ */
+export const getComponentBindableProperties = (asset, componentId) => {
+  if (!asset || !componentId) {
+    return []
+  }
+
+  // Ensure that the component exists and exposes context
+  const component = findComponent(asset.props, componentId)
+  const def = store.actions.components.getDefinition(component?._component)
+  if (!def?.context) {
+    return []
+  }
+
+  // Get the bindings for the component
+  return getProviderContextBindings(asset, component)
+}
+
 /**
  * Gets all data provider components above a component.
  */
@@ -82,13 +102,10 @@ export const getActionProviderComponents = (asset, componentId, actionType) => {
  * Gets a datasource object for a certain data provider component
  */
 export const getDatasourceForProvider = (asset, component) => {
-  const def = store.actions.components.getDefinition(component?._component)
+  const settings = getComponentSettings(component?._component)
-  if (!def) {
-    return null
-  }
 
   // If this component has a dataProvider setting, go up the stack and use it
-  const dataProviderSetting = def.settings.find(setting => {
+  const dataProviderSetting = settings.find(setting => {
     return setting.type === "dataProvider"
   })
   if (dataProviderSetting) {
@@ -100,7 +117,7 @@ export const getDatasourceForProvider = (asset, component) => {
 
   // Extract datasource from component instance
   const validSettingTypes = ["dataSource", "table", "schema"]
-  const datasourceSetting = def.settings.find(setting => {
+  const datasourceSetting = settings.find(setting => {
     return validSettingTypes.includes(setting.type)
   })
   if (!datasourceSetting) {
@@ -127,9 +144,26 @@ export const getDatasourceForProvider = (asset, component) => {
 const getContextBindings = (asset, componentId) => {
   // Extract any components which provide data contexts
   const dataProviders = getDataProviderComponents(asset, componentId)
-  let bindings = []
+
+  // Generate bindings for all matching components
+  return getProviderContextBindings(asset, dataProviders)
+}
+
+/**
+ * Gets the context bindings exposed by a set of data provider components.
+ */
+const getProviderContextBindings = (asset, dataProviders) => {
+  if (!asset || !dataProviders) {
+    return []
+  }
+
+  // Ensure providers is an array
+  if (!Array.isArray(dataProviders)) {
+    dataProviders = [dataProviders]
+  }
 
   // Create bindings for each data provider
+  let bindings = []
   dataProviders.forEach(component => {
     const def = store.actions.components.getDefinition(component._component)
     const contexts = Array.isArray(def.context) ? def.context : [def.context]
@@ -142,6 +176,7 @@ const getContextBindings = (asset, componentId) => {
 
       let schema
       let readablePrefix
+      let runtimeSuffix = context.suffix
 
       if (context.type === "form") {
         // Forms do not need table schemas
@@ -171,8 +206,14 @@ const getContextBindings = (asset, componentId) => {
 
       const keys = Object.keys(schema).sort()
 
+      // Generate safe unique runtime prefix
+      let runtimeId = component._id
+      if (runtimeSuffix) {
+        runtimeId += `-${runtimeSuffix}`
+      }
+      const safeComponentId = makePropSafe(runtimeId)
+
       // Create bindable properties for each schema field
-      const safeComponentId = makePropSafe(component._id)
       keys.forEach(key => {
         const fieldSchema = schema[key]
 
@@ -184,6 +225,7 @@ const getContextBindings = (asset, componentId) => {
         } else if (fieldSchema.type === "attachment") {
           runtimeBoundKey = `${key}_first`
         }
+
         const runtimeBinding = `${safeComponentId}.${makePropSafe(
           runtimeBoundKey
         )}`
@@ -374,8 +416,8 @@ const buildFormSchema = component => {
   if (!component) {
     return schema
   }
-  const def = store.actions.components.getDefinition(component._component)
+  const settings = getComponentSettings(component._component)
-  const fieldSetting = def?.settings?.find(
+  const fieldSetting = settings.find(
     setting => setting.key === "field" && setting.type.startsWith("field/")
   )
   if (fieldSetting && component.field) {

packages/builder/src/builderStore/store/frontend.js

@@ -25,6 +25,7 @@ import {
   findClosestMatchingComponent,
   findAllMatchingComponents,
   findComponent,
+  getComponentSettings,
 } from "../storeUtils"
 import { uuid } from "../uuid"
 import { removeBindings } from "../dataBinding"
@@ -368,14 +369,13 @@ export const getFrontendStore = () => {
         }
 
         // Generate default props
+        const settings = getComponentSettings(componentName)
         let props = { ...presetProps }
-        if (definition.settings) {
+        settings.forEach(setting => {
-          definition.settings.forEach(setting => {
+          if (setting.defaultValue !== undefined) {
-            if (setting.defaultValue !== undefined) {
+            props[setting.key] = setting.defaultValue
-              props[setting.key] = setting.defaultValue
+          }
-            }
+        })
-          })
-        }
 
         // Add any extra properties the component needs
         let extras = {}

packages/builder/src/builderStore/storeUtils.js

@@ -1,3 +1,5 @@
+import { store } from "./index"
+
 /**
  * Recursively searches for a specific component ID
  */
@@ -123,3 +125,20 @@ const searchComponentTree = (rootComponent, matchComponent) => {
   }
   return null
 }
+
+/**
+ * Searches a component's definition for a setting matching a certin predicate.
+ */
+export const getComponentSettings = componentType => {
+  const def = store.actions.components.getDefinition(componentType)
+  if (!def) {
+    return []
+  }
+  let settings = def.settings?.filter(setting => !setting.section) ?? []
+  def.settings
+    ?.filter(setting => setting.section)
+    .forEach(section => {
+      settings = settings.concat(section.settings || [])
+    })
+  return settings
+}

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -202,7 +202,7 @@
     display: inline-block;
   }
   .block {
-    width: 360px;
+    width: 480px;
     font-size: 16px;
     background-color: var(--background);
     border: 1px solid var(--spectrum-global-color-gray-300);

packages/builder/src/components/automation/AutomationPanel/CreateAutomationModal.svelte

@@ -8,10 +8,13 @@
 
   let name
   let selectedTrigger
+  let nameTouched = false
   let triggerVal
   export let webhookModal
 
   $: instanceId = $database._id
+  $: nameError =
+    nameTouched && !name ? "Please specify a name for the automation." : null
 
   async function createAutomation() {
     await automationStore.actions.create({
@@ -51,13 +54,18 @@
   confirmText="Save"
   size="M"
   onConfirm={createAutomation}
-  disabled={!selectedTrigger}
+  disabled={!selectedTrigger || !name}
 >
   <Body size="XS"
     >Please name your automation, then select a trigger. Every automation must
     start with a trigger.
   </Body>
-  <Input bind:value={name} label="Name" />
+  <Input
+    bind:value={name}
+    on:change={() => (nameTouched = true)}
+    bind:error={nameError}
+    label="Name"
+  />
 
   <Layout noPadding>
     <Body size="S">Triggers</Body>

packages/builder/src/components/backend/DataTable/DataTable.svelte

@@ -4,6 +4,7 @@
   import CreateRowButton from "./buttons/CreateRowButton.svelte"
   import CreateColumnButton from "./buttons/CreateColumnButton.svelte"
   import CreateViewButton from "./buttons/CreateViewButton.svelte"
+  import ExistingRelationshipButton from "./buttons/ExistingRelationshipButton.svelte"
   import ExportButton from "./buttons/ExportButton.svelte"
   import EditRolesButton from "./buttons/EditRolesButton.svelte"
   import ManageAccessButton from "./buttons/ManageAccessButton.svelte"
@@ -98,9 +99,7 @@
     on:updatecolumns={onUpdateColumns}
     on:updaterows={onUpdateRows}
   >
-    {#if isInternal}
+    <CreateColumnButton on:updatecolumns={onUpdateColumns} />
-      <CreateColumnButton on:updatecolumns={onUpdateColumns} />
-    {/if}
     {#if schema && Object.keys(schema).length > 0}
       {#if !isUsersTable}
         <CreateRowButton
@@ -116,6 +115,12 @@
       {#if isUsersTable}
         <EditRolesButton />
       {/if}
+      {#if !isInternal}
+        <ExistingRelationshipButton
+          table={$tables.selected}
+          on:updatecolumns={onUpdateColumns}
+        />
+      {/if}
       <HideAutocolumnButton bind:hideAutocolumns />
       <!-- always have the export last -->
       <ExportButton view={$tables.selected?._id} />

packages/builder/src/components/backend/DataTable/RowFieldControl.svelte

@@ -16,8 +16,8 @@
   export let value = defaultValue || (meta.type === "boolean" ? false : "")
   export let readonly
 
-  $: type = meta.type
+  $: type = meta?.type
-  $: label = capitalise(meta.name)
+  $: label = meta.name ? capitalise(meta.name) : ""
 </script>
 
 {#if type === "options"}

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -8,7 +8,11 @@
   import CreateEditRow from "./modals/CreateEditRow.svelte"
   import CreateEditUser from "./modals/CreateEditUser.svelte"
   import CreateEditColumn from "./modals/CreateEditColumn.svelte"
-  import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
+  import {
+    TableNames,
+    UNEDITABLE_USER_FIELDS,
+    UNSORTABLE_TYPES,
+  } from "constants"
   import RoleCell from "./cells/RoleCell.svelte"
 
   export let schema = {}
@@ -33,6 +37,15 @@
   $: isUsersTable = tableId === TableNames.USERS
   $: data && resetSelectedRows()
   $: editRowComponent = isUsersTable ? CreateEditUser : CreateEditRow
+  $: {
+    UNSORTABLE_TYPES.forEach(type => {
+      Object.values(schema).forEach(col => {
+        if (col.type === type) {
+          col.sortable = false
+        }
+      })
+    })
+  }
   $: {
     if (isUsersTable) {
       customRenderers = [
@@ -129,7 +142,7 @@
         bind:selectedRows
         allowSelectRows={allowEditing && !isUsersTable}
         allowEditRows={allowEditing}
-        allowEditColumns={allowEditing && isInternal}
+        allowEditColumns={allowEditing}
         showAutoColumns={!hideAutocolumns}
         on:editcolumn={e => editColumn(e.detail)}
         on:editrow={e => editRow(e.detail)}

packages/builder/src/components/backend/DataTable/buttons/ExistingRelationshipButton.svelte

@@ -0,0 +1,54 @@
+<script>
+  import { ActionButton, Modal, notifications } from "@budibase/bbui"
+  import CreateEditRelationship from "../../Datasources/CreateEditRelationship.svelte"
+  import { datasources, tables } from "../../../../stores/backend"
+  import { createEventDispatcher } from "svelte"
+
+  export let table
+  const dispatch = createEventDispatcher()
+
+  $: plusTables = datasource?.plus
+    ? Object.values(datasource?.entities || {})
+    : []
+  $: datasource = $datasources.list.find(
+    source => source._id === table?.sourceId
+  )
+
+  let modal
+
+  async function saveRelationship() {
+    try {
+      // Create datasource
+      await datasources.save(datasource)
+      notifications.success(`Relationship information saved.`)
+      const tableList = await tables.fetch()
+      await tables.select(tableList.find(tbl => tbl._id === table._id))
+      dispatch("updatecolumns")
+    } catch (err) {
+      notifications.error(`Error saving relationship info: ${err}`)
+    }
+  }
+</script>
+
+{#if table.sourceId}
+  <div>
+    <ActionButton
+      icon="DataCorrelated"
+      primary
+      size="S"
+      quiet
+      on:click={modal.show}
+    >
+      Define existing relationship
+    </ActionButton>
+  </div>
+  <Modal bind:this={modal}>
+    <CreateEditRelationship
+      {datasource}
+      save={saveRelationship}
+      close={modal.hide}
+      {plusTables}
+      selectedFromTable={table}
+    />
+  </Modal>
+{/if}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -31,6 +31,9 @@
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
   const LINK_TYPE = FIELDS.LINK.type
+  const STRING_TYPE = FIELDS.STRING.type
+  const NUMBER_TYPE = FIELDS.NUMBER.type
+
   const dispatch = createEventDispatcher()
   const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"]
   const { hide } = getContext(Context.Modal)
@@ -55,9 +58,7 @@
   let confirmDeleteDialog
   let deletion
 
-  $: tableOptions = $tables.list.filter(
+  $: checkConstraints(field)
-    table => table._id !== $tables.draft._id && table.type !== "external"
-  )
   $: required = !!field?.constraints?.presence || primaryDisplay
   $: uneditable =
     $tables.selected?._id === TableNames.USERS &&
@@ -83,6 +84,14 @@
   $: canBeRequired =
     field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_TYPE
   $: relationshipOptions = getRelationshipOptions(field)
+  $: external = table.type === "external"
+  // in the case of internal tables the sourceId will just be undefined
+  $: tableOptions = $tables.list.filter(
+    opt =>
+      opt._id !== $tables.draft._id &&
+      opt.type === table.type &&
+      table.sourceId === opt.sourceId
+  )
 
   async function saveColumn() {
     if (field.type === AUTO_TYPE) {
@@ -169,7 +178,7 @@
     if (!field || !field.tableId) {
       return null
     }
-    const linkTable = tableOptions.find(table => table._id === field.tableId)
+    const linkTable = tableOptions?.find(table => table._id === field.tableId)
     if (!linkTable) {
       return null
     }
@@ -193,6 +202,45 @@
       },
     ]
   }
+
+  function getAllowedTypes() {
+    if (!external) {
+      return [
+        ...Object.values(fieldDefinitions),
+        { name: "Auto Column", type: AUTO_TYPE },
+      ]
+    } else {
+      return [
+        FIELDS.STRING,
+        FIELDS.LONGFORM,
+        FIELDS.OPTIONS,
+        FIELDS.DATETIME,
+        FIELDS.NUMBER,
+        FIELDS.BOOLEAN,
+        FIELDS.ARRAY,
+        FIELDS.FORMULA,
+        FIELDS.LINK,
+      ]
+    }
+  }
+
+  function checkConstraints(fieldToCheck) {
+    // most types need this, just make sure its always present
+    if (fieldToCheck && !fieldToCheck.constraints) {
+      fieldToCheck.constraints = {}
+    }
+    // some string types may have been built by server, may not always have constraints
+    if (fieldToCheck.type === STRING_TYPE && !fieldToCheck.constraints.length) {
+      fieldToCheck.constraints.length = {}
+    }
+    // some number types made server-side will be missing constraints
+    if (
+      fieldToCheck.type === NUMBER_TYPE &&
+      !fieldToCheck.constraints.numericality
+    ) {
+      fieldToCheck.constraints.numericality = {}
+    }
+  }
 </script>
 
 <ModalContent
@@ -215,10 +263,7 @@
     label="Type"
     bind:value={field.type}
     on:change={handleTypeChange}
-    options={[
+    options={getAllowedTypes()}
-      ...Object.values(fieldDefinitions),
-      { name: "Auto Column", type: AUTO_TYPE },
-    ]}
     getOptionLabel={field => field.name}
     getOptionValue={field => field.type}
   />
@@ -245,7 +290,7 @@
     </div>
   {/if}
 
-  {#if canBeSearched}
+  {#if canBeSearched && !external}
     <div>
       <Label grey small>Search Indexes</Label>
       <Toggle

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -18,10 +18,19 @@
   export let fromRelationship = {}
   export let toRelationship = {}
   export let close
+  export let selectedFromTable
 
   let originalFromName = fromRelationship.name,
     originalToName = toRelationship.name
 
+  if (fromRelationship && !fromRelationship.relationshipType) {
+    fromRelationship.relationshipType = RelationshipTypes.MANY_TO_ONE
+  }
+
+  if (toRelationship && selectedFromTable) {
+    toRelationship.tableId = selectedFromTable._id
+  }
+
   function inSchema(table, prop, ogName) {
     if (!table || !prop || prop === ogName) {
       return false
@@ -114,6 +123,7 @@
     },
   ]
   $: updateRelationshipType(fromRelationship?.relationshipType)
+  $: tableChanged(fromTable, toTable)
 
   function updateRelationshipType(fromType) {
     if (fromType === RelationshipTypes.MANY_TO_MANY) {
@@ -205,7 +215,6 @@
     originalToName = toRelationship.name
     originalFromName = fromRelationship.name
     await save()
-    await tables.fetch()
   }
 
   async function deleteRelationship() {
@@ -215,10 +224,26 @@
     await tables.fetch()
     close()
   }
+
+  function tableChanged(fromTbl, toTbl) {
+    fromRelationship.name = toTbl?.name || ""
+    errors.fromCol = ""
+    toRelationship.name = fromTbl?.name || ""
+    errors.toCol = ""
+    if (toTbl || fromTbl) {
+      checkForErrors(
+        fromTable,
+        toTable,
+        through,
+        fromRelationship,
+        toRelationship
+      )
+    }
+  }
 </script>
 
 <ModalContent
-  title="Create Relationship"
+  title="Define Relationship"
   confirmText="Save"
   onConfirm={saveRelationship}
   disabled={!valid}
@@ -234,6 +259,7 @@
   <Select
     label="Select from table"
     options={tableOptions}
+    disabled={!!selectedFromTable}
     on:change={() => ($touched.from = true)}
     bind:error={errors.from}
     bind:value={toRelationship.tableId}

packages/builder/src/components/backend/TableNavigator/popovers/EditTablePopover.svelte

@@ -1,7 +1,7 @@
 <script>
   import { goto } from "@roxi/routify"
   import { allScreens, store } from "builderStore"
-  import { tables } from "stores/backend"
+  import { tables, datasources } from "stores/backend"
   import {
     ActionMenu,
     Icon,
@@ -40,7 +40,10 @@
     store.actions.screens.delete(templateScreens)
     await tables.fetch()
     notifications.success("Table deleted")
-    if (wasSelectedTable._id === table._id) {
+    if (table.type === "external") {
+      await datasources.fetch()
+    }
+    if (wasSelectedTable && wasSelectedTable._id === table._id) {
       $goto("./table")
     }
   }
@@ -64,9 +67,7 @@
     <Icon s hoverable name="MoreSmallList" />
   </div>
   <MenuItem icon="Edit" on:click={editorModal.show}>Edit</MenuItem>
-  {#if !external}
+  <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
-    <MenuItem icon="Delete" on:click={showDeleteModal}>Delete</MenuItem>
-  {/if}
 </ActionMenu>
 
 <Modal bind:this={editorModal}>

packages/builder/src/components/common/bindings/DrawerBindableCombobox.svelte

@@ -17,6 +17,7 @@
   export let disabled = false
   export let options
   export let allowJS = true
+  export let appendBindingsAsOptions = true
 
   const dispatch = createEventDispatcher()
   let bindingDrawer
@@ -24,15 +25,30 @@
   $: readableValue = runtimeToReadableBinding(bindings, value)
   $: tempValue = readableValue
   $: isJS = isJSBinding(value)
+  $: allOptions = buildOptions(options, bindings, appendBindingsAsOptions)
 
   const handleClose = () => {
     onChange(tempValue)
     bindingDrawer.hide()
   }
 
-  const onChange = value => {
+  const onChange = (value, optionPicked) => {
+    // Add HBS braces if picking binding
+    if (optionPicked && !options?.includes(value)) {
+      value = `{{ ${value} }}`
+    }
+
     dispatch("change", readableToRuntimeBinding(bindings, value))
   }
+
+  const buildOptions = (options, bindings, appendBindingsAsOptions) => {
+    if (!appendBindingsAsOptions) {
+      return options
+    }
+    return []
+      .concat(options || [])
+      .concat(bindings?.map(binding => binding.readableBinding) || [])
+  }
 </script>
 
 <div class="control">
@@ -40,12 +56,17 @@
     {label}
     {disabled}
     value={isJS ? "(JavaScript function)" : readableValue}
-    on:change={event => onChange(event.detail)}
+    on:type={e => onChange(e.detail, false)}
+    on:pick={e => onChange(e.detail, true)}
     {placeholder}
-    {options}
+    options={allOptions}
   />
   {#if !disabled}
-    <div class="icon" on:click={bindingDrawer.show}>
+    <div
+      class="icon"
+      on:click={bindingDrawer.show}
+      data-cy="text-binding-button"
+    >
       <Icon size="S" name="FlashOn" />
     </div>
   {/if}

packages/builder/src/components/design/AppPreview/CurrentItemPreview.svelte

@@ -32,6 +32,16 @@
     .component("@budibase/standard-components/screenslot")
     .instanceName("Content Placeholder")
     .json()
+  
+  // Messages that can be sent from the iframe preview to the builder
+  // Budibase events are and initalisation events
+  const MessageTypes = {
+    IFRAME_LOADED: "iframe-loaded",
+    READY: "ready",
+    ERROR: "error",
+    BUDIBASE: "type",
+    KEYDOWN: "keydown"
+  }
 
   // Construct iframe template
   $: template = iframeTemplate.replace(
@@ -80,46 +90,44 @@
   // Refresh the preview when required
   $: refreshContent(strippedJson)
 
-  onMount(() => {
+  function receiveMessage(message) {
-    // Initialise the app when mounted
+    const handlers = {
-    iframe.contentWindow.addEventListener(
+      [MessageTypes.READY]: () => {
-      "ready",
+        // Initialise the app when mounted
-      () => {
         // Display preview immediately if the intelligent loading feature
         // is not supported
+        if (!loading) return
+
         if (!$store.clientFeatures.intelligentLoading) {
           loading = false
         }
         refreshContent(strippedJson)
       },
-      { once: true }
+      [MessageTypes.ERROR]: event => {
-    )
+        // Catch any app errors
-
-    // Catch any app errors
-    iframe.contentWindow.addEventListener(
-      "error",
-      event => {
         loading = false
-        error = event.detail || "An unknown error occurred"
+        error = event.error || "An unknown error occurred"
       },
-      { once: true }
+      [MessageTypes.KEYDOWN]: handleKeydownEvent
-    )
+    }
 
-    // Add listener for events sent by client library in preview
+    const messageHandler = handlers[message.data.type] || handleBudibaseEvent
-    iframe.contentWindow.addEventListener("bb-event", handleBudibaseEvent)
+    messageHandler(message)
-    iframe.contentWindow.addEventListener("keydown", handleKeydownEvent)
+  }
+
+  onMount(() => {
+    window.addEventListener("message", receiveMessage)
   })
 
   // Remove all iframe event listeners on component destroy
   onDestroy(() => {
     if (iframe.contentWindow) {
-      iframe.contentWindow.removeEventListener("bb-event", handleBudibaseEvent)
+      window.removeEventListener("message", receiveMessage) //
-      iframe.contentWindow.removeEventListener("keydown", handleKeydownEvent)
     }
   })
 
   const handleBudibaseEvent = event => {
-    const { type, data } = event.detail
+    const { type, data } = event.data
     if (type === "select-component" && data.id) {
       store.actions.components.select({ _id: data.id })
     } else if (type === "update-prop") {
@@ -151,13 +159,14 @@
         store.actions.components.paste(destination, data.mode)
       }
     } else {
-      console.warning(`Client sent unknown event type: ${type}`)
+      console.warn(`Client sent unknown event type: ${type}`)
     }
   }
 
   const handleKeydownEvent = event => {
+    const { key } = event.data
     if (
-      (event.key === "Delete" || event.key === "Backspace") &&
+      (key === "Delete" || key === "Backspace") &&
       selectedComponentId &&
       ["input", "textarea"].indexOf(
         iframe.contentWindow.document.activeElement?.tagName.toLowerCase()

packages/builder/src/components/design/AppPreview/componentStructure.json

@@ -1,4 +1,12 @@
 [
+  {
+    "name": "Blocks",
+    "icon": "Article",
+    "children": [
+      "tablewithsearch",
+      "cardlistwithsearch"
+    ]
+  },
   "section",
   "container",
   "dataprovider",

packages/builder/src/components/design/AppPreview/iframeTemplate.js

@@ -54,7 +54,7 @@ export default `
         if (!parsed) {
           return
         }
-        
+
         // Extract data from message
         const {
           selectedComponentId,
@@ -84,17 +84,20 @@ export default `
           if (window.loadBudibase) {
             window.loadBudibase()
             document.documentElement.classList.add("loaded")
-            window.dispatchEvent(new Event("iframe-loaded"))
+            window.parent.postMessage({ type: "iframe-loaded" })
           } else {
             throw "The client library couldn't be loaded"
           }
         } catch (error) {
-          window.dispatchEvent(new CustomEvent("error", { detail: error }))
+          window.parent.postMessage({ type: "error", error })
         }
       }
 
       window.addEventListener("message", receiveMessage)
-      window.dispatchEvent(new Event("ready"))
+      window.addEventListener("keydown", evt => {
+        window.parent.postMessage({ type: "keydown", key: event.key })
+      })
+      window.parent.postMessage({ type: "ready" })
     </script>
   </head>
   <body/>

packages/builder/src/components/design/PropertiesPanel/ComponentSettingsSection.svelte

@@ -12,6 +12,7 @@
   export let componentInstance
   export let assetInstance
   export let bindings
+  export let componentBindings
 
   const layoutDefinition = []
   const screenDefinition = [
@@ -21,10 +22,24 @@
     { key: "layoutId", label: "Layout", control: LayoutSelect },
   ]
 
-  $: settings = componentDefinition?.settings ?? []
+  $: sections = getSections(componentDefinition)
   $: isLayout = assetInstance && assetInstance.favicon
   $: assetDefinition = isLayout ? layoutDefinition : screenDefinition
 
+  const getSections = definition => {
+    const settings = definition?.settings ?? []
+    const generalSettings = settings.filter(setting => !setting.section)
+    const customSections = settings.filter(setting => setting.section)
+    return [
+      {
+        name: "General",
+        info: componentDefinition?.info,
+        settings: generalSettings,
+      },
+      ...(customSections || []),
+    ]
+  }
+
   const updateProp = store.actions.components.updateProp
 
   const canRenderControl = setting => {
@@ -59,19 +74,18 @@
   }
 </script>
 
-<DetailSummary name="General" collapsible={false}>
+{#each sections as section, idx (section.name)}
-  {#if !componentInstance._component.endsWith("/layout")}
+  <DetailSummary name={section.name} collapsible={false}>
-    <PropertyControl
+    {#if idx === 0 && !componentInstance._component.endsWith("/layout")}
-      bindable={false}
+      <PropertyControl
-      control={Input}
+        control={Input}
-      label="Name"
+        label="Name"
-      key="_instanceName"
+        key="_instanceName"
-      value={componentInstance._instanceName}
+        value={componentInstance._instanceName}
-      onChange={val => updateProp("_instanceName", val)}
+        onChange={val => updateProp("_instanceName", val)}
-    />
+      />
-  {/if}
+    {/if}
-  {#if settings && settings.length > 0}
+    {#each section.settings as setting (setting.key)}
-    {#each settings as setting (setting.key)}
       {#if canRenderControl(setting)}
         <PropertyControl
           type={setting.type}
@@ -80,7 +94,7 @@
           key={setting.key}
           value={componentInstance[setting.key] ??
             componentInstance[setting.key]?.defaultValue}
-          {componentInstance}
+          nested={setting.nested}
           onChange={val => updateProp(setting.key, val)}
           props={{
             options: setting.options || [],
@@ -89,20 +103,22 @@
             max: setting.max || null,
           }}
           {bindings}
+          {componentBindings}
+          {componentInstance}
           {componentDefinition}
         />
       {/if}
     {/each}
-  {/if}
+    {#if idx === 0 && componentDefinition?.component?.endsWith("/fieldgroup")}
-  {#if componentDefinition?.component?.endsWith("/fieldgroup")}
+      <ResetFieldsButton {componentInstance} />
-    <ResetFieldsButton {componentInstance} />
+    {/if}
-  {/if}
+    {#if section?.info}
-  {#if componentDefinition?.info}
+      <div class="text">
-    <div class="text">
+        {@html section.info}
-      {@html componentDefinition?.info}
+      </div>
-    </div>
+    {/if}
-  {/if}
+  </DetailSummary>
-</DetailSummary>
+{/each}
 
 <style>
   .text {

packages/builder/src/components/design/PropertiesPanel/PropertiesPanel.svelte

@@ -6,13 +6,20 @@
   import DesignSection from "./DesignSection.svelte"
   import CustomStylesSection from "./CustomStylesSection.svelte"
   import ConditionalUISection from "./ConditionalUISection.svelte"
-  import { getBindableProperties } from "builderStore/dataBinding"
+  import {
+    getBindableProperties,
+    getComponentBindableProperties,
+  } from "builderStore/dataBinding"
 
   $: componentInstance = $selectedComponent
   $: componentDefinition = store.actions.components.getDefinition(
     $selectedComponent?._component
   )
   $: bindings = getBindableProperties($currentAsset, $store.selectedComponentId)
+  $: componentBindings = getComponentBindableProperties(
+    $currentAsset,
+    $store.selectedComponentId
+  )
 </script>
 
 <Tabs selected="Settings" noPadding>
@@ -28,6 +35,7 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          {componentBindings}
         />
         <DesignSection {componentInstance} {componentDefinition} {bindings} />
         <CustomStylesSection

packages/builder/src/components/design/PropertiesPanel/PropertyControls/Input.svelte

@@ -1,15 +0,0 @@
-<script>
-  import { Input } from "@budibase/bbui"
-  import { isJSBinding } from "@budibase/string-templates"
-
-  export let value
-
-  $: isJS = isJSBinding(value)
-</script>
-
-<Input
-  {...$$props}
-  value={isJS ? "(JavaScript function)" : value}
-  readonly={isJS}
-  on:change
-/>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/PropertyControl.svelte

@@ -1,11 +1,9 @@
 <script>
-  import { Button, Icon, Drawer, Label } from "@budibase/bbui"
+  import { Label } from "@budibase/bbui"
   import {
     readableToRuntimeBinding,
     runtimeToReadableBinding,
   } from "builderStore/dataBinding"
-  import BindingPanel from "components/common/bindings/BindingPanel.svelte"
-  import { capitalise } from "helpers"
 
   export let label = ""
   export let bindable = true
@@ -17,18 +15,19 @@
   export let props = {}
   export let onChange = () => {}
   export let bindings = []
+  export let componentBindings = []
+  export let nested = false
 
-  let bindingDrawer
+  $: allBindings = getAllBindings(bindings, componentBindings, nested)
-  let anchor
+  $: safeValue = getSafeValue(value, props.defaultValue, allBindings)
-  let valid
-
-  $: safeValue = getSafeValue(value, props.defaultValue, bindings)
   $: tempValue = safeValue
-  $: replaceBindings = val => readableToRuntimeBinding(bindings, val)
+  $: replaceBindings = val => readableToRuntimeBinding(allBindings, val)
 
-  const handleClose = () => {
+  const getAllBindings = (bindings, componentBindings, nested) => {
-    handleChange(tempValue)
+    if (!nested) {
-    bindingDrawer.hide()
+      return bindings
+    }
+    return [...(bindings || []), ...(componentBindings || [])]
   }
 
   // Handle a value change of any type
@@ -64,7 +63,7 @@
   }
 </script>
 
-<div class="property-control" bind:this={anchor} data-cy={`setting-${key}`}>
+<div class="property-control" data-cy={`setting-${key}`}>
   {#if type !== "boolean" && label}
     <div class="label">
       <Label>{label}</Label>
@@ -78,37 +77,12 @@
       updateOnChange={false}
       on:change={handleChange}
       onChange={handleChange}
-      {bindings}
+      bindings={allBindings}
       name={key}
       text={label}
       {type}
       {...props}
     />
-    {#if bindable && !key.startsWith("_") && type === "text"}
-      <div
-        class="icon"
-        data-cy={`${key}-binding-button`}
-        on:click={bindingDrawer.show}
-      >
-        <Icon size="S" name="FlashOn" />
-      </div>
-      <Drawer bind:this={bindingDrawer} title={capitalise(key)}>
-        <svelte:fragment slot="description">
-          Add the objects on the left to enrich your text.
-        </svelte:fragment>
-        <Button cta slot="buttons" disabled={!valid} on:click={handleClose}>
-          Save
-        </Button>
-        <BindingPanel
-          slot="body"
-          bind:valid
-          value={safeValue}
-          on:change={e => (tempValue = e.detail)}
-          bindableProperties={bindings}
-          allowJS
-        />
-      </Drawer>
-    {/if}
   </div>
 </div>
 
@@ -120,41 +94,10 @@
     justify-content: flex-start;
     align-items: stretch;
   }
-
   .label {
-    text-transform: capitalize;
     padding-bottom: var(--spectrum-global-dimension-size-65);
   }
-
   .control {
     position: relative;
   }
-
-  .icon {
-    right: 1px;
-    top: 1px;
-    bottom: 1px;
-    position: absolute;
-    justify-content: center;
-    align-items: center;
-    display: flex;
-    flex-direction: row;
-    box-sizing: border-box;
-    border-left: 1px solid var(--spectrum-alias-border-color);
-    border-top-right-radius: var(--spectrum-alias-border-radius-regular);
-    border-bottom-right-radius: var(--spectrum-alias-border-radius-regular);
-    width: 31px;
-    color: var(--spectrum-alias-text-color);
-    background-color: var(--spectrum-global-color-gray-75);
-    transition: background-color
-        var(--spectrum-global-animation-duration-100, 130ms),
-      box-shadow var(--spectrum-global-animation-duration-100, 130ms),
-      border-color var(--spectrum-global-animation-duration-100, 130ms);
-  }
-  .icon:hover {
-    cursor: pointer;
-    color: var(--spectrum-alias-text-color-hover);
-    background-color: var(--spectrum-global-color-gray-50);
-    border-color: var(--spectrum-alias-border-color-hover);
-  }
 </style>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/URLSelect.svelte

@@ -10,4 +10,10 @@
     .filter(x => x != null)
 </script>
 
-<DrawerBindableCombobox {value} {bindings} on:change options={urlOptions} />
+<DrawerBindableCombobox
+  {value}
+  {bindings}
+  on:change
+  options={urlOptions}
+  appendBindingsAsOptions={false}
+/>

packages/builder/src/components/design/PropertiesPanel/PropertyControls/componentSettings.js

@@ -15,10 +15,10 @@ import URLSelect from "./URLSelect.svelte"
 import OptionsEditor from "./OptionsEditor/OptionsEditor.svelte"
 import FormFieldSelect from "./FormFieldSelect.svelte"
 import ValidationEditor from "./ValidationEditor/ValidationEditor.svelte"
-import Input from "./Input.svelte"
+import DrawerBindableCombobox from "components/common/bindings/DrawerBindableCombobox.svelte"
 
 const componentMap = {
-  text: Input,
+  text: DrawerBindableCombobox,
   select: Select,
   dataSource: DataSourceSelect,
   dataProvider: DataProviderSelect,

packages/builder/src/components/design/PropertiesPanel/ScreenSettingsSection.svelte

@@ -54,7 +54,6 @@
   <DetailSummary name="Screen" collapsible={false}>
     {#each screenSettings as def (`${componentInstance._id}-${def.key}`)}
       <PropertyControl
-        bindable={false}
         control={def.control}
         label={def.label}
         key={def.key}

packages/builder/src/components/design/PropertiesPanel/StyleSection.svelte

@@ -30,7 +30,6 @@
     {#each properties as prop (`${componentInstance._id}-${prop.key}-${prop.label}`)}
       <div style="grid-column: {prop.column || 'auto'}">
         <PropertyControl
-          bindable={false}
           label={`${prop.label}${hasPropChanged(style, prop) ? " *" : ""}`}
           control={prop.control}
           key={prop.key}

packages/builder/src/components/design/PropertiesPanel/componentStyles.js

@@ -9,7 +9,6 @@ export const margin = {
       label: "Top",
       key: "margin-top",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -30,7 +29,6 @@ export const margin = {
       label: "Right",
       key: "margin-right",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -51,7 +49,6 @@ export const margin = {
       label: "Bottom",
       key: "margin-bottom",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -72,7 +69,6 @@ export const margin = {
       label: "Left",
       key: "margin-left",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -100,7 +96,6 @@ export const padding = {
       label: "Top",
       key: "padding-top",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -121,7 +116,6 @@ export const padding = {
       label: "Right",
       key: "padding-right",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -142,7 +136,6 @@ export const padding = {
       label: "Bottom",
       key: "padding-bottom",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },
@@ -163,7 +156,6 @@ export const padding = {
       label: "Left",
       key: "padding-left",
       control: Select,
-      bindable: false,
       placeholder: "None",
       options: [
         { label: "4px", value: "4px" },

packages/builder/src/components/start/CreateAppModal.svelte

@@ -9,7 +9,7 @@
     Checkbox,
   } from "@budibase/bbui"
   import { store, automationStore, hostingStore } from "builderStore"
-  import { admin } from "stores/portal"
+  import { admin, auth } from "stores/portal"
   import { string, mixed, object } from "yup"
   import api, { get, post } from "builderStore/api"
   import analytics, { Events } from "analytics"
@@ -139,6 +139,7 @@
       }
       const userResp = await api.post(`/api/users/metadata/self`, user)
       await userResp.json()
+      await auth.setInitInfo({})
       $goto(`/builder/app/${appJson.instance._id}`)
     } catch (error) {
       console.error(error)
@@ -146,6 +147,16 @@
       submitting = false
     }
   }
+
+  function getModalTitle() {
+    let title = "Create App"
+    if (template.fromFile) {
+      title = "Import App"
+    } else if (template.key) {
+      title = "Create app from template"
+    }
+    return title
+  }
 </script>
 
 {#if showTemplateSelection}
@@ -172,7 +183,7 @@
   </ModalContent>
 {:else}
   <ModalContent
-    title={template?.fromFile ? "Import app" : "Create app"}
+    title={getModalTitle()}
     confirmText={template?.fromFile ? "Import app" : "Create app"}
     onConfirm={createNewApp}
     onCancel={inline ? () => (template = null) : null}

packages/builder/src/constants/index.js

@@ -39,6 +39,8 @@ export const UNEDITABLE_USER_FIELDS = [
   "lastName",
 ]
 
+export const UNSORTABLE_TYPES = ["formula", "attachment", "array", "link"]
+
 export const LAYOUT_NAMES = {
   MASTER: {
     PRIVATE: "layout_private_master",

packages/builder/src/pages/builder/_layout.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { isActive, redirect } from "@roxi/routify"
+  import { isActive, redirect, params } from "@roxi/routify"
   import { admin, auth } from "stores/portal"
   import { onMount } from "svelte"
 
@@ -47,6 +47,10 @@
   }
 
   onMount(async () => {
+    if ($params["?template"]) {
+      await auth.setInitInfo({ init_template: $params["?template"] })
+    }
+
     await auth.checkAuth()
     await admin.init()
 

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/index.svelte

@@ -8,17 +8,18 @@
     Layout,
     Modal,
     InlineAlert,
+    ActionButton,
   } from "@budibase/bbui"
   import { datasources, integrations, queries, tables } from "stores/backend"
   import { notifications } from "@budibase/bbui"
   import IntegrationConfigForm from "components/backend/DatasourceNavigator/TableIntegrationMenu/IntegrationConfigForm.svelte"
-  import CreateEditRelationship from "./CreateEditRelationship/CreateEditRelationship.svelte"
+  import CreateEditRelationship from "components/backend/Datasources/CreateEditRelationship.svelte"
-  import DisplayColumnModal from "./modals/EditDisplayColumnsModal.svelte"
+  import CreateExternalTableModal from "./modals/CreateExternalTableModal.svelte"
   import ICONS from "components/backend/DatasourceNavigator/icons"
   import { capitalise } from "helpers"
 
   let relationshipModal
-  let displayColumnModal
+  let createExternalTableModal
   let selectedFromRelationship, selectedToRelationship
 
   $: datasource = $datasources.list.find(ds => ds._id === $datasources.selected)
@@ -78,6 +79,10 @@
     try {
       // Create datasource
       await datasources.save(datasource)
+      if (datasource?.plus) {
+        await tables.fetch()
+      }
+      await datasources.fetch()
       notifications.success(`Datasource ${name} updated successfully.`)
     } catch (err) {
       notifications.error(`Error saving datasource: ${err}`)
@@ -110,8 +115,8 @@
     relationshipModal.show()
   }
 
-  function openDisplayColumnModal() {
+  function createNewTable() {
-    displayColumnModal.show()
+    createExternalTableModal.show()
   }
 </script>
 
@@ -126,8 +131,8 @@
   />
 </Modal>
 
-<Modal bind:this={displayColumnModal}>
+<Modal bind:this={createExternalTableModal}>
-  <DisplayColumnModal {datasource} {plusTables} save={saveDatasource} />
+  <CreateExternalTableModal {datasource} />
 </Modal>
 
 {#if datasource && integration}
@@ -163,15 +168,15 @@
         <div class="query-header">
           <Heading size="S">Tables</Heading>
           <div class="table-buttons">
-            {#if plusTables && plusTables.length !== 0}
-              <Button primary on:click={openDisplayColumnModal}>
-                Update display columns
-              </Button>
-            {/if}
             <div>
-              <Button primary on:click={updateDatasourceSchema}>
+              <ActionButton
+                size="S"
+                quiet
+                icon="DataRefresh"
+                on:click={updateDatasourceSchema}
+              >
                 Fetch tables from database
-              </Button>
+              </ActionButton>
             </div>
           </div>
         </div>
@@ -196,14 +201,23 @@
               <p>→</p>
             </div>
           {/each}
+          <div class="add-table">
+            <Button cta on:click={createNewTable}>Create new table</Button>
+          </div>
         </div>
         {#if plusTables?.length !== 0}
           <Divider />
           <div class="query-header">
             <Heading size="S">Relationships</Heading>
-            <Button primary on:click={() => openRelationshipModal()}
+            <ActionButton
-              >Create relationship</Button
+              icon="DataCorrelated"
+              primary
+              size="S"
+              quiet
+              on:click={openRelationshipModal}
             >
+              Define existing relationship
+            </ActionButton>
           </div>
           <Body>
             Tell budibase how your tables are related to get even more smart
@@ -318,11 +332,14 @@
 
   .table-buttons {
     display: grid;
-    grid-gap: var(--spacing-l);
     grid-template-columns: 1fr 1fr;
   }
 
   .table-buttons div {
     grid-column-end: -1;
   }
+
+  .add-table {
+    margin-top: var(--spacing-m);
+  }
 </style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/modals/CreateExternalTableModal.svelte

@@ -0,0 +1,45 @@
+<script>
+  import { ModalContent, Body, Input } from "@budibase/bbui"
+  import { tables, datasources } from "stores/backend"
+  import { goto } from "@roxi/routify"
+
+  export let datasource
+
+  let name = ""
+  $: valid = name && name.length > 0 && !datasource?.entities[name]
+  $: error =
+    name && datasource?.entities[name] ? "Table name already in use." : null
+
+  function buildDefaultTable(tableName, datasourceId) {
+    return {
+      name: tableName,
+      type: "external",
+      primary: ["id"],
+      sourceId: datasourceId,
+      schema: {
+        id: {
+          autocolumn: true,
+          type: "number",
+        },
+      },
+    }
+  }
+
+  async function saveTable() {
+    const table = await tables.save(buildDefaultTable(name, datasource._id))
+    await datasources.fetch()
+    $goto(`../../table/${table._id}`)
+  }
+</script>
+
+<ModalContent
+  title="Create new table"
+  confirmText="Create"
+  onConfirm={saveTable}
+  disabled={!valid}
+>
+  <Body
+    >Provide a name for your new table; you can add columns once it is created.</Body
+  >
+  <Input label="Table Name" bind:error bind:value={name} />
+</ModalContent>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/modals/EditDisplayColumnsModal.svelte

@@ -1,43 +0,0 @@
-<script>
-  import { ModalContent, Select, Body } from "@budibase/bbui"
-  import { tables } from "stores/backend"
-
-  export let datasource
-  export let plusTables
-  export let save
-
-  async function saveDisplayColumns() {
-    // be explicit about copying over
-    for (let table of plusTables) {
-      datasource.entities[table.name].primaryDisplay = table.primaryDisplay
-    }
-    save()
-    await tables.fetch()
-  }
-
-  function getColumnOptions(table) {
-    if (!table || !table.schema) {
-      return []
-    }
-    return Object.entries(table.schema)
-      .filter(field => field[1].type !== "link")
-      .map(([fieldName]) => fieldName)
-  }
-</script>
-
-<ModalContent
-  title="Edit display columns"
-  confirmText="Save"
-  onConfirm={saveDisplayColumns}
->
-  <Body
-    >Select the columns that will be shown when displaying relationships.</Body
-  >
-  {#each plusTables as table}
-    <Select
-      label={table.name}
-      options={getColumnOptions(table)}
-      bind:value={table.primaryDisplay}
-    />
-  {/each}
-</ModalContent>

packages/builder/src/pages/builder/auth/login.svelte

@@ -44,7 +44,7 @@
       }
     } catch (err) {
       console.error(err)
-      notifications.error("Invalid credentials")
+      notifications.error(err.message ? err.message : "Invalid Credentials")
     }
   }
 

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -184,9 +184,27 @@
     }
   }
 
+  function createAppFromTemplateUrl(templateKey) {
+    // validate the template key just to make sure
+    const templateParts = templateKey.split("/")
+    if (templateParts.length === 2 && templateParts[0] === "app") {
+      template = {
+        key: templateKey,
+      }
+      initiateAppCreation()
+    } else {
+      notifications.error("Your Template URL is invalid. Please try another.")
+    }
+  }
+
   onMount(async () => {
     await apps.load()
     loaded = true
+    // if the portal is loaded from an external URL with a template param
+    const initInfo = await auth.getInitInfo()
+    if (initInfo.init_template) {
+      createAppFromTemplateUrl(initInfo.init_template)
+    }
   })
 </script>
 

packages/builder/src/stores/backend/tables.js

@@ -11,6 +11,7 @@ export function createTablesStore() {
     const tablesResponse = await api.get(`/api/tables`)
     const tables = await tablesResponse.json()
     update(state => ({ ...state, list: tables }))
+    return tables
   }
 
   async function select(table) {
@@ -62,6 +63,9 @@ export function createTablesStore() {
     const response = await api.post(`/api/tables`, updatedTable)
     const savedTable = await response.json()
     await fetch()
+    if (table.type === "external") {
+      await datasources.fetch()
+    }
     await select(savedTable)
     return savedTable
   }

packages/builder/src/stores/portal/auth.js

@@ -83,6 +83,13 @@ export function createAuthStore() {
   return {
     subscribe: store.subscribe,
     setOrganisation: setOrganisation,
+    getInitInfo: async () => {
+      const response = await api.get(`/api/global/auth/init`)
+      return await response.json()
+    },
+    setInitInfo: async info => {
+      await api.post(`/api/global/auth/init`, info)
+    },
     checkQueryString: async () => {
       const urlParams = new URLSearchParams(window.location.search)
       if (urlParams.has("tenantId")) {
@@ -112,7 +119,7 @@ export function createAuthStore() {
       if (response.status === 200) {
         setUser(json.user)
       } else {
-        throw "Invalid credentials"
+        throw new Error(json.message ? json.message : "Invalid credentials")
       }
       return json
     },

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/README.md

@@ -19,7 +19,7 @@ The object key is the name of the component, as exported by `index.js`.
 - **bindable** - whether the components provides a bindable value or not
 - **settings** - array of settings displayed in the builder
 
-###Settings Definitions
+### Settings Definitions
 
 The `type` field in each setting is used by the builder to know which component to use to display
 the setting, so it's important that this field is correct. The valid options are:

packages/client/manifest.json

@@ -2457,12 +2457,12 @@
     "settings": [
       {
         "type": "dataProvider",
-        "label": "Provider",
+        "label": "Data provider",
         "key": "dataProvider"
       },
       {
         "type": "number",
-        "label": "Row Count",
+        "label": "Row count",
         "key": "rowCount",
         "defaultValue": 8
       },
@@ -2496,9 +2496,36 @@
       },
       {
         "type": "boolean",
-        "label": "Auto Columns",
+        "label": "Show auto columns",
         "key": "showAutoColumns",
         "defaultValue": false
+      },
+      {
+        "type": "boolean",
+        "label": "Link table rows",
+        "key": "linkRows"
+      },
+      {
+        "type": "boolean",
+        "label": "Open link screens in modal",
+        "key": "linkPeek"
+      },
+      {
+        "type": "url",
+        "label": "Link screen",
+        "key": "linkURL"
+      },
+      {
+        "section": true,
+        "name": "Advanced",
+        "settings": [
+          {
+            "type": "field",
+            "label": "ID column for linking (appended to URL)",
+            "key": "linkColumn",
+            "placeholder": "Default"
+          }
+        ]
       }
     ],
     "context": {
@@ -2572,7 +2599,297 @@
         "type": "boolean",
         "key": "horizontal",
         "label": "Horizontal"
+      },
+      {
+        "type": "boolean",
+        "label": "Show button",
+        "key": "showButton"
+      },
+      {
+        "type": "text",
+        "key": "buttonText",
+        "label": "Button text"
+      },
+      {
+        "type": "event",
+        "label": "Button action",
+        "key": "buttonOnClick"
       }
     ]
+  },
+  "tablewithsearch": {
+    "block": true,
+    "name": "Table with search",
+    "icon": "Table",
+    "styles": ["size"],
+    "info": "Only the first 3 search columns will be used.",
+    "settings": [
+      {
+        "type": "text",
+        "label": "Title",
+        "key": "title"
+      },
+      {
+        "type": "dataSource",
+        "label": "Data",
+        "key": "dataSource"
+      },
+      {
+        "type": "multifield",
+        "label": "Search Columns",
+        "key": "searchColumns",
+        "placeholder": "Choose search columns"
+      },
+      {
+        "type": "filter",
+        "label": "Filtering",
+        "key": "filter"
+      },
+      {
+        "type": "field",
+        "label": "Sort Column",
+        "key": "sortColumn"
+      },
+      {
+        "type": "select",
+        "label": "Sort Order",
+        "key": "sortOrder",
+        "options": ["Ascending", "Descending"],
+        "defaultValue": "Descending"
+      },
+      {
+        "type": "select",
+        "label": "Size",
+        "key": "size",
+        "defaultValue": "spectrum--medium",
+        "options": [
+          {
+            "label": "Medium",
+            "value": "spectrum--medium"
+          },
+          {
+            "label": "Large",
+            "value": "spectrum--large"
+          }
+        ]
+      },
+      {
+        "type": "boolean",
+        "label": "Paginate",
+        "key": "paginate",
+        "defaultValue": true
+      },
+      {
+        "section": true,
+        "name": "Table",
+        "settings": [
+          {
+            "type": "number",
+            "label": "Row Count",
+            "key": "rowCount",
+            "defaultValue": 8
+          },
+          {
+            "type": "multifield",
+            "label": "Table Columns",
+            "key": "tableColumns",
+            "dependsOn": "dataSource",
+            "placeholder": "All columns"
+          },
+          {
+            "type": "boolean",
+            "label": "Quiet table variant",
+            "key": "quiet"
+          },
+          {
+            "type": "boolean",
+            "label": "Show auto columns",
+            "key": "showAutoColumns"
+          },
+          {
+            "type": "boolean",
+            "label": "Link table rows",
+            "key": "linkRows"
+          },
+          {
+            "type": "boolean",
+            "label": "Open link in modal",
+            "key": "linkPeek"
+          },
+          {
+            "type": "url",
+            "label": "Link screen",
+            "key": "linkURL"
+          }
+        ]
+      },
+      {
+        "section": true,
+        "name": "Title button",
+        "settings": [
+          {
+            "type": "boolean",
+            "key": "showTitleButton",
+            "label": "Show button",
+            "defaultValue": false
+          },
+          {
+            "type": "text",
+            "key": "titleButtonText",
+            "label": "Button text"
+          },
+          {
+            "type": "event",
+            "label": "Button action",
+            "key": "titleButtonOnClick"
+          }
+        ]
+      },
+      {
+        "section": true,
+        "name": "Advanced",
+        "settings": [
+          {
+            "type": "field",
+            "label": "ID column for linking (appended to URL)",
+            "key": "linkColumn",
+            "placeholder": "Default"
+          }
+        ]
+      }
+    ]
+  },
+  "cardlistwithsearch": {
+    "block": true,
+    "name": "Card list with search",
+    "icon": "Table",
+    "styles": ["size"],
+    "info": "Only the first 3 search columns will be used.",
+    "settings": [
+      {
+        "type": "text",
+        "label": "Title",
+        "key": "title"
+      },
+      {
+        "type": "dataSource",
+        "label": "Data",
+        "key": "dataSource"
+      },
+      {
+        "type": "multifield",
+        "label": "Search Columns",
+        "key": "searchColumns",
+        "placeholder": "Choose search columns"
+      },
+      {
+        "type": "filter",
+        "label": "Filtering",
+        "key": "filter"
+      },
+      {
+        "type": "field",
+        "label": "Sort Column",
+        "key": "sortColumn"
+      },
+      {
+        "type": "select",
+        "label": "Sort Order",
+        "key": "sortOrder",
+        "options": ["Ascending", "Descending"],
+        "defaultValue": "Descending"
+      },
+      {
+        "type": "number",
+        "label": "Limit",
+        "key": "limit",
+        "defaultValue": 8
+      },
+      {
+        "type": "boolean",
+        "label": "Paginate",
+        "key": "paginate"
+      },
+      {
+        "section": true,
+        "name": "Cards",
+        "settings": [
+          {
+            "type": "text",
+            "key": "cardTitle",
+            "label": "Title",
+            "nested": true
+          },
+          {
+            "type": "text",
+            "key": "cardSubtitle",
+            "label": "Subtitle",
+            "nested": true
+          },
+          {
+            "type": "text",
+            "key": "cardDescription",
+            "label": "Description",
+            "nested": true
+
+          },
+          {
+            "type": "text",
+            "key": "cardImageURL",
+            "label": "Image URL",
+            "nested": true
+
+          },
+          {
+            "type": "boolean",
+            "key": "cardHorizontal",
+            "label": "Horizontal"
+          },
+          {
+            "type": "boolean",
+            "label": "Show button",
+            "key": "showCardButton"
+          },
+          {
+            "type": "text",
+            "key": "cardButtonText",
+            "label": "Button text",
+            "nested": true
+
+          },
+          {
+            "type": "event",
+            "label": "Button action",
+            "key": "cardButtonOnClick",
+            "nested": true
+          }
+        ]
+      },
+      {
+        "section": true,
+        "name": "Title button",
+        "settings": [
+          {
+            "type": "boolean",
+            "key": "showTitleButton",
+            "label": "Show button"
+          },
+          {
+            "type": "text",
+            "key": "titleButtonText",
+            "label": "Button text"
+          },
+          {
+            "type": "event",
+            "label": "Button action",
+            "key": "titleButtonOnClick"
+          }
+        ]
+      }
+    ],
+    "context": {
+      "type": "schema",
+      "suffix": "repeater"
+    }
   }
 }

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.173-alpha.6",
+    "@budibase/bbui": "^0.9.180-alpha.6",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.173-alpha.6",
+    "@budibase/string-templates": "^0.9.180-alpha.6",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/client/src/api/datasources.js

@@ -1,8 +1,9 @@
 import { cloneDeep } from "lodash/fp"
-import { fetchTableData } from "./tables"
+import { fetchTableData, fetchTableDefinition } from "./tables"
 import { fetchViewData } from "./views"
 import { fetchRelationshipData } from "./relationships"
-import { executeQuery } from "./queries"
+import { FieldTypes } from "../constants"
+import { executeQuery, fetchQueryDefinition } from "./queries"
 
 /**
  * Fetches all rows for a particular Budibase data source.
@@ -28,7 +29,7 @@ export const fetchDatasource = async dataSource => {
       }
     }
     rows = await executeQuery({ queryId: dataSource._id, parameters })
-  } else if (type === "link") {
+  } else if (type === FieldTypes.LINK) {
     rows = await fetchRelationshipData({
       rowId: dataSource.rowId,
       tableId: dataSource.rowTableId,
@@ -39,3 +40,35 @@ export const fetchDatasource = async dataSource => {
   // Enrich the result is always an array
   return Array.isArray(rows) ? rows : []
 }
+
+/**
+ * Fetches the schema of any kind of datasource.
+ */
+export const fetchDatasourceSchema = async dataSource => {
+  if (!dataSource) {
+    return null
+  }
+  const { type } = dataSource
+
+  // Nested providers should already have exposed their own schema
+  if (type === "provider") {
+    return dataSource.value?.schema
+  }
+
+  // Tables, views and links can be fetched by table ID
+  if (
+    (type === "table" || type === "view" || type === "link") &&
+    dataSource.tableId
+  ) {
+    const table = await fetchTableDefinition(dataSource.tableId)
+    return table?.schema
+  }
+
+  // Queries can be fetched by query ID
+  if (type === "query" && dataSource._id) {
+    const definition = await fetchQueryDefinition(dataSource._id)
+    return definition?.schema
+  }
+
+  return null
+}

packages/client/src/api/queries.js

@@ -5,7 +5,7 @@ import API from "./api"
  * Executes a query against an external data connector.
  */
 export const executeQuery = async ({ queryId, parameters }) => {
-  const query = await API.get({ url: `/api/queries/${queryId}` })
+  const query = await fetchQueryDefinition(queryId)
   if (query?.datasourceId == null) {
     notificationStore.actions.error("That query couldn't be found")
     return
@@ -24,3 +24,10 @@ export const executeQuery = async ({ queryId, parameters }) => {
   }
   return res
 }
+
+/**
+ * Fetches the definition of an external query.
+ */
+export const fetchQueryDefinition = async queryId => {
+  return await API.get({ url: `/api/queries/${queryId}`, cache: true })
+}

packages/client/src/api/rows.js

@@ -1,6 +1,7 @@
 import { notificationStore, dataSourceStore } from "stores"
 import API from "./api"
 import { fetchTableDefinition } from "./tables"
+import { FieldTypes } from "../constants"
 
 /**
  * Fetches data about a certain row in a table.
@@ -129,7 +130,7 @@ export const enrichRows = async (rows, tableId) => {
         const keys = Object.keys(schema)
         for (let key of keys) {
           const type = schema[key].type
-          if (type === "link" && Array.isArray(row[key])) {
+          if (type === FieldTypes.LINK && Array.isArray(row[key])) {
             // Enrich row a string join of relationship fields
             row[`${key}_text`] =
               row[key]

packages/client/src/components/Block.svelte

@@ -0,0 +1,12 @@
+<script>
+  import { getContext, setContext } from "svelte"
+
+  const component = getContext("component")
+
+  // We need to set a block context to know we're inside a block, but also
+  // to be able to reference the actual component ID of the block from
+  // any depth
+  setContext("block", { id: $component.id })
+</script>
+
+<slot />

packages/client/src/components/BlockComponent.svelte

@@ -0,0 +1,35 @@
+<script>
+  import { getContext } from "svelte"
+  import { generate } from "shortid"
+  import Component from "components/Component.svelte"
+
+  export let type
+  export let props
+  export let styles
+  export let context
+
+  // ID is only exposed as a prop so that it can be bound to from parent
+  // block components
+  export let id
+
+  const block = getContext("block")
+  const rand = generate()
+
+  // Create a fake component instance so that we can use the core Component
+  // to render this part of the block, taking advantage of binding enrichment
+  $: id = `${block.id}-${context ?? rand}`
+  $: instance = {
+    _component: `@budibase/standard-components/${type}`,
+    _id: id,
+    _styles: {
+      normal: {
+        ...styles,
+      },
+    },
+    ...props,
+  }
+</script>
+
+<Component {instance}>
+  <slot />
+</Component>

packages/client/src/components/Component.svelte

@@ -1,3 +1,7 @@
+<script context="module">
+  let SettingsDefinitionCache = {}
+</script>
+
 <script>
   import { getContext, setContext } from "svelte"
   import { writable, get } from "svelte/store"
@@ -20,13 +24,13 @@
   // Any prop overrides that need to be applied due to conditional UI
   let conditionalSettings
 
-  // Props are hashed when inside the builder preview and used as a key, so that
+  // Settings are hashed when inside the builder preview and used as a key,
-  // components fully remount whenever any props change
+  // so that components fully remount whenever any settings change
-  let propsHash = 0
+  let hash = 0
 
   // Latest timestamp that we started a props update.
   // Due to enrichment now being async, we need to avoid overwriting newer
-  // props with old ones, depending on how long enrichment takes.
+  // settings with old ones, depending on how long enrichment takes.
   let latestUpdateTime
 
   // Keep track of stringified representations of context and instance
@@ -40,6 +44,7 @@
   // Get contexts
   const context = getContext("context")
   const insideScreenslot = !!getContext("screenslot")
+  const insideBlock = !!getContext("block")
 
   // Create component context
   const componentStore = writable({})
@@ -48,24 +53,59 @@
   // Extract component instance info
   $: constructor = getComponentConstructor(instance._component)
   $: definition = getComponentDefinition(instance._component)
+  $: settingsDefinition = getSettingsDefinition(definition)
   $: children = instance._children || []
   $: id = instance._id
   $: name = instance._instanceName
+
+  // Determine if the component is selected or is part of the critical path
+  // leading to the selected component
+  $: selected =
+    $builderStore.inBuilder && $builderStore.selectedComponentId === id
+  $: inSelectedPath = $builderStore.selectedComponentPath?.includes(id)
+
+  // Interactive components can be selected, dragged and highlighted inside
+  // the builder preview
   $: interactive =
     $builderStore.inBuilder &&
-    ($builderStore.previewType === "layout" || insideScreenslot)
+    ($builderStore.previewType === "layout" || insideScreenslot) &&
+    !insideBlock
+  $: draggable = interactive && !isLayout && !isScreen
+  $: droppable = interactive && !isLayout && !isScreen
+
+  // Empty components are those which accept children but do not have any.
+  // Empty states can be shown for these components, but can be disabled
+  // in the component manifest.
   $: empty = interactive && !children.length && definition?.hasChildren
   $: emptyState = empty && definition?.showEmptyState !== false
-  $: rawProps = getRawProps(instance)
+
-  $: instanceKey = JSON.stringify(rawProps)
+  // Raw props are all props excluding internal props and children
-  $: updateComponentProps(rawProps, instanceKey, $context)
+  $: rawSettings = getRawSettings(instance)
-  $: selected =
+  $: instanceKey = hashString(JSON.stringify(rawSettings))
-    $builderStore.inBuilder &&
+
-    $builderStore.selectedComponentId === instance._id
+  // Component settings are those which are intended for this component and
-  $: inSelectedPath = $builderStore.selectedComponentPath?.includes(id)
+  // which need to be enriched
+  $: componentSettings = getComponentSettings(rawSettings, settingsDefinition)
+  $: enrichComponentSettings(rawSettings, instanceKey, $context)
+
+  // Nested settings are those which are intended for child components inside
+  // blocks and which should not be enriched at this level
+  $: nestedSettings = getNestedSettings(rawSettings, settingsDefinition)
+
+  // Evaluate conditional UI settings and store any component setting changes
+  // which need to be made
   $: evaluateConditions(enrichedSettings?._conditions)
-  $: componentSettings = { ...enrichedSettings, ...conditionalSettings }
+
-  $: renderKey = `${propsHash}-${emptyState}`
+  // Build up the final settings object to be passed to the component
+  $: settings = {
+    ...enrichedSettings,
+    ...nestedSettings,
+    ...conditionalSettings,
+  }
+
+  // Render key is used when in the builder preview to fully remount
+  // components when settings are changed
+  $: renderKey = `${hash}-${emptyState}`
 
   // Update component context
   $: componentStore.set({
@@ -77,14 +117,14 @@
     name,
   })
 
-  const getRawProps = instance => {
+  const getRawSettings = instance => {
-    let validProps = {}
+    let validSettings = {}
     Object.entries(instance)
       .filter(([name]) => name === "_conditions" || !name.startsWith("_"))
       .forEach(([key, value]) => {
-        validProps[key] = value
+        validSettings[key] = value
       })
-    return validProps
+    return validSettings
   }
 
   // Gets the component constructor for the specified component
@@ -103,8 +143,47 @@
     return type ? Manifest[type] : null
   }
 
+  const getSettingsDefinition = definition => {
+    if (!definition) {
+      return []
+    }
+    if (SettingsDefinitionCache[definition.name]) {
+      return SettingsDefinitionCache[definition.name]
+    }
+    let settings = []
+    definition.settings?.forEach(setting => {
+      if (setting.section) {
+        settings = settings.concat(setting.settings || [])
+      } else {
+        settings.push(setting)
+      }
+    })
+    SettingsDefinitionCache[definition] = settings
+    return settings
+  }
+
+  const getComponentSettings = (rawSettings, settingsDefinition) => {
+    let clone = { ...rawSettings }
+    settingsDefinition?.forEach(setting => {
+      if (setting.nested) {
+        delete clone[setting.key]
+      }
+    })
+    return clone
+  }
+
+  const getNestedSettings = (rawSettings, settingsDefinition) => {
+    let clone = { ...rawSettings }
+    settingsDefinition?.forEach(setting => {
+      if (!setting.nested) {
+        delete clone[setting.key]
+      }
+    })
+    return clone
+  }
+
   // Enriches any string component props using handlebars
-  const updateComponentProps = (rawProps, instanceKey, context) => {
+  const enrichComponentSettings = (rawSettings, instanceKey, context) => {
     const instanceSame = instanceKey === lastInstanceKey
     const contextSame = context.key === lastContextKey
 
@@ -119,8 +198,8 @@
     latestUpdateTime = Date.now()
     const enrichmentTime = latestUpdateTime
 
-    // Enrich props with context
+    // Enrich settings with context
-    const enrichedProps = enrichProps(rawProps, context)
+    const newEnrichedSettings = enrichProps(rawSettings, context)
 
     // Abandon this update if a newer update has started
     if (enrichmentTime !== latestUpdateTime) {
@@ -130,7 +209,7 @@
     // Update the component props.
     // Most props are deeply compared so that svelte will only trigger reactive
     // statements on props that have actually changed.
-    if (!enrichedProps) {
+    if (!newEnrichedSettings) {
       return
     }
     let propsChanged = false
@@ -138,17 +217,17 @@
       enrichedSettings = {}
       propsChanged = true
     }
-    Object.keys(enrichedProps).forEach(key => {
+    Object.keys(newEnrichedSettings).forEach(key => {
-      if (!propsAreSame(enrichedProps[key], enrichedSettings[key])) {
+      if (!propsAreSame(newEnrichedSettings[key], enrichedSettings[key])) {
         propsChanged = true
-        enrichedSettings[key] = enrichedProps[key]
+        enrichedSettings[key] = newEnrichedSettings[key]
       }
     })
 
     // Update the hash if we're in the builder so we can fully remount this
     // component
     if (get(builderStore).inBuilder && propsChanged) {
-      propsHash = hashString(JSON.stringify(enrichedSettings))
+      hash = hashString(JSON.stringify(enrichedSettings))
     }
   }
 
@@ -171,14 +250,10 @@
     conditionalSettings = result.settingUpdates
     visible = nextVisible
   }
-
-  // Drag and drop helper tags
-  $: draggable = interactive && !isLayout && !isScreen
-  $: droppable = interactive && !isLayout && !isScreen
 </script>
 
 {#key renderKey}
-  {#if constructor && componentSettings && (visible || inSelectedPath)}
+  {#if constructor && settings && (visible || inSelectedPath)}
     <!-- The ID is used as a class because getElementsByClassName is O(1) -->
     <!-- and the performance matters for the selection indicators -->
     <div
@@ -190,13 +265,15 @@
       data-id={id}
       data-name={name}
     >
-      <svelte:component this={constructor} {...componentSettings}>
+      <svelte:component this={constructor} {...settings}>
         {#if children.length}
           {#each children as child (child._id)}
             <svelte:self instance={child} />
           {/each}
         {:else if emptyState}
           <Placeholder />
+        {:else if insideBlock}
+          <slot />
         {/if}
       </svelte:component>
     </div>

packages/client/src/components/app/Button.svelte

@@ -27,6 +27,9 @@
   button {
     width: fit-content;
     width: -moz-fit-content;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
   }
   .spectrum-Button--overBackground:hover {
     color: #555;

packages/client/src/components/app/DataProvider.svelte

@@ -35,28 +35,36 @@
   let pageNumber = 0
   let query = null
 
+  // Sorting can be overridden at run time, so we can't use the prop directly
+  let currentSortColumn = sortColumn
+  let currentSortOrder = sortOrder
+
   $: query = buildLuceneQuery(filter)
   $: internalTable = dataSource?.type === "table"
   $: nestedProvider = dataSource?.type === "provider"
   $: hasNextPage = bookmarks[pageNumber + 1] != null
   $: hasPrevPage = pageNumber > 0
   $: getSchema(dataSource)
-  $: sortType = getSortType(schema, sortColumn)
+  $: sortType = getSortType(schema, currentSortColumn)
+
+  // Wait until schema loads before loading data, so that we can determine
+  // the correct sort type first time
   $: {
-    // Wait until schema loads before loading data, so that we can determine
-    // the correct sort type first time
     if (schemaLoaded) {
       fetchData(
         dataSource,
+        schema,
         query,
         limit,
-        sortColumn,
+        currentSortColumn,
-        sortOrder,
+        currentSortOrder,
         sortType,
         paginate
       )
     }
   }
+
+  // Reactively filter and sort rows if required
   $: {
     if (internalTable) {
       // Internal tables are already processed server-side
@@ -65,10 +73,17 @@
       // For anything else we use client-side implementations to filter, sort
       // and limit
       const filtered = luceneQuery(allRows, query)
-      const sorted = luceneSort(filtered, sortColumn, sortOrder, sortType)
+      const sorted = luceneSort(
+        filtered,
+        currentSortColumn,
+        currentSortOrder,
+        sortType
+      )
       rows = luceneLimit(sorted, limit)
     }
   }
+
+  // Build our action context
   $: actions = [
     {
       type: ActionTypes.RefreshDatasource,
@@ -79,7 +94,20 @@
       type: ActionTypes.SetDataProviderQuery,
       callback: newQuery => (query = newQuery),
     },
+    {
+      type: ActionTypes.SetDataProviderSorting,
+      callback: ({ column, order }) => {
+        if (column) {
+          currentSortColumn = column
+        }
+        if (order) {
+          currentSortOrder = order
+        }
+      },
+    },
   ]
+
+  // Build our data context
   $: dataContext = {
     rows,
     schema,
@@ -88,7 +116,11 @@
     // Undocumented properties. These aren't supposed to be used in builder
     // bindings, but are used internally by other components
     id: $component?.id,
-    state: { query },
+    state: {
+      query,
+      sortColumn: currentSortColumn,
+      sortOrder: currentSortOrder,
+    },
     loaded,
   }
 
@@ -104,10 +136,11 @@
     if (schemaLoaded && !nestedProvider) {
       fetchData(
         dataSource,
+        schema,
         query,
         limit,
-        sortColumn,
+        currentSortColumn,
-        sortOrder,
+        currentSortOrder,
         sortType,
         paginate
       )
@@ -116,6 +149,7 @@
 
   const fetchData = async (
     dataSource,
+    schema,
     query,
     limit,
     sortColumn,
@@ -125,12 +159,16 @@
   ) => {
     loading = true
     if (dataSource?.type === "table") {
+      // Sanity check sort column, as using a non-existant column will prevent
+      // results coming back at all
+      const sort = schema?.[sortColumn] ? sortColumn : undefined
+
       // For internal tables we use server-side processing
       const res = await API.searchTable({
         tableId: dataSource.tableId,
         query,
         limit,
-        sort: sortColumn,
+        sort,
         sortOrder: sortOrder?.toLowerCase() ?? "ascending",
         sortType,
         paginate,
@@ -156,34 +194,24 @@
   }
 
   const getSchema = async dataSource => {
-    if (dataSource?.schema) {
+    let newSchema = (await API.fetchDatasourceSchema(dataSource)) || {}
-      schema = dataSource.schema
-    } else if (dataSource?.tableId) {
-      const definition = await API.fetchTableDefinition(dataSource.tableId)
-      schema = definition?.schema ?? {}
-    } else if (dataSource?.type === "provider") {
-      schema = dataSource.value?.schema ?? {}
-    } else {
-      schema = {}
-    }
 
     // Ensure there are "name" properties for all fields and that field schema
     // are objects
-    let fixedSchema = {}
+    Object.entries(newSchema).forEach(([fieldName, fieldSchema]) => {
-    Object.entries(schema || {}).forEach(([fieldName, fieldSchema]) => {
       if (typeof fieldSchema === "string") {
-        fixedSchema[fieldName] = {
+        newSchema[fieldName] = {
           type: fieldSchema,
           name: fieldName,
         }
       } else {
-        fixedSchema[fieldName] = {
+        newSchema[fieldName] = {
           ...fieldSchema,
           name: fieldName,
         }
       }
     })
-    schema = fixedSchema
+    schema = newSchema
     schemaLoaded = true
   }
 
@@ -191,13 +219,14 @@
     if (!hasNextPage || !internalTable) {
       return
     }
+    const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
     const res = await API.searchTable({
       tableId: dataSource?.tableId,
       query,
       bookmark: bookmarks[pageNumber + 1],
       limit,
-      sort: sortColumn,
+      sort,
-      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
       sortType,
       paginate: true,
     })
@@ -212,13 +241,14 @@
     if (!hasPrevPage || !internalTable) {
       return
     }
+    const sort = schema?.[currentSortColumn] ? currentSortColumn : undefined
     const res = await API.searchTable({
       tableId: dataSource?.tableId,
       query,
       bookmark: bookmarks[pageNumber - 1],
       limit,
-      sort: sortColumn,
+      sort,
-      sortOrder: sortOrder?.toLowerCase() ?? "ascending",
+      sortOrder: currentSortOrder?.toLowerCase() ?? "ascending",
       sortType,
       paginate: true,
     })
@@ -234,7 +264,7 @@
         <ProgressCircle />
       </div>
     {:else}
-      {#if !$component.children}
+      {#if $component.emptyState}
         <Placeholder />
       {:else}
         <slot />

packages/client/src/components/app/Layout.svelte

@@ -1,6 +1,7 @@
 <script>
   import { getContext } from "svelte"
   import { Heading, Icon } from "@budibase/bbui"
+  import { FieldTypes } from "../../constants"
   import active from "svelte-spa-router/active"
 
   const { routeStore, styleable, linkable, builderStore } = getContext("sdk")
@@ -108,7 +109,7 @@
             {#each validLinks as { text, url }}
               {#if isInternal(url)}
                 <a
-                  class="link"
+                  class={FieldTypes.LINK}
                   href={url}
                   use:linkable
                   on:click={close}
@@ -117,7 +118,11 @@
                   {text}
                 </a>
               {:else}
-                <a class="link" href={ensureExternal(url)} on:click={close}>
+                <a
+                  class={FieldTypes.LINK}
+                  href={ensureExternal(url)}
+                  on:click={close}
+                >
                   {text}
                 </a>
               {/if}

packages/client/src/components/app/Repeater.svelte

@@ -38,6 +38,7 @@
     padding: var(--spacing-l);
     display: grid;
     place-items: center;
+    grid-column: 1 / -1;
   }
   .noRows i {
     margin-bottom: var(--spacing-m);

packages/client/src/components/app/SpectrumCard.svelte

@@ -1,6 +1,7 @@
 <script>
   import "@spectrum-css/card/dist/index-vars.css"
   import { getContext } from "svelte"
+  import { Button } from "@budibase/bbui"
 
   export let title
   export let subtitle
@@ -8,6 +9,9 @@
   export let imageURL
   export let linkURL
   export let horizontal
+  export let showButton
+  export let buttonText
+  export let buttonOnClick
 
   const { styleable, linkable } = getContext("sdk")
   const component = getContext("component")
@@ -60,12 +64,17 @@
         {description}
       </div>
     {/if}
+    {#if showButton}
+      <div class="spectrum-Card-footer button-container">
+        <Button on:click={buttonOnClick} secondary>{buttonText || ""}</Button>
+      </div>
+    {/if}
   </div>
 </div>
 
 <style>
   .spectrum-Card {
-    width: 240px;
+    width: 300px;
     border-color: var(--spectrum-global-color-gray-300) !important;
     display: flex;
     flex-direction: column;
@@ -76,6 +85,9 @@
     flex-direction: row;
     width: 420px;
   }
+  .spectrum-Card-container {
+    padding: var(--spectrum-global-dimension-size-50) 0;
+  }
   .spectrum-Card-title :global(a) {
     text-overflow: ellipsis;
     overflow: hidden;
@@ -114,6 +126,19 @@
   .spectrum-Card-footer {
     border-top: none;
     padding-top: 0;
+    padding-bottom: 0;
     margin-top: -8px;
+    margin-bottom: var(
+      --spectrum-card-body-padding-bottom,
+      var(--spectrum-global-dimension-size-300)
+    );
+    display: -webkit-box;
+    -webkit-line-clamp: 2;
+    -webkit-box-orient: vertical;
+    overflow: hidden;
+  }
+
+  .button-container {
+    margin-bottom: var(--spectrum-global-dimension-size-300);
   }
 </style>

packages/client/src/components/app/blocks/CardListWithSearch.svelte

@@ -0,0 +1,238 @@
+<script>
+  import { onMount, getContext } from "svelte"
+  import Block from "components/Block.svelte"
+  import BlockComponent from "components/BlockComponent.svelte"
+  import { Heading } from "@budibase/bbui"
+
+  export let title
+  export let dataSource
+  export let searchColumns
+  export let filter
+  export let sortColumn
+  export let sortOrder
+  export let paginate
+  export let limit
+  export let showTitleButton
+  export let titleButtonText
+  export let titleButtonOnClick
+  export let cardTitle
+  export let cardSubtitle
+  export let cardDescription
+  export let cardImageURL
+  export let cardHorizontal
+  export let showCardButton
+  export let cardButtonText
+  export let cardButtonOnClick
+
+  const { API, styleable } = getContext("sdk")
+  const context = getContext("context")
+  const component = getContext("component")
+  const schemaComponentMap = {
+    string: "stringfield",
+    options: "optionsfield",
+    number: "numberfield",
+    datetime: "datetimefield",
+    boolean: "booleanfield",
+  }
+
+  let formId
+  let dataProviderId
+  let schema
+
+  $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema)
+  $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId)
+  $: cardWidth = cardHorizontal ? 420 : 300
+
+  // Enrich the default filter with the specified search fields
+  const enrichFilter = (filter, columns, formId) => {
+    let enrichedFilter = [...(filter || [])]
+    columns?.forEach(column => {
+      enrichedFilter.push({
+        field: column.name,
+        operator: column.type === "string" ? "string" : "equal",
+        type: "string",
+        valueType: "Binding",
+        value: `{{ [${formId}].[${column.name}] }}`,
+      })
+    })
+    return enrichedFilter
+  }
+
+  // Determine data types for search fields and only use those that are valid
+  const enrichSearchColumns = (searchColumns, schema) => {
+    let enrichedColumns = []
+    searchColumns?.forEach(column => {
+      const schemaType = schema?.[column]?.type
+      const componentType = schemaComponentMap[schemaType]
+      if (componentType) {
+        enrichedColumns.push({
+          name: column,
+          componentType,
+          type: schemaType,
+        })
+      }
+    })
+    return enrichedColumns.slice(0, 3)
+  }
+
+  // Load the datasource schema on mount so we can determine column types
+  onMount(async () => {
+    if (dataSource) {
+      schema = await API.fetchDatasourceSchema(dataSource)
+    }
+  })
+</script>
+
+<Block>
+  <div class="card-list" use:styleable={$component.styles}>
+    <BlockComponent type="form" bind:id={formId} props={{ dataSource }}>
+      {#if title || enrichedSearchColumns?.length || showTitleButton}
+        <div class="header" class:mobile={$context.device.mobile}>
+          <div class="title">
+            <Heading>{title || ""}</Heading>
+          </div>
+          <div class="controls">
+            {#if enrichedSearchColumns?.length}
+              <div
+                class="search"
+                style="--cols:{enrichedSearchColumns?.length}"
+              >
+                {#each enrichedSearchColumns as column}
+                  <BlockComponent
+                    type={column.componentType}
+                    props={{
+                      field: column.name,
+                      placeholder: column.name,
+                      text: column.name,
+                      autoWidth: true,
+                    }}
+                  />
+                {/each}
+              </div>
+            {/if}
+            {#if showTitleButton}
+              <BlockComponent
+                type="button"
+                props={{
+                  onClick: titleButtonOnClick,
+                  text: titleButtonText,
+                  type: "cta",
+                }}
+              />
+            {/if}
+          </div>
+        </div>
+      {/if}
+      <BlockComponent
+        type="dataprovider"
+        bind:id={dataProviderId}
+        props={{
+          dataSource,
+          filter: enrichedFilter,
+          sortColumn,
+          sortOrder,
+          paginate,
+          limit,
+        }}
+      >
+        <BlockComponent
+          type="repeater"
+          context="repeater"
+          props={{
+            dataProvider: `{{ literal [${dataProviderId}] }}`,
+            direction: "row",
+            hAlign: "stretch",
+            vAlign: "top",
+            gap: "M",
+            noRowsMessage: "No rows found",
+          }}
+          styles={{
+            display: "grid",
+            "grid-template-columns": `repeat(auto-fill, minmax(${cardWidth}px, 1fr))`,
+          }}
+        >
+          <BlockComponent
+            type="spectrumcard"
+            props={{
+              title: cardTitle,
+              subtitle: cardSubtitle,
+              description: cardDescription,
+              imageURL: cardImageURL,
+              horizontal: cardHorizontal,
+              showButton: showCardButton,
+              buttonText: cardButtonText,
+              buttonOnClick: cardButtonOnClick,
+            }}
+            styles={{
+              width: "auto",
+            }}
+          />
+        </BlockComponent>
+      </BlockComponent>
+    </BlockComponent>
+  </div>
+</Block>
+
+<style>
+  .header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    gap: 20px;
+    margin-bottom: 20px;
+  }
+
+  .title {
+    overflow: hidden;
+  }
+  .title :global(.spectrum-Heading) {
+    flex: 1 1 auto;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .controls {
+    flex: 0 1 auto;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    gap: 20px;
+  }
+  .controls :global(.spectrum-InputGroup .spectrum-InputGroup-input) {
+    width: 100%;
+  }
+
+  .search {
+    flex: 0 1 auto;
+    gap: 10px;
+    max-width: 100%;
+    display: grid;
+    grid-template-columns: repeat(var(--cols), minmax(120px, 200px));
+  }
+  .search :global(.spectrum-InputGroup) {
+    min-width: 0;
+  }
+
+  /* Mobile styles */
+  .mobile {
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .mobile .controls {
+    flex-direction: column-reverse;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .mobile .search {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    position: relative;
+    width: 100%;
+  }
+</style>

packages/client/src/components/app/blocks/TableWithSearch.svelte

@@ -0,0 +1,219 @@
+<script>
+  import { onMount, getContext } from "svelte"
+  import Block from "components/Block.svelte"
+  import BlockComponent from "components/BlockComponent.svelte"
+  import { Heading } from "@budibase/bbui"
+
+  export let title
+  export let dataSource
+  export let searchColumns
+  export let filter
+  export let sortColumn
+  export let sortOrder
+  export let paginate
+  export let tableColumns
+  export let showAutoColumns
+  export let rowCount
+  export let quiet
+  export let size
+  export let linkRows
+  export let linkURL
+  export let linkColumn
+  export let linkPeek
+  export let showTitleButton
+  export let titleButtonText
+  export let titleButtonOnClick
+
+  const { API, styleable } = getContext("sdk")
+  const context = getContext("context")
+  const component = getContext("component")
+  const schemaComponentMap = {
+    string: "stringfield",
+    options: "optionsfield",
+    number: "numberfield",
+    datetime: "datetimefield",
+    boolean: "booleanfield",
+  }
+
+  let formId
+  let dataProviderId
+  let schema
+
+  $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema)
+  $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId)
+
+  // Enrich the default filter with the specified search fields
+  const enrichFilter = (filter, columns, formId) => {
+    let enrichedFilter = [...(filter || [])]
+    columns?.forEach(column => {
+      enrichedFilter.push({
+        field: column.name,
+        operator: column.type === "string" ? "string" : "equal",
+        type: "string",
+        valueType: "Binding",
+        value: `{{ [${formId}].[${column.name}] }}`,
+      })
+    })
+    return enrichedFilter
+  }
+
+  // Determine data types for search fields and only use those that are valid
+  const enrichSearchColumns = (searchColumns, schema) => {
+    let enrichedColumns = []
+    searchColumns?.forEach(column => {
+      const schemaType = schema?.[column]?.type
+      const componentType = schemaComponentMap[schemaType]
+      if (componentType) {
+        enrichedColumns.push({
+          name: column,
+          componentType,
+          type: schemaType,
+        })
+      }
+    })
+    return enrichedColumns.slice(0, 3)
+  }
+
+  // Load the datasource schema on mount so we can determine column types
+  onMount(async () => {
+    if (dataSource) {
+      schema = await API.fetchDatasourceSchema(dataSource)
+    }
+  })
+</script>
+
+<Block>
+  <div class={size} use:styleable={$component.styles}>
+    <BlockComponent type="form" bind:id={formId} props={{ dataSource }}>
+      {#if title || enrichedSearchColumns?.length || showTitleButton}
+        <div class="header" class:mobile={$context.device.mobile}>
+          <div class="title">
+            <Heading>{title || ""}</Heading>
+          </div>
+          <div class="controls">
+            {#if enrichedSearchColumns?.length}
+              <div
+                class="search"
+                style="--cols:{enrichedSearchColumns?.length}"
+              >
+                {#each enrichedSearchColumns as column}
+                  <BlockComponent
+                    type={column.componentType}
+                    props={{
+                      field: column.name,
+                      placeholder: column.name,
+                      text: column.name,
+                      autoWidth: true,
+                    }}
+                  />
+                {/each}
+              </div>
+            {/if}
+            {#if showTitleButton}
+              <BlockComponent
+                type="button"
+                props={{
+                  onClick: titleButtonOnClick,
+                  text: titleButtonText,
+                  type: "cta",
+                }}
+              />
+            {/if}
+          </div>
+        </div>
+      {/if}
+      <BlockComponent
+        type="dataprovider"
+        bind:id={dataProviderId}
+        props={{
+          dataSource,
+          filter: enrichedFilter,
+          sortColumn,
+          sortOrder,
+          paginate,
+          limit: rowCount,
+        }}
+      >
+        <BlockComponent
+          type="table"
+          props={{
+            dataProvider: `{{ literal [${dataProviderId}] }}`,
+            columns: tableColumns,
+            showAutoColumns,
+            rowCount,
+            quiet,
+            size,
+            linkRows,
+            linkURL,
+            linkColumn,
+            linkPeek,
+          }}
+        />
+      </BlockComponent>
+    </BlockComponent>
+  </div>
+</Block>
+
+<style>
+  .header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    gap: 20px;
+    margin-bottom: 20px;
+  }
+
+  .title {
+    overflow: hidden;
+  }
+  .title :global(.spectrum-Heading) {
+    flex: 1 1 auto;
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
+
+  .controls {
+    flex: 0 1 auto;
+    display: flex;
+    flex-direction: row;
+    justify-content: flex-end;
+    align-items: center;
+    gap: 20px;
+  }
+  .controls :global(.spectrum-InputGroup .spectrum-InputGroup-input) {
+    width: 100%;
+  }
+
+  .search {
+    flex: 0 1 auto;
+    gap: 10px;
+    max-width: 100%;
+    display: grid;
+    grid-template-columns: repeat(var(--cols), minmax(120px, 200px));
+  }
+  .search :global(.spectrum-InputGroup) {
+    min-width: 0;
+  }
+
+  /* Mobile styles */
+  .mobile {
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .mobile .controls {
+    flex-direction: column-reverse;
+    justify-content: flex-start;
+    align-items: stretch;
+  }
+  .mobile .search {
+    display: flex;
+    flex-direction: column;
+    justify-content: flex-start;
+    align-items: stretch;
+    position: relative;
+    width: 100%;
+  }
+</style>

packages/client/src/components/app/blocks/index.js

@@ -0,0 +1,2 @@
+export { default as tablewithsearch } from "./TableWithSearch.svelte"
+export { default as cardlistwithsearch } from "./CardListWithSearch.svelte"

packages/client/src/components/app/forms/Form.svelte

@@ -34,25 +34,34 @@
     return closestContext || {}
   }
 
-  // Fetches the form schema from this form's dataSource, if one exists
+  // Fetches the form schema from this form's dataSource
   const fetchSchema = async () => {
-    if (!dataSource?.tableId) {
+    if (!dataSource) {
       schema = {}
-      table = null
-    } else {
-      table = await API.fetchTableDefinition(dataSource?.tableId)
-      if (table) {
-        if (dataSource?.type === "query") {
-          schema = {}
-          const params = table.parameters || []
-          params.forEach(param => {
-            schema[param.name] = { ...param, type: "string" }
-          })
-        } else {
-          schema = table.schema || {}
-        }
-      }
     }
+
+    // If the datasource is a query, then we instead use a schema of the query
+    // parameters rather than the output schema
+    else if (
+      dataSource.type === "query" &&
+      dataSource._id &&
+      actionType === "Create"
+    ) {
+      const query = await API.fetchQueryDefinition(dataSource._id)
+      let paramSchema = {}
+      const params = query.parameters || []
+      params.forEach(param => {
+        paramSchema[param.name] = { ...param, type: "string" }
+      })
+      schema = paramSchema
+    }
+
+    // For all other cases, just grab the normal schema
+    else {
+      const dataSourceSchema = await API.fetchDatasourceSchema(dataSource)
+      schema = dataSourceSchema || {}
+    }
+
     loaded = true
   }
 

packages/client/src/components/app/forms/RelationshipField.svelte

@@ -2,6 +2,7 @@
   import { CoreSelect, CoreMultiselect } from "@budibase/bbui"
   import { getContext } from "svelte"
   import Field from "./Field.svelte"
+  import { FieldTypes } from "../../../constants"
 
   const { API } = getContext("sdk")
 
@@ -68,7 +69,7 @@
   {field}
   {disabled}
   {validation}
-  type="link"
+  type={FieldTypes.LINK}
   bind:fieldState
   bind:fieldApi
   bind:fieldSchema

packages/client/src/components/app/forms/validation.js

@@ -1,4 +1,5 @@
 import flatpickr from "flatpickr"
+import { FieldTypes } from "../../../constants"
 
 /**
  * Creates a validation function from a combination of schema-level constraints
@@ -154,7 +155,7 @@ const parseType = (value, type) => {
   }
 
   // Parse as string
-  if (type === "string") {
+  if (type === FieldTypes.STRING) {
     if (typeof value === "string" || Array.isArray(value)) {
       return value
     }
@@ -165,7 +166,7 @@ const parseType = (value, type) => {
   }
 
   // Parse as number
-  if (type === "number") {
+  if (type === FieldTypes.NUMBER) {
     if (isNaN(value)) {
       return null
     }
@@ -173,7 +174,7 @@ const parseType = (value, type) => {
   }
 
   // Parse as date
-  if (type === "datetime") {
+  if (type === FieldTypes.DATETIME) {
     if (value instanceof Date) {
       return value.getTime()
     }
@@ -182,7 +183,7 @@ const parseType = (value, type) => {
   }
 
   // Parse as boolean
-  if (type === "boolean") {
+  if (type === FieldTypes.BOOLEAN) {
     if (typeof value === "string") {
       return value.toLowerCase() === "true"
     }
@@ -190,7 +191,7 @@ const parseType = (value, type) => {
   }
 
   // Parse attachments, treating no elements as null
-  if (type === "attachment") {
+  if (type === FieldTypes.ATTACHMENT) {
     if (!Array.isArray(value) || !value.length) {
       return null
     }
@@ -198,14 +199,14 @@ const parseType = (value, type) => {
   }
 
   // Parse links, treating no elements as null
-  if (type === "link") {
+  if (type === FieldTypes.LINK) {
     if (!Array.isArray(value) || !value.length) {
       return null
     }
     return value
   }
 
-  if (type === "array") {
+  if (type === FieldTypes.ARRAY) {
     if (!Array.isArray(value) || !value.length) {
       return null
     }

packages/client/src/components/app/index.js

@@ -32,6 +32,7 @@ export { default as spectrumcard } from "./SpectrumCard.svelte"
 export * from "./charts"
 export * from "./forms"
 export * from "./table"
+export * from "./blocks"
 
 // Deprecated component left for compatibility in old apps
 export { default as navigation } from "./deprecated/Navigation.svelte"

packages/client/src/components/app/table/Table.svelte

@@ -2,6 +2,7 @@
   import { getContext } from "svelte"
   import { Table } from "@budibase/bbui"
   import SlotRenderer from "./SlotRenderer.svelte"
+  import { UnsortableTypes } from "../../../constants"
 
   export let dataProvider
   export let columns
@@ -9,9 +10,17 @@
   export let rowCount
   export let quiet
   export let size
+  export let linkRows
+  export let linkURL
+  export let linkColumn
+  export let linkPeek
 
   const component = getContext("component")
-  const { styleable } = getContext("sdk")
+  const { styleable, getAction, ActionTypes, routeStore } = getContext("sdk")
+  const setSorting = getAction(
+    dataProvider?.id,
+    ActionTypes.SetDataProviderSorting
+  )
   const customColumnKey = `custom-${Math.random()}`
   const customRenderers = [
     {
@@ -65,11 +74,35 @@
         divider: true,
       }
     }
+
     fields.forEach(field => {
       newSchema[field] = schema[field]
+      if (schema[field] && UnsortableTypes.indexOf(schema[field].type) !== -1) {
+        newSchema[field].sortable = false
+      }
     })
     return newSchema
   }
+
+  const onSort = e => {
+    setSorting({
+      column: e.detail.column,
+      order: e.detail.order,
+    })
+  }
+
+  const onClick = e => {
+    if (!linkRows || !linkURL) {
+      return
+    }
+    const col = linkColumn || "_id"
+    const id = e.detail?.[col]
+    if (!id) {
+      return
+    }
+    const split = linkURL.split("/:")
+    routeStore.actions.navigate(`${split[0]}/${id}`, linkPeek)
+  }
 </script>
 
 <div use:styleable={$component.styles} class={size}>
@@ -84,6 +117,9 @@
     allowEditRows={false}
     allowEditColumns={false}
     showAutoColumns={true}
+    disableSorting
+    on:sort={onSort}
+    on:click={onClick}
   >
     <slot />
   </Table>

packages/client/src/components/overlay/PeekScreenDisplay.svelte

@@ -8,6 +8,12 @@
   import { Modal, ModalContent, ActionButton } from "@budibase/bbui"
   import { onDestroy } from "svelte"
 
+  const MessageTypes = {
+    NOTIFICATION: "notification",
+    CLOSE_SCREEN_MODAL: "close-screen-modal",
+    INVALIDATE_DATASOURCE: "invalidate-datasource",
+  }
+
   let iframe
   let listenersAttached = false
 
@@ -21,32 +27,33 @@
     notificationStore.actions.send(message, type, icon)
   }
 
+  function receiveMessage(message) {
+    const handlers = {
+      [MessageTypes.NOTIFICATION]: () => {
+        proxyNotification(message.data)
+      },
+      [MessageTypes.CLOSE_SCREEN_MODAL]: peekStore.actions.hidePeek,
+      [MessageTypes.INVALIDATE_DATASOURCE]: () => {
+        invalidateDataSource(message.data)
+      },
+    }
+
+    const messageHandler = handlers[message.data.type]
+    if (messageHandler) {
+      messageHandler(message)
+    } else {
+      console.warning("Unknown event type", message?.data?.type)
+    }
+  }
+
   const attachListeners = () => {
     // Mirror datasource invalidation to keep the parent window up to date
-    iframe.contentWindow.addEventListener(
+    window.addEventListener("message", receiveMessage)
-      "invalidate-datasource",
-      invalidateDataSource
-    )
-    // Listen for a close event to close the screen peek
-    iframe.contentWindow.addEventListener(
-      "close-screen-modal",
-      peekStore.actions.hidePeek
-    )
-    // Proxy notifications back to the parent window instead of iframe
-    iframe.contentWindow.addEventListener("notification", proxyNotification)
   }
 
   const handleCancel = () => {
     peekStore.actions.hidePeek()
-    iframe.contentWindow.removeEventListener(
+    window.removeEventListener("message", receiveMessage)
-      "invalidate-datasource",
-      invalidateDataSource
-    )
-    iframe.contentWindow.removeEventListener(
-      "close-screen-modal",
-      peekStore.actions.hidePeek
-    )
-    iframe.contentWindow.removeEventListener("notification", proxyNotification)
   }
 
   const handleFullscreen = () => {

packages/client/src/constants.js

@@ -2,10 +2,31 @@ export const TableNames = {
   USERS: "ta_users",
 }
 
+export const FieldTypes = {
+  STRING: "string",
+  LONGFORM: "longform",
+  OPTIONS: "options",
+  NUMBER: "number",
+  BOOLEAN: "boolean",
+  ARRAY: "array",
+  DATETIME: "datetime",
+  ATTACHMENT: "attachment",
+  LINK: "link",
+  FORMULA: "formula",
+}
+
+export const UnsortableTypes = [
+  FieldTypes.FORMULA,
+  FieldTypes.ATTACHMENT,
+  FieldTypes.ARRAY,
+  FieldTypes.LINK,
+]
+
 export const ActionTypes = {
   ValidateForm: "ValidateForm",
   RefreshDatasource: "RefreshDatasource",
   SetDataProviderQuery: "SetDataProviderQuery",
+  SetDataProviderSorting: "SetDataProviderSorting",
   ClearForm: "ClearForm",
   ChangeFormStep: "ChangeFormStep",
 }

packages/client/src/stores/builder.js

@@ -4,11 +4,7 @@ import { findComponentById, findComponentPathById } from "../utils/components"
 import { pingEndUser } from "../api"
 
 const dispatchEvent = (type, data = {}) => {
-  window.dispatchEvent(
+  window.parent.postMessage({ type, data })
-    new CustomEvent("bb-event", {
-      detail: { type, data },
-    })
-  )
 }
 
 const createBuilderStore = () => {

packages/client/src/stores/context.js

@@ -1,4 +1,5 @@
 import { writable, derived } from "svelte/store"
+import { hashString } from "../utils/helpers"
 
 export const createContextStore = oldContext => {
   const newContext = writable({})
@@ -9,7 +10,7 @@ export const createContextStore = oldContext => {
     for (let i = 0; i < $contexts.length - 1; i++) {
       key += $contexts[i].key
     }
-    key += JSON.stringify($contexts[$contexts.length - 1])
+    key = hashString(key + JSON.stringify($contexts[$contexts.length - 1]))
 
     // Reduce global state
     const reducer = (total, context) => ({ ...total, ...context })

packages/client/src/stores/dataSource.js

@@ -1,5 +1,6 @@
 import { writable, get } from "svelte/store"
 import { fetchTableDefinition } from "../api"
+import { FieldTypes } from "../constants"
 
 export const createDataSourceStore = () => {
   const store = writable([])
@@ -20,7 +21,7 @@ export const createDataSourceStore = () => {
 
     // Only one side of the relationship is required as a trigger, as it will
     // automatically invalidate related table IDs
-    else if (dataSource.type === "link") {
+    else if (dataSource.type === FieldTypes.LINK) {
       dataSourceId = dataSource.tableId || dataSource.rowTableId
     }
 
@@ -59,11 +60,10 @@ export const createDataSourceStore = () => {
 
     // Emit this as a window event, so parent screens which are iframing us in
     // can also invalidate the same datasource
-    window.dispatchEvent(
+    window.parent.postMessage({
-      new CustomEvent("invalidate-datasource", {
+      type: "close-screen-modal",
-        detail: { dataSourceId },
+      detail: { dataSourceId },
-      })
+    })
-    )
 
     let invalidations = [dataSourceId]
 
@@ -73,7 +73,7 @@ export const createDataSourceStore = () => {
     if (schema) {
       Object.values(schema).forEach(fieldSchema => {
         if (
-          fieldSchema.type === "link" &&
+          fieldSchema.type === FieldTypes.LINK &&
           fieldSchema.tableId &&
           !fieldSchema.autocolumn
         ) {

packages/client/src/stores/notification.js

@@ -26,11 +26,14 @@ const createNotificationStore = () => {
 
     // If peeking, pass notifications back to parent window
     if (get(routeStore).queryParams?.peek) {
-      window.dispatchEvent(
+      window.parent.postMessage({
-        new CustomEvent("notification", {
+        type: "notification",
-          detail: { message, type, icon },
+        detail: {
-        })
+          message,
-      )
+          type,
+          icon,
+        },
+      })
       return
     }
 

packages/client/src/stores/routes.js

@@ -1,6 +1,8 @@
-import { writable } from "svelte/store"
+import { get, writable } from "svelte/store"
 import { push } from "svelte-spa-router"
 import * as API from "../api"
+import { peekStore } from "./peek"
+import { builderStore } from "./builder"
 
 const createRouteStore = () => {
   const initialState = {
@@ -59,7 +61,25 @@ const createRouteStore = () => {
       return state
     })
   }
-  const navigate = push
+  const navigate = (url, peek) => {
+    if (get(builderStore).inBuilder) {
+      return
+    }
+    if (url) {
+      // If we're already peeking, don't peek again
+      const isPeeking = get(store).queryParams?.peek
+      if (peek && !isPeeking) {
+        peekStore.actions.showPeek(url)
+      } else {
+        const external = !url.startsWith("/")
+        if (external) {
+          window.location.href = url
+        } else {
+          push(url)
+        }
+      }
+    }
+  }
   const setRouterLoaded = () => {
     store.update(state => ({ ...state, routerLoaded: true }))
   }

packages/client/src/utils/buttonActions.js

@@ -4,7 +4,6 @@ import {
   builderStore,
   confirmationStore,
   authStore,
-  peekStore,
   stateStore,
 } from "stores"
 import { saveRow, deleteRow, executeQuery, triggerAutomation } from "api"
@@ -42,20 +41,7 @@ const triggerAutomationHandler = async action => {
 
 const navigationHandler = action => {
   const { url, peek } = action.parameters
-  if (url) {
+  routeStore.actions.navigate(url, peek)
-    // If we're already peeking, don't peek again
-    const isPeeking = get(routeStore).queryParams?.peek
-    if (peek && !isPeeking) {
-      peekStore.actions.showPeek(url)
-    } else {
-      const external = !url.startsWith("/")
-      if (external) {
-        window.location.href = url
-      } else {
-        routeStore.actions.navigate(action.parameters.url)
-      }
-    }
-  }
 }
 
 const queryExecutionHandler = async action => {
@@ -120,7 +106,7 @@ const changeFormStepHandler = async (action, context) => {
 const closeScreenModalHandler = () => {
   // Emit this as a window event, so parent screens which are iframing us in
   // can close the modal
-  window.dispatchEvent(new Event("close-screen-modal"))
+  window.parent.postMessage({ type: "close-screen-modal" })
 }
 
 const updateStateHandler = action => {
@@ -161,9 +147,15 @@ const confirmTextMap = {
  */
 export const enrichButtonActions = (actions, context) => {
   // Prevent button actions in the builder preview
-  if (get(builderStore).inBuilder) {
+  if (!actions || get(builderStore).inBuilder) {
     return () => {}
   }
+
+  // If this is a function then it has already been enriched
+  if (typeof actions === "function") {
+    return actions
+  }
+
   const handlers = actions.map(def => handlerMap[def["##eventHandlerType"]])
   return async () => {
     for (let i = 0; i < handlers.length; i++) {

packages/client/src/utils/componentProps.js

@@ -36,17 +36,19 @@ export const enrichProps = (props, context) => {
   let enrichedProps = enrichDataBindings(props, totalContext)
 
   // Enrich click actions if they exist
-  if (enrichedProps.onClick) {
+  Object.keys(enrichedProps).forEach(prop => {
-    enrichedProps.onClick = enrichButtonActions(
+    if (prop?.toLowerCase().includes("onclick")) {
-      enrichedProps.onClick,
+      enrichedProps[prop] = enrichButtonActions(
-      totalContext
+        enrichedProps[prop],
-    )
+        totalContext
-  }
+      )
+    }
+  })
 
   // Enrich any click actions in conditions
   if (enrichedProps._conditions) {
     enrichedProps._conditions.forEach(condition => {
-      if (condition.setting === "onClick") {
+      if (condition.setting?.toLowerCase().includes("onclick")) {
         condition.settingValue = enrichButtonActions(
           condition.settingValue,
           totalContext

packages/client/src/utils/styleable.js

@@ -1,4 +1,3 @@
-import { get } from "svelte/store"
 import { builderStore } from "stores"
 
 /**
@@ -64,9 +63,7 @@ export const styleable = (node, styles = {}) => {
     // Handler to select a component in the builder when clicking it in the
     // builder preview
     selectComponent = event => {
-      if (newStyles.interactive) {
+      builderStore.actions.selectComponent(componentId)
-        builderStore.actions.selectComponent(componentId)
-      }
       event.preventDefault()
       event.stopPropagation()
       return false
@@ -77,7 +74,7 @@ export const styleable = (node, styles = {}) => {
     node.addEventListener("mouseout", applyNormalStyles)
 
     // Add builder preview click listener
-    if (get(builderStore).inBuilder) {
+    if (newStyles.interactive) {
       node.addEventListener("click", selectComponent, false)
     }
 
@@ -89,11 +86,7 @@ export const styleable = (node, styles = {}) => {
   const removeListeners = () => {
     node.removeEventListener("mouseover", applyHoverStyles)
     node.removeEventListener("mouseout", applyNormalStyles)
-
+    node.removeEventListener("click", selectComponent)
-    // Remove builder preview click listener
-    if (get(builderStore).inBuilder) {
-      node.removeEventListener("click", selectComponent)
-    }
   }
 
   // Apply initial styles

packages/server/__mocks__/mssql.ts

@@ -15,7 +15,7 @@ module MsSqlMock {
   mssql.ConnectionPool = jest.fn(() => ({
     connect: jest.fn(() => ({
       request: jest.fn(() => ({
-        query: jest.fn(() => ({})),
+        query: jest.fn(sql => ({ recordset: [ sql ] })),
       })),
     })),
   }))

packages/server/__mocks__/pg.ts

@@ -21,15 +21,18 @@ module PgMock {
   function Pool() {
   }
 
+  const on = jest.fn()
   Pool.prototype.query = query
   Pool.prototype.connect = jest.fn(() => {
     // @ts-ignore
     return new Client()
   })
+  Pool.prototype.on = on
 
   pg.Client = Client
   pg.Pool = Pool
   pg.queryMock = query
+  pg.on = on
 
   module.exports = pg
 }

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.173-alpha.6",
+  "version": "0.9.180-alpha.6",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.173-alpha.6",
+    "@budibase/auth": "^0.9.180-alpha.6",
-    "@budibase/client": "^0.9.173-alpha.6",
+    "@budibase/client": "^0.9.180-alpha.6",
-    "@budibase/string-templates": "^0.9.173-alpha.6",
+    "@budibase/string-templates": "^0.9.180-alpha.6",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/scripts/integrations/mssql/data/Dockerfile

@@ -0,0 +1,9 @@
+FROM mcr.microsoft.com/mssql/server
+
+ENV ACCEPT_EULA=Y
+ENV SA_PASSWORD=Passw0rd
+
+COPY ./data /
+
+ENTRYPOINT [ "/bin/bash", "entrypoint.sh" ]
+CMD [ "/opt/mssql/bin/sqlservr" ]

packages/server/scripts/integrations/mssql/data/entrypoint.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+set -e
+
+if [ "$1" = '/opt/mssql/bin/sqlservr' ]; then
+  # If this is the container's first run, initialize the application database
+  if [ ! -f /tmp/app-initialized ]; then
+    # Initialize the application database asynchronously in a background process. This allows a) the SQL Server process to be the main process in the container, which allows graceful shutdown and other goodies, and b) us to only start the SQL Server process once, as opposed to starting, stopping, then starting it again.
+    function initialize_app_database() {
+      # Wait a bit for SQL Server to start. SQL Server's process doesn't provide a clever way to check if it's up or not, and it needs to be up before we can import the application database
+      sleep 30s
+
+      echo "RUNNING BUDIBASE SETUP"
+
+      #run the setup script to create the DB and the schema in the DB
+      /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P Passw0rd -i setup.sql
+
+      # Note that the container has been initialized so future starts won't wipe changes to the data
+      touch /tmp/app-initialized
+    }
+    initialize_app_database &
+  fi
+fi
+
+exec "$@"

packages/server/scripts/integrations/mssql/data/setup.sql

@@ -0,0 +1,54 @@
+USE master;
+  
+IF OBJECT_ID ('dbo.products', 'U') IS NOT NULL  
+  DROP TABLE products;  
+GO  
+CREATE TABLE products 
+(  
+  id int IDENTITY(1,1),  
+  name varchar (20),  
+  description varchar(30),
+  CONSTRAINT pk_products PRIMARY KEY NONCLUSTERED (id)
+);
+
+IF OBJECT_ID ('dbo.tasks', 'U') IS NOT NULL
+  DROP TABLE tasks;
+GO
+CREATE TABLE tasks
+(
+  taskid int IDENTITY(1,1),
+  taskname varchar (20),
+  productid int,
+  CONSTRAINT pk_tasks PRIMARY KEY NONCLUSTERED (taskid),
+  CONSTRAINT fk_products FOREIGN KEY (productid) REFERENCES products (id),
+);
+
+IF OBJECT_ID ('dbo.people', 'U') IS NOT NULL
+  DROP TABLE people;
+GO
+CREATE TABLE people
+(
+  name varchar(30),
+  age varchar(20),
+  CONSTRAINT pk_people PRIMARY KEY NONCLUSTERED (name, age)
+);
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Bananas', 'Fruit thing');  
+  
+INSERT products
+  (name, description)  
+VALUES  
+  ('Meat', 'Animal thing');
+
+INSERT tasks
+  (taskname, productid)
+VALUES
+  ('Processing', 1);
+
+INSERT people
+  (name, age)
+VALUES
+  ('Bob', '30');

packages/server/scripts/integrations/mssql/docker-compose.yaml

@@ -0,0 +1,12 @@
+version: "3.8"
+services:
+  # password: Passw0rd
+  # user: sa
+  # database: master
+  mssql:
+    image: bb/mssql
+    build:
+      context: .
+      dockerfile: data/Dockerfile
+    ports:
+      - "1433:1433"

packages/server/scripts/integrations/mssql/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/src/api/controllers/application.js

@@ -44,6 +44,7 @@ const {
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { syncGlobalUsers } = require("./user")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -328,6 +329,8 @@ exports.sync = async ctx => {
   if (!isDevAppID(appId)) {
     ctx.throw(400, "This action cannot be performed for production apps")
   }
+
+  // replicate prod to dev
   const prodAppId = getDeployedAppID(appId)
   const replication = new Replication({
     source: prodAppId,
@@ -343,6 +346,10 @@ exports.sync = async ctx => {
   } catch (err) {
     error = err
   }
+
+  // sync the users
+  await syncGlobalUsers(appId)
+
   if (error) {
     ctx.throw(400, error)
   } else {

packages/server/src/api/controllers/auth.js

@@ -14,6 +14,8 @@ exports.fetchSelf = async ctx => {
   }
 
   const user = await getFullUser(ctx, userId)
+  // this shouldn't be returned by the app self
+  delete user.roles
 
   if (appId) {
     const db = new CouchDB(appId)
@@ -36,6 +38,7 @@ exports.fetchSelf = async ctx => {
       // user has a role of some sort, return them
       else if (err.status === 404) {
         const metadata = {
+          ...user,
           _id: userId,
         }
         const dbResp = await db.put(metadata)

packages/server/src/api/controllers/datasource.js

@@ -9,7 +9,7 @@ const {
 } = require("../../db/utils")
 const { BuildSchemaErrors } = require("../../constants")
 const { integrations } = require("../../integrations")
-const { makeExternalQuery } = require("./row/utils")
+const { getDatasourceAndQuery } = require("./row/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -138,7 +138,7 @@ exports.find = async function (ctx) {
 exports.query = async function (ctx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await makeExternalQuery(ctx.appId, queryJson)
+    ctx.body = await getDatasourceAndQuery(ctx.appId, queryJson)
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -152,6 +152,19 @@ const buildSchemaHelper = async datasource => {
   await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
+  // make sure they all have a display name selected
+  for (let entity of Object.values(datasource.entities)) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+
   const errors = connector.schemaErrors
   let error = null
   if (errors && Object.keys(errors).length > 0) {

packages/server/src/api/controllers/query.js

@@ -23,9 +23,6 @@ function formatResponse(resp) {
     try {
       resp = JSON.parse(resp)
     } catch (err) {
-      console.error(
-        "Error parsing JSON response. Returning string in array instead."
-      )
       resp = { response: resp }
     }
   }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -36,7 +36,7 @@ interface RunConfig {
 }
 
 module External {
-  const { makeExternalQuery } = require("./utils")
+  const { getDatasourceAndQuery } = require("./utils")
   const {
     DataSourceOperation,
     FieldTypes,
@@ -46,6 +46,7 @@ module External {
   const { processObjectSync } = require("@budibase/string-templates")
   const { cloneDeep } = require("lodash/fp")
   const CouchDB = require("../../../db")
+  const { processFormulas } = require("../../../utilities/rowProcessor/utils")
 
   function buildFilters(
     id: string | undefined,
@@ -162,8 +163,8 @@ module External {
     }
   }
 
-  function basicProcessing(row: Row, table: Table) {
+  function basicProcessing(row: Row, table: Table): Row {
-    const thisRow: { [key: string]: any } = {}
+    const thisRow: Row = {}
     // filter the row down to what is actually the row (not joined)
     for (let fieldName of Object.keys(table.schema)) {
       const value = row[`${table.name}.${fieldName}`] || row[fieldName]
@@ -178,6 +179,23 @@ module External {
     return thisRow
   }
 
+  function fixArrayTypes(row: Row, table: Table) {
+    for (let [fieldName, schema] of Object.entries(table.schema)) {
+      if (
+        schema.type === FieldTypes.ARRAY &&
+        typeof row[fieldName] === "string"
+      ) {
+        try {
+          row[fieldName] = JSON.parse(row[fieldName])
+        } catch (err) {
+          // couldn't convert back to array, ignore
+          delete row[fieldName]
+        }
+      }
+    }
+    return row
+  }
+
   function isMany(field: FieldSchema) {
     return (
       field.relationshipType && field.relationshipType.split("-")[0] === "many"
@@ -225,7 +243,12 @@ module External {
         manyRelationships: ManyRelationship[] = []
       for (let [key, field] of Object.entries(table.schema)) {
         // if set already, or not set just skip it
-        if ((!row[key] && row[key] !== "") || newRow[key] || field.autocolumn) {
+        if (
+          row[key] == null ||
+          newRow[key] ||
+          field.autocolumn ||
+          field.type === FieldTypes.FORMULA
+        ) {
           continue
         }
         // if its an empty string then it means return the column to null (if possible)
@@ -336,7 +359,7 @@ module External {
       table: Table,
       relationships: RelationshipsJson[]
     ) {
-      if (rows[0].read === true) {
+      if (!rows || rows.length === 0 || rows[0].read === true) {
         return []
       }
       let finalRows: { [key: string]: Row } = {}
@@ -352,7 +375,10 @@ module External {
           )
           continue
         }
-        const thisRow = basicProcessing(row, table)
+        const thisRow = fixArrayTypes(basicProcessing(row, table), table)
+        if (thisRow._id == null) {
+          throw "Unable to generate row ID for SQL rows"
+        }
         finalRows[thisRow._id] = thisRow
         // do this at end once its been added to the final rows
         finalRows = this.updateRelationshipColumns(
@@ -361,7 +387,7 @@ module External {
           relationships
         )
       }
-      return Object.values(finalRows)
+      return processFormulas(table, Object.values(finalRows))
     }
 
     /**
@@ -428,7 +454,7 @@ module External {
         const tableId = isMany ? field.through : field.tableId
         const manyKey = field.throughFrom || primaryKey
         const fieldName = isMany ? manyKey : field.fieldName
-        const response = await makeExternalQuery(this.appId, {
+        const response = await getDatasourceAndQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
@@ -479,7 +505,7 @@ module External {
           : DataSourceOperation.CREATE
         if (!found) {
           promises.push(
-            makeExternalQuery(appId, {
+            getDatasourceAndQuery(appId, {
               endpoint: getEndpoint(tableId, operation),
               // if we're doing many relationships then we're writing, only one response
               body,
@@ -509,7 +535,7 @@ module External {
               : DataSourceOperation.UPDATE
             const body = isMany ? null : { [colName]: null }
             promises.push(
-              makeExternalQuery(this.appId, {
+              getDatasourceAndQuery(this.appId, {
                 endpoint: getEndpoint(tableId, op),
                 body,
                 filters,
@@ -532,16 +558,17 @@ module External {
       table: Table,
       includeRelations: IncludeRelationships = IncludeRelationships.INCLUDE
     ) {
-      function extractNonLinkFieldNames(table: Table, existing: string[] = []) {
+      function extractRealFields(table: Table, existing: string[] = []) {
         return Object.entries(table.schema)
           .filter(
             column =>
               column[1].type !== FieldTypes.LINK &&
+              column[1].type !== FieldTypes.FORMULA &&
               !existing.find((field: string) => field === column[0])
           )
           .map(column => `${table.name}.${column[0]}`)
       }
-      let fields = extractNonLinkFieldNames(table)
+      let fields = extractRealFields(table)
       for (let field of Object.values(table.schema)) {
         if (field.type !== FieldTypes.LINK || !includeRelations) {
           continue
@@ -549,7 +576,7 @@ module External {
         const { tableName: linkTableName } = breakExternalTableId(field.tableId)
         const linkTable = this.tables[linkTableName]
         if (linkTable) {
-          const linkedFields = extractNonLinkFieldNames(linkTable, fields)
+          const linkedFields = extractRealFields(linkTable, fields)
           fields = fields.concat(linkedFields)
         }
       }
@@ -609,7 +636,7 @@ module External {
         },
       }
       // can't really use response right now
-      const response = await makeExternalQuery(appId, json)
+      const response = await getDatasourceAndQuery(appId, json)
       // handle many to many relationships now if we know the ID (could be auto increment)
       if (
         operation !== DataSourceOperation.READ &&

packages/server/src/api/controllers/row/utils.js

@@ -4,8 +4,8 @@ const CouchDB = require("../../../db")
 const { InternalTables } = require("../../../db/utils")
 const userController = require("../user")
 const { FieldTypes } = require("../../../constants")
-const { integrations } = require("../../../integrations")
 const { processStringSync } = require("@budibase/string-templates")
+const { makeExternalQuery } = require("../../../integrations/base/utils")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value) {
@@ -17,18 +17,11 @@ validateJs.extend(validateJs.validators.datetime, {
   },
 })
 
-exports.makeExternalQuery = async (appId, json) => {
+exports.getDatasourceAndQuery = async (appId, json) => {
   const datasourceId = json.endpoint.datasourceId
   const db = new CouchDB(appId)
   const datasource = await db.get(datasourceId)
-  const Integration = integrations[datasource.source]
+  return makeExternalQuery(datasource, json)
-  // query is the opinionated function
-  if (Integration.prototype.query) {
-    const integration = new Integration(datasource.config)
-    return integration.query(json)
-  } else {
-    throw "Datasource does not support query."
-  }
 }
 
 exports.findRow = async (ctx, db, tableId, rowId) => {
@@ -68,7 +61,11 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (table.schema[fieldName].type === FieldTypes.ARRAY) {
+    if (
+      table.schema[fieldName].type === FieldTypes.ARRAY &&
+      row[fieldName] &&
+      row[fieldName].length
+    ) {
       row[fieldName].map(val => {
         if (!constraints.inclusion.includes(val)) {
           errors[fieldName] = "Field not in list"

packages/server/src/api/controllers/table/external.js

@@ -0,0 +1,276 @@
+const CouchDB = require("../../../db")
+const {
+  buildExternalTableId,
+  breakExternalTableId,
+} = require("../../../integrations/utils")
+const {
+  getTable,
+  generateForeignKey,
+  generateJunctionTableName,
+  foreignKeyStructure,
+} = require("./utils")
+const {
+  DataSourceOperation,
+  FieldTypes,
+  RelationshipTypes,
+} = require("../../../constants")
+const { makeExternalQuery } = require("../../../integrations/base/utils")
+const { cloneDeep } = require("lodash/fp")
+
+async function makeTableRequest(
+  datasource,
+  operation,
+  table,
+  tables,
+  oldTable = null
+) {
+  const json = {
+    endpoint: {
+      datasourceId: datasource._id,
+      entityId: table._id,
+      operation,
+    },
+    meta: {
+      tables,
+    },
+    table,
+  }
+  if (oldTable) {
+    json.meta.table = oldTable
+  }
+  return makeExternalQuery(datasource, json)
+}
+
+function cleanupRelationships(table, tables, oldTable = null) {
+  const tableToIterate = oldTable ? oldTable : table
+  // clean up relationships in couch table schemas
+  for (let [key, schema] of Object.entries(tableToIterate.schema)) {
+    if (
+      schema.type === FieldTypes.LINK &&
+      (!oldTable || table.schema[key] == null)
+    ) {
+      const relatedTable = Object.values(tables).find(
+        table => table._id === schema.tableId
+      )
+      const foreignKey = schema.foreignKey
+      if (!relatedTable || !foreignKey) {
+        continue
+      }
+      for (let [relatedKey, relatedSchema] of Object.entries(
+        relatedTable.schema
+      )) {
+        if (
+          relatedSchema.type === FieldTypes.LINK &&
+          relatedSchema.fieldName === foreignKey
+        ) {
+          delete relatedTable.schema[relatedKey]
+        }
+      }
+    }
+  }
+}
+
+function getDatasourceId(table) {
+  if (!table) {
+    throw "No table supplied"
+  }
+  if (table.sourceId) {
+    return table.sourceId
+  }
+  return breakExternalTableId(table._id).datasourceId
+}
+
+function otherRelationshipType(type) {
+  if (type === RelationshipTypes.MANY_TO_MANY) {
+    return RelationshipTypes.MANY_TO_MANY
+  }
+  return type === RelationshipTypes.ONE_TO_MANY
+    ? RelationshipTypes.MANY_TO_ONE
+    : RelationshipTypes.ONE_TO_MANY
+}
+
+function generateManyLinkSchema(datasource, column, table, relatedTable) {
+  const primary = table.name + table.primary[0]
+  const relatedPrimary = relatedTable.name + relatedTable.primary[0]
+  const jcTblName = generateJunctionTableName(column, table, relatedTable)
+  // first create the new table
+  const junctionTable = {
+    _id: buildExternalTableId(datasource._id, jcTblName),
+    name: jcTblName,
+    primary: [primary, relatedPrimary],
+    constrained: [primary, relatedPrimary],
+    schema: {
+      [primary]: foreignKeyStructure(primary, {
+        toTable: table.name,
+        toKey: table.primary[0],
+      }),
+      [relatedPrimary]: foreignKeyStructure(relatedPrimary, {
+        toTable: relatedTable.name,
+        toKey: relatedTable.primary[0],
+      }),
+    },
+  }
+  column.through = junctionTable._id
+  column.throughFrom = primary
+  column.throughTo = relatedPrimary
+  column.fieldName = relatedPrimary
+  return junctionTable
+}
+
+function generateLinkSchema(column, table, relatedTable, type) {
+  const isOneSide = type === RelationshipTypes.ONE_TO_MANY
+  const primary = isOneSide ? relatedTable.primary[0] : table.primary[0]
+  // generate a foreign key
+  const foreignKey = generateForeignKey(column, relatedTable)
+  column.relationshipType = type
+  column.foreignKey = isOneSide ? foreignKey : primary
+  column.fieldName = isOneSide ? primary : foreignKey
+  return foreignKey
+}
+
+function generateRelatedSchema(linkColumn, table, relatedTable, columnName) {
+  // generate column for other table
+  const relatedSchema = cloneDeep(linkColumn)
+  // swap them from the main link
+  if (linkColumn.foreignKey) {
+    relatedSchema.fieldName = linkColumn.foreignKey
+    relatedSchema.foreignKey = linkColumn.fieldName
+  }
+  // is many to many
+  else {
+    // don't need to copy through, already got it
+    relatedSchema.fieldName = linkColumn.throughFrom
+    relatedSchema.throughTo = linkColumn.throughFrom
+    relatedSchema.throughFrom = linkColumn.throughTo
+  }
+  relatedSchema.relationshipType = otherRelationshipType(
+    linkColumn.relationshipType
+  )
+  relatedSchema.tableId = relatedTable._id
+  relatedSchema.name = columnName
+  table.schema[columnName] = relatedSchema
+}
+
+function isRelationshipSetup(column) {
+  return column.foreignKey || column.through
+}
+
+exports.save = async function (ctx) {
+  const appId = ctx.appId
+  const table = ctx.request.body
+  // can't do this
+  delete table.dataImport
+  const datasourceId = getDatasourceId(ctx.request.body)
+  let tableToSave = {
+    type: "table",
+    _id: buildExternalTableId(datasourceId, table.name),
+    ...table,
+  }
+
+  let oldTable
+  if (ctx.request.body && ctx.request.body._id) {
+    oldTable = await getTable(appId, ctx.request.body._id)
+  }
+
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  const oldTables = cloneDeep(datasource.entities)
+  const tables = datasource.entities
+
+  const extraTablesToUpdate = []
+
+  // check if relations need setup
+  for (let schema of Object.values(tableToSave.schema)) {
+    if (schema.type !== FieldTypes.LINK || isRelationshipSetup(schema)) {
+      continue
+    }
+    const relatedTable = Object.values(tables).find(
+      table => table._id === schema.tableId
+    )
+    const relatedColumnName = schema.fieldName
+    const relationType = schema.relationshipType
+    if (relationType === RelationshipTypes.MANY_TO_MANY) {
+      const junctionTable = generateManyLinkSchema(
+        datasource,
+        schema,
+        table,
+        relatedTable
+      )
+      if (tables[junctionTable.name]) {
+        throw "Junction table already exists, cannot create another relationship."
+      }
+      tables[junctionTable.name] = junctionTable
+      extraTablesToUpdate.push(junctionTable)
+    } else {
+      const fkTable =
+        relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable
+      const foreignKey = generateLinkSchema(
+        schema,
+        table,
+        relatedTable,
+        relationType
+      )
+      fkTable.schema[foreignKey] = foreignKeyStructure(foreignKey)
+      if (fkTable.constrained == null) {
+        fkTable.constrained = []
+      }
+      if (fkTable.constrained.indexOf(foreignKey) === -1) {
+        fkTable.constrained.push(foreignKey)
+      }
+      // foreign key is in other table, need to save it to external
+      if (fkTable._id !== table._id) {
+        extraTablesToUpdate.push(fkTable)
+      }
+    }
+    generateRelatedSchema(schema, relatedTable, table, relatedColumnName)
+    schema.main = true
+  }
+
+  cleanupRelationships(tableToSave, tables, oldTable)
+
+  const operation = oldTable
+    ? DataSourceOperation.UPDATE_TABLE
+    : DataSourceOperation.CREATE_TABLE
+  await makeTableRequest(datasource, operation, tableToSave, tables, oldTable)
+  // update any extra tables (like foreign keys in other tables)
+  for (let extraTable of extraTablesToUpdate) {
+    const oldExtraTable = oldTables[extraTable.name]
+    let op = oldExtraTable
+      ? DataSourceOperation.UPDATE_TABLE
+      : DataSourceOperation.CREATE_TABLE
+    await makeTableRequest(datasource, op, extraTable, tables, oldExtraTable)
+  }
+
+  // make sure the constrained list, all still exist
+  if (Array.isArray(tableToSave.constrained)) {
+    tableToSave.constrained = tableToSave.constrained.filter(constraint =>
+      Object.keys(tableToSave.schema).includes(constraint)
+    )
+  }
+
+  // store it into couch now for budibase reference
+  datasource.entities[tableToSave.name] = tableToSave
+  await db.put(datasource)
+
+  return tableToSave
+}
+
+exports.destroy = async function (ctx) {
+  const appId = ctx.appId
+  const tableToDelete = await getTable(appId, ctx.params.tableId)
+  const datasourceId = getDatasourceId(tableToDelete)
+
+  const db = new CouchDB(appId)
+  const datasource = await db.get(datasourceId)
+  const tables = datasource.entities
+
+  const operation = DataSourceOperation.DELETE_TABLE
+  await makeTableRequest(datasource, operation, tableToDelete, tables)
+
+  cleanupRelationships(tableToDelete, tables)
+
+  delete datasource.entities[tableToDelete.name]
+  await db.put(datasource)
+
+  return tableToDelete
+}

packages/server/src/api/controllers/table/index.js

@@ -1,16 +1,28 @@
 const CouchDB = require("../../../db")
-const linkRows = require("../../../db/linkedRows")
+const internal = require("./internal")
+const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
+const { isExternalTable } = require("../../../integrations/utils")
 const {
-  getRowParams,
   getTableParams,
-  generateTableID,
   getDatasourceParams,
   BudibaseInternalDB,
 } = require("../../../db/utils")
-const { FieldTypes } = require("../../../constants")
+const { getTable } = require("./utils")
-const { TableSaveFunctions, getTable } = require("./utils")
 
+function pickApi({ tableId, table }) {
+  if (table && !tableId) {
+    tableId = table._id
+  }
+  if (table && table.type === "external") {
+    return external
+  } else if (tableId && isExternalTable(tableId)) {
+    return external
+  }
+  return internal
+}
+
+// covers both internal and external
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -50,143 +62,23 @@ exports.find = async function (ctx) {
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const table = ctx.request.body
-  const { dataImport, ...rest } = ctx.request.body
+  const savedTable = await pickApi({ table }).save(ctx)
-  let tableToSave = {
-    type: "table",
-    _id: generateTableID(),
-    views: {},
-    ...rest,
-  }
-
-  // if the table obj had an _id then it will have been retrieved
-  let oldTable
-  if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await db.get(ctx.request.body._id)
-  }
-
-  // saving a table is a complex operation, involving many different steps, this
-  // has been broken out into a utility to make it more obvious/easier to manipulate
-  const tableSaveFunctions = new TableSaveFunctions({
-    db,
-    ctx,
-    oldTable,
-    dataImport,
-  })
-  tableToSave = await tableSaveFunctions.before(tableToSave)
-
-  // make sure that types don't change of a column, have to remove
-  // the column if you want to change the type
-  if (oldTable && oldTable.schema) {
-    for (let propKey of Object.keys(tableToSave.schema)) {
-      let column = tableToSave.schema[propKey]
-      let oldColumn = oldTable.schema[propKey]
-      if (oldColumn && oldColumn.type === "internal") {
-        oldColumn.type = "auto"
-      }
-      if (oldColumn && oldColumn.type !== column.type) {
-        ctx.throw(400, "Cannot change the type of a column")
-      }
-    }
-  }
-
-  // Don't rename if the name is the same
-  let { _rename } = tableToSave
-  /* istanbul ignore next */
-  if (_rename && _rename.old === _rename.updated) {
-    _rename = null
-    delete tableToSave._rename
-  }
-
-  // rename row fields when table column is renamed
-  /* istanbul ignore next */
-  if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
-    ctx.throw(400, "Cannot rename a linked column.")
-  }
-
-  tableToSave = await tableSaveFunctions.mid(tableToSave)
-
-  // update schema of non-statistics views when new columns are added
-  for (let view in tableToSave.views) {
-    const tableView = tableToSave.views[view]
-    if (!tableView) continue
-
-    if (tableView.schema.group || tableView.schema.field) continue
-    tableView.schema = tableToSave.schema
-  }
-
-  // update linked rows
-  try {
-    const linkResp = await linkRows.updateLinks({
-      appId,
-      eventType: oldTable
-        ? linkRows.EventType.TABLE_UPDATED
-        : linkRows.EventType.TABLE_SAVE,
-      table: tableToSave,
-      oldTable: oldTable,
-    })
-    if (linkResp != null && linkResp._rev) {
-      tableToSave._rev = linkResp._rev
-    }
-  } catch (err) {
-    ctx.throw(400, err)
-  }
-
-  // don't perform any updates until relationships have been
-  // checked by the updateLinks function
-  const updatedRows = tableSaveFunctions.getUpdatedRows()
-  if (updatedRows && updatedRows.length !== 0) {
-    await db.bulkDocs(updatedRows)
-  }
-  const result = await db.put(tableToSave)
-  tableToSave._rev = result.rev
-
-  tableToSave = await tableSaveFunctions.after(tableToSave)
-
-  ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, tableToSave)
-
   ctx.status = 200
-  ctx.message = `Table ${ctx.request.body.name} saved successfully.`
+  ctx.message = `Table ${table.name} saved successfully.`
-  ctx.body = tableToSave
+  ctx.eventEmitter &&
+    ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
+  ctx.body = savedTable
 }
 
 exports.destroy = async function (ctx) {
   const appId = ctx.appId
-  const db = new CouchDB(appId)
+  const tableId = ctx.params.tableId
-  const tableToDelete = await db.get(ctx.params.tableId)
+  const deletedTable = await pickApi({ tableId }).destroy(ctx)
-
-  // Delete all rows for that table
-  const rows = await db.allDocs(
-    getRowParams(ctx.params.tableId, null, {
-      include_docs: true,
-    })
-  )
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
-
-  // update linked rows
-  await linkRows.updateLinks({
-    appId,
-    eventType: linkRows.EventType.TABLE_DELETE,
-    table: tableToDelete,
-  })
-
-  // don't remove the table itself until very end
-  await db.remove(tableToDelete)
-
-  // remove table search index
-  const currentIndexes = await db.getIndexes()
-  const existingIndex = currentIndexes.indexes.find(
-    existing => existing.name === `search:${ctx.params.tableId}`
-  )
-  if (existingIndex) {
-    await db.deleteIndex(existingIndex)
-  }
-
   ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:delete`, appId, tableToDelete)
+    ctx.eventEmitter.emitTable(`table:delete`, appId, deletedTable)
   ctx.status = 200
-  ctx.body = { message: `Table ${ctx.params.tableId} deleted.` }
+  ctx.body = { message: `Table ${tableId} deleted.` }
 }
 
 exports.validateCSVSchema = async function (ctx) {

packages/server/src/api/controllers/table/internal.js

@@ -0,0 +1,138 @@
+const CouchDB = require("../../../db")
+const linkRows = require("../../../db/linkedRows")
+const { getRowParams, generateTableID } = require("../../../db/utils")
+const { FieldTypes } = require("../../../constants")
+const { TableSaveFunctions } = require("./utils")
+
+exports.save = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const { dataImport, ...rest } = ctx.request.body
+  let tableToSave = {
+    type: "table",
+    _id: generateTableID(),
+    views: {},
+    ...rest,
+  }
+
+  // if the table obj had an _id then it will have been retrieved
+  let oldTable
+  if (ctx.request.body && ctx.request.body._id) {
+    oldTable = await db.get(ctx.request.body._id)
+  }
+
+  // saving a table is a complex operation, involving many different steps, this
+  // has been broken out into a utility to make it more obvious/easier to manipulate
+  const tableSaveFunctions = new TableSaveFunctions({
+    db,
+    ctx,
+    oldTable,
+    dataImport,
+  })
+  tableToSave = await tableSaveFunctions.before(tableToSave)
+
+  // make sure that types don't change of a column, have to remove
+  // the column if you want to change the type
+  if (oldTable && oldTable.schema) {
+    for (let propKey of Object.keys(tableToSave.schema)) {
+      let column = tableToSave.schema[propKey]
+      let oldColumn = oldTable.schema[propKey]
+      if (oldColumn && oldColumn.type === "internal") {
+        oldColumn.type = "auto"
+      }
+      if (oldColumn && oldColumn.type !== column.type) {
+        ctx.throw(400, "Cannot change the type of a column")
+      }
+    }
+  }
+
+  // Don't rename if the name is the same
+  let { _rename } = tableToSave
+  /* istanbul ignore next */
+  if (_rename && _rename.old === _rename.updated) {
+    _rename = null
+    delete tableToSave._rename
+  }
+
+  // rename row fields when table column is renamed
+  /* istanbul ignore next */
+  if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
+    ctx.throw(400, "Cannot rename a linked column.")
+  }
+
+  tableToSave = await tableSaveFunctions.mid(tableToSave)
+
+  // update schema of non-statistics views when new columns are added
+  for (let view in tableToSave.views) {
+    const tableView = tableToSave.views[view]
+    if (!tableView) continue
+
+    if (tableView.schema.group || tableView.schema.field) continue
+    tableView.schema = tableToSave.schema
+  }
+
+  // update linked rows
+  try {
+    const linkResp = await linkRows.updateLinks({
+      appId,
+      eventType: oldTable
+        ? linkRows.EventType.TABLE_UPDATED
+        : linkRows.EventType.TABLE_SAVE,
+      table: tableToSave,
+      oldTable: oldTable,
+    })
+    if (linkResp != null && linkResp._rev) {
+      tableToSave._rev = linkResp._rev
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+
+  // don't perform any updates until relationships have been
+  // checked by the updateLinks function
+  const updatedRows = tableSaveFunctions.getUpdatedRows()
+  if (updatedRows && updatedRows.length !== 0) {
+    await db.bulkDocs(updatedRows)
+  }
+  const result = await db.put(tableToSave)
+  tableToSave._rev = result.rev
+
+  tableToSave = await tableSaveFunctions.after(tableToSave)
+
+  return tableToSave
+}
+
+exports.destroy = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const tableToDelete = await db.get(ctx.params.tableId)
+
+  // Delete all rows for that table
+  const rows = await db.allDocs(
+    getRowParams(ctx.params.tableId, null, {
+      include_docs: true,
+    })
+  )
+  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+
+  // update linked rows
+  await linkRows.updateLinks({
+    appId,
+    eventType: linkRows.EventType.TABLE_DELETE,
+    table: tableToDelete,
+  })
+
+  // don't remove the table itself until very end
+  await db.remove(tableToDelete)
+
+  // remove table search index
+  const currentIndexes = await db.getIndexes()
+  const existingIndex = currentIndexes.indexes.find(
+    existing => existing.name === `search:${ctx.params.tableId}`
+  )
+  if (existingIndex) {
+    await db.deleteIndex(existingIndex)
+  }
+
+  return tableToDelete
+}

packages/server/src/api/controllers/table/utils.js

@@ -315,4 +315,24 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
   }
 }
 
+exports.generateForeignKey = (column, relatedTable) => {
+  return `fk_${relatedTable.name}_${column.fieldName}`
+}
+
+exports.generateJunctionTableName = (column, table, relatedTable) => {
+  return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
+}
+
+exports.foreignKeyStructure = (keyName, meta = null) => {
+  const structure = {
+    type: FieldTypes.NUMBER,
+    constraints: {},
+    name: keyName,
+  }
+  if (meta) {
+    structure.meta = meta
+  }
+  return structure
+}
+
 exports.TableSaveFunctions = TableSaveFunctions