Merge branch 'develop' into feature/offline-license

charts/budibase/templates/proxy-service-deployment.yaml

@@ -40,6 +40,24 @@ spec:
       - image: budibase/proxy:{{ .Values.globals.appVersion | default .Chart.AppVersion }}
         imagePullPolicy: Always
         name: proxy-service
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.services.proxy.port }}
+          initialDelaySeconds: 0
+          periodSeconds: 5
+          successThreshold: 1
+          failureThreshold: 2
+          timeoutSeconds: 3
+        readinessProbe:
+          httpGet:
+            path: /health
+            port: {{ .Values.services.proxy.port }}
+          initialDelaySeconds: 0
+          periodSeconds: 5
+          successThreshold: 1
+          failureThreshold: 2
+          timeoutSeconds: 3
         ports:
         - containerPort: {{ .Values.services.proxy.port }}
         env:

docs/CONTRIBUTING.md

@@ -231,18 +231,33 @@ An overview of the CI pipelines can be found [here](../.github/workflows/README.
 
 ### Pro
 
-@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you require to update code inside the pro package it can be cloned to the same root level as budibase, e.g.
+@budibase/pro is the closed source package that supports licensed features in budibase. By default the package will be pulled from NPM and will not normally need to be touched in local development. If you need to make an update to pro and have access to the repo, then you can update your submodule within the mono-repo by running `git submodule update --init` - from here you can use normal submodule flow to develop a change within pro.
+
+Once you have updated to use the pro submodule, it will be linked into all of your local dependencies by NX as with all other monorepo packages. If you have been using the NPM version of `@budibase/pro` then you may need to run a `git reset --hard` to fix all of the pro versions back to `0.0.0` to be monorepo aware.
+
+From here - to develop a change in pro, you can follow the below flow:
 
 ```
-.
-|_ budibase
-|_ budibase-pro
+# enter the pro submodule
+cd packages/pro
+# get the base branch you are working from (same as monorepo)
+git fetch
+git checkout <develop | master>
+# create a branch, named the same as the branch in your monorepo
+git checkout -b <some branch>
+... make changes
+# commit the changes you've made, with a message for pro
+git commit <something>
+# within the monorepo, add the pro reference to your branch, commit it with a message like "Update pro ref"
+cd ../..
+git add packages/pro
+git commit <add the new reference to main repo>
 ```
 
+From here, you will have created a branch in the pro repository and commited the reference to your branch on the monorepo. When you eventually PR this work back into the mainline branch, you will need to first merge your pro PR to the pro mainline, then go into your PR in the monorepo and update the reference again to the new mainline.
+
 Note that only budibase maintainers will be able to access the pro repo.
 
-By default, NX will make sure that dependencies are replaced with local source aware version. This is achieved using the `yarn link` command. To see specifically how dependencies are linked see [scripts/link-dependencies.sh](../scripts/link-dependencies.sh). The same link script is used to link dependencies to account-portal in local dev.
-
 ### Troubleshooting
 
 Sometimes, things go wrong. This can be due to incompatible updates on the budibase platform. To clear down your development environment and start again follow **Step 6. Cleanup**, then proceed from **Step 3. Install and Build** in the setup guide above to create a fresh Budibase installation.

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.18-alpha.0",
+  "version": "2.8.22-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/cache/appMetadata.ts

@@ -2,9 +2,14 @@ import { getAppClient } from "../redis/init"
 import { doWithDB, DocumentType } from "../db"
 import { Database, App } from "@budibase/types"
 
-const AppState = {
-  INVALID: "invalid",
+export enum AppState {
+  INVALID = "invalid",
 }
+
+export interface DeletedApp {
+  state: AppState
+}
+
 const EXPIRY_SECONDS = 3600
 
 /**
@@ -31,7 +36,7 @@ function isInvalid(metadata?: { state: string }) {
  * @param {string} appId the id of the app to get metadata from.
  * @returns {object} the app metadata.
  */
-export async function getAppMetadata(appId: string) {
+export async function getAppMetadata(appId: string): Promise<App | DeletedApp> {
   const client = await getAppClient()
   // try cache
   let metadata = await client.get(appId)
@@ -61,11 +66,8 @@ export async function getAppMetadata(appId: string) {
     }
     await client.store(appId, metadata, expiry)
   }
-  // we've stored in the cache an object to tell us that it is currently invalid
-  if (isInvalid(metadata)) {
-    throw { status: 404, message: "No app metadata found" }
-  }
-  return metadata as App
+
+  return metadata
 }
 
 /**

packages/backend-core/src/db/utils.ts

@@ -2,7 +2,7 @@ import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
 import { getTenantId, getGlobalDBName } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
-import { getAppMetadata } from "../cache/appMetadata"
+import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
 import { App, Database } from "@budibase/types"
 import { getStartEndKeyURL } from "../docIds"
@@ -101,7 +101,9 @@ export async function getAllApps({
     const response = await Promise.allSettled(appPromises)
     const apps = response
       .filter(
-        (result: any) => result.status === "fulfilled" && result.value != null
+        (result: any) =>
+          result.status === "fulfilled" &&
+          result.value?.state !== AppState.INVALID
       )
       .map(({ value }: any) => value)
     if (!all) {
@@ -126,7 +128,11 @@ export async function getAppsByIDs(appIds: string[]) {
   )
   // have to list the apps which exist, some may have been deleted
   return settled
-    .filter(promise => promise.status === "fulfilled")
+    .filter(
+      promise =>
+        promise.status === "fulfilled" &&
+        (promise.value as DeletedApp).state !== AppState.INVALID
+    )
     .map(promise => (promise as PromiseFulfilledResult<App>).value)
 }
 

packages/backend-core/src/logging/pino/logger.ts

@@ -12,29 +12,44 @@ import { localFileDestination } from "../system"
 
 let pinoInstance: pino.Logger | undefined
 if (!env.DISABLE_PINO_LOGGER) {
+  const level = env.LOG_LEVEL
   const pinoOptions: LoggerOptions = {
-    level: env.LOG_LEVEL,
+    level,
     formatters: {
-      level: label => {
-        return { level: label.toUpperCase() }
+      level: level => {
+        return { level: level.toUpperCase() }
       },
       bindings: () => {
-        return {
-          service: env.SERVICE_NAME,
+        if (env.SELF_HOSTED) {
+          // "service" is being injected in datadog using the pod names,
+          // so we should leave it blank to allow the default behaviour if it's not running self-hosted
+          return {
+            service: env.SERVICE_NAME,
+          }
+        } else {
+          return {}
         }
       },
     },
     timestamp: () => `,"timestamp":"${new Date(Date.now()).toISOString()}"`,
   }
 
-  const destinations: pino.DestinationStream[] = []
+  const destinations: pino.StreamEntry[] = []
 
-  if (env.isDev()) {
-    destinations.push(pinoPretty({ singleLine: true }))
-  }
+  destinations.push(
+    env.isDev()
+      ? {
+          stream: pinoPretty({ singleLine: true }),
+          level: level as pino.Level,
+        }
+      : { stream: process.stdout, level: level as pino.Level }
+  )
 
   if (env.SELF_HOSTED) {
-    destinations.push(localFileDestination())
+    destinations.push({
+      stream: localFileDestination(),
+      level: level as pino.Level,
+    })
   }
 
   pinoInstance = destinations.length

packages/bbui/src/Form/Core/CheckboxGroup.svelte

@@ -12,23 +12,24 @@
   export let getOptionValue = option => option
 
   const dispatch = createEventDispatcher()
+
   const onChange = e => {
-    let tempValue = value
-    let isChecked = e.target.checked
-    if (!tempValue.includes(e.target.value) && isChecked) {
-      tempValue.push(e.target.value)
+    const optionValue = e.target.value
+    if (e.target.checked && !value.includes(optionValue)) {
+      dispatch("change", [...value, optionValue])
+    } else {
+      dispatch(
+        "change",
+        value.filter(x => x !== optionValue)
+      )
     }
-    value = tempValue
-    dispatch(
-      "change",
-      tempValue.filter(val => val !== e.target.value || isChecked)
-    )
   }
 </script>
 
 <div class={`spectrum-FieldGroup spectrum-FieldGroup--${direction}`}>
   {#if options && Array.isArray(options)}
     {#each options as option}
+      {@const optionValue = getOptionValue(option)}
       <div
         title={getOptionLabel(option)}
         class="spectrum-Checkbox spectrum-FieldGroup-item"
@@ -39,11 +40,11 @@
         >
           <input
             on:change={onChange}
-            value={getOptionValue(option)}
             type="checkbox"
             class="spectrum-Checkbox-input"
+            value={optionValue}
+            checked={value.includes(optionValue)}
             {disabled}
-            checked={value.includes(getOptionValue(option))}
           />
           <span class="spectrum-Checkbox-box">
             <svg

packages/bbui/src/Icon/Icon.svelte

@@ -47,7 +47,7 @@
   </svg>
   {#if tooltip && showTooltip}
     <div class="tooltip" in:fade={{ duration: 130, delay: 250 }}>
-      <Tooltip textWrapping direction="bottom" text={tooltip} />
+      <Tooltip textWrapping direction="top" text={tooltip} />
     </div>
   {/if}
 </div>
@@ -80,15 +80,14 @@
     position: absolute;
     pointer-events: none;
     left: 50%;
-    top: calc(100% + 4px);
-    width: 100vw;
-    max-width: 150px;
+    bottom: calc(100% + 4px);
     transform: translateX(-50%);
     text-align: center;
+    z-index: 1;
   }
 
   .spectrum-Icon--sizeXS {
-    width: 10px;
-    height: 10px;
+    width: var(--spectrum-global-dimension-size-150);
+    height: var(--spectrum-global-dimension-size-150);
   }
 </style>

packages/builder/src/components/backend/DataTable/Table.svelte

@@ -109,6 +109,7 @@
         {disableSorting}
         {customPlaceholder}
         allowEditRows={allowEditing}
+        allowEditColumns={allowEditing}
         showAutoColumns={!hideAutocolumns}
         {allowClickRows}
         on:clickrelationship={e => selectRelationship(e.detail)}

packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte

@@ -18,7 +18,7 @@
   import { TableNames, UNEDITABLE_USER_FIELDS } from "constants"
   import {
     FIELDS,
-    RelationshipTypes,
+    RelationshipType,
     ALLOWABLE_STRING_OPTIONS,
     ALLOWABLE_NUMBER_OPTIONS,
     ALLOWABLE_STRING_TYPES,
@@ -33,6 +33,7 @@
   import { getBindings } from "components/backend/DataTable/formula"
   import { getContext } from "svelte"
   import JSONSchemaModal from "./JSONSchemaModal.svelte"
+  import { ValidColumnNameRegex } from "@budibase/shared-core"
 
   const AUTO_TYPE = "auto"
   const FORMULA_TYPE = FIELDS.FORMULA.type
@@ -183,7 +184,7 @@
       dispatch("updatecolumns")
       if (
         saveColumn.type === LINK_TYPE &&
-        saveColumn.relationshipType === RelationshipTypes.MANY_TO_MANY
+        saveColumn.relationshipType === RelationshipType.MANY_TO_MANY
       ) {
         // Fetching the new tables
         tables.fetch()
@@ -237,7 +238,7 @@
 
     // Default relationships many to many
     if (editableColumn.type === LINK_TYPE) {
-      editableColumn.relationshipType = RelationshipTypes.MANY_TO_MANY
+      editableColumn.relationshipType = RelationshipType.MANY_TO_MANY
     }
     if (editableColumn.type === FORMULA_TYPE) {
       editableColumn.formulaType = "dynamic"
@@ -285,17 +286,17 @@
       {
         name: `Many ${thisName} rows → many ${linkName} rows`,
         alt: `Many ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipTypes.MANY_TO_MANY,
+        value: RelationshipType.MANY_TO_MANY,
       },
       {
         name: `One ${linkName} row → many ${thisName} rows`,
         alt: `One ${linkTable.name} rows → many ${table.name} rows`,
-        value: RelationshipTypes.ONE_TO_MANY,
+        value: RelationshipType.ONE_TO_MANY,
       },
       {
         name: `One ${thisName} row → many ${linkName} rows`,
         alt: `One ${table.name} rows → many ${linkTable.name} rows`,
-        value: RelationshipTypes.MANY_TO_ONE,
+        value: RelationshipType.MANY_TO_ONE,
       },
     ]
   }
@@ -375,7 +376,7 @@
     const newError = {}
     if (!external && fieldInfo.name?.startsWith("_")) {
       newError.name = `Column name cannot start with an underscore.`
-    } else if (fieldInfo.name && !fieldInfo.name.match(/^[_a-zA-Z0-9\s]*$/g)) {
+    } else if (fieldInfo.name && !fieldInfo.name.match(ValidColumnNameRegex)) {
       newError.name = `Illegal character; must be alpha-numeric.`
     } else if (PROHIBITED_COLUMN_NAMES.some(name => fieldInfo.name === name)) {
       newError.name = `${PROHIBITED_COLUMN_NAMES.join(

packages/builder/src/components/backend/DataTable/modals/CreateEditUser.svelte

@@ -95,9 +95,9 @@
   {#if !creating}
     <div>
       A user's email, role, first and last names cannot be changed from within
-      the app builder. Please go to the <Link
-        on:click={$goto("/builder/portal/manage/users")}>user portal</Link
-      > to do this.
+      the app builder. Please go to the
+      <Link on:click={$goto("/builder/portal/users/users")}>user portal</Link>
+      to do this.
     </div>
   {/if}
   <RowFieldControl

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -1,5 +1,5 @@
 <script>
-  import { RelationshipTypes } from "constants/backend"
+  import { RelationshipType } from "constants/backend"
   import {
     keepOpen,
     Button,
@@ -25,11 +25,11 @@
   const relationshipTypes = [
     {
       label: "One to Many",
-      value: RelationshipTypes.MANY_TO_ONE,
+      value: RelationshipType.MANY_TO_ONE,
     },
     {
       label: "Many to Many",
-      value: RelationshipTypes.MANY_TO_MANY,
+      value: RelationshipType.MANY_TO_MANY,
     },
   ]
 
@@ -58,8 +58,8 @@
     value: table._id,
   }))
   $: valid = getErrorCount(errors) === 0 && allRequiredAttributesSet()
-  $: isManyToMany = relationshipType === RelationshipTypes.MANY_TO_MANY
-  $: isManyToOne = relationshipType === RelationshipTypes.MANY_TO_ONE
+  $: isManyToMany = relationshipType === RelationshipType.MANY_TO_MANY
+  $: isManyToOne = relationshipType === RelationshipType.MANY_TO_ONE
 
   function getTable(id) {
     return plusTables.find(table => table._id === id)
@@ -116,7 +116,7 @@
 
   function allRequiredAttributesSet() {
     const base = getTable(fromId) && getTable(toId) && fromColumn && toColumn
-    if (relationshipType === RelationshipTypes.MANY_TO_ONE) {
+    if (relationshipType === RelationshipType.MANY_TO_ONE) {
       return base && fromPrimary && fromForeign
     } else {
       return base && getTable(throughId) && throughFromKey && throughToKey
@@ -181,12 +181,12 @@
   }
 
   function otherRelationshipType(type) {
-    if (type === RelationshipTypes.MANY_TO_ONE) {
-      return RelationshipTypes.ONE_TO_MANY
-    } else if (type === RelationshipTypes.ONE_TO_MANY) {
-      return RelationshipTypes.MANY_TO_ONE
-    } else if (type === RelationshipTypes.MANY_TO_MANY) {
-      return RelationshipTypes.MANY_TO_MANY
+    if (type === RelationshipType.MANY_TO_ONE) {
+      return RelationshipType.ONE_TO_MANY
+    } else if (type === RelationshipType.ONE_TO_MANY) {
+      return RelationshipType.MANY_TO_ONE
+    } else if (type === RelationshipType.MANY_TO_MANY) {
+      return RelationshipType.MANY_TO_MANY
     }
   }
 
@@ -218,7 +218,7 @@
 
     // if any to many only need to check from
     const manyToMany =
-      relateFrom.relationshipType === RelationshipTypes.MANY_TO_MANY
+      relateFrom.relationshipType === RelationshipType.MANY_TO_MANY
 
     if (!manyToMany) {
       delete relateFrom.through
@@ -253,7 +253,7 @@
       }
       relateTo = {
         ...relateTo,
-        relationshipType: RelationshipTypes.ONE_TO_MANY,
+        relationshipType: RelationshipType.ONE_TO_MANY,
         foreignKey: relateFrom.fieldName,
         fieldName: fromPrimary,
       }
@@ -321,7 +321,7 @@
       fromColumn = toRelationship.name
     }
     relationshipType =
-      fromRelationship.relationshipType || RelationshipTypes.MANY_TO_ONE
+      fromRelationship.relationshipType || RelationshipType.MANY_TO_ONE
     if (selectedFromTable) {
       fromId = selectedFromTable._id
       fromColumn = selectedFromTable.name

packages/builder/src/components/backend/Datasources/relationshipErrors.js

@@ -1,4 +1,4 @@
-import { RelationshipTypes } from "constants/backend"
+import { RelationshipType } from "constants/backend"
 
 const typeMismatch = "Column type of the foreign key must match the primary key"
 const columnBeingUsed = "Column name cannot be an existing column"
@@ -40,7 +40,7 @@ export class RelationshipErrorChecker {
   }
 
   isMany() {
-    return this.type === RelationshipTypes.MANY_TO_MANY
+    return this.type === RelationshipType.MANY_TO_MANY
   }
 
   relationshipTypeSet(type) {

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,17 +1,9 @@
 <script>
-  import { Select } from "@budibase/bbui"
+  import { Select, Icon } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
   import { parseFile } from "./utils"
 
-  let fileInput
-  let error = null
-  let fileName = null
-
-  let loading = false
-  let validation = {}
-  let validateHash = ""
-
   export let rows = []
   export let schema = {}
   export let allValid = true
@@ -49,6 +41,27 @@
     },
   ]
 
+  let fileInput
+  let error = null
+  let fileName = null
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let errors = {}
+
+  $: displayColumnOptions = Object.keys(schema || {}).filter(column => {
+    return validation[column]
+  })
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+    validateHash = newValidateHash
+  }
+  $: openFileUpload(promptUpload, fileInput)
+
   async function handleFile(e) {
     loading = true
     error = null
@@ -67,34 +80,23 @@
 
   async function validate(rows, schema) {
     loading = true
-    error = null
-    validation = {}
-    allValid = false
-
     try {
       if (rows.length > 0) {
         const response = await API.validateNewTableImport({ rows, schema })
         validation = response.schemaValidation
         allValid = response.allValid
+        errors = response.errors
+        error = null
       }
     } catch (e) {
       error = e.message
+      validation = {}
+      allValid = false
+      errors = {}
     }
-
     loading = false
   }
 
-  $: {
-    // binding in consumer is causing double renders here
-    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
-
-    if (newValidateHash !== validateHash) {
-      validate(rows, schema)
-    }
-
-    validateHash = newValidateHash
-  }
-
   const handleChange = (name, e) => {
     schema[name].type = e.detail
     schema[name].constraints = FIELDS[e.detail.toUpperCase()].constraints
@@ -106,7 +108,13 @@
     }
   }
 
-  $: openFileUpload(promptUpload, fileInput)
+  const deleteColumn = name => {
+    if (loading) {
+      return
+    }
+    delete schema[name]
+    schema = schema
+  }
 </script>
 
 <div class="dropzone">
@@ -119,10 +127,8 @@
     on:change={handleFile}
   />
   <label for="file-upload" class:uploaded={rows.length > 0}>
-    {#if loading}
-      loading...
-    {:else if error}
-      error: {error}
+    {#if error}
+      Error: {error}
     {:else if fileName}
       {fileName}
     {:else}
@@ -142,23 +148,26 @@
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={loading}
         />
         <span
-          class={loading || validation[column.name]
+          class={validation[column.name]
             ? "fieldStatusSuccess"
             : "fieldStatusFailure"}
         >
-          {validation[column.name] ? "Success" : "Failure"}
+          {#if validation[column.name]}
+            Success
+          {:else}
+            Failure
+            {#if errors[column.name]}
+              <Icon name="Help" tooltip={errors[column.name]} />
+            {/if}
+          {/if}
         </span>
-        <i
-          class={`omit-button ri-close-circle-fill ${
-            loading ? "omit-button-disabled" : ""
-          }`}
-          on:click={() => {
-            delete schema[column.name]
-            schema = schema
-          }}
+        <Icon
+          size="S"
+          name="Close"
+          hoverable
+          on:click={() => deleteColumn(column.name)}
         />
       </div>
     {/each}
@@ -167,7 +176,7 @@
     <Select
       label="Display Column"
       bind:value={displayColumn}
-      options={Object.keys(schema)}
+      options={displayColumnOptions}
       sort
     />
   </div>
@@ -235,23 +244,16 @@
     justify-self: center;
     font-weight: 600;
   }
-
   .fieldStatusFailure {
     color: var(--red);
     justify-self: center;
     font-weight: 600;
+    display: flex;
+    align-items: center;
+    gap: 4px;
   }
-
-  .omit-button {
-    font-size: 1.2em;
-    color: var(--grey-7);
-    cursor: pointer;
-    justify-self: flex-end;
-  }
-
-  .omit-button-disabled {
-    pointer-events: none;
-    opacity: 70%;
+  .fieldStatusFailure :global(.spectrum-Icon) {
+    width: 12px;
   }
 
   .display-column {

packages/builder/src/components/common/HelpMenu.svelte

@@ -4,7 +4,8 @@
   import { licensing } from "stores/portal"
   import { isEnabled, TENANT_FEATURE_FLAGS } from "helpers/featureFlags"
 
-  $: isPremiumUser = $licensing.license && !$licensing.isFreePlan
+  $: isBusinessAndAbove =
+    $licensing.isBusinessPlan || $licensing.isEnterprisePlan
 
   let show
   let hide
@@ -55,22 +56,22 @@
       <div class="divider" />
       {#if isEnabled(TENANT_FEATURE_FLAGS.LICENSING)}
         <a
-          href={isPremiumUser
+          href={isBusinessAndAbove
             ? "mailto:support@budibase.com"
             : "/builder/portal/account/usage"}
         >
-          <div class="premiumLinkContent" class:disabled={!isPremiumUser}>
+          <div class="premiumLinkContent" class:disabled={!isBusinessAndAbove}>
             <div class="icon">
               <FontAwesomeIcon name="fa-solid fa-envelope" />
             </div>
             <Body size="S">Email support</Body>
           </div>
-          {#if !isPremiumUser}
+          {#if !isBusinessAndAbove}
             <div class="premiumBadge">
               <div class="icon">
                 <FontAwesomeIcon name="fa-solid fa-lock" />
               </div>
-              <Body size="XS">Premium</Body>
+              <Body size="XS">Business</Body>
             </div>
           {/if}
         </a>

packages/builder/src/components/common/NavItem.svelte

@@ -132,7 +132,6 @@
     flex-direction: row;
     justify-content: flex-start;
     align-items: stretch;
-    position: relative;
   }
   .nav-item.scrollable {
     flex-direction: column;

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -419,16 +419,22 @@
     if (query && !query.fields.pagination) {
       query.fields.pagination = {}
     }
-    dynamicVariables = getDynamicVariables(
-      datasource,
-      query._id,
-      (variable, queryId) => variable.queryId === queryId
-    )
-    globalDynamicBindings = getDynamicVariables(
-      datasource,
-      query._id,
-      (variable, queryId) => variable.queryId !== queryId
-    )
+    // if query doesn't have ID then its new - don't try to copy existing dynamic variables
+    if (!queryId) {
+      dynamicVariables = []
+      globalDynamicBindings = getDynamicVariables(datasource)
+    } else {
+      dynamicVariables = getDynamicVariables(
+        datasource,
+        query._id,
+        (variable, queryId) => variable.queryId === queryId
+      )
+      globalDynamicBindings = getDynamicVariables(
+        datasource,
+        query._id,
+        (variable, queryId) => variable.queryId !== queryId
+      )
+    }
 
     prettifyQueryRequestBody(
       query,

packages/builder/src/constants/backend/index.js

@@ -151,7 +151,7 @@ export function isAutoColumnUserRelationship(subtype) {
   )
 }
 
-export const RelationshipTypes = {
+export const RelationshipType = {
   MANY_TO_MANY: "many-to-many",
   ONE_TO_MANY: "one-to-many",
   MANY_TO_ONE: "many-to-one",

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/EditDatasourceConfigButton.svelte

@@ -74,11 +74,12 @@
     border: 1px solid var(--spectrum-global-color-gray-300);
     border-radius: 5px;
     width: 100%;
-    background-color: #00000047;
+    background: var(--spectrum-global-color-gray-50);
     color: white;
     overflow: hidden;
     padding: 12px 16px;
     box-sizing: border-box;
+    transition: background 130ms ease-out;
   }
   .left {
     flex: 1;
@@ -94,7 +95,7 @@
   }
   .button:hover {
     cursor: pointer;
-    filter: brightness(1.2);
+    background: var(--spectrum-global-color-gray-100);
   }
   .connected {
     display: flex;

packages/builder/src/pages/builder/app/[application]/data/datasource/[datasourceId]/_components/panels/Variables/ViewDynamicVariables.svelte

@@ -18,7 +18,7 @@
   const onClick = dynamicVariable => {
     const queryId = dynamicVariable.queryId
     queries.select({ _id: queryId })
-    $goto(`./${queryId}`)
+    $goto(`../../query/${queryId}`)
   }
 
   /**

packages/client/manifest.json

@@ -2341,10 +2341,6 @@
             "label": "Left",
             "value": "left"
           },
-          {
-            "label": "Right",
-            "value": "right"
-          },
           {
             "label": "Above",
             "value": "above"

packages/client/src/components/app/forms/MultiFieldSelect.svelte

@@ -38,7 +38,7 @@
       return []
     }
     if (Array.isArray(values)) {
-      return values
+      return values.slice()
     }
     return values.split(",").map(value => value.trim())
   }

packages/pro

@@ -1 +1 @@
-Subproject commit 1b8fd8ed445c4c25210f8faf07ff404c41fbc805
+Subproject commit 5cf188ab64dfe679a300a06d3c6609060196c4ad

packages/server/specs/resources/table.ts

@@ -1,8 +1,4 @@
-import {
-  FieldTypes,
-  RelationshipTypes,
-  FormulaTypes,
-} from "../../src/constants"
+import { FieldTypes, RelationshipType, FormulaTypes } from "../../src/constants"
 import { object } from "./utils"
 import Resource from "./utils/Resource"
 
@@ -100,7 +96,7 @@ const tableSchema = {
               },
               relationshipType: {
                 type: "string",
-                enum: Object.values(RelationshipTypes),
+                enum: Object.values(RelationshipType),
                 description:
                   "Defines the type of relationship that this column will be used for.",
               },

packages/server/src/api/controllers/datasource.ts

@@ -1,34 +1,33 @@
 import {
-  generateDatasourceID,
-  getDatasourceParams,
-  getQueryParams,
   DocumentType,
-  BudibaseInternalDB,
+  generateDatasourceID,
+  getQueryParams,
   getTableParams,
 } from "../../db/utils"
 import { destroy as tableDestroy } from "./table/internal"
 import { BuildSchemaErrors, InvalidColumns } from "../../constants"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
-import { db as dbCore, context, events } from "@budibase/backend-core"
+import { context, db as dbCore, events } from "@budibase/backend-core"
 import {
-  UserCtx,
-  Datasource,
-  Row,
-  CreateDatasourceResponse,
-  UpdateDatasourceResponse,
   CreateDatasourceRequest,
-  VerifyDatasourceRequest,
-  VerifyDatasourceResponse,
+  CreateDatasourceResponse,
+  Datasource,
+  DatasourcePlus,
   FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
-  DatasourcePlus,
+  RestConfig,
   SourceName,
+  UpdateDatasourceResponse,
+  UserCtx,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
 import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
+import { areRESTVariablesValid } from "../../sdk/app/datasources/datasources"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -119,46 +118,7 @@ async function buildFilteredSchema(datasource: Datasource, filter?: string[]) {
 }
 
 export async function fetch(ctx: UserCtx) {
-  // Get internal tables
-  const db = context.getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-
-  const internal = internalTables.rows.reduce((acc: any, row: Row) => {
-    const sourceId = row.doc.sourceId || "bb_internal"
-    acc[sourceId] = acc[sourceId] || []
-    acc[sourceId].push(row.doc)
-    return acc
-  }, {})
-
-  const bbInternalDb = {
-    ...BudibaseInternalDB,
-  }
-
-  // Get external datasources
-  const datasources = (
-    await db.allDocs(
-      getDatasourceParams(null, {
-        include_docs: true,
-      })
-    )
-  ).rows.map(row => row.doc)
-
-  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
-    bbInternalDb,
-    ...datasources,
-  ])
-
-  for (let datasource of allDatasources) {
-    if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id!]
-    }
-  }
-
-  ctx.body = [bbInternalDb, ...datasources]
+  ctx.body = await sdk.datasources.fetch()
 }
 
 export async function verify(
@@ -290,6 +250,14 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
     datasource.config!.auth = auth
   }
 
+  // check all variables are unique
+  if (
+    datasource.source === SourceName.REST &&
+    !sdk.datasources.areRESTVariablesValid(datasource)
+  ) {
+    ctx.throw(400, "Duplicate dynamic/static variable names are invalid.")
+  }
+
   const response = await db.put(
     sdk.tables.populateExternalTableSchemas(datasource)
   )

packages/server/src/api/controllers/query/index.ts

@@ -1,4 +1,4 @@
-import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils"
+import { generateQueryID } from "../../../db/utils"
 import { BaseQueryVerbs, FieldTypes } from "../../../constants"
 import { Thread, ThreadType } from "../../../threads"
 import { save as saveDatasource } from "../datasource"
@@ -28,15 +28,7 @@ function enrichQueries(input: any) {
 }
 
 export async function fetch(ctx: any) {
-  const db = context.getAppDB()
-
-  const body = await db.allDocs(
-    getQueryParams(null, {
-      include_docs: true,
-    })
-  )
-
-  ctx.body = enrichQueries(body.rows.map((row: any) => row.doc))
+  ctx.body = await sdk.queries.fetch()
 }
 
 const _import = async (ctx: any) => {
@@ -103,14 +95,8 @@ export async function save(ctx: any) {
 }
 
 export async function find(ctx: any) {
-  const db = context.getAppDB()
-  const query = enrichQueries(await db.get(ctx.params.queryId))
-  // remove properties that could be dangerous in real app
-  if (isProdAppID(ctx.appId)) {
-    delete query.fields
-    delete query.parameters
-  }
-  ctx.body = query
+  const queryId = ctx.params.queryId
+  ctx.body = await sdk.queries.find(queryId)
 }
 
 //Required to discern between OIDC OAuth config entries

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -7,7 +7,7 @@ import {
   Operation,
   PaginationJson,
   RelationshipsJson,
-  RelationshipTypes,
+  RelationshipType,
   Row,
   SearchFilters,
   SortJson,
@@ -577,7 +577,7 @@ export class ExternalRequest {
       ) {
         continue
       }
-      const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
+      const isMany = field.relationshipType === RelationshipType.MANY_TO_MANY
       const tableId = isMany ? field.through : field.tableId
       const { tableName: relatedTableName } = breakExternalTableId(tableId)
       // @ts-ignore

packages/server/src/api/controllers/table/external.ts

@@ -20,7 +20,7 @@ import {
   FieldSchema,
   Operation,
   QueryJson,
-  RelationshipTypes,
+  RelationshipType,
   RenameColumn,
   Table,
   TableRequest,
@@ -103,12 +103,12 @@ function getDatasourceId(table: Table) {
 }
 
 function otherRelationshipType(type?: string) {
-  if (type === RelationshipTypes.MANY_TO_MANY) {
-    return RelationshipTypes.MANY_TO_MANY
+  if (type === RelationshipType.MANY_TO_MANY) {
+    return RelationshipType.MANY_TO_MANY
   }
-  return type === RelationshipTypes.ONE_TO_MANY
-    ? RelationshipTypes.MANY_TO_ONE
-    : RelationshipTypes.ONE_TO_MANY
+  return type === RelationshipType.ONE_TO_MANY
+    ? RelationshipType.MANY_TO_ONE
+    : RelationshipType.ONE_TO_MANY
 }
 
 function generateManyLinkSchema(
@@ -151,12 +151,12 @@ function generateLinkSchema(
   column: FieldSchema,
   table: Table,
   relatedTable: Table,
-  type: RelationshipTypes
+  type: RelationshipType
 ) {
   if (!table.primary || !relatedTable.primary) {
     throw new Error("Unable to generate link schema, no primary keys")
   }
-  const isOneSide = type === RelationshipTypes.ONE_TO_MANY
+  const isOneSide = type === RelationshipType.ONE_TO_MANY
   const primary = isOneSide ? relatedTable.primary[0] : table.primary[0]
   // generate a foreign key
   const foreignKey = generateForeignKey(column, relatedTable)
@@ -251,7 +251,7 @@ export async function save(ctx: UserCtx) {
     }
     const relatedColumnName = schema.fieldName!
     const relationType = schema.relationshipType!
-    if (relationType === RelationshipTypes.MANY_TO_MANY) {
+    if (relationType === RelationshipType.MANY_TO_MANY) {
       const junctionTable = generateManyLinkSchema(
         datasource,
         schema,
@@ -265,7 +265,7 @@ export async function save(ctx: UserCtx) {
       extraTablesToUpdate.push(junctionTable)
     } else {
       const fkTable =
-        relationType === RelationshipTypes.ONE_TO_MANY
+        relationType === RelationshipType.ONE_TO_MANY
           ? tableToSave
           : relatedTable
       const foreignKey = generateLinkSchema(

packages/server/src/constants/index.ts

@@ -1,6 +1,6 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
 import { FieldType as FieldTypes } from "@budibase/types"
-export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"
+export { FieldType as FieldTypes, RelationshipType } from "@budibase/types"
 
 export enum FilterTypes {
   STRING = "string",

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -7,7 +7,7 @@ import { employeeImport } from "./employeeImport"
 import { jobsImport } from "./jobsImport"
 import { expensesImport } from "./expensesImport"
 import { db as dbCore } from "@budibase/backend-core"
-import { Table, Row, RelationshipTypes } from "@budibase/types"
+import { Table, Row, RelationshipType } from "@budibase/types"
 
 export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
 export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
@@ -299,7 +299,7 @@ export const DEFAULT_EMPLOYEE_TABLE_SCHEMA: Table = {
       },
       fieldName: "Assigned",
       name: "Jobs",
-      relationshipType: RelationshipTypes.MANY_TO_MANY,
+      relationshipType: RelationshipType.MANY_TO_MANY,
       tableId: DEFAULT_JOBS_TABLE_ID,
     },
     "Start Date": {
@@ -458,7 +458,7 @@ export const DEFAULT_JOBS_TABLE_SCHEMA: Table = {
       type: FieldTypes.LINK,
       tableId: DEFAULT_EMPLOYEE_TABLE_ID,
       fieldName: "Jobs",
-      relationshipType: RelationshipTypes.MANY_TO_MANY,
+      relationshipType: RelationshipType.MANY_TO_MANY,
       // sortable: true,
     },
     "Works End": {

packages/server/src/db/linkedRows/LinkController.ts

@@ -8,7 +8,7 @@ import {
   Database,
   FieldSchema,
   LinkDocumentValue,
-  RelationshipTypes,
+  RelationshipType,
   Row,
   Table,
 } from "@budibase/types"
@@ -136,16 +136,16 @@ class LinkController {
   handleRelationshipType(linkerField: FieldSchema, linkedField: FieldSchema) {
     if (
       !linkerField.relationshipType ||
-      linkerField.relationshipType === RelationshipTypes.MANY_TO_MANY
+      linkerField.relationshipType === RelationshipType.MANY_TO_MANY
     ) {
-      linkedField.relationshipType = RelationshipTypes.MANY_TO_MANY
+      linkedField.relationshipType = RelationshipType.MANY_TO_MANY
       // make sure by default all are many to many (if not specified)
-      linkerField.relationshipType = RelationshipTypes.MANY_TO_MANY
-    } else if (linkerField.relationshipType === RelationshipTypes.MANY_TO_ONE) {
+      linkerField.relationshipType = RelationshipType.MANY_TO_MANY
+    } else if (linkerField.relationshipType === RelationshipType.MANY_TO_ONE) {
       // Ensure that the other side of the relationship is locked to one record
-      linkedField.relationshipType = RelationshipTypes.ONE_TO_MANY
-    } else if (linkerField.relationshipType === RelationshipTypes.ONE_TO_MANY) {
-      linkedField.relationshipType = RelationshipTypes.MANY_TO_ONE
+      linkedField.relationshipType = RelationshipType.ONE_TO_MANY
+    } else if (linkerField.relationshipType === RelationshipType.ONE_TO_MANY) {
+      linkedField.relationshipType = RelationshipType.MANY_TO_ONE
     }
     return { linkerField, linkedField }
   }
@@ -200,9 +200,7 @@ class LinkController {
 
         // iterate through the link IDs in the row field, see if any don't exist already
         for (let linkId of rowField) {
-          if (
-            linkedSchema?.relationshipType === RelationshipTypes.ONE_TO_MANY
-          ) {
+          if (linkedSchema?.relationshipType === RelationshipType.ONE_TO_MANY) {
             let links = (
               (await getLinkDocuments({
                 tableId: field.tableId,

packages/server/src/db/tests/linkController.spec.js

@@ -2,7 +2,7 @@ const TestConfig = require("../../tests/utilities/TestConfiguration")
 const { basicRow, basicLinkedRow, basicTable } = require("../../tests/utilities/structures")
 const LinkController = require("../linkedRows/LinkController").default
 const { context } = require("@budibase/backend-core")
-const { RelationshipTypes } = require("../../constants")
+const { RelationshipType } = require("../../constants")
 const { cloneDeep } = require("lodash/fp")
 
 describe("test the link controller", () => {
@@ -16,7 +16,7 @@ describe("test the link controller", () => {
 
   beforeEach(async () => {
     const { _id } = await config.createTable()
-    table2 = await config.createLinkedTable(RelationshipTypes.MANY_TO_MANY, ["link", "link2"])
+    table2 = await config.createLinkedTable(RelationshipType.MANY_TO_MANY, ["link", "link2"])
     // update table after creating link
     table1 = await config.getTable(_id)
   })
@@ -57,17 +57,17 @@ describe("test the link controller", () => {
     const controller = await createLinkController(table1)
     // empty case
     let output = controller.handleRelationshipType({}, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipTypes.MANY_TO_MANY)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipTypes.MANY_TO_MANY)
-    output = controller.handleRelationshipType({ relationshipType: RelationshipTypes.MANY_TO_MANY }, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipTypes.MANY_TO_MANY)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipTypes.MANY_TO_MANY)
-    output = controller.handleRelationshipType({ relationshipType: RelationshipTypes.MANY_TO_ONE }, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipTypes.ONE_TO_MANY)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipTypes.MANY_TO_ONE)
-    output = controller.handleRelationshipType({ relationshipType: RelationshipTypes.ONE_TO_MANY }, {})
-    expect(output.linkedField.relationshipType).toEqual(RelationshipTypes.MANY_TO_ONE)
-    expect(output.linkerField.relationshipType).toEqual(RelationshipTypes.ONE_TO_MANY)
+    expect(output.linkedField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
+    expect(output.linkerField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
+    output = controller.handleRelationshipType({ relationshipType: RelationshipType.MANY_TO_MANY }, {})
+    expect(output.linkedField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
+    expect(output.linkerField.relationshipType).toEqual(RelationshipType.MANY_TO_MANY)
+    output = controller.handleRelationshipType({ relationshipType: RelationshipType.MANY_TO_ONE }, {})
+    expect(output.linkedField.relationshipType).toEqual(RelationshipType.ONE_TO_MANY)
+    expect(output.linkerField.relationshipType).toEqual(RelationshipType.MANY_TO_ONE)
+    output = controller.handleRelationshipType({ relationshipType: RelationshipType.ONE_TO_MANY }, {})
+    expect(output.linkedField.relationshipType).toEqual(RelationshipType.MANY_TO_ONE)
+    expect(output.linkerField.relationshipType).toEqual(RelationshipType.ONE_TO_MANY)
   })
 
   it("should be able to delete a row", async () => {
@@ -157,7 +157,7 @@ describe("test the link controller", () => {
 
   it("should throw an error when overwriting a link column", async () => {
     const update = cloneDeep(table1)
-    update.schema.link.relationshipType = RelationshipTypes.MANY_TO_ONE
+    update.schema.link.relationshipType = RelationshipType.MANY_TO_ONE
     let error
     try {
       const controller = await createLinkController(update)
@@ -183,7 +183,7 @@ describe("test the link controller", () => {
 
   it("shouldn't allow one to many having many relationships against it", async () => {
     const firstTable = await config.createTable()
-    const { _id } = await config.createLinkedTable(RelationshipTypes.MANY_TO_ONE, ["link"])
+    const { _id } = await config.createLinkedTable(RelationshipType.MANY_TO_ONE, ["link"])
     const linkTable = await config.getTable(_id)
     // an initial row to link around
     const row = await createLinkedRow("link", linkTable, firstTable)

packages/server/src/integration-test/postgres.spec.ts

@@ -10,7 +10,7 @@ import * as setup from "../api/routes/tests/utilities"
 import {
   Datasource,
   FieldType,
-  RelationshipTypes,
+  RelationshipType,
   Row,
   SourceName,
   Table,
@@ -101,17 +101,17 @@ describe("postgres integrations", () => {
     oneToManyRelationshipInfo = {
       table: await createAuxTable("o2m"),
       fieldName: "oneToManyRelation",
-      relationshipType: RelationshipTypes.ONE_TO_MANY,
+      relationshipType: RelationshipType.ONE_TO_MANY,
     }
     manyToOneRelationshipInfo = {
       table: await createAuxTable("m2o"),
       fieldName: "manyToOneRelation",
-      relationshipType: RelationshipTypes.MANY_TO_ONE,
+      relationshipType: RelationshipType.MANY_TO_ONE,
     }
     manyToManyRelationshipInfo = {
       table: await createAuxTable("m2m"),
       fieldName: "manyToManyRelation",
-      relationshipType: RelationshipTypes.MANY_TO_MANY,
+      relationshipType: RelationshipType.MANY_TO_MANY,
     }
 
     primaryPostgresTable = await config.createTable({
@@ -143,7 +143,7 @@ describe("postgres integrations", () => {
           },
           fieldName: oneToManyRelationshipInfo.fieldName,
           name: "oneToManyRelation",
-          relationshipType: RelationshipTypes.ONE_TO_MANY,
+          relationshipType: RelationshipType.ONE_TO_MANY,
           tableId: oneToManyRelationshipInfo.table._id,
           main: true,
         },
@@ -154,7 +154,7 @@ describe("postgres integrations", () => {
           },
           fieldName: manyToOneRelationshipInfo.fieldName,
           name: "manyToOneRelation",
-          relationshipType: RelationshipTypes.MANY_TO_ONE,
+          relationshipType: RelationshipType.MANY_TO_ONE,
           tableId: manyToOneRelationshipInfo.table._id,
           main: true,
         },
@@ -165,7 +165,7 @@ describe("postgres integrations", () => {
           },
           fieldName: manyToManyRelationshipInfo.fieldName,
           name: "manyToManyRelation",
-          relationshipType: RelationshipTypes.MANY_TO_MANY,
+          relationshipType: RelationshipType.MANY_TO_MANY,
           tableId: manyToManyRelationshipInfo.table._id,
           main: true,
         },
@@ -193,12 +193,12 @@ describe("postgres integrations", () => {
   type ForeignTableInfo = {
     table: Table
     fieldName: string
-    relationshipType: RelationshipTypes
+    relationshipType: RelationshipType
   }
 
   type ForeignRowsInfo = {
     row: Row
-    relationshipType: RelationshipTypes
+    relationshipType: RelationshipType
   }
 
   async function createPrimaryRow(opts: {
@@ -263,7 +263,7 @@ describe("postgres integrations", () => {
       rowData[manyToOneRelationshipInfo.fieldName].push(foreignRow._id)
       foreignRows.push({
         row: foreignRow,
-        relationshipType: RelationshipTypes.MANY_TO_ONE,
+        relationshipType: RelationshipType.MANY_TO_ONE,
       })
     }
 
@@ -281,7 +281,7 @@ describe("postgres integrations", () => {
       rowData[manyToManyRelationshipInfo.fieldName].push(foreignRow._id)
       foreignRows.push({
         row: foreignRow,
-        relationshipType: RelationshipTypes.MANY_TO_MANY,
+        relationshipType: RelationshipType.MANY_TO_MANY,
       })
     }
 
@@ -559,7 +559,7 @@ describe("postgres integrations", () => {
           expect(res.status).toBe(200)
 
           const one2ManyForeignRows = foreignRows.filter(
-            x => x.relationshipType === RelationshipTypes.ONE_TO_MANY
+            x => x.relationshipType === RelationshipType.ONE_TO_MANY
           )
           expect(one2ManyForeignRows).toHaveLength(1)
 
@@ -921,7 +921,7 @@ describe("postgres integrations", () => {
             (row: Row) => row.id === 2
           )
           expect(m2mRow1).toEqual({
-            ...foreignRowsByType[RelationshipTypes.MANY_TO_MANY][0].row,
+            ...foreignRowsByType[RelationshipType.MANY_TO_MANY][0].row,
             [m2mFieldName]: [
               {
                 _id: row._id,
@@ -930,7 +930,7 @@ describe("postgres integrations", () => {
             ],
           })
           expect(m2mRow2).toEqual({
-            ...foreignRowsByType[RelationshipTypes.MANY_TO_MANY][1].row,
+            ...foreignRowsByType[RelationshipType.MANY_TO_MANY][1].row,
             [m2mFieldName]: [
               {
                 _id: row._id,
@@ -940,24 +940,24 @@ describe("postgres integrations", () => {
           })
           expect(res.body[m2oFieldName]).toEqual([
             {
-              ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][0].row,
+              ...foreignRowsByType[RelationshipType.MANY_TO_ONE][0].row,
               [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
                 row.id,
             },
             {
-              ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][1].row,
+              ...foreignRowsByType[RelationshipType.MANY_TO_ONE][1].row,
               [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
                 row.id,
             },
             {
-              ...foreignRowsByType[RelationshipTypes.MANY_TO_ONE][2].row,
+              ...foreignRowsByType[RelationshipType.MANY_TO_ONE][2].row,
               [`fk_${manyToOneRelationshipInfo.table.name}_${manyToOneRelationshipInfo.fieldName}`]:
                 row.id,
             },
           ])
           expect(res.body[o2mFieldName]).toEqual([
             {
-              ...foreignRowsByType[RelationshipTypes.ONE_TO_MANY][0].row,
+              ...foreignRowsByType[RelationshipType.ONE_TO_MANY][0].row,
               _id: expect.any(String),
               _rev: expect.any(String),
             },

packages/server/src/integrations/base/sqlTable.ts

@@ -3,7 +3,7 @@ import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-import { FieldTypes, RelationshipTypes } from "../../constants"
+import { FieldTypes, RelationshipType } from "../../constants"
 
 function generateSchema(
   schema: CreateTableBuilder,
@@ -70,8 +70,8 @@ function generateSchema(
       case FieldTypes.LINK:
         // this side of the relationship doesn't need any SQL work
         if (
-          column.relationshipType !== RelationshipTypes.MANY_TO_ONE &&
-          column.relationshipType !== RelationshipTypes.MANY_TO_MANY
+          column.relationshipType !== RelationshipType.MANY_TO_ONE &&
+          column.relationshipType !== RelationshipType.MANY_TO_MANY
         ) {
           if (!column.foreignKey || !column.tableId) {
             throw "Invalid relationship schema"

packages/server/src/sdk/app/datasources/datasources.ts

@@ -1,4 +1,4 @@
-import { context } from "@budibase/backend-core"
+import { context, db as dbCore } from "@budibase/backend-core"
 import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
@@ -8,15 +8,88 @@ import {
   RestAuthConfig,
   RestAuthType,
   RestBasicAuthConfig,
+  Row,
+  RestConfig,
   SourceName,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
 import { getDefinitions, getDefinition } from "../../../integrations"
 import _ from "lodash"
+import {
+  BudibaseInternalDB,
+  getDatasourceParams,
+  getTableParams,
+} from "../../../db/utils"
+import sdk from "../../index"
 
 const ENV_VAR_PREFIX = "env."
 
+export async function fetch() {
+  // Get internal tables
+  const db = context.getAppDB()
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+
+  const internal = internalTables.rows.reduce((acc: any, row: Row) => {
+    const sourceId = row.doc.sourceId || "bb_internal"
+    acc[sourceId] = acc[sourceId] || []
+    acc[sourceId].push(row.doc)
+    return acc
+  }, {})
+
+  const bbInternalDb = {
+    ...BudibaseInternalDB,
+  }
+
+  // Get external datasources
+  const datasources = (
+    await db.allDocs(
+      getDatasourceParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
+
+  for (let datasource of allDatasources) {
+    if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
+      datasource.entities = internal[datasource._id!]
+    }
+  }
+
+  return [bbInternalDb, ...datasources]
+}
+
+export function areRESTVariablesValid(datasource: Datasource) {
+  const restConfig = datasource.config as RestConfig
+  const varNames: string[] = []
+  if (restConfig.dynamicVariables) {
+    for (let variable of restConfig.dynamicVariables) {
+      if (varNames.includes(variable.name)) {
+        return false
+      }
+      varNames.push(variable.name)
+    }
+  }
+  if (restConfig.staticVariables) {
+    for (let name of Object.keys(restConfig.staticVariables)) {
+      if (varNames.includes(name)) {
+        return false
+      }
+      varNames.push(name)
+    }
+  }
+  return true
+}
+
 export function checkDatasourceTypes(schema: Integration, config: any) {
   for (let key of Object.keys(config)) {
     if (!schema.datasource[key]) {

packages/server/src/sdk/app/queries/queries.ts

@@ -1,5 +1,49 @@
 import { getEnvironmentVariables } from "../../utils"
 import { processStringSync } from "@budibase/string-templates"
+import { context } from "@budibase/backend-core"
+import { getQueryParams, isProdAppID } from "../../../db/utils"
+import { BaseQueryVerbs } from "../../../constants"
+
+// simple function to append "readable" to all read queries
+function enrichQueries(input: any) {
+  const wasArray = Array.isArray(input)
+  const queries = wasArray ? input : [input]
+  for (let query of queries) {
+    if (query.queryVerb === BaseQueryVerbs.READ) {
+      query.readable = true
+    }
+  }
+  return wasArray ? queries : queries[0]
+}
+
+export async function find(queryId: string) {
+  const db = context.getAppDB()
+  const appId = context.getAppId()
+  const query = enrichQueries(await db.get(queryId))
+  // remove properties that could be dangerous in real app
+  if (isProdAppID(appId)) {
+    delete query.fields
+    delete query.parameters
+  }
+  return query
+}
+
+export async function fetch(opts: { enrich: boolean } = { enrich: true }) {
+  const db = context.getAppDB()
+
+  const body = await db.allDocs(
+    getQueryParams(null, {
+      include_docs: true,
+    })
+  )
+
+  const queries = body.rows.map((row: any) => row.doc)
+  if (opts.enrich) {
+    return enrichQueries(queries)
+  } else {
+    return queries
+  }
+}
 
 export async function enrichContext(
   fields: Record<string, any>,

packages/server/src/sdk/app/tables/validation.ts

@@ -3,7 +3,7 @@ import {
   Datasource,
   FieldSchema,
   FieldType,
-  RelationshipTypes,
+  RelationshipType,
 } from "@budibase/types"
 import { FieldTypes } from "../../../constants"
 
@@ -19,14 +19,14 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
       column => column.type === FieldType.LINK
     )
     relationships.forEach(relationship => {
-      if (relationship.relationshipType === RelationshipTypes.MANY_TO_MANY) {
+      if (relationship.relationshipType === RelationshipType.MANY_TO_MANY) {
         const tableId = relationship.through!
         foreignKeys.push({ key: relationship.throughTo!, tableId })
         foreignKeys.push({ key: relationship.throughFrom!, tableId })
       } else {
         const fk = relationship.foreignKey!
         const oneSide =
-          relationship.relationshipType === RelationshipTypes.ONE_TO_MANY
+          relationship.relationshipType === RelationshipType.ONE_TO_MANY
         foreignKeys.push({
           tableId: oneSide ? table._id! : relationship.tableId!,
           key: fk,

packages/server/src/utilities/schema.ts

@@ -1,4 +1,5 @@
 import { FieldTypes } from "../constants"
+import { ValidColumnNameRegex } from "@budibase/shared-core"
 
 interface SchemaColumn {
   readonly name: string
@@ -27,6 +28,7 @@ interface ValidationResults {
   schemaValidation: SchemaValidation
   allValid: boolean
   invalidColumns: Array<string>
+  errors: Record<string, string>
 }
 
 const PARSERS: any = {
@@ -69,6 +71,7 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
     schemaValidation: {},
     allValid: false,
     invalidColumns: [],
+    errors: {},
   }
 
   rows.forEach(row => {
@@ -79,6 +82,11 @@ export function validate(rows: Rows, schema: Schema): ValidationResults {
       // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
       if (typeof columnType !== "string") {
         results.invalidColumns.push(columnName)
+      } else if (!columnName.match(ValidColumnNameRegex)) {
+        // Check for special characters in column names
+        results.schemaValidation[columnName] = false
+        results.errors[columnName] =
+          "Column names can't contain special characters"
       } else if (
         columnData == null &&
         !schema[columnName].constraints?.presence

packages/shared-core/src/constants.ts

@@ -94,3 +94,4 @@ export enum BuilderSocketEvent {
 }
 
 export const SocketSessionTTL = 60
+export const ValidColumnNameRegex = /^[_a-zA-Z0-9\s]*$/g

packages/types/src/documents/app/datasource.ts

@@ -7,9 +7,7 @@ export interface Datasource extends Document {
   name?: string
   source: SourceName
   // the config is defined by the schema
-  config?: {
-    [key: string]: string | number | boolean | any[]
-  }
+  config?: Record<string, any>
   plus?: boolean
   entities?: {
     [key: string]: Table

packages/types/src/documents/app/table.ts

@@ -1,97 +0,0 @@
-import { Document } from "../document"
-import { View } from "./view"
-import { RenameColumn } from "../../sdk"
-import { FieldType } from "./row"
-
-export enum RelationshipTypes {
-  ONE_TO_MANY = "one-to-many",
-  MANY_TO_ONE = "many-to-one",
-  MANY_TO_MANY = "many-to-many",
-}
-
-export enum AutoReason {
-  FOREIGN_KEY = "foreign_key",
-}
-
-export interface FieldSchema {
-  type: FieldType
-  externalType?: string
-  fieldName?: string
-  name: string
-  sortable?: boolean
-  tableId?: string
-  relationshipType?: RelationshipTypes
-  through?: string
-  foreignKey?: string
-  icon?: string
-  autocolumn?: boolean
-  autoReason?: AutoReason
-  subtype?: string
-  throughFrom?: string
-  throughTo?: string
-  formula?: string
-  formulaType?: string
-  main?: boolean
-  ignoreTimezones?: boolean
-  timeOnly?: boolean
-  lastID?: number
-  useRichText?: boolean | null
-  order?: number
-  width?: number
-  meta?: {
-    toTable: string
-    toKey: string
-  }
-  constraints?: {
-    type?: string
-    email?: boolean
-    inclusion?: string[]
-    length?: {
-      minimum?: string | number | null
-      maximum?: string | number | null
-    }
-    numericality?: {
-      greaterThanOrEqualTo: string | null
-      lessThanOrEqualTo: string | null
-    }
-    presence?:
-      | boolean
-      | {
-          allowEmpty?: boolean
-        }
-    datetime?: {
-      latest: string
-      earliest: string
-    }
-  }
-}
-
-export interface TableSchema {
-  [key: string]: FieldSchema
-}
-
-export interface Table extends Document {
-  type?: string
-  views?: { [key: string]: View }
-  name: string
-  primary?: string[]
-  schema: TableSchema
-  primaryDisplay?: string
-  sourceId?: string
-  relatedFormula?: string[]
-  constrained?: string[]
-  sql?: boolean
-  indexes?: { [key: string]: any }
-  rows?: { [key: string]: any }
-  created?: boolean
-  rowHeight?: number
-}
-
-export interface ExternalTable extends Table {
-  sourceId: string
-}
-
-export interface TableRequest extends Table {
-  _rename?: RenameColumn
-  created?: boolean
-}

packages/types/src/documents/app/table/constants.ts

@@ -0,0 +1,9 @@
+export enum RelationshipType {
+  ONE_TO_MANY = "one-to-many",
+  MANY_TO_ONE = "many-to-one",
+  MANY_TO_MANY = "many-to-many",
+}
+
+export enum AutoReason {
+  FOREIGN_KEY = "foreign_key",
+}

packages/types/src/documents/app/table/index.ts

@@ -0,0 +1,3 @@
+export * from "./table"
+export * from "./schema"
+export * from "./constants"

packages/types/src/documents/app/table/schema.ts

@@ -0,0 +1,98 @@
+// all added by grid/table when defining the
+// column size, position and whether it can be viewed
+import { FieldType } from "../row"
+import { AutoReason, RelationshipType } from "./constants"
+
+export interface UIFieldMetadata {
+  order?: number
+  width?: number
+  visible?: boolean
+  icon?: string
+}
+
+export interface RelationshipFieldMetadata {
+  main?: boolean
+  fieldName?: string
+  tableId?: string
+  // below is used for SQL relationships, needed to define the foreign keys
+  // or the tables used for many-to-many relationships (through)
+  relationshipType?: RelationshipType
+  through?: string
+  foreignKey?: string
+  throughFrom?: string
+  throughTo?: string
+}
+
+export interface AutoColumnFieldMetadata {
+  autocolumn?: boolean
+  subtype?: string
+  lastID?: number
+  // if the column was turned to an auto-column for SQL, explains why (primary, foreign etc)
+  autoReason?: AutoReason
+}
+
+export interface NumberFieldMetadata {
+  // used specifically when Budibase generates external tables, this denotes if a number field
+  // is a foreign key used for a many-to-many relationship
+  meta?: {
+    toTable: string
+    toKey: string
+  }
+}
+
+export interface DateFieldMetadata {
+  ignoreTimezones?: boolean
+  timeOnly?: boolean
+}
+
+export interface StringFieldMetadata {
+  useRichText?: boolean | null
+}
+
+export interface FormulaFieldMetadata {
+  formula?: string
+  formulaType?: string
+}
+
+export interface FieldConstraints {
+  type?: string
+  email?: boolean
+  inclusion?: string[]
+  length?: {
+    minimum?: string | number | null
+    maximum?: string | number | null
+  }
+  numericality?: {
+    greaterThanOrEqualTo: string | null
+    lessThanOrEqualTo: string | null
+  }
+  presence?:
+    | boolean
+    | {
+        allowEmpty?: boolean
+      }
+  datetime?: {
+    latest: string
+    earliest: string
+  }
+}
+
+export interface FieldSchema
+  extends UIFieldMetadata,
+    DateFieldMetadata,
+    RelationshipFieldMetadata,
+    AutoColumnFieldMetadata,
+    StringFieldMetadata,
+    FormulaFieldMetadata,
+    NumberFieldMetadata {
+  type: FieldType
+  name: string
+  sortable?: boolean
+  // only used by external databases, to denote the real type
+  externalType?: string
+  constraints?: FieldConstraints
+}
+
+export interface TableSchema {
+  [key: string]: FieldSchema
+}

packages/types/src/documents/app/table/table.ts

@@ -0,0 +1,30 @@
+import { Document } from "../../document"
+import { View } from "../view"
+import { RenameColumn } from "../../../sdk"
+import { TableSchema } from "./schema"
+
+export interface Table extends Document {
+  type?: string
+  views?: { [key: string]: View }
+  name: string
+  primary?: string[]
+  schema: TableSchema
+  primaryDisplay?: string
+  sourceId?: string
+  relatedFormula?: string[]
+  constrained?: string[]
+  sql?: boolean
+  indexes?: { [key: string]: any }
+  rows?: { [key: string]: any }
+  created?: boolean
+  rowHeight?: number
+}
+
+export interface ExternalTable extends Table {
+  sourceId: string
+}
+
+export interface TableRequest extends Table {
+  _rename?: RenameColumn
+  created?: boolean
+}

packages/worker/src/api/controllers/global/configs.ts

@@ -507,17 +507,17 @@ export async function configChecklist(ctx: Ctx) {
           smtp: {
             checked: !!smtpConfig,
             label: "Set up email",
-            link: "/builder/portal/manage/email",
+            link: "/builder/portal/settings/email",
           },
           adminUser: {
             checked: userExists,
             label: "Create your first user",
-            link: "/builder/portal/manage/users",
+            link: "/builder/portal/users/users",
           },
           sso: {
             checked: !!googleConfig || !!oidcConfig,
             label: "Set up single sign-on",
-            link: "/builder/portal/manage/auth",
+            link: "/builder/portal/settings/auth",
           },
         }
       }