Merge branch 'develop' of github.com:Budibase/budibase into rest-pagination

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/README.md

@@ -4,7 +4,7 @@ This library contains core functionality, like auth and security features
 which are shared between backend services.
 
 #### Note about top level JS files
-For the purposes of being able to do say `require("@budibase/auth/permissions")` we need to
+For the purposes of being able to do say `require("@budibase/backend-core/permissions")` we need to
 specify the exports at the top-level of the module.
 
 For these files they should be limited to a single `require` of the file that should

packages/backend-core/auth.js

@@ -0,0 +1 @@
+module.exports = require("./src/auth")

packages/backend-core/middleware.js

@@ -0,0 +1 @@
+module.exports = require("./src/middleware")

packages/backend-core/objectStore.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require("./src/objectStore"),
+  ...require("./src/objectStore/utils"),
+}

packages/backend-core/package.json

@@ -1,7 +1,7 @@
 {
-  "name": "@budibase/auth",
+  "name": "@budibase/backend-core",
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
-  "description": "Authentication middlewares for budibase builder and apps",
+  "description": "Budibase backend core libraries used in server and worker",
   "main": "src/index.js",
   "author": "Budibase",
   "license": "GPL-3.0",

packages/backend-core/src/auth.js

@@ -1,7 +1,6 @@
 const passport = require("koa-passport")
 const LocalStrategy = require("passport-local").Strategy
 const JwtStrategy = require("passport-jwt").Strategy
-const { StaticDatabases } = require("./db/utils")
 const { getGlobalDB } = require("./tenancy")
 const {
   jwt,
@@ -14,8 +13,6 @@ const {
   appTenancy,
   authError,
 } = require("./middleware")
-const { setDB } = require("./db")
-const userCache = require("./cache/user")
 
 // Strategies
 passport.use(new LocalStrategy(local.options, local.authenticate))
@@ -36,36 +33,13 @@ passport.deserializeUser(async (user, done) => {
 })
 
 module.exports = {
-  init(pouch) {
+  buildAuthMiddleware: authenticated,
-    setDB(pouch)
+  passport,
-  },
+  google,
-  db: require("./db/utils"),
+  oidc,
-  redis: {
+  jwt: require("jsonwebtoken"),
-    Client: require("./redis"),
+  buildTenancyMiddleware: tenancy,
-    utils: require("./redis/utils"),
+  buildAppTenancyMiddleware: appTenancy,
-  },
+  auditLog,
-  objectStore: {
+  authError,
-    ...require("./objectStore"),
-    ...require("./objectStore/utils"),
-  },
-  utils: {
-    ...require("./utils"),
-    ...require("./hashing"),
-  },
-  auth: {
-    buildAuthMiddleware: authenticated,
-    passport,
-    google,
-    oidc,
-    jwt: require("jsonwebtoken"),
-    buildTenancyMiddleware: tenancy,
-    buildAppTenancyMiddleware: appTenancy,
-    auditLog,
-    authError,
-  },
-  cache: {
-    user: userCache,
-  },
-  StaticDatabases,
-  constants: require("./constants"),
 }

packages/backend-core/src/db/utils.js

@@ -224,8 +224,15 @@ exports.getAllDbs = async () => {
     }
   }
   let couchUrl = `${exports.getCouchUrl()}/_all_dbs`
-  if (env.MULTI_TENANCY) {
+  let tenantId = getTenantId()
-    let tenantId = getTenantId()
+  if (!env.MULTI_TENANCY || tenantId == DEFAULT_TENANT_ID) {
+    // just get all DBs when:
+    // - single tenancy
+    // - default tenant
+    //    - apps dbs don't contain tenant id
+    //    - non-default tenant dbs are filtered out application side in getAllApps
+    await addDbs(couchUrl)
+  } else {
     // get prod apps
     await addDbs(
       exports.getStartEndKeyURL(couchUrl, DocumentTypes.APP, tenantId)
@@ -236,9 +243,6 @@ exports.getAllDbs = async () => {
     )
     // add global db name
     dbs.push(getGlobalDBName(tenantId))
-  } else {
-    // just get all DBs in self host
-    await addDbs(couchUrl)
   }
   return dbs
 }

packages/backend-core/src/index.js

@@ -0,0 +1,17 @@
+const { setDB } = require("./db")
+
+module.exports = {
+  init(pouch) {
+    setDB(pouch)
+  },
+  // some default exports from the library, however these ideally shouldn't
+  // be used, instead the syntax require("@budibase/backend-core/db") should be used
+  StaticDatabases: require("./db/utils").StaticDatabases,
+  db: require("../db"),
+  redis: require("../redis"),
+  objectStore: require("../objectStore"),
+  utils: require("../utils"),
+  cache: require("../cache"),
+  auth: require("../auth"),
+  constants: require("../constants"),
+}

packages/backend-core/src/objectStore/index.js

@@ -206,6 +206,34 @@ exports.retrieveToTmp = async (bucketName, filepath) => {
   return outputPath
 }
 
+/**
+ * Delete a single file.
+ */
+exports.deleteFile = async (bucketName, filepath) => {
+  const objectStore = exports.ObjectStore(bucketName)
+  await exports.makeSureBucketExists(objectStore, bucketName)
+  const params = {
+    Bucket: bucketName,
+    Key: filepath,
+  }
+  return objectStore.deleteObject(params)
+}
+
+exports.deleteFiles = async (bucketName, filepaths) => {
+  const objectStore = exports.ObjectStore(bucketName)
+  await exports.makeSureBucketExists(objectStore, bucketName)
+  const params = {
+    Bucket: bucketName,
+    Delete: {
+      Objects: filepaths.map(path => ({ Key: path })),
+    },
+  }
+  return objectStore.deleteObjects(params).promise()
+}
+
+/**
+ * Delete a path, including everything within.
+ */
 exports.deleteFolder = async (bucketName, folder) => {
   bucketName = sanitizeBucket(bucketName)
   folder = sanitizeKey(folder)

packages/backend-core/utils.js

@@ -0,0 +1,4 @@
+module.exports = {
+  ...require("./src/utils"),
+  ...require("./src/hashing"),
+}

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/bbui/src/Tabs/Tabs.svelte

@@ -10,6 +10,8 @@
   export let noHorizPadding = false
   export let quiet = false
   export let emphasized = false
+  // overlay content from the tab bar onto tabs e.g. for a dropdown
+  export let onTop = false
 
   let thisSelected = undefined
 
@@ -78,6 +80,7 @@
     'spectrum-Tabs--quiet'} spectrum-Tabs--{vertical
     ? 'vertical'
     : 'horizontal'}"
+  class:onTop
 >
   <slot />
   {#if $tab.info}
@@ -98,7 +101,9 @@
   .quiet {
     border-bottom: none !important;
   }
-
+  .onTop {
+    z-index: 20;
+  }
   .spectrum-Tabs {
     padding-left: var(--spacing-xl);
     padding-right: var(--spacing-xl);

packages/bbui/yarn.lock

@@ -2076,9 +2076,9 @@ postcss@^7.0.0, postcss@^7.0.1, postcss@^7.0.27:
     supports-color "^6.1.0"
 
 postcss@^8.2.9:
-  version "8.2.10"
+  version "8.2.13"
-  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.10.tgz#ca7a042aa8aff494b334d0ff3e9e77079f6f702b"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.2.13.tgz#dbe043e26e3c068e45113b1ed6375d2d37e2129f"
-  integrity sha512-b/h7CPV7QEdrqIxtAf2j31U5ef05uBDuvoXv6L51Q4rcS1jdlXAVKJv+atCFdUXYl9dyTHGyoMzIepwowRJjFw==
+  integrity sha512-FCE5xLH+hjbzRdpbRb1IMCvPv9yZx2QnDarBEYSN0N0HYk+TcXsEhwdFcFb+SRWOKzKGErhIEbBK2ogyLdTtfQ==
   dependencies:
     colorette "^1.2.2"
     nanoid "^3.1.22"

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.27-alpha.5",
+    "@budibase/bbui": "^1.0.27-alpha.13",
-    "@budibase/client": "^1.0.27-alpha.5",
+    "@budibase/client": "^1.0.27-alpha.13",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^1.0.27-alpha.5",
+    "@budibase/string-templates": "^1.0.27-alpha.13",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/datasource.js

@@ -23,10 +23,10 @@ function prepareData(config) {
   return datasource
 }
 
-export async function saveDatasource(config) {
+export async function saveDatasource(config, skipFetch = false) {
   const datasource = prepareData(config)
   // Create datasource
-  const resp = await datasources.save(datasource, datasource.plus)
+  const resp = await datasources.save(datasource, !skipFetch && datasource.plus)
 
   // update the tables incase data source plus
   await tables.fetch()

packages/builder/src/components/automation/AutomationBuilder/FlowChart/FlowItem.svelte

@@ -9,6 +9,7 @@
     Modal,
     Button,
     StatusLight,
+    ActionButton,
   } from "@budibase/bbui"
   import AutomationBlockSetup from "../../SetupPanel/AutomationBlockSetup.svelte"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
@@ -27,7 +28,7 @@
   let blockComplete
 
   $: testResult = $automationStore.selectedAutomation.testResults?.steps.filter(
-    step => step.stepId === block.stepId
+    step => (block.id ? step.id === block.id : step.stepId === block.stepId)
   )
   $: isTrigger = block.type === "TRIGGER"
 
@@ -119,19 +120,13 @@
     <div class="blockSection">
       <Layout noPadding gap="S">
         <div class="splitHeader">
-          <div
+          <ActionButton
-            on:click|stopPropagation={() => {
+            on:click={() => (setupToggled = !setupToggled)}
-              setupToggled = !setupToggled
+            quiet
-            }}
+            icon={setupToggled ? "ChevronDown" : "ChevronRight"}
-            class="center-items"
           >
-            {#if setupToggled}
-              <Icon size="M" name="ChevronDown" />
-            {:else}
-              <Icon size="M" name="ChevronRight" />
-            {/if}
             <Detail size="S">Setup</Detail>
-          </div>
+          </ActionButton>
           {#if !isTrigger}
             <div on:click={() => deleteStep()}>
               <Icon name="DeleteOutline" />
@@ -187,6 +182,7 @@
   .splitHeader {
     display: flex;
     justify-content: space-between;
+    align-items: center;
   }
   .iconAlign {
     padding: 0 0 0 var(--spacing-m);

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -10,6 +10,7 @@
     ActionButton,
     Drawer,
     Modal,
+    Detail,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
 
@@ -37,6 +38,7 @@
   let drawer
   let tempFilters = lookForFilters(schemaProperties) || []
   let fillWidth = true
+  let codeBindingOpen = false
 
   $: stepId = block.stepId
   $: bindings = getAvailableBindings(
@@ -233,7 +235,16 @@
         <SchemaSetup on:change={e => onChange(e, key)} value={inputData[key]} />
       {:else if value.customType === "code"}
         <CodeEditorModal>
-          <pre>{JSON.stringify(bindings, null, 2)}</pre>
+          <ActionButton
+            on:click={() => (codeBindingOpen = !codeBindingOpen)}
+            quiet
+            icon={codeBindingOpen ? "ChevronDown" : "ChevronRight"}
+          >
+            <Detail size="S">Bindings</Detail>
+          </ActionButton>
+          {#if codeBindingOpen}
+            <pre>{JSON.stringify(bindings, null, 2)}</pre>
+          {/if}
           <Editor
             mode="javascript"
             on:change={e => {

packages/builder/src/components/backend/DatasourceNavigator/DatasourceNavigator.svelte

@@ -137,7 +137,7 @@
             selected={$queries.selected === query._id}
             on:click={() => onClickQuery(query)}
           >
-            <EditQueryPopover {query} />
+            <EditQueryPopover {query} {onClickQuery} />
           </NavItem>
         {/each}
       {/if}

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/PlusConfigForm.svelte

@@ -199,18 +199,18 @@
   <Body>
     Tell budibase how your tables are related to get even more smart features.
   </Body>
-{/if}
+  {#if relationshipInfo && relationshipInfo.length > 0}
-{#if relationshipInfo && relationshipInfo.length > 0}
+    <Table
-  <Table
+      on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
-    on:click={({ detail }) => openRelationshipModal(detail.from, detail.to)}
+      schema={relationshipSchema}
-    schema={relationshipSchema}
+      data={relationshipInfo}
-    data={relationshipInfo}
+      allowEditColumns={false}
-    allowEditColumns={false}
+      allowEditRows={false}
-    allowEditRows={false}
+      allowSelectRows={false}
-    allowSelectRows={false}
+    />
-  />
+  {:else}
-{:else}
+    <Body size="S"><i>No relationships configured.</i></Body>
-  <Body size="S"><i>No relationships configured.</i></Body>
+  {/if}
 {/if}
 
 <style>

packages/builder/src/components/backend/DatasourceNavigator/TableIntegrationMenu/rest/auth/RestAuthenticationBuilder.svelte

@@ -58,7 +58,7 @@
     />
   {/if}
   <div>
-    <ActionButton on:click={() => openConfigModal()} con="Add"
+    <ActionButton on:click={() => openConfigModal()} icon="Add"
       >Add authentication</ActionButton
     >
   </div>

packages/builder/src/components/backend/DatasourceNavigator/modals/DatasourceConfigModal.svelte

@@ -5,22 +5,28 @@
   import { IntegrationNames } from "constants/backend"
   import cloneDeep from "lodash/cloneDeepWith"
   import { saveDatasource as save } from "builderStore/datasource"
+  import { onMount } from "svelte"
 
   export let integration
   export let modal
 
   // kill the reference so the input isn't saved
   let datasource = cloneDeep(integration)
+  let skipFetch = false
 
   async function saveDatasource() {
     try {
-      const resp = await save(datasource)
+      const resp = await save(datasource, skipFetch)
       $goto(`./datasource/${resp._id}`)
       notifications.success(`Datasource updated successfully.`)
     } catch (err) {
       notifications.error(`Error saving datasource: ${err}`)
     }
   }
+
+  onMount(() => {
+    skipFetch = false
+  })
 </script>
 
 <ModalContent
@@ -28,9 +34,16 @@
   onConfirm={() => saveDatasource()}
   onCancel={() => modal.show()}
   confirmText={datasource.plus
-    ? "Fetch tables from database"
+    ? "Save and fetch tables"
     : "Save and continue to query"}
   cancelText="Back"
+  showSecondaryButton={datasource.plus}
+  secondaryButtonText={datasource.plus ? "Skip table fetch" : undefined}
+  secondaryAction={() => {
+    skipFetch = true
+    saveDatasource()
+    return true
+  }}
   size="L"
 >
   <Layout noPadding>

packages/builder/src/components/backend/DatasourceNavigator/popovers/EditQueryPopover.svelte

@@ -5,22 +5,29 @@
   import { datasources, queries } from "stores/backend"
 
   export let query
+  export let onClickQuery
 
   let confirmDeleteDialog
 
   async function deleteQuery() {
     const wasSelectedQuery = $queries.selected
-    const selectedDatasource = $datasources.selected
+    // need to calculate this before the query is deleted
+    const navigateToDatasource = wasSelectedQuery === query._id
+
     await queries.delete(query)
-    if (wasSelectedQuery === query._id) {
+    await datasources.fetch()
-      $goto(`./datasource/${selectedDatasource}`)
+
+    if (navigateToDatasource) {
+      await datasources.select(query.datasourceId)
+      $goto(`./datasource/${query.datasourceId}`)
     }
     notifications.success("Query deleted")
   }
 
   async function duplicateQuery() {
     try {
-      await queries.duplicate(query)
+      const newQuery = await queries.duplicate(query)
+      onClickQuery(newQuery)
     } catch (e) {
       notifications.error(e.message)
     }

packages/builder/src/components/start/AppCard.svelte

@@ -1,126 +0,0 @@
-<script>
-  import {
-    Heading,
-    Icon,
-    Body,
-    Layout,
-    ActionMenu,
-    MenuItem,
-    StatusLight,
-  } from "@budibase/bbui"
-  import { gradient } from "actions"
-  import { processStringSync } from "@budibase/string-templates"
-
-  export let app
-  export let exportApp
-  export let viewApp
-  export let editApp
-  export let updateApp
-  export let deleteApp
-  export let unpublishApp
-  export let releaseLock
-</script>
-
-<div class="wrapper">
-  <Layout noPadding gap="XS" alignContent="start">
-    <div class="preview" use:gradient={{ seed: app.name }} />
-    <div class="title">
-      {#if app.lockedBy}
-        <Icon name="LockClosed" />
-      {/if}
-      <div class="name" on:click={() => editApp(app)}>
-        <Heading size="XS">
-          {app.name}
-        </Heading>
-      </div>
-      <ActionMenu align="right">
-        <Icon slot="control" name="More" hoverable />
-        {#if app.deployed}
-          <MenuItem on:click={() => viewApp(app)} icon="GlobeOutline">
-            View published app
-          </MenuItem>
-        {/if}
-        {#if app.lockedYou}
-          <MenuItem on:click={() => releaseLock(app)} icon="LockOpen">
-            Release lock
-          </MenuItem>
-        {/if}
-        <MenuItem on:click={() => exportApp(app)} icon="Download">
-          Export
-        </MenuItem>
-        {#if app.deployed}
-          <MenuItem on:click={() => unpublishApp(app)} icon="GlobeRemove">
-            Unpublish
-          </MenuItem>
-        {/if}
-        {#if !app.deployed}
-          <MenuItem on:click={() => updateApp(app)} icon="Edit">Edit</MenuItem>
-          <MenuItem on:click={() => deleteApp(app)} icon="Delete">
-            Delete
-          </MenuItem>
-        {/if}
-      </ActionMenu>
-    </div>
-    <div class="status">
-      <Body size="S">
-        {#if app.updatedAt}
-          {processStringSync("Updated {{ duration time 'millisecond' }} ago", {
-            time: new Date().getTime() - new Date(app.updatedAt).getTime(),
-          })}
-        {:else}
-          Never updated
-        {/if}
-      </Body>
-      <StatusLight active={app.deployed} neutral={!app.deployed}>
-        {#if app.deployed}Published{:else}Unpublished{/if}
-      </StatusLight>
-    </div>
-  </Layout>
-</div>
-
-<style>
-  .wrapper {
-    overflow: hidden;
-  }
-  .wrapper :global(.spectrum-StatusLight) {
-    padding: 0;
-    min-height: 0;
-  }
-
-  .preview {
-    height: 135px;
-    border-radius: var(--border-radius-s);
-    margin-bottom: var(--spacing-m);
-  }
-
-  .status {
-    display: flex;
-    flex-direction: row;
-    justify-content: space-between;
-    align-items: center;
-  }
-
-  .name {
-    flex: 1 1 auto;
-  }
-
-  .title {
-    display: flex;
-    flex-direction: row;
-    justify-content: space-between;
-    align-items: flex-start;
-    gap: var(--spacing-m);
-  }
-  .title :global(.spectrum-Icon) {
-    flex: 0 0 auto;
-  }
-  .title :global(h1) {
-    overflow: hidden;
-    text-overflow: ellipsis;
-  }
-  .title :global(h1:hover) {
-    color: var(--spectrum-global-color-blue-600);
-    cursor: pointer;
-    transition: color 130ms ease;
-  }
-</style>

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/rest/[query]/index.svelte

@@ -234,8 +234,12 @@
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString
     breakQs = restUtils.breakQueryString(qs)
-    if (datasourceUrl && !query.fields.path?.startsWith(datasourceUrl)) {
+    const path = query.fields.path
-      const path = query.fields.path
+    if (
+      datasourceUrl &&
+      !path?.startsWith("http") &&
+      !path?.startsWith("{{") // don't substitute the datasource url when query starts with a variable e.g. the upgrade path
+    ) {
       query.fields.path = `${datasource.config.url}/${path ? path : ""}`
     }
     url = buildUrl(query.fields.path, breakQs)
@@ -311,7 +315,7 @@
           </div>
           <Button cta disabled={!url} on:click={runQuery}>Send</Button>
         </div>
-        <Tabs selected="Bindings" quiet noPadding noHorizPadding>
+        <Tabs selected="Bindings" quiet noPadding noHorizPadding onTop>
           <Tab title="Bindings">
             <KeyValueBuilder
               bind:object={bindings}
@@ -491,7 +495,7 @@
                 <Layout noPadding gap="S">
                   <Body size="S">
                     Create dynamic variables based on response body or headers
-                    from other queries.
+                    from this query.
                   </Body>
                   <KeyValueBuilder
                     bind:object={dynamicVariables}

packages/builder/src/pages/builder/apps/index.svelte

@@ -12,7 +12,7 @@
     Modal,
   } from "@budibase/bbui"
   import { onMount } from "svelte"
-  import { apps, organisation, auth } from "stores/portal"
+  import { apps, organisation, auth, admin } from "stores/portal"
   import { goto } from "@roxi/routify"
   import { AppStatus } from "constants"
   import { gradient } from "actions"
@@ -34,12 +34,16 @@
   const publishedAppsOnly = app => app.status === AppStatus.DEPLOYED
 
   $: publishedApps = $apps.filter(publishedAppsOnly)
-
+  $: isCloud = $admin.cloud
   $: userApps = $auth.user?.builder?.global
     ? publishedApps
     : publishedApps.filter(app =>
         Object.keys($auth.user?.roles).includes(app.prodId)
       )
+
+  function getUrl(app) {
+    return !isCloud ? `/app/${encodeURIComponent(app.name)}` : `/${app.prodId}`
+  }
 </script>
 
 {#if $auth.user && loaded}
@@ -93,7 +97,7 @@
             <div class="group">
               <Layout gap="S" noPadding>
                 {#each userApps as app, idx (app.appId)}
-                  <a class="app" target="_blank" href={`/${app.prodId}`}>
+                  <a class="app" target="_blank" href={getUrl(app)}>
                     <div class="preview" use:gradient={{ seed: app.name }} />
                     <div class="app-info">
                       <Heading size="XS">{app.name}</Heading>

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -47,6 +47,7 @@
   $: filteredApps = enrichedApps.filter(app =>
     app?.name?.toLowerCase().includes(searchTerm.toLowerCase())
   )
+  $: isCloud = $admin.cloud
 
   const enrichApps = (apps, user, sortBy) => {
     const enrichedApps = apps.map(app => ({
@@ -158,8 +159,13 @@
   }
 
   const viewApp = app => {
-    const id = app.deployed ? app.prodId : app.devId
+    if (!isCloud && app.deployed) {
-    window.open(`/${id}`, "_blank")
+      // special case to use the short form name if self hosted
+      window.open(`/app/${encodeURIComponent(app.name)}`)
+    } else {
+      const id = app.deployed ? app.prodId : app.devId
+      window.open(`/${id}`, "_blank")
+    }
   }
 
   const editApp = app => {

packages/builder/src/stores/backend/queries.js

@@ -134,7 +134,7 @@ export function createQueriesStore() {
         list.map(q => q.name)
       )
 
-      actions.save(datasourceId, newQuery)
+      return actions.save(datasourceId, newQuery)
     },
   }
 

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^1.0.27-alpha.5",
+    "@budibase/bbui": "^1.0.27-alpha.13",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^1.0.27-alpha.5",
+    "@budibase/string-templates": "^1.0.27-alpha.13",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.5",
+  "version": "1.0.27-alpha.13",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/auth": "^1.0.27-alpha.5",
+    "@budibase/backend-core": "^1.0.27-alpha.13",
-    "@budibase/client": "^1.0.27-alpha.5",
+    "@budibase/client": "^1.0.27-alpha.13",
-    "@budibase/string-templates": "^1.0.27-alpha.5",
+    "@budibase/string-templates": "^1.0.27-alpha.13",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/scripts/replicateApp.js

@@ -7,7 +7,7 @@
 
 const CouchDB = require("../src/db")
 const { DocumentTypes } = require("../src/db/utils")
-const { getAllDbs } = require("@budibase/auth/db")
+const { getAllDbs } = require("@budibase/backend-core/db")
 
 const appName = process.argv[2].toLowerCase()
 const remoteUrl = process.argv[3]

packages/server/src/api/controllers/apikeys.js

@@ -1,5 +1,5 @@
-const { StaticDatabases } = require("@budibase/auth/db")
+const { StaticDatabases } = require("@budibase/backend-core/db")
-const { getGlobalDB } = require("@budibase/auth/tenancy")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
 
 const KEYS_DOC = StaticDatabases.GLOBAL.docs.apiKeys
 

packages/server/src/api/controllers/application.js

@@ -19,7 +19,10 @@ const {
   DocumentTypes,
   AppStatus,
 } = require("../../db/utils")
-const { BUILTIN_ROLE_IDS, AccessController } = require("@budibase/auth/roles")
+const {
+  BUILTIN_ROLE_IDS,
+  AccessController,
+} = require("@budibase/backend-core/roles")
 const { BASE_LAYOUTS } = require("../../constants/layouts")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
@@ -28,7 +31,7 @@ const {
   isDevAppID,
   getDeployedAppID,
   Replication,
-} = require("@budibase/auth/db")
+} = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
   getDeployedApps,
@@ -41,9 +44,9 @@ const {
   backupClientLibrary,
   revertClientLibrary,
 } = require("../../utilities/fileSystem/clientLibrary")
-const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
+const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
 const { syncGlobalUsers } = require("./user")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g

packages/server/src/api/controllers/auth.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../db")
 const { outputProcessing } = require("../../utilities/rowProcessor")
 const { InternalTables } = require("../../db/utils")
 const { getFullUser } = require("../../utilities/users")
-const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 
 exports.fetchSelf = async ctx => {
   const appId = ctx.appId

packages/server/src/api/controllers/cloud.js

@@ -1,5 +1,5 @@
 const env = require("../../environment")
-const { getAllApps } = require("@budibase/auth/db")
+const { getAllApps } = require("@budibase/backend-core/db")
 const CouchDB = require("../../db")
 const {
   exportDB,
@@ -7,7 +7,10 @@ const {
   readFileSync,
 } = require("../../utilities/fileSystem")
 const { stringToReadStream } = require("../../utilities")
-const { getGlobalDBName, getGlobalDB } = require("@budibase/auth/tenancy")
+const {
+  getGlobalDBName,
+  getGlobalDB,
+} = require("@budibase/backend-core/tenancy")
 const { create } = require("./application")
 const { getDocParams, DocumentTypes, isDevAppID } = require("../../db/utils")
 

packages/server/src/api/controllers/datasource.js

@@ -10,6 +10,7 @@ const {
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
+const { invalidateDynamicVariables } = require("../../threads/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -57,10 +58,43 @@ exports.buildSchemaFromDb = async function (ctx) {
   ctx.body = response
 }
 
+/**
+ * Check for variables that have been updated or removed and invalidate them.
+ */
+const invalidateVariables = async (existingDatasource, updatedDatasource) => {
+  const existingVariables = existingDatasource.config.dynamicVariables
+  const updatedVariables = updatedDatasource.config.dynamicVariables
+  const toInvalidate = []
+
+  if (!existingVariables) {
+    return
+  }
+
+  if (!updatedVariables) {
+    // invalidate all
+    toInvalidate.push(...existingVariables)
+  } else {
+    // invaldate changed / removed
+    existingVariables.forEach(existing => {
+      const unchanged = updatedVariables.find(
+        updated =>
+          existing.name === updated.name &&
+          existing.queryId === updated.queryId &&
+          existing.value === updated.value
+      )
+      if (!unchanged) {
+        toInvalidate.push(existing)
+      }
+    })
+  }
+  await invalidateDynamicVariables(toInvalidate)
+}
+
 exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
+  await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
 
   const response = await db.put(datasource)

packages/server/src/api/controllers/deploy/index.js

@@ -1,12 +1,12 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication, getDeployedAppID } = require("@budibase/auth/db")
+const { Replication, getDeployedAppID } = require("@budibase/backend-core/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000

packages/server/src/api/controllers/dev.js

@@ -4,9 +4,9 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication } = require("@budibase/auth").db
+const { Replication } = require("@budibase/backend-core/db")
 const { DocumentTypes } = require("../../db/utils")
-const { app: appCache } = require("@budibase/auth/cache")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params

packages/server/src/api/controllers/hosting.js

@@ -1,7 +1,7 @@
 const CouchDB = require("../../db")
 const { getDeployedApps } = require("../../utilities/workerRequests")
-const { getScopedConfig } = require("@budibase/auth/db")
+const { getScopedConfig } = require("@budibase/backend-core/db")
-const { Configs } = require("@budibase/auth").constants
+const { Configs } = require("@budibase/backend-core/constants")
 const { checkSlashesInUrl } = require("../../utilities")
 
 exports.fetchUrls = async ctx => {