Merge branch 'master' into fix/collect-data

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.14.4",
+  "version": "2.14.5",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/account-portal

@@ -1 +1 @@
-Subproject commit bcd86d9034ba954f013da4c10171bf495ab88189
+Subproject commit b23fb3b17961fb04badd9487913a683fcf26dbe6

packages/builder/src/builderStore/store/frontend.js

@@ -85,7 +85,6 @@ const INITIAL_FRONTEND_STATE = {
   selectedScreenId: null,
   selectedComponentId: null,
   selectedLayoutId: null,
-  hoverComponentId: null,
 
   // Client state
   selectedComponentInstance: null,
@@ -93,6 +92,9 @@ const INITIAL_FRONTEND_STATE = {
   // Onboarding
   onboarding: false,
   tourNodes: null,
+
+  // UI state
+  hoveredComponentId: null,
 }
 
 export const getFrontendStore = () => {
@@ -1413,6 +1415,18 @@ export const getFrontendStore = () => {
           return state
         })
       },
+      hover: (componentId, notifyClient = true) => {
+        if (componentId === get(store).hoveredComponentId) {
+          return
+        }
+        store.update(state => {
+          state.hoveredComponentId = componentId
+          return state
+        })
+        if (notifyClient) {
+          store.actions.preview.sendEvent("hover-component", componentId)
+        }
+      },
     },
     links: {
       save: async (url, title) => {

packages/builder/src/components/automation/AutomationBuilder/FlowChart/ActionModal.svelte

@@ -152,7 +152,7 @@
             {#if isDisabled && !syncAutomationsEnabled && action.stepId === ActionStepID.COLLECT}
               <div class="tag-color">
                 <Tags>
-                  <Tag icon="LockClosed">Business</Tag>
+                  <Tag icon="LockClosed">Premium</Tag>
                 </Tags>
               </div>
             {:else if isDisabled}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -41,7 +41,7 @@
       { label: "False", value: "false" },
     ]}
   />
-{:else if schema.type === "array"}
+{:else if schemaHasOptions(schema) && schema.type === "array"}
   <Multiselect
     bind:value={value[field]}
     options={schema.constraints.inclusion}
@@ -77,7 +77,7 @@
     on:change={e => onChange(e, field)}
     useLabel={false}
   />
-{:else if ["string", "number", "bigint", "barcodeqr"].includes(schema.type)}
+{:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
     this={isTestModal ? ModalBindableInput : DrawerBindableInput}
     panel={AutomationBindingPanel}

packages/builder/src/components/design/settings/controls/RelationshipFilterEditor.svelte

@@ -1,6 +1,9 @@
 <script>
-  import { currentAsset } from "builderStore"
+  import { currentAsset, store } from "builderStore"
-  import { findClosestMatchingComponent } from "builderStore/componentUtils"
+  import {
+    findClosestMatchingComponent,
+    findComponent,
+  } from "builderStore/componentUtils"
   import {
     getDatasourceForProvider,
     getSchemaForDatasource,
@@ -20,8 +23,23 @@
     component => component._component.endsWith("/form")
   )
 
+  const resolveDatasource = (currentAsset, componentInstance, form) => {
+    if (!form && componentInstance._id != $store.selectedComponentId) {
+      const block = findComponent(
+        currentAsset.props,
+        $store.selectedComponentId
+      )
+      const def = store.actions.components.getDefinition(block._component)
+      return def?.block === true
+        ? getDatasourceForProvider(currentAsset, block)
+        : {}
+    } else {
+      return getDatasourceForProvider(currentAsset, form)
+    }
+  }
+
   // Get that form's schema
-  $: datasource = getDatasourceForProvider($currentAsset, form)
+  $: datasource = resolveDatasource($currentAsset, componentInstance, form)
   $: formSchema = getSchemaForDatasource($currentAsset, datasource)?.schema
 
   // Get the schema for the relationship field that this picker is using

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsPanel.svelte

@@ -108,6 +108,8 @@
           {componentInstance}
           {componentDefinition}
           {bindings}
+          iconTooltip={componentName}
+          componentTitle={title}
         />
       {/if}
       {#if section == "conditions"}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/CustomStylesSection.svelte

@@ -5,6 +5,9 @@
     Drawer,
     Button,
     notifications,
+    AbsTooltip,
+    Icon,
+    Body,
   } from "@budibase/bbui"
   import { selectedScreen, store } from "builderStore"
   import ClientBindingPanel from "components/common/bindings/ClientBindingPanel.svelte"
@@ -15,6 +18,9 @@
   } from "builderStore/dataBinding"
 
   export let componentInstance
+  export let componentDefinition
+  export let iconTooltip
+  export let componentTitle
 
   let tempValue
   let drawer
@@ -24,6 +30,8 @@
     $store.selectedComponentId
   )
 
+  $: icon = componentDefinition?.icon
+
   const openDrawer = () => {
     tempValue = runtimeToReadableBinding(
       bindings,
@@ -54,7 +62,19 @@
 {#key componentInstance?._id}
   <Drawer bind:this={drawer} title="Custom CSS">
     <svelte:fragment slot="description">
-      Custom CSS overrides all other component styles.
+      <div class="header">
+        Your CSS will overwrite styles for:
+        {#if icon}
+          <AbsTooltip type="info" text={iconTooltip}>
+            <Icon
+              color={`var(--spectrum-global-color-gray-600)`}
+              size="S"
+              name={icon}
+            />
+          </AbsTooltip>
+          <Body size="S"><b>{componentTitle || ""}</b></Body>
+        {/if}
+      </div>
     </svelte:fragment>
     <Button cta slot="buttons" on:click={save}>Save</Button>
     <svelte:component
@@ -68,3 +88,13 @@
     />
   </Drawer>
 {/key}
+
+<style>
+  .header {
+    display: flex;
+    flex-direction: row;
+    justify-content: space-between;
+    align-items: center;
+    gap: var(--spacing-m);
+  }
+</style>

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte

@@ -36,14 +36,12 @@
 
   // Determine selected component ID
   $: selectedComponentId = $store.selectedComponentId
-  $: hoverComponentId = $store.hoverComponentId
 
   $: previewData = {
     appId: $store.appId,
     layout,
     screen,
     selectedComponentId,
-    hoverComponentId,
     theme: $store.theme,
     customTheme: $store.customTheme,
     previewDevice: $store.previewDevice,
@@ -119,8 +117,8 @@
       error = event.error || "An unknown error occurred"
     } else if (type === "select-component" && data.id) {
       $store.selectedComponentId = data.id
-    } else if (type === "hover-component" && data.id) {
+    } else if (type === "hover-component") {
-      $store.hoverComponentId = data.id
+      store.actions.components.hover(data.id, false)
     } else if (type === "update-prop") {
       await store.actions.components.updateSetting(data.prop, data.value)
     } else if (type === "update-styles") {

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/ComponentTree.svelte

@@ -90,16 +90,7 @@
     return findComponentPath($selectedComponent, component._id)?.length > 0
   }
 
-  const handleMouseover = componentId => {
+  const hover = store.actions.components.hover
-    if ($store.hoverComponentId !== componentId) {
-      $store.hoverComponentId = componentId
-    }
-  }
-  const handleMouseout = componentId => {
-    if ($store.hoverComponentId === componentId) {
-      $store.hoverComponentId = null
-    }
-  }
 </script>
 
 <ul>
@@ -120,9 +111,9 @@
         on:dragover={dragover(component, index)}
         on:iconClick={() => toggleNodeOpen(component._id)}
         on:drop={onDrop}
-        hovering={$store.hoverComponentId === component._id}
+        hovering={$store.hoveredComponentId === component._id}
-        on:mouseenter={() => handleMouseover(component._id)}
+        on:mouseenter={() => hover(component._id)}
-        on:mouseleave={() => handleMouseout(component._id)}
+        on:mouseleave={() => hover(null)}
         text={getComponentText(component)}
         icon={getComponentIcon(component)}
         iconTooltip={getComponentName(component)}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/ComponentList/index.svelte

@@ -12,6 +12,9 @@
 
   let scrolling = false
 
+  $: screenComponentId = `${$store.selectedScreenId}-screen`
+  $: navComponentId = `${$store.selectedScreenId}-navigation`
+
   const toNewComponentRoute = () => {
     if ($isActive(`./:componentId/new`)) {
       $goto(`./:componentId`)
@@ -33,16 +36,7 @@
     scrolling = e.target.scrollTop !== 0
   }
 
-  const handleMouseover = componentId => {
+  const hover = store.actions.components.hover
-    if ($store.hoverComponentId !== componentId) {
-      $store.hoverComponentId = componentId
-    }
-  }
-  const handleMouseout = componentId => {
-    if ($store.hoverComponentId === componentId) {
-      $store.hoverComponentId = null
-    }
-  }
 </script>
 
 <div class="components">
@@ -65,46 +59,31 @@
             scrollable
             icon="WebPage"
             on:drop={onDrop}
-            on:click={() => {
+            on:click={() => ($store.selectedComponentId = screenComponentId)}
-              $store.selectedComponentId = `${$store.selectedScreenId}-screen`
+            hovering={$store.hoveredComponentId === screenComponentId}
-            }}
+            on:mouseenter={() => hover(screenComponentId)}
-            hovering={$store.hoverComponentId ===
+            on:mouseleave={() => hover(null)}
-              `${$store.selectedScreenId}-screen`}
+            id="component-screen"
-            on:mouseenter={() =>
+            selectedBy={$userSelectedResourceMap[screenComponentId]}
-              handleMouseover(`${$store.selectedScreenId}-screen`)}
-            on:mouseleave={() =>
-              handleMouseout(`${$store.selectedScreenId}-screen`)}
-            id={`component-screen`}
-            selectedBy={$userSelectedResourceMap[
-              `${$store.selectedScreenId}-screen`
-            ]}
           >
             <ScreenslotDropdownMenu component={$selectedScreen?.props} />
           </NavItem>
           <NavItem
             text="Navigation"
             indentLevel={0}
-            selected={$store.selectedComponentId ===
+            selected={$store.selectedComponentId === navComponentId}
-              `${$store.selectedScreenId}-navigation`}
             opened
             scrollable
             icon={$selectedScreen.showNavigation
               ? "Visibility"
               : "VisibilityOff"}
             on:drop={onDrop}
-            on:click={() => {
+            on:click={() => ($store.selectedComponentId = navComponentId)}
-              $store.selectedComponentId = `${$store.selectedScreenId}-navigation`
+            hovering={$store.hoveredComponentId === navComponentId}
-            }}
+            on:mouseenter={() => hover(navComponentId)}
-            hovering={$store.hoverComponentId ===
+            on:mouseleave={() => hover(null)}
-              `${$store.selectedScreenId}-navigation`}
+            id="component-nav"
-            on:mouseenter={() =>
+            selectedBy={$userSelectedResourceMap[navComponentId]}
-              handleMouseover(`${$store.selectedScreenId}-navigation`)}
-            on:mouseleave={() =>
-              handleMouseout(`${$store.selectedScreenId}-navigation`)}
-            id={`component-nav`}
-            selectedBy={$userSelectedResourceMap[
-              `${$store.selectedScreenId}-navigation`
-            ]}
           />
           <ComponentTree
             level={0}

packages/builder/src/pages/builder/portal/settings/branding.svelte

@@ -214,7 +214,7 @@
         <Heading size="M">Branding</Heading>
         {#if !isCloud && !brandingEnabled}
           <Tags>
-            <Tag icon="LockClosed">Business</Tag>
+            <Tag icon="LockClosed">Premium</Tag>
           </Tags>
         {/if}
         {#if isCloud && !brandingEnabled}

packages/builder/src/pages/builder/portal/users/groups/index.svelte

@@ -97,7 +97,7 @@
       <Heading size="M">Groups</Heading>
       {#if !$licensing.groupsEnabled}
         <Tags>
-          <Tag icon="LockClosed">Business</Tag>
+          <Tag icon="LockClosed">Enterpise</Tag>
         </Tags>
       {/if}
     </div>

packages/client/src/components/preview/HoverIndicator.svelte

@@ -1,9 +1,9 @@
 <script>
   import { onMount, onDestroy } from "svelte"
   import IndicatorSet from "./IndicatorSet.svelte"
-  import { builderStore, dndIsDragging } from "stores"
+  import { builderStore, dndIsDragging, hoverStore } from "stores"
 
-  $: componentId = $builderStore.hoverComponentId
+  $: componentId = $hoverStore.hoveredComponentId
   $: zIndex = componentId === $builderStore.selectedComponentId ? 900 : 920
 
   const onMouseOver = e => {
@@ -23,12 +23,12 @@
     }
 
     if (newId !== componentId) {
-      builderStore.actions.hoverComponent(newId)
+      hoverStore.actions.hoverComponent(newId)
     }
   }
 
   const onMouseLeave = () => {
-    builderStore.actions.hoverComponent(null)
+    hoverStore.actions.hoverComponent(null)
   }
 
   onMount(() => {

packages/client/src/index.js

@@ -8,6 +8,7 @@ import {
   environmentStore,
   dndStore,
   eventStore,
+  hoverStore,
 } from "./stores"
 import loadSpectrumIcons from "@budibase/bbui/spectrum-icons-rollup.js"
 import { get } from "svelte/store"
@@ -33,7 +34,6 @@ const loadBudibase = async () => {
     layout: window["##BUDIBASE_PREVIEW_LAYOUT##"],
     screen: window["##BUDIBASE_PREVIEW_SCREEN##"],
     selectedComponentId: window["##BUDIBASE_SELECTED_COMPONENT_ID##"],
-    hoverComponentId: window["##BUDIBASE_HOVER_COMPONENT_ID##"],
     previewId: window["##BUDIBASE_PREVIEW_ID##"],
     theme: window["##BUDIBASE_PREVIEW_THEME##"],
     customTheme: window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"],
@@ -84,6 +84,8 @@ const loadBudibase = async () => {
       } else {
         dndStore.actions.reset()
       }
+    } else if (type === "hover-component") {
+      hoverStore.actions.hoverComponent(data)
     } else if (type === "builder-meta") {
       builderStore.actions.setMetadata(data)
     }

packages/client/src/stores/builder.js

@@ -8,7 +8,6 @@ const createBuilderStore = () => {
     inBuilder: false,
     screen: null,
     selectedComponentId: null,
-    hoverComponentId: null,
     editMode: false,
     previewId: null,
     theme: null,
@@ -25,16 +24,6 @@ const createBuilderStore = () => {
   }
   const store = writable(initialState)
   const actions = {
-    hoverComponent: id => {
-      if (id === get(store).hoverComponentId) {
-        return
-      }
-      store.update(state => ({
-        ...state,
-        hoverComponentId: id,
-      }))
-      eventStore.actions.dispatchEvent("hover-component", { id })
-    },
     selectComponent: id => {
       if (id === get(store).selectedComponentId) {
         return

packages/client/src/stores/hover.js

@@ -0,0 +1,25 @@
+import { get, writable } from "svelte/store"
+import { eventStore } from "./events.js"
+
+const createHoverStore = () => {
+  const store = writable({
+    hoveredComponentId: null,
+  })
+
+  const hoverComponent = id => {
+    if (id === get(store).hoveredComponentId) {
+      return
+    }
+    store.set({ hoveredComponentId: id })
+    eventStore.actions.dispatchEvent("hover-component", { id })
+  }
+
+  return {
+    ...store,
+    actions: {
+      hoverComponent,
+    },
+  }
+}
+
+export const hoverStore = createHoverStore()

packages/client/src/stores/index.js

@@ -27,6 +27,7 @@ export {
   dndIsDragging,
 } from "./dnd"
 export { sidePanelStore } from "./sidePanel"
+export { hoverStore } from "./hover"
 
 // Context stores are layered and duplicated, so it is not a singleton
 export { createContextStore } from "./context"

packages/server/package.json

@@ -80,7 +80,7 @@
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",
-    "koa-send": "5.0.0",
+    "koa-send": "5.0.1",
     "koa-useragent": "^4.1.0",
     "koa2-ratelimit": "1.1.1",
     "lodash": "4.17.21",
@@ -120,6 +120,7 @@
     "@types/jest": "29.5.5",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
+    "@types/koa-send": "^4.1.6",
     "@types/lodash": "4.14.200",
     "@types/mssql": "9.1.4",
     "@types/node-fetch": "2.6.4",

packages/server/src/api/controllers/datasource.ts

@@ -1,13 +1,10 @@
-import {
+import { getQueryParams, getTableParams } from "../../db/utils"
-  DocumentType,
-  generateDatasourceID,
-  getQueryParams,
-  getTableParams,
-} from "../../db/utils"
 import { getIntegration } from "../../integrations"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { context, db as dbCore, events } from "@budibase/backend-core"
 import {
+  BuildSchemaFromSourceRequest,
+  BuildSchemaFromSourceResponse,
   CreateDatasourceRequest,
   CreateDatasourceResponse,
   Datasource,
@@ -22,7 +19,6 @@ import {
 } from "@budibase/types"
 import sdk from "../../sdk"
 import { builderSocket } from "../../websockets"
-import { setupCreationAuth as googleSetupCreationAuth } from "../../integrations/googlesheets"
 import { isEqual } from "lodash"
 
 export async function fetch(ctx: UserCtx) {
@@ -67,22 +63,16 @@ export async function information(
   }
 }
 
-export async function buildSchemaFromDb(ctx: UserCtx) {
+export async function buildSchemaFromSource(
-  const db = context.getAppDB()
+  ctx: UserCtx<BuildSchemaFromSourceRequest, BuildSchemaFromSourceResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
   const tablesFilter = ctx.request.body.tablesFilter
-  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
 
-  const { tables, errors } = await sdk.datasources.buildFilteredSchema(
+  const { datasource, errors } = await sdk.datasources.buildSchemaFromSource(
-    datasource,
+    datasourceId,
     tablesFilter
   )
-  datasource.entities = tables
-
-  setDefaultDisplayColumns(datasource)
-  const dbResp = await db.put(
-    sdk.tables.populateExternalTableSchemas(datasource)
-  )
-  datasource._rev = dbResp.rev
 
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
@@ -90,24 +80,6 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   }
 }
 
-/**
- * Make sure all datasource entities have a display name selected
- */
-function setDefaultDisplayColumns(datasource: Datasource) {
-  //
-  for (let entity of Object.values(datasource.entities || {})) {
-    if (entity.primaryDisplay) {
-      continue
-    }
-    const notAutoColumn = Object.values(entity.schema).find(
-      schema => !schema.autocolumn
-    )
-    if (notAutoColumn) {
-      entity.primaryDisplay = notAutoColumn.name
-    }
-  }
-}
-
 /**
  * Check for variables that have been updated or removed and invalidate them.
  */
@@ -205,54 +177,18 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   }
 }
 
-const preSaveAction: Partial<Record<SourceName, any>> = {
-  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
-    await googleSetupCreationAuth(datasource.config as any)
-  },
-}
-
 export async function save(
   ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
 ) {
-  const db = context.getAppDB()
+  const {
-  const plus = ctx.request.body.datasource.plus
+    datasource: datasourceData,
-  const fetchSchema = ctx.request.body.fetchSchema
+    fetchSchema,
-  const tablesFilter = ctx.request.body.tablesFilter
+    tablesFilter,
-
+  } = ctx.request.body
-  const datasource = {
+  const { datasource, errors } = await sdk.datasources.save(datasourceData, {
-    _id: generateDatasourceID({ plus }),
+    fetchSchema,
-    ...ctx.request.body.datasource,
+    tablesFilter,
-    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
+  })
-  }
-
-  let errors: Record<string, string> = {}
-  if (fetchSchema) {
-    const schema = await sdk.datasources.buildFilteredSchema(
-      datasource,
-      tablesFilter
-    )
-    datasource.entities = schema.tables
-    setDefaultDisplayColumns(datasource)
-    errors = schema.errors
-  }
-
-  if (preSaveAction[datasource.source]) {
-    await preSaveAction[datasource.source](datasource)
-  }
-
-  const dbResp = await db.put(
-    sdk.tables.populateExternalTableSchemas(datasource)
-  )
-  await events.datasource.created(datasource)
-  datasource._rev = dbResp.rev
-
-  // Drain connection pools when configuration is changed
-  if (datasource.source) {
-    const source = await getIntegration(datasource.source)
-    if (source && source.pool) {
-      await source.pool.end()
-    }
-  }
 
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),

packages/server/src/api/controllers/static/templates/preview.hbs

@@ -63,7 +63,6 @@
       // Extract data from message
       const {
         selectedComponentId,
-        hoverComponentId,
         layout,
         screen,
         appId,
@@ -82,7 +81,6 @@
       window["##BUDIBASE_PREVIEW_LAYOUT##"] = layout
       window["##BUDIBASE_PREVIEW_SCREEN##"] = screen
       window["##BUDIBASE_SELECTED_COMPONENT_ID##"] = selectedComponentId
-      window["##BUDIBASE_HOVER_COMPONENT_ID##"] = hoverComponentId
       window["##BUDIBASE_PREVIEW_ID##"] = Math.random()
       window["##BUDIBASE_PREVIEW_THEME##"] = theme
       window["##BUDIBASE_PREVIEW_CUSTOM_THEME##"] = customTheme

packages/server/src/api/routes/datasource.ts

@@ -53,7 +53,7 @@ router
   .post(
     "/api/datasources/:datasourceId/schema",
     authorized(permissions.BUILDER),
-    datasourceController.buildSchemaFromDb
+    datasourceController.buildSchemaFromSource
   )
   .post(
     "/api/datasources",

packages/server/src/automations/steps/queryRows.ts

@@ -103,8 +103,7 @@ function typeCoercion(filters: SearchFilters, table: Table) {
     return filters
   }
   for (let key of Object.keys(filters)) {
-    // @ts-ignore
+    const searchParam = filters[key as keyof SearchFilters]
-    const searchParam = filters[key]
     if (typeof searchParam === "object") {
       for (let [property, value] of Object.entries(searchParam)) {
         // We need to strip numerical prefixes here, so that we can look up
@@ -117,7 +116,13 @@ function typeCoercion(filters: SearchFilters, table: Table) {
           continue
         }
         if (column.type === FieldTypes.NUMBER) {
-          searchParam[property] = parseFloat(value)
+          if (key === "oneOf") {
+            searchParam[property] = value
+              .split(",")
+              .map(item => parseFloat(item))
+          } else {
+            searchParam[property] = parseFloat(value)
+          }
         }
       }
     }

packages/server/src/integrations/postgres.ts

@@ -149,8 +149,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   private index: number = 1
   private open: boolean
 
-  COLUMNS_SQL!: string
-
   PRIMARY_KEYS_SQL = () => `
   SELECT pg_namespace.nspname table_schema
      , pg_class.relname table_name
@@ -171,6 +169,11 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   JOIN pg_catalog.pg_namespace n ON n.oid = t.typnamespace;
   `
 
+  COLUMNS_SQL = () => `
+    select * from information_schema.columns where table_schema = ANY(current_schemas(false)) 
+      AND pg_table_is_visible(to_regclass(format('%I.%I', table_schema, table_name)));
+  `
+
   constructor(config: PostgresConfig) {
     super(SqlClient.POSTGRES)
     this.config = config
@@ -224,8 +227,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       .split(",")
       .map(item => `"${item.trim()}"`)
     await this.client.query(`SET search_path TO ${search_path.join(",")};`)
-    this.COLUMNS_SQL = `select * from information_schema.columns where table_schema = ANY(current_schemas(false))
-    AND pg_table_is_visible(to_regclass(table_schema || '.' || table_name));`
     this.open = true
   }
 
@@ -312,7 +313,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
 
     try {
       const columnsResponse: { rows: PostgresColumn[] } =
-        await this.client.query(this.COLUMNS_SQL)
+        await this.client.query(this.COLUMNS_SQL())
 
       const tables: { [key: string]: Table } = {}
 
@@ -382,7 +383,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     try {
       await this.openConnection()
       const columnsResponse: { rows: PostgresColumn[] } =
-        await this.client.query(this.COLUMNS_SQL)
+        await this.client.query(this.COLUMNS_SQL())
       const names = columnsResponse.rows.map(row => row.table_name)
       return [...new Set(names)]
     } finally {

packages/server/src/jsRunner.ts

@@ -12,51 +12,49 @@ export function init() {
       const perRequestLimit = env.JS_PER_REQUEST_TIME_LIMIT_MS
       let track: TrackerFn = f => f()
       if (perRequestLimit) {
-        tracer.trace<any>("runJS.setupTracker", {}, span => {
+        const bbCtx = tracer.trace("runJS.getCurrentContext", {}, span =>
-          const bbCtx = context.getCurrentContext()
+          context.getCurrentContext()
-          if (bbCtx) {
+        )
-            if (!bbCtx.jsExecutionTracker) {
+        if (bbCtx) {
-              span?.addTags({
+          if (!bbCtx.jsExecutionTracker) {
-                createdExecutionTracker: true,
-              })
-              bbCtx.jsExecutionTracker =
-                timers.ExecutionTimeTracker.withLimit(perRequestLimit)
-            }
             span?.addTags({
-              js: {
+              createdExecutionTracker: true,
-                limitMS: bbCtx.jsExecutionTracker.limitMs,
-                elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
-              },
             })
-            // We call checkLimit() here to prevent paying the cost of creating
+            bbCtx.jsExecutionTracker = tracer.trace(
-            // a new VM context below when we don't need to.
+              "runJS.createExecutionTimeTracker",
-            bbCtx.jsExecutionTracker.checkLimit()
+              {},
-            track = bbCtx.jsExecutionTracker.track.bind(
+              span => timers.ExecutionTimeTracker.withLimit(perRequestLimit)
-              bbCtx.jsExecutionTracker
             )
           }
-        })
+          span?.addTags({
+            js: {
+              limitMS: bbCtx.jsExecutionTracker.limitMs,
+              elapsedMS: bbCtx.jsExecutionTracker.elapsedMS,
+            },
+          })
+          // We call checkLimit() here to prevent paying the cost of creating
+          // a new VM context below when we don't need to.
+          tracer.trace("runJS.checkLimitAndBind", {}, span => {
+            bbCtx.jsExecutionTracker!.checkLimit()
+            track = bbCtx.jsExecutionTracker!.track.bind(
+              bbCtx.jsExecutionTracker
+            )
+          })
+        }
       }
 
-      ctx = tracer.trace("runJS.ctxClone", {}, span => {
+      ctx = {
-        return {
+        ...ctx,
-          ...ctx,
+        alert: undefined,
-          alert: undefined,
+        setInterval: undefined,
-          setInterval: undefined,
+        setTimeout: undefined,
-          setTimeout: undefined,
+      }
-        }
-      })
-
-      tracer.trace("runJS.vm.createContext", {}, span => {
-        vm.createContext(ctx)
-      })
 
+      vm.createContext(ctx)
       return track(() =>
-        tracer.trace("runJS.vm.runInNewContext", {}, span =>
+        vm.runInNewContext(js, ctx, {
-          vm.runInNewContext(js, ctx, {
+          timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
-            timeout: env.JS_PER_EXECUTION_TIME_LIMIT_MS,
+        })
-          })
-        )
       )
     })
   })

packages/server/src/sdk/app/datasources/datasources.ts

@@ -1,4 +1,4 @@
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, events } from "@budibase/backend-core"
 import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
@@ -14,16 +14,22 @@ import {
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
-import { getDefinitions, getDefinition } from "../../../integrations"
+import {
+  getDefinitions,
+  getDefinition,
+  getIntegration,
+} from "../../../integrations"
 import merge from "lodash/merge"
 import {
   BudibaseInternalDB,
+  generateDatasourceID,
   getDatasourceParams,
   getDatasourcePlusParams,
   getTableParams,
+  DocumentType,
 } from "../../../db/utils"
 import sdk from "../../index"
-import datasource from "../../../api/routes/datasource"
+import { setupCreationAuth as googleSetupCreationAuth } from "../../../integrations/googlesheets"
 
 const ENV_VAR_PREFIX = "env."
 
@@ -273,3 +279,75 @@ export async function getExternalDatasources(): Promise<Datasource[]> {
 
   return externalDatasources.rows.map(r => r.doc!)
 }
+
+export async function save(
+  datasource: Datasource,
+  opts?: { fetchSchema?: boolean; tablesFilter?: string[] }
+): Promise<{ datasource: Datasource; errors: Record<string, string> }> {
+  const db = context.getAppDB()
+  const plus = datasource.plus
+
+  const fetchSchema = opts?.fetchSchema || false
+  const tablesFilter = opts?.tablesFilter || []
+
+  datasource = {
+    _id: generateDatasourceID({ plus }),
+    ...datasource,
+    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
+  }
+
+  let errors: Record<string, string> = {}
+  if (fetchSchema) {
+    const schema = await sdk.datasources.buildFilteredSchema(
+      datasource,
+      tablesFilter
+    )
+    datasource.entities = schema.tables
+    setDefaultDisplayColumns(datasource)
+    errors = schema.errors
+  }
+
+  if (preSaveAction[datasource.source]) {
+    await preSaveAction[datasource.source](datasource)
+  }
+
+  const dbResp = await db.put(
+    sdk.tables.populateExternalTableSchemas(datasource)
+  )
+  await events.datasource.created(datasource)
+  datasource._rev = dbResp.rev
+
+  // Drain connection pools when configuration is changed
+  if (datasource.source) {
+    const source = await getIntegration(datasource.source)
+    if (source && source.pool) {
+      await source.pool.end()
+    }
+  }
+
+  return { datasource, errors }
+}
+
+const preSaveAction: Partial<Record<SourceName, any>> = {
+  [SourceName.GOOGLE_SHEETS]: async (datasource: Datasource) => {
+    await googleSetupCreationAuth(datasource.config as any)
+  },
+}
+
+/**
+ * Make sure all datasource entities have a display name selected
+ */
+export function setDefaultDisplayColumns(datasource: Datasource) {
+  //
+  for (let entity of Object.values(datasource.entities || {})) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+}

packages/server/src/sdk/app/datasources/plus.ts

@@ -5,7 +5,9 @@ import {
   Schema,
 } from "@budibase/types"
 import * as datasources from "./datasources"
+import tableSdk from "../tables"
 import { getIntegration } from "../../../integrations"
+import { context } from "@budibase/backend-core"
 
 export async function buildFilteredSchema(
   datasource: Datasource,
@@ -60,3 +62,24 @@ export async function getAndMergeDatasource(datasource: Datasource) {
   }
   return await datasources.enrich(datasource)
 }
+
+export async function buildSchemaFromSource(
+  datasourceId: string,
+  tablesFilter?: string[]
+) {
+  const db = context.getAppDB()
+
+  const datasource = await datasources.get(datasourceId)
+
+  const { tables, errors } = await buildFilteredSchema(datasource, tablesFilter)
+  datasource.entities = tables
+
+  datasources.setDefaultDisplayColumns(datasource)
+  const dbResp = await db.put(tableSdk.populateExternalTableSchemas(datasource))
+  datasource._rev = dbResp.rev
+
+  return {
+    datasource,
+    errors,
+  }
+}

packages/shared-core/src/filters.ts

@@ -143,100 +143,104 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => {
     oneOf: {},
     containsAny: {},
   }
-  if (Array.isArray(filter)) {
+
-    filter.forEach(expression => {
+  if (!Array.isArray(filter)) {
-      let { operator, field, type, value, externalType, onEmptyFilter } =
+    return query
-        expression
+  }
-      const isHbs =
+
-        typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+  filter.forEach(expression => {
-      // Parse all values into correct types
+    let { operator, field, type, value, externalType, onEmptyFilter } =
-      if (operator === "allOr") {
+      expression
-        query.allOr = true
+    const isHbs =
+      typeof value === "string" && (value.match(HBS_REGEX) || []).length > 0
+    // Parse all values into correct types
+    if (operator === "allOr") {
+      query.allOr = true
+      return
+    }
+    if (onEmptyFilter) {
+      query.onEmptyFilter = onEmptyFilter
+      return
+    }
+    if (
+      type === "datetime" &&
+      !isHbs &&
+      operator !== "empty" &&
+      operator !== "notEmpty"
+    ) {
+      // Ensure date value is a valid date and parse into correct format
+      if (!value) {
         return
       }
-      if (onEmptyFilter) {
+      try {
-        query.onEmptyFilter = onEmptyFilter
+        value = new Date(value).toISOString()
+      } catch (error) {
         return
       }
-      if (
+    }
-        type === "datetime" &&
+    if (type === "number" && typeof value === "string" && !isHbs) {
-        !isHbs &&
+      if (operator === "oneOf") {
-        operator !== "empty" &&
+        value = value.split(",").map(item => parseFloat(item))
-        operator !== "notEmpty"
+      } else {
+        value = parseFloat(value)
+      }
+    }
+    if (type === "boolean") {
+      value = `${value}`?.toLowerCase() === "true"
+    }
+    if (
+      ["contains", "notContains", "containsAny"].includes(operator) &&
+      type === "array" &&
+      typeof value === "string"
+    ) {
+      value = value.split(",")
+    }
+    if (operator.startsWith("range") && query.range) {
+      const minint =
+        SqlNumberTypeRangeMap[
+          externalType as keyof typeof SqlNumberTypeRangeMap
+        ]?.min || Number.MIN_SAFE_INTEGER
+      const maxint =
+        SqlNumberTypeRangeMap[
+          externalType as keyof typeof SqlNumberTypeRangeMap
+        ]?.max || Number.MAX_SAFE_INTEGER
+      if (!query.range[field]) {
+        query.range[field] = {
+          low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
+          high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
+        }
+      }
+      if ((operator as any) === "rangeLow" && value != null && value !== "") {
+        query.range[field].low = value
+      } else if (
+        (operator as any) === "rangeHigh" &&
+        value != null &&
+        value !== ""
       ) {
-        // Ensure date value is a valid date and parse into correct format
+        query.range[field].high = value
-        if (!value) {
-          return
-        }
-        try {
-          value = new Date(value).toISOString()
-        } catch (error) {
-          return
-        }
-      }
-      if (type === "number" && typeof value === "string") {
-        if (operator === "oneOf") {
-          value = value.split(",").map(item => parseFloat(item))
-        } else if (!isHbs) {
-          value = parseFloat(value)
-        }
       }
+    } else if (query[operator] && operator !== "onEmptyFilter") {
       if (type === "boolean") {
-        value = `${value}`?.toLowerCase() === "true"
+        // Transform boolean filters to cope with null.
-      }
+        // "equals false" needs to be "not equals true"
-      if (
+        // "not equals false" needs to be "equals true"
-        ["contains", "notContains", "containsAny"].includes(operator) &&
+        if (operator === "equal" && value === false) {
-        type === "array" &&
+          query.notEqual = query.notEqual || {}
-        typeof value === "string"
+          query.notEqual[field] = true
-      ) {
+        } else if (operator === "notEqual" && value === false) {
-        value = value.split(",")
+          query.equal = query.equal || {}
-      }
+          query.equal[field] = true
-      if (operator.startsWith("range") && query.range) {
-        const minint =
-          SqlNumberTypeRangeMap[
-            externalType as keyof typeof SqlNumberTypeRangeMap
-          ]?.min || Number.MIN_SAFE_INTEGER
-        const maxint =
-          SqlNumberTypeRangeMap[
-            externalType as keyof typeof SqlNumberTypeRangeMap
-          ]?.max || Number.MAX_SAFE_INTEGER
-        if (!query.range[field]) {
-          query.range[field] = {
-            low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
-            high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z",
-          }
-        }
-        if ((operator as any) === "rangeLow" && value != null && value !== "") {
-          query.range[field].low = value
-        } else if (
-          (operator as any) === "rangeHigh" &&
-          value != null &&
-          value !== ""
-        ) {
-          query.range[field].high = value
-        }
-      } else if (query[operator] && operator !== "onEmptyFilter") {
-        if (type === "boolean") {
-          // Transform boolean filters to cope with null.
-          // "equals false" needs to be "not equals true"
-          // "not equals false" needs to be "equals true"
-          if (operator === "equal" && value === false) {
-            query.notEqual = query.notEqual || {}
-            query.notEqual[field] = true
-          } else if (operator === "notEqual" && value === false) {
-            query.equal = query.equal || {}
-            query.equal[field] = true
-          } else {
-            query[operator] = query[operator] || {}
-            query[operator]![field] = value
-          }
         } else {
           query[operator] = query[operator] || {}
           query[operator]![field] = value
         }
+      } else {
+        query[operator] = query[operator] || {}
+        query[operator]![field] = value
       }
-    })
+    }
-  }
+  })
+
   return query
 }
 

packages/shared-core/src/tests/filters.test.ts

@@ -1,6 +1,11 @@
-import { SearchQuery, SearchQueryOperators } from "@budibase/types"
+import {
-import { runLuceneQuery } from "../filters"
+  SearchQuery,
-import { expect, describe, it } from "vitest"
+  SearchQueryOperators,
+  FieldType,
+  SearchFilter,
+} from "@budibase/types"
+import { buildLuceneQuery, runLuceneQuery } from "../filters"
+import { expect, describe, it, test } from "vitest"
 
 describe("runLuceneQuery", () => {
   const docs = [
@@ -167,4 +172,186 @@ describe("runLuceneQuery", () => {
     })
     expect(runLuceneQuery(docs, query).map(row => row.order_id)).toEqual([2, 3])
   })
+
+  test.each([[523, 259], "523,259"])(
+    "should return rows with matches on numeric oneOf filter",
+    input => {
+      let query = buildQuery("oneOf", {
+        customer_id: input,
+      })
+      expect(runLuceneQuery(docs, query).map(row => row.customer_id)).toEqual([
+        259, 523,
+      ])
+    }
+  )
+})
+
+describe("buildLuceneQuery", () => {
+  it("should return a basic search query template if the input is not an array", () => {
+    const filter: any = "NOT_AN_ARRAY"
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {},
+      containsAny: {},
+    })
+  })
+
+  it("should parseFloat if the type is a number, but the value is a numeric string", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "1212",
+      },
+      {
+        operator: SearchQueryOperators.ONE_OF,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "1000,1212,3400",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {
+        customer_id: 1212,
+      },
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {
+        customer_id: [1000, 1212, 3400],
+      },
+      containsAny: {},
+    })
+  })
+
+  it("should not parseFloat if the type is a number, but the value is a handlebars binding string", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "{{ customer_id }}",
+      },
+      {
+        operator: SearchQueryOperators.ONE_OF,
+        field: "customer_id",
+        type: FieldType.NUMBER,
+        value: "{{ list_of_customer_ids }}",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {
+        customer_id: "{{ customer_id }}",
+      },
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {
+        customer_id: "{{ list_of_customer_ids }}",
+      },
+      containsAny: {},
+    })
+  })
+
+  it("should cast string to boolean if the type is boolean", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "a",
+        type: FieldType.BOOLEAN,
+        value: "not_true",
+      },
+      {
+        operator: SearchQueryOperators.NOT_EQUAL,
+        field: "b",
+        type: FieldType.BOOLEAN,
+        value: "not_true",
+      },
+      {
+        operator: SearchQueryOperators.EQUAL,
+        field: "c",
+        type: FieldType.BOOLEAN,
+        value: "true",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {
+        b: true,
+        c: true,
+      },
+      notEqual: {
+        a: true,
+      },
+      empty: {},
+      notEmpty: {},
+      contains: {},
+      notContains: {},
+      oneOf: {},
+      containsAny: {},
+    })
+  })
+
+  it("should split the string for contains operators", () => {
+    const filter: SearchFilter[] = [
+      {
+        operator: SearchQueryOperators.CONTAINS,
+        field: "description",
+        type: FieldType.ARRAY,
+        value: "Large box,Heavy box,Small box",
+      },
+      {
+        operator: SearchQueryOperators.NOT_CONTAINS,
+        field: "description",
+        type: FieldType.ARRAY,
+        value: "Large box,Heavy box,Small box",
+      },
+      {
+        operator: SearchQueryOperators.CONTAINS_ANY,
+        field: "description",
+        type: FieldType.ARRAY,
+        value: "Large box,Heavy box,Small box",
+      },
+    ]
+    expect(buildLuceneQuery(filter)).toEqual({
+      string: {},
+      fuzzy: {},
+      range: {},
+      equal: {},
+      notEqual: {},
+      empty: {},
+      notEmpty: {},
+      contains: {
+        description: ["Large box", "Heavy box", "Small box"],
+      },
+      notContains: {
+        description: ["Large box", "Heavy box", "Small box"],
+      },
+      oneOf: {},
+      containsAny: {
+        description: ["Large box", "Heavy box", "Small box"],
+      },
+    })
+  })
 })

packages/types/src/api/web/app/datasource.ts

@@ -35,3 +35,12 @@ export interface FetchDatasourceInfoResponse {
 export interface UpdateDatasourceRequest extends Datasource {
   datasource: Datasource
 }
+
+export interface BuildSchemaFromSourceRequest {
+  tablesFilter?: string[]
+}
+
+export interface BuildSchemaFromSourceResponse {
+  datasource: Datasource
+  errors: Record<string, string>
+}

qa-core/src/account-api/tests/licensing/license.manage.spec.ts

@@ -39,10 +39,10 @@ describe("license management", () => {
     let premiumPriceId = null
     let businessPriceId = ""
     for (const plan of planBody) {
-      if (plan.type === PlanType.PREMIUM) {
+      if (plan.type === PlanType.PREMIUM_PLUS) {
         premiumPriceId = plan.prices[0].priceId
       }
-      if (plan.type === PlanType.BUSINESS) {
+      if (plan.type === PlanType.ENTERPRISE_BASIC) {
         businessPriceId = plan.prices[0].priceId
       }
     }
@@ -97,7 +97,7 @@ describe("license management", () => {
       await config.loginAsAccount(createAccountRequest)
       await config.api.stripe.linkStripeCustomer(account.accountId, customer.id)
       const [_, selfBodyPremium] = await config.api.accounts.self()
-      expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM)
+      expect(selfBodyPremium.license.plan.type).toBe(PlanType.PREMIUM_PLUS)
 
       // Create portal session - Check URL
       const [portalRes, portalSessionBody] =
@@ -109,7 +109,7 @@ describe("license management", () => {
 
       // License updated to Business
       const [selfRes, selfBodyBusiness] = await config.api.accounts.self()
-      expect(selfBodyBusiness.license.plan.type).toBe(PlanType.BUSINESS)
+      expect(selfBodyBusiness.license.plan.type).toBe(PlanType.ENTERPRISE_BASIC)
     })
   })
 })