Merge branch 'master' into fix/app-title-alignment

.github/workflows/budibase_ci.yml

@@ -108,7 +108,7 @@ jobs:
       - name: Pull testcontainers images
         run: |
           docker pull testcontainers/ryuk:0.5.1 &
-          docker pull budibase/couchdb:v3.2.1-sql &
+          docker pull budibase/couchdb:v3.2.1-sqs &
           docker pull redis &
 
           wait $(jobs -p)

lerna.json

@@ -1,6 +1,6 @@
 {
   "$schema": "node_modules/lerna/schemas/lerna-schema.json",
-  "version": "2.29.15",
+  "version": "2.29.20",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/db/couch/connections.ts

@@ -25,7 +25,10 @@ export const getCouchInfo = (connection?: string) => {
   }
   const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
   let sqlUrl = env.COUCH_DB_SQL_URL
-  if (!sqlUrl && urlInfo.url) {
+  // default for dev
+  if (env.isDev() && !sqlUrl) {
+    sqlUrl = "http://localhost:4006"
+  } else if (!sqlUrl && urlInfo.url) {
     const parsed = new URL(urlInfo.url)
     // attempt to connect on default port
     sqlUrl = urlInfo.url.replace(parsed.port, "4984")

packages/backend-core/src/db/utils.ts

@@ -1,6 +1,6 @@
 import env from "../environment"
 import { DEFAULT_TENANT_ID, SEPARATOR, DocumentType } from "../constants"
-import { getTenantId, getGlobalDBName } from "../context"
+import { getTenantId, getGlobalDBName, isMultiTenant } from "../context"
 import { doWithDB, directCouchAllDbs } from "./db"
 import { AppState, DeletedApp, getAppMetadata } from "../cache/appMetadata"
 import { isDevApp, isDevAppID, getProdAppID } from "../docIds/conversions"
@@ -206,3 +206,34 @@ export function pagination<T>(
     nextPage,
   }
 }
+
+export function isSqsEnabledForTenant(): boolean {
+  const tenantId = getTenantId()
+  if (!env.SQS_SEARCH_ENABLE) {
+    return false
+  }
+
+  // single tenant (self host and dev) always enabled if flag set
+  if (!isMultiTenant()) {
+    return true
+  }
+
+  // This is to guard against the situation in tests where tests pass because
+  // we're not actually using SQS, we're using Lucene and the tests pass due to
+  // parity.
+  if (env.isTest() && env.SQS_SEARCH_ENABLE_TENANTS.length === 0) {
+    throw new Error(
+      "to enable SQS you must specify a list of tenants in the SQS_SEARCH_ENABLE_TENANTS env var"
+    )
+  }
+
+  // Special case to enable all tenants, for testing in QA.
+  if (
+    env.SQS_SEARCH_ENABLE_TENANTS.length === 1 &&
+    env.SQS_SEARCH_ENABLE_TENANTS[0] === "*"
+  ) {
+    return true
+  }
+
+  return env.SQS_SEARCH_ENABLE_TENANTS.includes(tenantId)
+}

packages/backend-core/src/environment.ts

@@ -114,8 +114,11 @@ const environment = {
   ENCRYPTION_KEY: process.env.ENCRYPTION_KEY,
   API_ENCRYPTION_KEY: getAPIEncryptionKey(),
   COUCH_DB_URL: process.env.COUCH_DB_URL || "http://localhost:4005",
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || "http://localhost:4006",
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
   SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
+  SQS_SEARCH_ENABLE_TENANTS:
+    process.env.SQS_SEARCH_ENABLE_TENANTS?.split(",") || [],
+  SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE,
   COUCH_DB_USERNAME: process.env.COUCH_DB_USER,
   COUCH_DB_PASSWORD: process.env.COUCH_DB_PASSWORD,
   GOOGLE_CLIENT_ID: process.env.GOOGLE_CLIENT_ID,

packages/backend-core/src/redis/tests/redis.spec.ts

@@ -2,6 +2,7 @@ import { GenericContainer, StartedTestContainer } from "testcontainers"
 import { generator, structures } from "../../../tests"
 import RedisWrapper from "../redis"
 import { env } from "../.."
+import { randomUUID } from "crypto"
 
 jest.setTimeout(30000)
 
@@ -52,10 +53,10 @@ describe("redis", () => {
   describe("bulkStore", () => {
     function createRandomObject(
       keyLength: number,
-      valueGenerator: () => any = () => generator.word()
+      valueGenerator: () => any = () => randomUUID()
     ) {
       return generator
-        .unique(() => generator.word(), keyLength)
+        .unique(() => randomUUID(), keyLength)
         .reduce((acc, key) => {
           acc[key] = valueGenerator()
           return acc

packages/backend-core/src/users/users.ts

@@ -18,9 +18,10 @@ import {
   CouchFindOptions,
   DatabaseQueryOpts,
   SearchFilters,
-  SearchFilterOperator,
   SearchUsersRequest,
   User,
+  BasicOperator,
+  ArrayOperator,
 } from "@budibase/types"
 import * as context from "../context"
 import { getGlobalDB } from "../context"
@@ -46,9 +47,9 @@ function removeUserPassword(users: User | User[]) {
 
 export function isSupportedUserSearch(query: SearchFilters) {
   const allowed = [
-    { op: SearchFilterOperator.STRING, key: "email" },
-    { op: SearchFilterOperator.EQUAL, key: "_id" },
-    { op: SearchFilterOperator.ONE_OF, key: "_id" },
+    { op: BasicOperator.STRING, key: "email" },
+    { op: BasicOperator.EQUAL, key: "_id" },
+    { op: ArrayOperator.ONE_OF, key: "_id" },
   ]
   for (let [key, operation] of Object.entries(query)) {
     if (typeof operation !== "object") {

packages/builder/src/components/automation/AutomationPanel/AutomationPanel.svelte

@@ -30,6 +30,16 @@
       return lowerA > lowerB ? 1 : -1
     })
 
+  $: groupedAutomations = filteredAutomations.reduce((acc, auto) => {
+    acc[auto.definition.trigger.event] ??= {
+      icon: auto.definition.trigger.icon,
+      name: (auto.definition.trigger?.name || "").toUpperCase(),
+      entries: [],
+    }
+    acc[auto.definition.trigger.event].entries.push(auto)
+    return acc
+  }, {})
+
   $: showNoResults = searchString && !filteredAutomations.length
 
   onMount(async () => {
@@ -55,16 +65,25 @@
     />
   </div>
   <div class="side-bar-nav">
-    {#each filteredAutomations as automation}
-      <NavItem
-        text={automation.name}
-        selected={automation._id === selectedAutomationId}
-        on:click={() => selectAutomation(automation._id)}
-        selectedBy={$userSelectedResourceMap[automation._id]}
-        disabled={automation.disabled}
-      >
-        <EditAutomationPopover {automation} />
-      </NavItem>
+    {#each Object.values(groupedAutomations || {}) as triggerGroup}
+      <div class="nav-group">
+        <div class="nav-group-header" title={triggerGroup?.name}>
+          {triggerGroup?.name}
+        </div>
+        {#each triggerGroup.entries as automation}
+          <NavItem
+            icon={triggerGroup.icon}
+            iconColor={"var(--spectrum-global-color-gray-900)"}
+            text={automation.name}
+            selected={automation._id === selectedAutomationId}
+            on:click={() => selectAutomation(automation._id)}
+            selectedBy={$userSelectedResourceMap[automation._id]}
+            disabled={automation.disabled}
+          >
+            <EditAutomationPopover {automation} />
+          </NavItem>
+        {/each}
+      </div>
     {/each}
 
     {#if showNoResults}
@@ -82,6 +101,17 @@
 </Modal>
 
 <style>
+  .nav-group {
+    padding-top: var(--spacing-l);
+  }
+  .nav-group-header {
+    color: var(--spectrum-global-color-gray-600);
+    padding: 0px calc(var(--spacing-l) + 4px);
+    padding-bottom: var(--spacing-l);
+    white-space: nowrap;
+    overflow: hidden;
+    text-overflow: ellipsis;
+  }
   .side-bar {
     flex: 0 0 260px;
     display: flex;
@@ -104,7 +134,7 @@
     justify-content: flex-start;
     align-items: center;
     gap: var(--spacing-l);
-    padding: 0 var(--spacing-l);
+    padding: 0 calc(var(--spacing-l) + 4px);
   }
   .side-bar-nav {
     flex: 1 1 auto;

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -250,6 +250,7 @@
                   },
                 }))
               },
+              placeholder: false,
               getOptionLabel: type => type.name,
               getOptionValue: type => type.id,
               options: [
@@ -280,14 +281,14 @@
                   tableId: inputData["row"].tableId,
                 },
                 meta: {
-                  fields: inputData["meta"].oldFields || {},
+                  fields: inputData["meta"]?.oldFields || {},
                 },
                 onChange: e => {
                   onChange({
                     oldRow: e.detail.row,
                     meta: {
-                      fields: inputData["meta"].fields,
-                      oldFields: e.detail.meta.fields,
+                      fields: inputData["meta"]?.fields || {},
+                      oldFields: e.detail?.meta?.fields || {},
                     },
                   })
                 },
@@ -304,7 +305,15 @@
               row: inputData["row"],
               meta: inputData["meta"] || {},
               onChange: e => {
-                onChange(e.detail)
+                onChange({
+                  row: e.detail.row,
+                  meta: {
+                    fields: e.detail?.meta?.fields || {},
+                    ...(isTestModal
+                      ? { oldFields: inputData["meta"]?.oldFields || {} }
+                      : {}),
+                  },
+                })
               },
               ...baseProps,
             },
@@ -452,7 +461,6 @@
    */
   const onChange = Utils.sequential(async update => {
     const request = cloneDeep(update)
-
     // Process app trigger updates
     if (isTrigger && !isTestModal) {
       // Row trigger

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -288,29 +288,27 @@
 {/each}
 
 {#if table && schemaFields}
-  {#key editableFields}
-    <div
-      class="add-fields-btn"
-      class:empty={Object.is(editableFields, {})}
-      bind:this={popoverAnchor}
-    >
-      <ActionButton
-        icon="Add"
-        fullWidth
-        on:click={() => {
-          customPopover.show()
-        }}
-        disabled={!schemaFields}
-        >Add fields
-      </ActionButton>
-    </div>
-  {/key}
+  <div
+    class="add-fields-btn"
+    class:empty={Object.is(editableFields, {})}
+    bind:this={popoverAnchor}
+  >
+    <ActionButton
+      icon="Add"
+      fullWidth
+      on:click={() => {
+        customPopover.show()
+      }}
+      disabled={!schemaFields}
+      >Add fields
+    </ActionButton>
+  </div>
 {/if}
 
 <Popover
   align="center"
   bind:this={customPopover}
-  anchor={popoverAnchor}
+  anchor={editableFields ? popoverAnchor : null}
   useAnchorWidth
   maxHeight={300}
   resizable={false}

packages/frontend-core/src/components/FilterBuilder.svelte

@@ -11,7 +11,7 @@
     Label,
     Multiselect,
   } from "@budibase/bbui"
-  import { FieldType, SearchFilterOperator } from "@budibase/types"
+  import { ArrayOperator, FieldType } from "@budibase/types"
   import { generate } from "shortid"
   import { QueryUtils, Constants } from "@budibase/frontend-core"
   import { getContext } from "svelte"
@@ -268,7 +268,7 @@
                 <slot name="binding" {filter} />
               {:else if [FieldType.STRING, FieldType.LONGFORM, FieldType.NUMBER, FieldType.BIGINT, FieldType.FORMULA].includes(filter.type)}
                 <Input disabled={filter.noValue} bind:value={filter.value} />
-              {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === SearchFilterOperator.ONE_OF)}
+              {:else if filter.type === FieldType.ARRAY || (filter.type === FieldType.OPTIONS && filter.operator === ArrayOperator.ONE_OF)}
                 <Multiselect
                   disabled={filter.noValue}
                   options={getFieldOptions(filter.field)}

packages/server/src/api/controllers/deploy/Deployment.ts

@@ -1,5 +1,4 @@
-import newid from "../../../db/newid"
-import { context } from "@budibase/backend-core"
+import { context, utils } from "@budibase/backend-core"
 
 /**
  * This is used to pass around information about the deployment that is occurring
@@ -12,7 +11,7 @@ export default class Deployment {
   appUrl?: string
 
   constructor(id = null) {
-    this._id = id || newid()
+    this._id = id || utils.newid()
   }
 
   setVerification(verification: any) {

packages/server/src/api/controllers/row/utils/sqlUtils.ts

@@ -22,6 +22,20 @@ export function isManyToMany(
   return !!(field as ManyToManyRelationshipFieldMetadata).through
 }
 
+function isCorrectRelationship(
+  relationship: RelationshipsJson,
+  table1: Table,
+  table2: Table,
+  row: Row
+): boolean {
+  const junctionTableId = generateJunctionTableID(table1._id!, table2._id!)
+  const possibleColumns = [
+    `${junctionTableId}.doc1.fieldName`,
+    `${junctionTableId}.doc2.fieldName`,
+  ]
+  return !!possibleColumns.find(col => row[col] === relationship.column)
+}
+
 /**
  * This iterates through the returned rows and works out what elements of the rows
  * actually match up to another row (based on primary keys) - this is pretty specific
@@ -64,7 +78,12 @@ export async function updateRelationshipColumns(
     if (!linked._id) {
       continue
     }
-    columns[relationship.column] = linked
+    if (
+      !opts?.sqs ||
+      isCorrectRelationship(relationship, table, linkedTable, row)
+    ) {
+      columns[relationship.column] = linked
+    }
   }
   for (let [column, related] of Object.entries(columns)) {
     if (!row._id) {

packages/server/src/api/controllers/table/utils.ts

@@ -15,7 +15,7 @@ import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
-import { events, context } from "@budibase/backend-core"
+import { events, context, db as dbCore } from "@budibase/backend-core"
 import {
   AutoFieldSubType,
   ContextUser,
@@ -324,7 +324,7 @@ class TableSaveFunctions {
       importRows: this.importRows,
       user: this.user,
     })
-    if (env.SQS_SEARCH_ENABLE) {
+    if (dbCore.isSqsEnabledForTenant()) {
       await sdk.tables.sqs.addTable(table)
     }
     return table
@@ -518,7 +518,7 @@ export async function internalTableCleanup(table: Table, rows?: Row[]) {
   if (rows) {
     await AttachmentCleanup.tableDelete(table, rows)
   }
-  if (env.SQS_SEARCH_ENABLE) {
+  if (dbCore.isSqsEnabledForTenant()) {
     await sdk.tables.sqs.removeTable(table)
   }
 }

packages/server/src/api/routes/tests/permissions.spec.ts

@@ -203,7 +203,7 @@ describe("/permission", () => {
       // replicate changes before checking permissions
       await config.publish()
 
-      await config.api.viewV2.publicSearch(view.id, undefined, { status: 403 })
+      await config.api.viewV2.publicSearch(view.id, undefined, { status: 401 })
     })
 
     it("should ignore the view permissions if the flag is not on", async () => {
@@ -221,7 +221,7 @@ describe("/permission", () => {
       await config.publish()
 
       await config.api.viewV2.publicSearch(view.id, undefined, {
-        status: 403,
+        status: 401,
       })
     })
 
@@ -250,8 +250,8 @@ describe("/permission", () => {
         .send(basicRow(table._id))
         .set(config.publicHeaders())
         .expect("Content-Type", /json/)
-        .expect(403)
-      expect(res.status).toEqual(403)
+        .expect(401)
+      expect(res.status).toEqual(401)
     })
   })
 

packages/server/src/api/routes/tests/row.spec.ts

@@ -819,7 +819,10 @@ describe.each([
         const table = await config.api.table.save(tableRequest)
 
         const stringValue = generator.word()
-        const naturalValue = generator.integer({ min: 0, max: 1000 })
+
+        // MySQL and MariaDB auto-increment fields have a minimum value of 1. If
+        // you try to save a row with a value of 0 it will use 1 instead.
+        const naturalValue = generator.integer({ min: 1, max: 1000 })
 
         const existing = await config.api.row.save(table._id!, {
           string: stringValue,
@@ -1457,17 +1460,12 @@ describe.each([
         delete tableRequest.schema.id
 
         const table = await config.api.table.save(tableRequest)
+        const toCreate = generator
+          .unique(() => generator.integer({ min: 0, max: 10000 }), 10)
+          .map(number => ({ number, string: generator.word({ length: 30 }) }))
 
         const rows = await Promise.all(
-          generator
-            .unique(
-              () => ({
-                string: generator.word({ length: 30 }),
-                number: generator.integer({ min: 0, max: 10000 }),
-              }),
-              10
-            )
-            .map(d => config.api.row.save(table._id!, d))
+          toCreate.map(d => config.api.row.save(table._id!, d))
         )
 
         const res = await config.api.row.exportRows(table._id!, {

packages/server/src/api/routes/tests/search.spec.ts

@@ -54,10 +54,13 @@ describe.each([
   let rows: Row[]
 
   beforeAll(async () => {
+    await config.withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init())
     if (isSqs) {
-      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+      envCleanup = config.setCoreEnv({
+        SQS_SEARCH_ENABLE: "true",
+        SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
+      })
     }
-    await config.init()
 
     if (config.app?.appId) {
       config.app = await config.api.application.update(config.app?.appId, {
@@ -780,6 +783,46 @@ describe.each([
       it("fails to find nonexistent row", async () => {
         await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()
       })
+
+      it("can have multiple values for same column", async () => {
+        await expectQuery({
+          oneOf: {
+            name: ["foo", "bar"],
+          },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+      })
+
+      it("splits comma separated strings", async () => {
+        await expectQuery({
+          oneOf: {
+            // @ts-ignore
+            name: "foo,bar",
+          },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+      })
+
+      it("trims whitespace", async () => {
+        await expectQuery({
+          oneOf: {
+            // @ts-ignore
+            name: "foo, bar",
+          },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+      })
+
+      it("empty arrays returns all when onEmptyFilter is set to return 'all'", async () => {
+        await expectQuery({
+          onEmptyFilter: EmptyFilterOption.RETURN_ALL,
+          oneOf: { name: [] },
+        }).toContainExactly([{ name: "foo" }, { name: "bar" }])
+      })
+
+      it("empty arrays returns all when onEmptyFilter is set to return 'none'", async () => {
+        await expectQuery({
+          onEmptyFilter: EmptyFilterOption.RETURN_NONE,
+          oneOf: { name: [] },
+        }).toContainExactly([])
+      })
     })
 
     describe("fuzzy", () => {
@@ -1002,6 +1045,32 @@ describe.each([
       it("fails to find nonexistent row", async () => {
         await expectQuery({ oneOf: { age: [2] } }).toFindNothing()
       })
+
+      // I couldn't find a way to make this work in Lucene and given that
+      // we're getting rid of Lucene soon I wasn't inclined to spend time on
+      // it.
+      !isLucene &&
+        it("can convert from a string", async () => {
+          await expectQuery({
+            oneOf: {
+              // @ts-ignore
+              age: "1",
+            },
+          }).toContainExactly([{ age: 1 }])
+        })
+
+      // I couldn't find a way to make this work in Lucene and given that
+      // we're getting rid of Lucene soon I wasn't inclined to spend time on
+      // it.
+      !isLucene &&
+        it("can find multiple values for same column", async () => {
+          await expectQuery({
+            oneOf: {
+              // @ts-ignore
+              age: "1,10",
+            },
+          }).toContainExactly([{ age: 1 }, { age: 10 }])
+        })
     })
 
     describe("range", () => {
@@ -2085,6 +2154,106 @@ describe.each([
       })
     })
 
+  isInternal &&
+    describe("relations to same table", () => {
+      let relatedTable: Table, relatedRows: Row[]
+
+      beforeAll(async () => {
+        relatedTable = await createTable(
+          {
+            name: { name: "name", type: FieldType.STRING },
+          },
+          "productCategory"
+        )
+        table = await createTable({
+          name: { name: "name", type: FieldType.STRING },
+          related1: {
+            type: FieldType.LINK,
+            name: "related1",
+            fieldName: "main1",
+            tableId: relatedTable._id!,
+            relationshipType: RelationshipType.MANY_TO_MANY,
+          },
+          related2: {
+            type: FieldType.LINK,
+            name: "related2",
+            fieldName: "main2",
+            tableId: relatedTable._id!,
+            relationshipType: RelationshipType.MANY_TO_MANY,
+          },
+        })
+        relatedRows = await Promise.all([
+          config.api.row.save(relatedTable._id!, { name: "foo" }),
+          config.api.row.save(relatedTable._id!, { name: "bar" }),
+          config.api.row.save(relatedTable._id!, { name: "baz" }),
+          config.api.row.save(relatedTable._id!, { name: "boo" }),
+        ])
+        await Promise.all([
+          config.api.row.save(table._id!, {
+            name: "test",
+            related1: [relatedRows[0]._id!],
+            related2: [relatedRows[1]._id!],
+          }),
+          config.api.row.save(table._id!, {
+            name: "test2",
+            related1: [relatedRows[2]._id!],
+            related2: [relatedRows[3]._id!],
+          }),
+        ])
+      })
+
+      it("should be able to relate to same table", async () => {
+        await expectSearch({
+          query: {},
+        }).toContainExactly([
+          {
+            name: "test",
+            related1: [{ _id: relatedRows[0]._id }],
+            related2: [{ _id: relatedRows[1]._id }],
+          },
+          {
+            name: "test2",
+            related1: [{ _id: relatedRows[2]._id }],
+            related2: [{ _id: relatedRows[3]._id }],
+          },
+        ])
+      })
+
+      isSqs &&
+        it("should be able to filter down to second row with equal", async () => {
+          await expectSearch({
+            query: {
+              equal: {
+                ["related1.name"]: "baz",
+              },
+            },
+          }).toContainExactly([
+            {
+              name: "test2",
+              related1: [{ _id: relatedRows[2]._id }],
+            },
+          ])
+        })
+
+      isSqs &&
+        it("should be able to filter down to first row with not equal", async () => {
+          await expectSearch({
+            query: {
+              notEqual: {
+                ["1:related2.name"]: "bar",
+                ["2:related2.name"]: "baz",
+                ["3:related2.name"]: "boo",
+              },
+            },
+          }).toContainExactly([
+            {
+              name: "test",
+              related1: [{ _id: relatedRows[0]._id }],
+            },
+          ])
+        })
+    })
+
   isInternal &&
     describe("no column error backwards compat", () => {
       beforeAll(async () => {

packages/server/src/api/routes/tests/templates.spec.ts

@@ -86,9 +86,10 @@ describe("/templates", () => {
       async source => {
         const env = {
           SQS_SEARCH_ENABLE: source === "sqs" ? "true" : "false",
+          SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
         }
 
-        await config.withEnv(env, async () => {
+        await config.withCoreEnv(env, async () => {
           const name = generator.guid().replaceAll("-", "")
           const url = `/${name}`
 

packages/server/src/api/routes/tests/utilities/TestFunctions.ts

@@ -151,7 +151,7 @@ export const checkPermissionsEndpoint = async ({
   await exports
     .createRequest(config.request, method, url, body)
     .set(failHeader)
-    .expect(403)
+    .expect(401)
 }
 
 export const getDB = () => {

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -9,7 +9,6 @@ import {
   QuotaUsageType,
   Row,
   SaveTableRequest,
-  SearchFilterOperator,
   SortOrder,
   SortType,
   StaticQuotaName,
@@ -19,6 +18,7 @@ import {
   ViewUIFieldMetadata,
   ViewV2,
   SearchResponse,
+  BasicOperator,
 } from "@budibase/types"
 import { generator, mocks } from "@budibase/backend-core/tests"
 import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
@@ -88,10 +88,16 @@ describe.each([
   }
 
   beforeAll(async () => {
+    await config.withCoreEnv(
+      { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" },
+      () => config.init()
+    )
     if (isSqs) {
-      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+      envCleanup = config.setCoreEnv({
+        SQS_SEARCH_ENABLE: "true",
+        SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
+      })
     }
-    await config.init()
 
     if (dsProvider) {
       datasource = await config.createDatasource({
@@ -149,7 +155,7 @@ describe.each([
         primaryDisplay: "id",
         query: [
           {
-            operator: SearchFilterOperator.EQUAL,
+            operator: BasicOperator.EQUAL,
             field: "field",
             value: "value",
           },
@@ -561,7 +567,7 @@ describe.each([
         ...view,
         query: [
           {
-            operator: SearchFilterOperator.EQUAL,
+            operator: BasicOperator.EQUAL,
             field: "newField",
             value: "thatValue",
           },
@@ -589,7 +595,7 @@ describe.each([
         primaryDisplay: "Price",
         query: [
           {
-            operator: SearchFilterOperator.EQUAL,
+            operator: BasicOperator.EQUAL,
             field: generator.word(),
             value: generator.word(),
           },
@@ -673,7 +679,7 @@ describe.each([
           tableId: generator.guid(),
           query: [
             {
-              operator: SearchFilterOperator.EQUAL,
+              operator: BasicOperator.EQUAL,
               field: "newField",
               value: "thatValue",
             },
@@ -1194,7 +1200,7 @@ describe.each([
           name: generator.guid(),
           query: [
             {
-              operator: SearchFilterOperator.EQUAL,
+              operator: BasicOperator.EQUAL,
               field: "two",
               value: "bar2",
             },
@@ -1484,7 +1490,7 @@ describe.each([
       it("does not allow public users to fetch by default", async () => {
         await config.publish()
         await config.api.viewV2.publicSearch(view.id, undefined, {
-          status: 403,
+          status: 401,
         })
       })
 
@@ -1528,7 +1534,7 @@ describe.each([
         await config.publish()
 
         await config.api.viewV2.publicSearch(view.id, undefined, {
-          status: 403,
+          status: 401,
         })
       })
     })

packages/server/src/appMigrations/migrations.ts

@@ -1,6 +1,6 @@
 // This file should never be manually modified, use `yarn add-app-migration` in order to add a new one
 
-import env from "../environment"
+import { env } from "@budibase/backend-core"
 import { AppMigration } from "."
 
 import m20240604153647_initial_sqs from "./migrations/20240604153647_initial_sqs"

packages/server/src/appMigrations/migrations/20240604153647_initial_sqs.ts

@@ -1,8 +1,7 @@
-import { context } from "@budibase/backend-core"
+import { context, env } from "@budibase/backend-core"
 import { allLinkDocs } from "../../db/utils"
 import LinkDocumentImpl from "../../db/linkedRows/LinkDocument"
 import sdk from "../../sdk"
-import env from "../../environment"
 
 const migration = async () => {
   const linkDocs = await allLinkDocs()

packages/server/src/appMigrations/migrations/tests/20240604153647_initial_sqs.spec.ts

@@ -69,11 +69,14 @@ function oldLinkDocument(): Omit<LinkDocument, "tableId"> {
 type SQSEnvVar = "SQS_MIGRATION_ENABLE" | "SQS_SEARCH_ENABLE"
 
 async function sqsDisabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
-  await config.withEnv({ [envVar]: "" }, cb)
+  await config.withCoreEnv({ [envVar]: "", SQS_SEARCH_ENABLE_TENANTS: [] }, cb)
 }
 
 async function sqsEnabled(envVar: SQSEnvVar, cb: () => Promise<void>) {
-  await config.withEnv({ [envVar]: "1" }, cb)
+  await config.withCoreEnv(
+    { [envVar]: "1", SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()] },
+    cb
+  )
 }
 
 describe.each(["SQS_MIGRATION_ENABLE", "SQS_SEARCH_ENABLE"] as SQSEnvVar[])(

packages/server/src/automations/utils.ts

@@ -1,10 +1,9 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
 import { automationQueue } from "./bullboard"
-import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
 import { MetadataTypes } from "../constants"
-import { db as dbCore, context } from "@budibase/backend-core"
+import { db as dbCore, context, utils } from "@budibase/backend-core"
 import { getAutomationMetadataParams } from "../db/utils"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
@@ -207,7 +206,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
       )
     }
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
-    const jobId = `${appId}_cron_${newid()}`
+    const jobId = `${appId}_cron_${utils.newid()}`
     const job: any = await automationQueue.add(
       {
         automation,

packages/server/src/constants/index.ts

@@ -24,16 +24,6 @@ export enum FilterTypes {
   ONE_OF = "oneOf",
 }
 
-export const NoEmptyFilterStrings = [
-  FilterTypes.STRING,
-  FilterTypes.FUZZY,
-  FilterTypes.EQUAL,
-  FilterTypes.NOT_EQUAL,
-  FilterTypes.CONTAINS,
-  FilterTypes.NOT_CONTAINS,
-  FilterTypes.CONTAINS_ANY,
-]
-
 export const CanSwitchTypes = [
   [FieldType.JSON, FieldType.ARRAY],
   [

packages/server/src/db/inMemoryView.ts

@@ -1,9 +1,8 @@
-import newid from "./newid"
 import { Row, Document, DBView } from "@budibase/types"
 
 // bypass the main application db config
 // use in memory pouchdb directly
-import { db as dbCore } from "@budibase/backend-core"
+import { db as dbCore, utils } from "@budibase/backend-core"
 
 const Pouch = dbCore.getPouch({ inMemory: true })
 
@@ -16,7 +15,7 @@ export async function runView(
   // use a different ID each time for the DB, make sure they
   // are always unique for each query, don't want overlap
   // which could cause 409s
-  const db = new Pouch(newid())
+  const db = new Pouch(utils.newid())
   try {
     // write all the docs to the in memory Pouch (remove revs)
     await db.bulkDocs(

packages/server/src/db/newid.ts

@@ -1,5 +0,0 @@
-import { v4 } from "uuid"
-
-export default function (): string {
-  return v4().replace(/-/g, "")
-}

packages/server/src/db/utils.ts

@@ -1,5 +1,4 @@
-import newid from "./newid"
-import { context, db as dbCore } from "@budibase/backend-core"
+import { context, db as dbCore, utils } from "@budibase/backend-core"
 import {
   DatabaseQueryOpts,
   Datasource,
@@ -15,6 +14,8 @@ import {
 
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
 
+const newid = utils.newid
+
 type Optional = string | null
 
 export const enum AppStatus {

packages/server/src/environment.ts

@@ -28,7 +28,6 @@ const DEFAULTS = {
   PLUGINS_DIR: "/plugins",
   FORKED_PROCESS_NAME: "main",
   JS_RUNNER_MEMORY_LIMIT: 64,
-  COUCH_DB_SQL_URL: "http://localhost:4006",
 }
 
 const QUERY_THREAD_TIMEOUT =
@@ -44,7 +43,7 @@ const environment = {
   // important - prefer app port to generic port
   PORT: process.env.APP_PORT || process.env.PORT,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
-  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL || DEFAULTS.COUCH_DB_SQL_URL,
+  COUCH_DB_SQL_URL: process.env.COUCH_DB_SQL_URL,
   MINIO_URL: process.env.MINIO_URL,
   WORKER_URL: process.env.WORKER_URL,
   AWS_REGION: process.env.AWS_REGION,
@@ -87,8 +86,6 @@ const environment = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
   SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE,
-  SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
-  SQS_MIGRATION_ENABLE: process.env.SQS_MIGRATION_ENABLE,
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,

packages/server/src/middleware/authorized.ts

@@ -96,7 +96,7 @@ const authorized =
     }
 
     if (!ctx.user) {
-      return ctx.throw(403, "No user info found")
+      return ctx.throw(401, "No user info found")
     }
 
     // get the resource roles
@@ -148,7 +148,7 @@ const authorized =
 
     // check authenticated
     if (!ctx.isAuthenticated) {
-      return ctx.throw(403, "Session not authenticated")
+      return ctx.throw(401, "Session not authenticated")
     }
 
     // check general builder stuff, this middleware is a good way

packages/server/src/middleware/tests/authorized.spec.ts

@@ -105,7 +105,7 @@ describe("Authorization middleware", () => {
     it("throws when no user data is present in context", async () => {
       await config.executeMiddleware()
 
-      expect(config.throw).toHaveBeenCalledWith(403, "No user info found")
+      expect(config.throw).toHaveBeenCalledWith(401, "No user info found")
     })
 
     it("passes on to next() middleware if user is an admin", async () => {
@@ -157,7 +157,7 @@ describe("Authorization middleware", () => {
 
       await config.executeMiddleware()
       expect(config.throw).toHaveBeenCalledWith(
-        403,
+        401,
         "Session not authenticated"
       )
     })

packages/server/src/sdk/app/rows/search.ts

@@ -2,21 +2,18 @@ import {
   EmptyFilterOption,
   Row,
   RowSearchParams,
-  SearchFilterOperator,
-  SearchFilters,
   SearchResponse,
   SortOrder,
 } from "@budibase/types"
 import { isExternalTableID } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
-import { NoEmptyFilterStrings } from "../../../constants"
 import * as sqs from "./search/sqs"
-import env from "../../../environment"
 import { ExportRowsParams, ExportRowsResult } from "./search/types"
 import { dataFilters } from "@budibase/shared-core"
 import sdk from "../../index"
 import { searchInputMapping } from "./search/utils"
+import { db as dbCore } from "@budibase/backend-core"
 
 export { isValidFilter } from "../../../integrations/utils"
 
@@ -33,70 +30,12 @@ function pickApi(tableId: any) {
   return internal
 }
 
-function isEmptyArray(value: any) {
-  return Array.isArray(value) && value.length === 0
-}
-
-// don't do a pure falsy check, as 0 is included
-// https://github.com/Budibase/budibase/issues/10118
-export function removeEmptyFilters(filters: SearchFilters) {
-  for (let filterField of NoEmptyFilterStrings) {
-    if (!filters[filterField]) {
-      continue
-    }
-
-    for (let filterType of Object.keys(filters)) {
-      if (filterType !== filterField) {
-        continue
-      }
-      // don't know which one we're checking, type could be anything
-      const value = filters[filterType] as unknown
-      if (typeof value === "object") {
-        for (let [key, value] of Object.entries(
-          filters[filterType] as object
-        )) {
-          if (value == null || value === "" || isEmptyArray(value)) {
-            // @ts-ignore
-            delete filters[filterField][key]
-          }
-        }
-      }
-    }
-  }
-  return filters
-}
-
-// The frontend can send single values for array fields sometimes, so to handle
-// this we convert them to arrays at the controller level so that nothing below
-// this has to worry about the non-array values.
-function fixupFilterArrays(filters: SearchFilters) {
-  const arrayFields = [
-    SearchFilterOperator.ONE_OF,
-    SearchFilterOperator.CONTAINS,
-    SearchFilterOperator.NOT_CONTAINS,
-    SearchFilterOperator.CONTAINS_ANY,
-  ]
-  for (const searchField of arrayFields) {
-    const field = filters[searchField]
-    if (field == null) {
-      continue
-    }
-
-    for (const key of Object.keys(field)) {
-      if (!Array.isArray(field[key])) {
-        field[key] = [field[key]]
-      }
-    }
-  }
-  return filters
-}
-
 export async function search(
   options: RowSearchParams
 ): Promise<SearchResponse<Row>> {
   const isExternalTable = isExternalTableID(options.tableId)
-  options.query = removeEmptyFilters(options.query || {})
-  options.query = fixupFilterArrays(options.query)
+  options.query = dataFilters.cleanupQuery(options.query || {})
+  options.query = dataFilters.fixupFilterArrays(options.query)
   if (
     !dataFilters.hasFilters(options.query) &&
     options.query.onEmptyFilter === EmptyFilterOption.RETURN_NONE
@@ -115,7 +54,7 @@ export async function search(
 
   if (isExternalTable) {
     return external.search(options, table)
-  } else if (env.SQS_SEARCH_ENABLE) {
+  } else if (dbCore.isSqsEnabledForTenant()) {
     return sqs.search(options, table)
   } else {
     return internal.search(options, table)

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -5,6 +5,7 @@ import {
   Operation,
   QueryJson,
   RelationshipFieldMetadata,
+  RelationshipsJson,
   Row,
   RowSearchParams,
   SearchFilters,
@@ -30,7 +31,10 @@ import {
   SQLITE_DESIGN_DOC_ID,
   SQS_DATASOURCE_INTERNAL,
 } from "@budibase/backend-core"
-import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
+import {
+  CONSTANT_INTERNAL_ROW_COLS,
+  generateJunctionTableID,
+} from "../../../../db/utils"
 import AliasTables from "../sqlAlias"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
 import pick from "lodash/pick"
@@ -41,39 +45,43 @@ import {
   getTableIDList,
 } from "./filters"
 import { dataFilters } from "@budibase/shared-core"
-import { DEFAULT_TABLE_IDS } from "../../../../constants"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
-const USER_COLUMN_PREFIX_REGEX = new RegExp(
-  `no such column: .+${USER_COLUMN_PREFIX}`
-)
+const MISSING_TABLE_REGX = new RegExp(`no such table: .+`)
 
 function buildInternalFieldList(
   table: Table,
   tables: Table[],
-  opts: { relationships: boolean } = { relationships: true }
+  opts?: { relationships?: RelationshipsJson[] }
 ) {
   let fieldList: string[] = []
+  const addJunctionFields = (relatedTable: Table, fields: string[]) => {
+    fields.forEach(field => {
+      fieldList.push(
+        `${generateJunctionTableID(table._id!, relatedTable._id!)}.${field}`
+      )
+    })
+  }
   fieldList = fieldList.concat(
     CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
   )
   for (let col of Object.values(table.schema)) {
     const isRelationship = col.type === FieldType.LINK
-    if (!opts.relationships && isRelationship) {
+    if (!opts?.relationships && isRelationship) {
       continue
     }
     if (isRelationship) {
       const linkCol = col as RelationshipFieldMetadata
       const relatedTable = tables.find(table => table._id === linkCol.tableId)!
-      fieldList = fieldList.concat(
-        buildInternalFieldList(relatedTable, tables, { relationships: false })
-      )
+      // no relationships provided, don't go more than a layer deep
+      fieldList = fieldList.concat(buildInternalFieldList(relatedTable, tables))
+      addJunctionFields(relatedTable, ["doc1.fieldName", "doc2.fieldName"])
     } else {
       fieldList.push(`${table._id}.${mapToUserColumn(col.name)}`)
     }
   }
-  return fieldList
+  return [...new Set(fieldList)]
 }
 
 function cleanupFilters(
@@ -165,18 +173,27 @@ function reverseUserColumnMapping(rows: Row[]) {
   })
 }
 
-function runSqlQuery(json: QueryJson, tables: Table[]): Promise<Row[]>
 function runSqlQuery(
   json: QueryJson,
   tables: Table[],
+  relationships: RelationshipsJson[]
+): Promise<Row[]>
+function runSqlQuery(
+  json: QueryJson,
+  tables: Table[],
+  relationships: RelationshipsJson[],
   opts: { countTotalRows: true }
 ): Promise<number>
 async function runSqlQuery(
   json: QueryJson,
   tables: Table[],
+  relationships: RelationshipsJson[],
   opts?: { countTotalRows?: boolean }
 ) {
-  const alias = new AliasTables(tables.map(table => table.name))
+  const relationshipJunctionTableIds = relationships.map(rel => rel.through!)
+  const alias = new AliasTables(
+    tables.map(table => table.name).concat(relationshipJunctionTableIds)
+  )
   if (opts?.countTotalRows) {
     json.endpoint.operation = Operation.COUNT
   }
@@ -193,8 +210,13 @@ async function runSqlQuery(
     let bindings = query.bindings
 
     // quick hack for docIds
-    sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
-    sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
+
+    const fixJunctionDocs = (field: string) =>
+      ["doc1", "doc2"].forEach(doc => {
+        sql = sql.replaceAll(`\`${doc}\`.\`${field}\``, `\`${doc}.${field}\``)
+      })
+    fixJunctionDocs("rowId")
+    fixJunctionDocs("fieldName")
 
     if (Array.isArray(query)) {
       throw new Error("SQS cannot currently handle multiple queries")
@@ -215,10 +237,10 @@ async function runSqlQuery(
 function resyncDefinitionsRequired(status: number, message: string) {
   // pre data_ prefix on column names, need to resync
   return (
-    (status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) ||
-    // default tables aren't included in definition
-    (status === 400 &&
-      DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) ||
+    // there are tables missing - try a resync
+    (status === 400 && message.match(MISSING_TABLE_REGX)) ||
+    // there are columns missing - try a resync
+    (status === 400 && message.match(MISSING_COLUMN_REGEX)) ||
     // no design document found, needs a full sync
     (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID))
   )
@@ -226,7 +248,8 @@ function resyncDefinitionsRequired(status: number, message: string) {
 
 export async function search(
   options: RowSearchParams,
-  table: Table
+  table: Table,
+  opts?: { retrying?: boolean }
 ): Promise<SearchResponse<Row>> {
   let { paginate, query, ...params } = options
 
@@ -260,7 +283,7 @@ export async function search(
       columnPrefix: USER_COLUMN_PREFIX,
     },
     resource: {
-      fields: buildInternalFieldList(table, allTables),
+      fields: buildInternalFieldList(table, allTables, { relationships }),
     },
     relationships,
   }
@@ -292,11 +315,11 @@ export async function search(
 
   try {
     const queries: Promise<Row[] | number>[] = []
-    queries.push(runSqlQuery(request, allTables))
+    queries.push(runSqlQuery(request, allTables, relationships))
     if (options.countRows) {
       // get the total count of rows
       queries.push(
-        runSqlQuery(request, allTables, {
+        runSqlQuery(request, allTables, relationships, {
           countTotalRows: true,
         })
       )
@@ -351,9 +374,9 @@ export async function search(
     return response
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
-    if (resyncDefinitionsRequired(err.status, msg)) {
+    if (!opts?.retrying && resyncDefinitionsRequired(err.status, msg)) {
       await sdk.tables.sqs.syncDefinition()
-      return search(options, table)
+      return search(options, table, { retrying: true })
     }
     // previously the internal table didn't error when a column didn't exist in search
     if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) {

packages/server/src/sdk/app/rows/search/tests/search.spec.ts

@@ -31,10 +31,17 @@ describe.each([
   let rows: Row[]
 
   beforeAll(async () => {
+    await config.withCoreEnv(
+      { SQS_SEARCH_ENABLE: isSqs ? "true" : "false" },
+      () => config.init()
+    )
+
     if (isSqs) {
-      envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
+      envCleanup = config.setCoreEnv({
+        SQS_SEARCH_ENABLE: "true",
+        SQS_SEARCH_ENABLE_TENANTS: [config.getTenantId()],
+      })
     }
-    await config.init()
 
     if (dsProvider) {
       datasource = await config.createDatasource({

packages/server/src/sdk/app/rows/sqlAlias.ts

@@ -111,7 +111,8 @@ export default class AliasTables {
   aliasField(field: string) {
     const tableNames = this.tableNames
     if (field.includes(".")) {
-      const [tableName, column] = field.split(".")
+      const [tableName, ...rest] = field.split(".")
+      const column = rest.join(".")
       const foundTableName = tableNames.find(name => {
         const idx = tableName.indexOf(name)
         if (idx === -1 || idx > 1) {

packages/server/src/sdk/app/tables/getters.ts

@@ -1,4 +1,4 @@
-import { context } from "@budibase/backend-core"
+import { context, db as dbCore, env } from "@budibase/backend-core"
 import { getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
@@ -15,7 +15,6 @@ import {
 } from "@budibase/types"
 import datasources from "../datasources"
 import sdk from "../../../sdk"
-import env from "../../../environment"
 
 export function processTable(table: Table): Table {
   if (!table) {
@@ -34,7 +33,7 @@ export function processTable(table: Table): Table {
       sourceId: table.sourceId || INTERNAL_TABLE_SOURCE_ID,
       sourceType: TableSourceType.INTERNAL,
     }
-    if (env.SQS_SEARCH_ENABLE) {
+    if (dbCore.isSqsEnabledForTenant()) {
       processed.sql = !!env.SQS_SEARCH_ENABLE
     }
     return processed

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -127,9 +127,14 @@ function mapTable(table: Table): SQLiteTables {
 // nothing exists, need to iterate though existing tables
 async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
-  const defaultTables = DEFAULT_TABLES
+  for (const defaultTable of DEFAULT_TABLES) {
+    // the default table doesn't exist in Couch, use the in-memory representation
+    if (!tables.find(table => table._id === defaultTable._id)) {
+      tables.push(defaultTable)
+    }
+  }
   const definition = sql.designDoc.base("tableId")
-  for (let table of tables.concat(defaultTables)) {
+  for (let table of tables) {
     definition.sql.tables = {
       ...definition.sql.tables,
       ...mapTable(table),
@@ -176,9 +181,22 @@ export async function addTable(table: Table) {
 export async function removeTable(table: Table) {
   const db = context.getAppDB()
   try {
-    const definition = await db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
-    if (definition.sql?.tables?.[table._id!]) {
-      delete definition.sql.tables[table._id!]
+    const [tables, definition] = await Promise.all([
+      tablesSdk.getAllInternalTables(),
+      db.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID),
+    ])
+    const tableIds = tables
+      .map(tbl => tbl._id!)
+      .filter(id => !id.includes(table._id!))
+    let cleanup = false
+    for (let tableKey of Object.keys(definition.sql?.tables || {})) {
+      // there are no tables matching anymore
+      if (!tableIds.find(id => tableKey.includes(id))) {
+        delete definition.sql.tables[tableKey]
+        cleanup = true
+      }
+    }
+    if (cleanup) {
       await db.put(definition)
       // make sure SQS is cleaned up, tables removed
       await db.sqlDiskCleanup()

packages/server/src/sdk/tests/attachments.spec.ts

@@ -39,7 +39,9 @@ describe("should be able to re-write attachment URLs", () => {
     }
 
     const db = dbCore.getDB(config.getAppId())
-    await sdk.backups.updateAttachmentColumns(db.name, db)
+    await config.doInContext(config.getAppId(), () =>
+      sdk.backups.updateAttachmentColumns(db.name, db)
+    )
 
     return {
       db,

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -26,6 +26,7 @@ import {
   roles,
   sessions,
   tenancy,
+  utils,
 } from "@budibase/backend-core"
 import {
   app as appController,
@@ -40,7 +41,6 @@ import {
 } from "./controllers"
 
 import { cleanup } from "../../utilities/fileSystem"
-import newid from "../../db/newid"
 import { generateUserMetadataID } from "../../db/utils"
 import { startup } from "../../startup"
 import supertest from "supertest"
@@ -74,6 +74,8 @@ import { cloneDeep } from "lodash"
 import jwt, { Secret } from "jsonwebtoken"
 import { Server } from "http"
 
+const newid = utils.newid
+
 mocks.licenses.init(pro)
 
 // use unlimited license by default
@@ -245,10 +247,10 @@ export default class TestConfiguration {
     }
   }
 
-  async withEnv(newEnvVars: Partial<typeof env>, f: () => Promise<void>) {
+  async withEnv<T>(newEnvVars: Partial<typeof env>, f: () => Promise<T>) {
     let cleanup = this.setEnv(newEnvVars)
     try {
-      await f()
+      return await f()
     } finally {
       cleanup()
     }
@@ -273,13 +275,13 @@ export default class TestConfiguration {
     }
   }
 
-  async withCoreEnv(
+  async withCoreEnv<T>(
     newEnvVars: Partial<typeof coreEnv>,
-    f: () => Promise<void>
+    f: () => Promise<T>
   ) {
     let cleanup = this.setCoreEnv(newEnvVars)
     try {
-      await f()
+      return await f()
     } finally {
       cleanup()
     }

packages/shared-core/src/filters.ts

@@ -6,6 +6,7 @@ import {
   SearchFilter,
   SearchFilters,
   SearchQueryFields,
+  ArrayOperator,
   SearchFilterOperator,
   SortType,
   FieldConstraints,
@@ -14,11 +15,13 @@ import {
   EmptyFilterOption,
   SearchResponse,
   Table,
+  BasicOperator,
+  RangeOperator,
 } from "@budibase/types"
 import dayjs from "dayjs"
 import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
 import { deepGet, schema } from "./helpers"
-import _ from "lodash"
+import { isPlainObject, isEmpty } from "lodash"
 
 const HBS_REGEX = /{{([^{].*?)}}/g
 
@@ -103,31 +106,49 @@ export const NoEmptyFilterStrings = [
   OperatorOptions.NotEquals.value,
   OperatorOptions.Contains.value,
   OperatorOptions.NotContains.value,
+  OperatorOptions.ContainsAny.value,
+  OperatorOptions.In.value,
 ] as (keyof SearchQueryFields)[]
 
 /**
  * Removes any fields that contain empty strings that would cause inconsistent
  * behaviour with how backend tables are filtered (no value means no filter).
+ *
+ * don't do a pure falsy check, as 0 is included
+ * https://github.com/Budibase/budibase/issues/10118
  */
-const cleanupQuery = (query: SearchFilters) => {
+export const cleanupQuery = (query: SearchFilters) => {
   if (!query) {
     return query
   }
   for (let filterField of NoEmptyFilterStrings) {
-    const operator = filterField as SearchFilterOperator
-    if (!query[operator]) {
+    if (!query[filterField]) {
       continue
     }
 
-    for (let [key, value] of Object.entries(query[operator]!)) {
-      if (value == null || value === "") {
-        delete query[operator]![key]
+    for (let filterType of Object.keys(query)) {
+      if (filterType !== filterField) {
+        continue
+      }
+      // don't know which one we're checking, type could be anything
+      const value = query[filterType] as unknown
+      if (typeof value === "object") {
+        for (let [key, value] of Object.entries(query[filterType] as object)) {
+          if (value == null || value === "" || isEmptyArray(value)) {
+            // @ts-ignore
+            delete query[filterField][key]
+          }
+        }
       }
     }
   }
   return query
 }
 
+function isEmptyArray(value: any) {
+  return Array.isArray(value) && value.length === 0
+}
+
 /**
  * Removes a numeric prefix on field names designed to give fields uniqueness
  */
@@ -323,6 +344,32 @@ export const buildQuery = (filter: SearchFilter[]) => {
   return query
 }
 
+// The frontend can send single values for array fields sometimes, so to handle
+// this we convert them to arrays at the controller level so that nothing below
+// this has to worry about the non-array values.
+export function fixupFilterArrays(filters: SearchFilters) {
+  for (const searchField of Object.values(ArrayOperator)) {
+    const field = filters[searchField]
+    if (field == null || !isPlainObject(field)) {
+      continue
+    }
+
+    for (const key of Object.keys(field)) {
+      if (Array.isArray(field[key])) {
+        continue
+      }
+
+      const value = field[key] as any
+      if (typeof value === "string") {
+        field[key] = value.split(",").map((x: string) => x.trim())
+      } else {
+        field[key] = [value]
+      }
+    }
+  }
+  return filters
+}
+
 export const search = (
   docs: Record<string, any>[],
   query: RowSearchParams
@@ -356,6 +403,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
   }
 
   query = cleanupQuery(query)
+  query = fixupFilterArrays(query)
 
   if (
     !hasFilters(query) &&
@@ -382,7 +430,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
     }
 
   const stringMatch = match(
-    SearchFilterOperator.STRING,
+    BasicOperator.STRING,
     (docValue: any, testValue: any) => {
       if (!(typeof docValue === "string")) {
         return false
@@ -395,7 +443,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
   )
 
   const fuzzyMatch = match(
-    SearchFilterOperator.FUZZY,
+    BasicOperator.FUZZY,
     (docValue: any, testValue: any) => {
       if (!(typeof docValue === "string")) {
         return false
@@ -408,17 +456,17 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
   )
 
   const rangeMatch = match(
-    SearchFilterOperator.RANGE,
+    RangeOperator.RANGE,
     (docValue: any, testValue: any) => {
       if (docValue == null || docValue === "") {
         return false
       }
 
-      if (_.isObject(testValue.low) && _.isEmpty(testValue.low)) {
+      if (isPlainObject(testValue.low) && isEmpty(testValue.low)) {
         testValue.low = undefined
       }
 
-      if (_.isObject(testValue.high) && _.isEmpty(testValue.high)) {
+      if (isPlainObject(testValue.high) && isEmpty(testValue.high)) {
         testValue.high = undefined
       }
 
@@ -497,11 +545,8 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
     (...args: T): boolean =>
       !f(...args)
 
-  const equalMatch = match(SearchFilterOperator.EQUAL, _valueMatches)
-  const notEqualMatch = match(
-    SearchFilterOperator.NOT_EQUAL,
-    not(_valueMatches)
-  )
+  const equalMatch = match(BasicOperator.EQUAL, _valueMatches)
+  const notEqualMatch = match(BasicOperator.NOT_EQUAL, not(_valueMatches))
 
   const _empty = (docValue: any) => {
     if (typeof docValue === "string") {
@@ -516,26 +561,24 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
     return docValue == null
   }
 
-  const emptyMatch = match(SearchFilterOperator.EMPTY, _empty)
-  const notEmptyMatch = match(SearchFilterOperator.NOT_EMPTY, not(_empty))
+  const emptyMatch = match(BasicOperator.EMPTY, _empty)
+  const notEmptyMatch = match(BasicOperator.NOT_EMPTY, not(_empty))
 
-  const oneOf = match(
-    SearchFilterOperator.ONE_OF,
-    (docValue: any, testValue: any) => {
-      if (typeof testValue === "string") {
-        testValue = testValue.split(",")
-        if (typeof docValue === "number") {
-          testValue = testValue.map((item: string) => parseFloat(item))
-        }
-      }
-
-      if (!Array.isArray(testValue)) {
-        return false
-      }
-
-      return testValue.some(item => _valueMatches(docValue, item))
+  const oneOf = match(ArrayOperator.ONE_OF, (docValue: any, testValue: any) => {
+    if (typeof testValue === "string") {
+      testValue = testValue.split(",")
     }
-  )
+
+    if (typeof docValue === "number") {
+      testValue = testValue.map((item: string) => parseFloat(item))
+    }
+
+    if (!Array.isArray(testValue)) {
+      return false
+    }
+
+    return testValue.some(item => _valueMatches(docValue, item))
+  })
 
   const _contains =
     (f: "some" | "every") => (docValue: any, testValue: any) => {
@@ -562,7 +605,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
     }
 
   const contains = match(
-    SearchFilterOperator.CONTAINS,
+    ArrayOperator.CONTAINS,
     (docValue: any, testValue: any) => {
       if (Array.isArray(testValue) && testValue.length === 0) {
         return true
@@ -571,7 +614,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
     }
   )
   const notContains = match(
-    SearchFilterOperator.NOT_CONTAINS,
+    ArrayOperator.NOT_CONTAINS,
     (docValue: any, testValue: any) => {
       // Not sure if this is logically correct, but at the time this code was
       // written the search endpoint behaved this way and we wanted to make this
@@ -582,10 +625,7 @@ export const runQuery = (docs: Record<string, any>[], query: SearchFilters) => {
       return not(_contains("every"))(docValue, testValue)
     }
   )
-  const containsAny = match(
-    SearchFilterOperator.CONTAINS_ANY,
-    _contains("some")
-  )
+  const containsAny = match(ArrayOperator.CONTAINS_ANY, _contains("some"))
 
   const docMatch = (doc: Record<string, any>) => {
     const filterFunctions = {

packages/types/src/sdk/search.ts

@@ -3,20 +3,28 @@ import { Row, Table, DocumentType } from "../documents"
 import { SortOrder, SortType } from "../api"
 import { Knex } from "knex"
 
-export enum SearchFilterOperator {
-  STRING = "string",
-  FUZZY = "fuzzy",
-  RANGE = "range",
+export enum BasicOperator {
   EQUAL = "equal",
   NOT_EQUAL = "notEqual",
   EMPTY = "empty",
   NOT_EMPTY = "notEmpty",
-  ONE_OF = "oneOf",
+  FUZZY = "fuzzy",
+  STRING = "string",
+}
+
+export enum ArrayOperator {
   CONTAINS = "contains",
   NOT_CONTAINS = "notContains",
   CONTAINS_ANY = "containsAny",
+  ONE_OF = "oneOf",
 }
 
+export enum RangeOperator {
+  RANGE = "range",
+}
+
+export type SearchFilterOperator = BasicOperator | ArrayOperator | RangeOperator
+
 export enum InternalSearchFilterOperator {
   COMPLEX_ID_OPERATOR = "_complexIdOperator",
 }
@@ -52,17 +60,17 @@ export interface SearchFilters {
   // allows just fuzzy to be or - all the fuzzy/like parameters
   fuzzyOr?: boolean
   onEmptyFilter?: EmptyFilterOption
-  [SearchFilterOperator.STRING]?: BasicFilter<string>
-  [SearchFilterOperator.FUZZY]?: BasicFilter<string>
-  [SearchFilterOperator.RANGE]?: RangeFilter
-  [SearchFilterOperator.EQUAL]?: BasicFilter
-  [SearchFilterOperator.NOT_EQUAL]?: BasicFilter
-  [SearchFilterOperator.EMPTY]?: BasicFilter
-  [SearchFilterOperator.NOT_EMPTY]?: BasicFilter
-  [SearchFilterOperator.ONE_OF]?: ArrayFilter
-  [SearchFilterOperator.CONTAINS]?: ArrayFilter
-  [SearchFilterOperator.NOT_CONTAINS]?: ArrayFilter
-  [SearchFilterOperator.CONTAINS_ANY]?: ArrayFilter
+  [BasicOperator.STRING]?: BasicFilter<string>
+  [BasicOperator.FUZZY]?: BasicFilter<string>
+  [RangeOperator.RANGE]?: RangeFilter
+  [BasicOperator.EQUAL]?: BasicFilter
+  [BasicOperator.NOT_EQUAL]?: BasicFilter
+  [BasicOperator.EMPTY]?: BasicFilter
+  [BasicOperator.NOT_EMPTY]?: BasicFilter
+  [ArrayOperator.ONE_OF]?: ArrayFilter
+  [ArrayOperator.CONTAINS]?: ArrayFilter
+  [ArrayOperator.NOT_CONTAINS]?: ArrayFilter
+  [ArrayOperator.CONTAINS_ANY]?: ArrayFilter
   // specific to SQS/SQLite search on internal tables this can be used
   // to make sure the documents returned are always filtered down to a
   // specific document type (such as just rows)

packages/worker/src/api/controllers/system/environment.ts

@@ -1,6 +1,6 @@
 import { Ctx, MaintenanceType } from "@budibase/types"
 import env from "../../../environment"
-import { env as coreEnv } from "@budibase/backend-core"
+import { env as coreEnv, db as dbCore } from "@budibase/backend-core"
 import nodeFetch from "node-fetch"
 
 let sqsAvailable: boolean
@@ -12,7 +12,12 @@ async function isSqsAvailable() {
   }
 
   try {
-    await nodeFetch(coreEnv.COUCH_DB_SQL_URL, {
+    const couchInfo = dbCore.getCouchInfo()
+    if (!couchInfo.sqlUrl) {
+      sqsAvailable = false
+      return false
+    }
+    await nodeFetch(couchInfo.sqlUrl, {
       timeout: 1000,
     })
     sqsAvailable = true
@@ -24,7 +29,7 @@ async function isSqsAvailable() {
 }
 
 async function isSqsMissing() {
-  return env.SQS_SEARCH_ENABLE && !(await isSqsAvailable())
+  return coreEnv.SQS_SEARCH_ENABLE && !(await isSqsAvailable())
 }
 
 export const fetch = async (ctx: Ctx) => {

packages/worker/src/api/index.ts

@@ -5,8 +5,7 @@ const compress = require("koa-compress")
 import zlib from "zlib"
 import { routes } from "./routes"
 import { middleware as pro, sdk } from "@budibase/pro"
-import { auth, middleware } from "@budibase/backend-core"
-import env from "../environment"
+import { auth, middleware, env } from "@budibase/backend-core"
 
 if (env.SQS_SEARCH_ENABLE) {
   sdk.auditLogs.useSQLSearch()

packages/worker/src/environment.ts

@@ -46,7 +46,6 @@ const environment = {
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
   DISABLE_DEVELOPER_LICENSE: process.env.DISABLE_DEVELOPER_LICENSE,
-  SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   // smtp
   SMTP_USER: process.env.SMTP_USER,