Merge pull request #14114 from Budibase/fix/default-table-sqs-indexing

Adding default (sample) tables in SQS indexing

packages/server/src/api/routes/tests/search.spec.ts

@@ -30,6 +30,7 @@ import { encodeJSBinding } from "@budibase/string-templates"
 import { dataFilters } from "@budibase/shared-core"
 import { Knex } from "knex"
 import { structures } from "@budibase/backend-core/tests"
+import { DEFAULT_EMPLOYEE_TABLE_SCHEMA } from "../../../db/defaultData/datasource_bb_default"
 
 describe.each([
   ["in-memory", undefined],
@@ -2167,6 +2168,24 @@ describe.each([
     }
   )
 
+  isInternal &&
+    describe("sample data", () => {
+      beforeAll(async () => {
+        await config.api.application.addSampleData(config.appId!)
+        table = DEFAULT_EMPLOYEE_TABLE_SCHEMA
+      })
+
+      it("should be able to search sample data", async () => {
+        await expectSearch({
+          query: {},
+        }).toContain([
+          {
+            "First Name": "Mandy",
+          },
+        ])
+      })
+    })
+
   describe.each([
     { low: "2024-07-03T00:00:00.000Z", high: "9999-00-00T00:00:00.000Z" },
     { low: "2024-07-03T00:00:00.000Z", high: "9998-00-00T00:00:00.000Z" },

packages/server/src/constants/index.ts

@@ -173,3 +173,9 @@ export const DEFAULT_INVENTORY_TABLE_ID = constants.DEFAULT_INVENTORY_TABLE_ID
 export const DEFAULT_EXPENSES_TABLE_ID = constants.DEFAULT_EXPENSES_TABLE_ID
 export const DEFAULT_EMPLOYEE_TABLE_ID = constants.DEFAULT_EMPLOYEE_TABLE_ID
 export const DEFAULT_BB_DATASOURCE_ID = constants.DEFAULT_BB_DATASOURCE_ID
+export const DEFAULT_TABLE_IDS = [
+  DEFAULT_JOBS_TABLE_ID,
+  DEFAULT_INVENTORY_TABLE_ID,
+  DEFAULT_EXPENSES_TABLE_ID,
+  DEFAULT_EMPLOYEE_TABLE_ID,
+]

packages/server/src/db/defaultData/datasource_bb_default.ts

@@ -619,6 +619,13 @@ export const DEFAULT_EXPENSES_TABLE_SCHEMA: Table = {
   },
 }
 
+export const DEFAULT_TABLES: Table[] = [
+  DEFAULT_INVENTORY_TABLE_SCHEMA,
+  DEFAULT_EMPLOYEE_TABLE_SCHEMA,
+  DEFAULT_JOBS_TABLE_SCHEMA,
+  DEFAULT_EXPENSES_TABLE_SCHEMA,
+]
+
 export async function buildDefaultDocs() {
   const inventoryData = await tableImport(
     DEFAULT_INVENTORY_TABLE_SCHEMA,

packages/server/src/sdk/app/rows/search/sqs.ts

@@ -41,6 +41,7 @@ import {
   getTableIDList,
 } from "./filters"
 import { dataFilters } from "@budibase/shared-core"
+import { DEFAULT_TABLE_IDS } from "../../../../constants"
 
 const builder = new sql.Sql(SqlClient.SQL_LITE)
 const MISSING_COLUMN_REGEX = new RegExp(`no such column: .+`)
@@ -211,6 +212,18 @@ async function runSqlQuery(
   return response
 }
 
+function resyncDefinitionsRequired(status: number, message: string) {
+  // pre data_ prefix on column names, need to resync
+  return (
+    (status === 400 && message?.match(USER_COLUMN_PREFIX_REGEX)) ||
+    // default tables aren't included in definition
+    (status === 400 &&
+      DEFAULT_TABLE_IDS.find(tableId => message?.includes(tableId))) ||
+    // no design document found, needs a full sync
+    (status === 404 && message?.includes(SQLITE_DESIGN_DOC_ID))
+  )
+}
+
 export async function search(
   options: RowSearchParams,
   table: Table
@@ -338,10 +351,7 @@ export async function search(
     return response
   } catch (err: any) {
     const msg = typeof err === "string" ? err : err.message
-    const syncAndRepeat =
-      (err.status === 400 && msg?.match(USER_COLUMN_PREFIX_REGEX)) ||
-      (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID))
-    if (syncAndRepeat) {
+    if (resyncDefinitionsRequired(err.status, msg)) {
       await sdk.tables.sqs.syncDefinition()
       return search(options, table)
     }

packages/server/src/sdk/app/tables/internal/sqs.ts

@@ -15,6 +15,7 @@ import {
   generateJunctionTableID,
 } from "../../../../db/utils"
 import { isEqual } from "lodash"
+import { DEFAULT_TABLES } from "../../../../db/defaultData/datasource_bb_default"
 
 const FieldTypeMap: Record<FieldType, SQLiteType> = {
   [FieldType.BOOLEAN]: SQLiteType.NUMERIC,
@@ -126,8 +127,9 @@ function mapTable(table: Table): SQLiteTables {
 // nothing exists, need to iterate though existing tables
 async function buildBaseDefinition(): Promise<PreSaveSQLiteDefinition> {
   const tables = await tablesSdk.getAllInternalTables()
+  const defaultTables = DEFAULT_TABLES
   const definition = sql.designDoc.base("tableId")
-  for (let table of tables) {
+  for (let table of tables.concat(defaultTables)) {
     definition.sql.tables = {
       ...definition.sql.tables,
       ...mapTable(table),

packages/server/src/tests/utilities/api/application.ts

@@ -149,4 +149,8 @@ export class ApplicationAPI extends TestAPI {
       query: { status },
     })
   }
+
+  addSampleData = async (appId: string): Promise<void> => {
+    await this._post(`/api/applications/${appId}/sample`)
+  }
 }