Merge pull request #11352 from Budibase/BUDI-7189/return_default_row_data

Return default row data

packages/backend-core/src/db/constants.ts

@@ -0,0 +1,10 @@
+export const CONSTANT_INTERNAL_ROW_COLS = [
+  "_id",
+  "_rev",
+  "type",
+  "createdAt",
+  "updatedAt",
+  "tableId",
+] as const
+
+export const CONSTANT_EXTERNAL_ROW_COLS = ["_id", "_rev", "tableId"] as const

packages/backend-core/src/db/couch/index.ts

@@ -2,3 +2,4 @@ export * from "./connections"
 export * from "./DatabaseImpl"
 export * from "./utils"
 export { init, getPouch, getPouchDB, closePouchDB } from "./pouchDB"
+export * from "../constants"

packages/backend-core/tests/core/utilities/jestUtils.ts

@@ -1,3 +1,5 @@
+import { db } from "../../../src"
+
 export function expectFunctionWasCalledTimesWith(
   jestFunction: any,
   times: number,
@@ -7,3 +9,22 @@ export function expectFunctionWasCalledTimesWith(
     jestFunction.mock.calls.filter((call: any) => call[0] === argument).length
   ).toBe(times)
 }
+
+export const expectAnyInternalColsAttributes: {
+  [K in (typeof db.CONSTANT_INTERNAL_ROW_COLS)[number]]: any
+} = {
+  tableId: expect.anything(),
+  type: expect.anything(),
+  _id: expect.anything(),
+  _rev: expect.anything(),
+  createdAt: expect.anything(),
+  updatedAt: expect.anything(),
+}
+
+export const expectAnyExternalColsAttributes: {
+  [K in (typeof db.CONSTANT_EXTERNAL_ROW_COLS)[number]]: any
+} = {
+  tableId: expect.anything(),
+  _id: expect.anything(),
+  _rev: expect.anything(),
+}

packages/server/src/api/routes/tests/row.spec.ts

@@ -5,7 +5,7 @@ tk.freeze(timestamp)
 import { outputProcessing } from "../../../utilities/rowProcessor"
 import * as setup from "./utilities"
 const { basicRow } = setup.structures
-import { context, tenancy } from "@budibase/backend-core"
+import { context, db, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   QuotaUsageType,
@@ -17,7 +17,11 @@ import {
   SortType,
   SortOrder,
 } from "@budibase/types"
-import { generator, structures } from "@budibase/backend-core/tests"
+import {
+  expectAnyInternalColsAttributes,
+  generator,
+  structures,
+} from "@budibase/backend-core/tests"
 
 describe("/rows", () => {
   let request = setup.getRequest()
@@ -969,7 +973,7 @@ describe("/rows", () => {
       }
     )
 
-    it("when schema is defined, no other columns are returned", async () => {
+    it("when schema is defined, defined columns and row attributes are returned", async () => {
       const table = await config.createTable(userTable())
       const rows = []
       for (let i = 0; i < 10; i++) {
@@ -989,7 +993,12 @@ describe("/rows", () => {
 
       expect(response.body.rows).toHaveLength(10)
       expect(response.body.rows).toEqual(
-        expect.arrayContaining(rows.map(r => ({ name: r.name })))
+        expect.arrayContaining(
+          rows.map(r => ({
+            ...expectAnyInternalColsAttributes,
+            name: r.name,
+          }))
+        )
       )
     })
 

packages/server/src/sdk/app/rows/search.ts

@@ -3,7 +3,6 @@ import { isExternalTable } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
 import { Format } from "../../../api/controllers/view/exporters"
-import _ from "lodash"
 
 export interface SearchParams {
   tableId: string
@@ -37,12 +36,7 @@ export async function search(options: SearchParams): Promise<{
   hasNextPage?: boolean
   bookmark?: number | null
 }> {
-  const result = await pickApi(options.tableId).search(options)
-
-  if (options.fields) {
-    result.rows = result.rows.map((r: any) => _.pick(r, options.fields!))
-  }
-  return result
+  return pickApi(options.tableId).search(options)
 }
 
 export interface ExportRowsParams {

packages/server/src/sdk/app/rows/search/external.ts

@@ -14,7 +14,8 @@ import { breakExternalTableId } from "../../../../integrations/utils"
 import { cleanExportRows } from "../utils"
 import { utils } from "@budibase/shared-core"
 import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
-import { HTTPError } from "@budibase/backend-core"
+import { HTTPError, db } from "@budibase/backend-core"
+import pick from "lodash/pick"
 
 export async function search(options: SearchParams) {
   const { tableId } = options
@@ -48,7 +49,7 @@ export async function search(options: SearchParams) {
     }
   }
   try {
-    const rows = (await handleRequest(Operation.READ, tableId, {
+    let rows = (await handleRequest(Operation.READ, tableId, {
       filters: query,
       sort,
       paginate: paginateObj as PaginationJson,
@@ -67,6 +68,12 @@ export async function search(options: SearchParams) {
       })) as Row[]
       hasNextPage = nextRows.length > 0
     }
+
+    if (options.fields) {
+      const fields = [...options.fields, ...db.CONSTANT_EXTERNAL_ROW_COLS]
+      rows = rows.map((r: any) => pick(r, fields))
+    }
+
     // need wrapper object for bookmarks etc when paginating
     return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
   } catch (err: any) {

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,5 +1,6 @@
 import {
   context,
+  db,
   SearchParams as InternalSearchParams,
 } from "@budibase/backend-core"
 import env from "../../../../environment"
@@ -28,6 +29,7 @@ import {
 } from "../../../../api/controllers/view/utils"
 import sdk from "../../../../sdk"
 import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import pick from "lodash/pick"
 
 export async function search(options: SearchParams) {
   const { tableId } = options
@@ -72,6 +74,12 @@ export async function search(options: SearchParams) {
       response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
     table = table || (await sdk.tables.getTable(tableId))
+
+    if (options.fields) {
+      const fields = [...options.fields, ...db.CONSTANT_INTERNAL_ROW_COLS]
+      response.rows = response.rows.map((r: any) => pick(r, fields))
+    }
+
     response.rows = await outputProcessing(table, response.rows)
   }
 

packages/server/src/sdk/app/rows/search/tests/external.spec.ts

@@ -0,0 +1,143 @@
+import { GenericContainer } from "testcontainers"
+
+import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
+import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
+import { SearchParams } from "../../search"
+import { search } from "../external"
+import {
+  expectAnyExternalColsAttributes,
+  generator,
+} from "@budibase/backend-core/tests"
+
+jest.unmock("mysql2/promise")
+
+jest.setTimeout(30000)
+
+describe("external", () => {
+  const config = new TestConfiguration()
+
+  let externalDatasource: Datasource
+
+  const tableData: Table = {
+    name: generator.word(),
+    type: "external",
+    primary: ["id"],
+    schema: {
+      id: {
+        name: "id",
+        type: FieldType.AUTO,
+        autocolumn: true,
+      },
+      name: {
+        name: "name",
+        type: FieldType.STRING,
+      },
+      surname: {
+        name: "surname",
+        type: FieldType.STRING,
+      },
+      age: {
+        name: "age",
+        type: FieldType.NUMBER,
+      },
+      address: {
+        name: "address",
+        type: FieldType.STRING,
+      },
+    },
+  }
+
+  beforeAll(async () => {
+    const container = await new GenericContainer("mysql")
+      .withExposedPorts(3306)
+      .withEnv("MYSQL_ROOT_PASSWORD", "admin")
+      .withEnv("MYSQL_DATABASE", "db")
+      .withEnv("MYSQL_USER", "user")
+      .withEnv("MYSQL_PASSWORD", "password")
+      .start()
+
+    const host = container.getContainerIpAddress()
+    const port = container.getMappedPort(3306)
+
+    await config.init()
+
+    externalDatasource = await config.createDatasource({
+      datasource: {
+        type: "datasource",
+        name: "Test",
+        source: SourceName.MYSQL,
+        plus: true,
+        config: {
+          host,
+          port,
+          user: "user",
+          database: "db",
+          password: "password",
+          rejectUnauthorized: true,
+        },
+      },
+    })
+  })
+
+  describe("search", () => {
+    const rows: Row[] = []
+    beforeAll(async () => {
+      const table = await config.createTable({
+        ...tableData,
+        sourceId: externalDatasource._id,
+      })
+      for (let i = 0; i < 10; i++) {
+        rows.push(
+          await config.createRow({
+            tableId: table._id,
+            name: generator.first(),
+            surname: generator.last(),
+            age: generator.age(),
+            address: generator.address(),
+          })
+        )
+      }
+    })
+
+    it("default search returns all the data", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {},
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(10)
+        expect(result.rows).toEqual(
+          expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
+        )
+      })
+    })
+
+    it("querying by fields will always return data attribute columns", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {},
+          fields: ["name", "age"],
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(10)
+        expect(result.rows).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              ...expectAnyExternalColsAttributes,
+              name: r.name,
+              age: r.age,
+            }))
+          )
+        )
+      })
+    })
+  })
+})

packages/server/src/sdk/app/rows/search/tests/internal.spec.ts

@@ -0,0 +1,109 @@
+import { FieldType, Row, Table } from "@budibase/types"
+import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
+import { SearchParams } from "../../search"
+import { search } from "../internal"
+import {
+  expectAnyInternalColsAttributes,
+  generator,
+} from "@budibase/backend-core/tests"
+
+describe("internal", () => {
+  const config = new TestConfiguration()
+
+  const tableData: Table = {
+    name: generator.word(),
+    type: "table",
+    schema: {
+      name: {
+        name: "name",
+        type: FieldType.STRING,
+        constraints: {
+          type: FieldType.STRING,
+        },
+      },
+      surname: {
+        name: "surname",
+        type: FieldType.STRING,
+        constraints: {
+          type: FieldType.STRING,
+        },
+      },
+      age: {
+        name: "age",
+        type: FieldType.NUMBER,
+        constraints: {
+          type: FieldType.NUMBER,
+        },
+      },
+      address: {
+        name: "address",
+        type: FieldType.STRING,
+        constraints: {
+          type: FieldType.STRING,
+        },
+      },
+    },
+  }
+
+  beforeAll(async () => {
+    await config.init()
+  })
+
+  describe("search", () => {
+    const rows: Row[] = []
+    beforeAll(async () => {
+      await config.createTable(tableData)
+      for (let i = 0; i < 10; i++) {
+        rows.push(
+          await config.createRow({
+            name: generator.first(),
+            surname: generator.last(),
+            age: generator.age(),
+            address: generator.address(),
+          })
+        )
+      }
+    })
+
+    it("default search returns all the data", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {},
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(10)
+        expect(result.rows).toEqual(
+          expect.arrayContaining(rows.map(r => expect.objectContaining(r)))
+        )
+      })
+    })
+
+    it("querying by fields will always return data attribute columns", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tableId = config.table!._id!
+
+        const searchParams: SearchParams = {
+          tableId,
+          query: {},
+          fields: ["name", "age"],
+        }
+        const result = await search(searchParams)
+
+        expect(result.rows).toHaveLength(10)
+        expect(result.rows).toEqual(
+          expect.arrayContaining(
+            rows.map(r => ({
+              ...expectAnyInternalColsAttributes,
+              name: r.name,
+              age: r.age,
+            }))
+          )
+        )
+      })
+    })
+  })
+})