Merge pull request #11250 from Budibase/BUDI-7189/clean_ctx_dependencies_from_sdk

Clean ctx dependencies from rows SDK

packages/server/src/api/controllers/row/external.ts

@@ -49,6 +49,7 @@ export async function handleRequest(
       }
     }
   }
+
   return new ExternalRequest(operation, tableId, opts?.datasource).run(
     opts || {}
   )

packages/server/src/api/controllers/row/index.ts

@@ -6,6 +6,8 @@ import { Ctx } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
 import sdk from "../../../sdk"
+import * as exporters from "../view/exporters"
+import { apiFileReturn } from "../../../utilities/fileSystem"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -65,14 +67,26 @@ export const save = async (ctx: any) => {
 }
 export async function fetchView(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), {
+  const viewName = decodeURIComponent(ctx.params.viewName)
+
+  const { calculation, group, field } = ctx.query
+
+  ctx.body = await quotas.addQuery(
+    () =>
+      sdk.rows.fetchView(tableId, viewName, {
+        calculation,
+        group,
+        field,
+      }),
+    {
       datasourceId: tableId,
-  })
+    }
+  )
 }
 
 export async function fetch(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), {
+  ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId), {
     datasourceId: tableId,
   })
 }
@@ -120,9 +134,14 @@ export async function destroy(ctx: any) {
 
 export async function search(ctx: any) {
   const tableId = utils.getTableId(ctx)
-  ctx.status = 200
 
-  ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), {
+  const searchParams = {
+    ...ctx.request.body,
+    tableId,
+  }
+
+  ctx.status = 200
+  ctx.body = await quotas.addQuery(() => sdk.rows.search(searchParams), {
     datasourceId: tableId,
   })
 }
@@ -152,7 +171,33 @@ export async function fetchEnrichedRow(ctx: any) {
 
 export const exportRows = async (ctx: any) => {
   const tableId = utils.getTableId(ctx)
-  ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), {
-    datasourceId: tableId,
+
+  const format = ctx.query.format
+
+  const { rows, columns, query } = ctx.request.body
+  if (typeof format !== "string" || !exporters.isFormat(format)) {
+    ctx.throw(
+      400,
+      `Format ${format} not valid. Valid values: ${Object.values(
+        exporters.Format
+      )}`
+    )
+  }
+
+  ctx.body = await quotas.addQuery(
+    async () => {
+      const { fileName, content } = await sdk.rows.exportRows({
+        tableId,
+        format,
+        rowIds: rows,
+        columns,
+        query,
       })
+      ctx.attachment(fileName)
+      return apiFileReturn(content)
+    },
+    {
+      datasourceId: tableId,
+    }
+  )
 }

packages/server/src/api/controllers/view/index.ts

@@ -162,7 +162,10 @@ export async function exportView(ctx: Ctx) {
   let rows = ctx.body as Row[]
 
   let schema: TableSchema = view && view.meta && view.meta.schema
-  const tableId = ctx.params.tableId || view.meta.tableId
+  const tableId =
+    ctx.params.tableId ||
+    view?.meta?.tableId ||
+    (viewName.startsWith(DocumentType.TABLE) && viewName)
   const table: Table = await sdk.tables.getTable(tableId)
   if (!schema) {
     schema = table.schema

packages/server/src/sdk/app/rows/search.ts

@@ -1,17 +1,26 @@
-import { Ctx, SearchFilters } from "@budibase/types"
+import { SearchFilters } from "@budibase/types"
 import { isExternalTable } from "../../../integrations/utils"
 import * as internal from "./search/internal"
 import * as external from "./search/external"
+import { Format } from "../../../api/controllers/view/exporters"
 
 export interface SearchParams {
   tableId: string
-  paginate: boolean
-  query?: SearchFilters
-  bookmark?: number
-  limit: number
+  paginate?: boolean
+  query: SearchFilters
+  bookmark?: string
+  limit?: number
   sort?: string
   sortOrder?: string
   sortType?: string
+  version?: string
+  disableEscaping?: boolean
+}
+
+export interface ViewParams {
+  calculation: string
+  group: string
+  field: string
 }
 
 function pickApi(tableId: any) {
@@ -21,18 +30,37 @@ function pickApi(tableId: any) {
   return internal
 }
 
-export async function search(tableId: string, ctx: Ctx) {
-  return pickApi(tableId).search(ctx)
+export async function search(options: SearchParams) {
+  return pickApi(options.tableId).search(options)
 }
 
-export async function exportRows(tableId: string, ctx: Ctx) {
-  return pickApi(tableId).exportRows(ctx)
+export interface ExportRowsParams {
+  tableId: string
+  format: Format
+  rowIds?: string[]
+  columns?: string[]
+  query: SearchFilters
 }
 
-export async function fetch(tableId: string, ctx: Ctx) {
-  return pickApi(tableId).fetch(ctx)
+export interface ExportRowsResult {
+  fileName: string
+  content: string
 }
 
-export async function fetchView(tableId: string, ctx: Ctx) {
-  return pickApi(tableId).fetchView(ctx)
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  return pickApi(options.tableId).exportRows(options)
+}
+
+export async function fetch(tableId: string) {
+  return pickApi(tableId).fetch(tableId)
+}
+
+export async function fetchView(
+  tableId: string,
+  viewName: string,
+  params: ViewParams
+) {
+  return pickApi(tableId).fetchView(viewName, params)
 }

packages/server/src/sdk/app/rows/search/external.ts

@@ -5,21 +5,23 @@ import {
   PaginationJson,
   IncludeRelationship,
   Row,
-  Ctx,
+  SearchFilters,
 } from "@budibase/types"
 import * as exporters from "../../../../api/controllers/view/exporters"
 import sdk from "../../../../sdk"
 import { handleRequest } from "../../../../api/controllers/row/external"
 import { breakExternalTableId } from "../../../../integrations/utils"
 import { cleanExportRows } from "../utils"
-import { apiFileReturn } from "../../../../utilities/fileSystem"
 import { utils } from "@budibase/shared-core"
+import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+import { HTTPError } from "@budibase/backend-core"
 
-export async function search(ctx: Ctx) {
-  const tableId = ctx.params.tableId
-  const { paginate, query, ...params } = ctx.request.body
-  let { bookmark, limit } = params
-  if (!bookmark && paginate) {
+export async function search(options: SearchParams) {
+  const { tableId } = options
+  const { paginate, query, ...params } = options
+  const { limit } = params
+  let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
+  if (paginate && !bookmark) {
     bookmark = 1
   }
   let paginateObj = {}
@@ -59,14 +61,14 @@ export async function search(ctx: Ctx) {
         sort,
         paginate: {
           limit: 1,
-          page: bookmark * limit + 1,
+          page: bookmark! * limit + 1,
         },
         includeSqlRelationships: IncludeRelationship.INCLUDE,
       })) as Row[]
       hasNextPage = nextRows.length > 0
     }
     // need wrapper object for bookmarks etc when paginating
-    return { rows, hasNextPage, bookmark: bookmark + 1 }
+    return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 }
   } catch (err: any) {
     if (err.message && err.message.includes("does not exist")) {
       throw new Error(
@@ -78,43 +80,38 @@ export async function search(ctx: Ctx) {
   }
 }
 
-export async function exportRows(ctx: Ctx) {
-  const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
-  const format = ctx.query.format as string
-  const { columns } = ctx.request.body
-  const datasource = await sdk.datasources.get(datasourceId!)
-  if (!datasource || !datasource.entities) {
-    ctx.throw(400, "Datasource has not been configured for plus API.")
-  }
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  const { tableId, format, columns, rowIds } = options
+  const { datasourceId, tableName } = breakExternalTableId(tableId)
 
-  if (!exporters.isFormat(format)) {
-    ctx.throw(
-      400,
-      `Format ${format} not valid. Valid values: ${Object.values(
-        exporters.Format
-      )}`
-    )
-  }
-
-  if (ctx.request.body.rows) {
-    ctx.request.body = {
-      query: {
+  let query: SearchFilters = {}
+  if (rowIds?.length) {
+    query = {
       oneOf: {
-          _id: ctx.request.body.rows.map((row: string) => {
+        _id: rowIds.map((row: string) => {
           const ids = JSON.parse(
             decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
           )
           if (ids.length > 1) {
-              ctx.throw(400, "Export data does not support composite keys.")
+            throw new HTTPError(
+              "Export data does not support composite keys.",
+              400
+            )
           }
           return ids[0]
         }),
       },
-      },
     }
   }
 
-  let result = await search(ctx)
+  const datasource = await sdk.datasources.get(datasourceId!)
+  if (!datasource || !datasource.entities) {
+    throw new HTTPError("Datasource has not been configured for plus API.", 400)
+  }
+
+  let result = await search({ tableId, query })
   let rows: Row[] = []
 
   // Filter data to only specified columns if required
@@ -131,14 +128,14 @@ export async function exportRows(ctx: Ctx) {
   }
 
   if (!tableName) {
-    ctx.throw(400, "Could not find table name.")
+    throw new HTTPError("Could not find table name.", 400)
   }
-  let schema = datasource.entities[tableName].schema
+  const schema = datasource.entities[tableName].schema
   let exportRows = cleanExportRows(rows, schema, format, columns)
 
   let headers = Object.keys(schema)
 
-  let content
+  let content: string
   switch (format) {
     case exporters.Format.CSV:
       content = exporters.csv(headers, exportRows)
@@ -150,28 +147,26 @@ export async function exportRows(ctx: Ctx) {
       content = exporters.jsonWithSchema(schema, exportRows)
       break
     default:
-      utils.unreachable(format)
-      break
+      throw utils.unreachable(format)
   }
 
-  const filename = `export.${format}`
-
-  // send down the file
-  ctx.attachment(filename)
-  return apiFileReturn(content)
+  const fileName = `export.${format}`
+  return {
+    fileName,
+    content,
+  }
 }
 
-export async function fetch(ctx: Ctx) {
-  const tableId = ctx.params.tableId
+export async function fetch(tableId: string) {
   return handleRequest(Operation.READ, tableId, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
 }
 
-export async function fetchView(ctx: Ctx) {
+export async function fetchView(viewName: string) {
   // there are no views in external datasources, shouldn't ever be called
   // for now just fetch
-  const split = ctx.params.viewName.split("all_")
-  ctx.params.tableId = split[1] ? split[1] : split[0]
-  return fetch(ctx)
+  const split = viewName.split("all_")
+  const tableId = split[1] ? split[1] : split[0]
+  return fetch(tableId)
 }

packages/server/src/sdk/app/rows/search/internal.ts

@@ -1,4 +1,7 @@
-import { context } from "@budibase/backend-core"
+import {
+  context,
+  SearchParams as InternalSearchParams,
+} from "@budibase/backend-core"
 import env from "../../../../environment"
 import { fullSearch, paginatedSearch } from "./internalSearch"
 import {
@@ -8,7 +11,7 @@ import {
 } from "../../../../db/utils"
 import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
 import { outputProcessing } from "../../../../utilities/rowProcessor"
-import { Ctx, Database, Row } from "@budibase/types"
+import { Database, Row } from "@budibase/types"
 import { cleanExportRows } from "../utils"
 import {
   Format,
@@ -16,7 +19,6 @@ import {
   json,
   jsonWithSchema,
 } from "../../../../api/controllers/view/exporters"
-import { apiFileReturn } from "../../../../utilities/fileSystem"
 import * as inMemoryViews from "../../../../db/inMemoryView"
 import {
   migrateToInMemoryView,
@@ -25,25 +27,36 @@ import {
   getFromMemoryDoc,
 } from "../../../../api/controllers/view/utils"
 import sdk from "../../../../sdk"
+import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
+
+export async function search(options: SearchParams) {
+  const { tableId } = options
 
-export async function search(ctx: Ctx) {
   // Fetch the whole table when running in cypress, as search doesn't work
   if (!env.COUCH_DB_URL && env.isCypress()) {
-    return { rows: await fetch(ctx) }
+    return { rows: await fetch(tableId) }
   }
 
-  const { tableId } = ctx.params
   const db = context.getAppDB()
-  const { paginate, query, ...params } = ctx.request.body
-  params.version = ctx.version
-  params.tableId = tableId
+  const { paginate, query } = options
+
+  const params: InternalSearchParams<any> = {
+    tableId: options.tableId,
+    sort: options.sort,
+    sortOrder: options.sortOrder,
+    sortType: options.sortType,
+    limit: options.limit,
+    bookmark: options.bookmark,
+    version: options.version,
+    disableEscaping: options.disableEscaping,
+  }
 
   let table
   if (params.sort && !params.sortType) {
     table = await db.get(tableId)
     const schema = table.schema
     const sortField = schema[params.sort]
-    params.sortType = sortField.type == "number" ? "number" : "string"
+    params.sortType = sortField.type === "number" ? "number" : "string"
   }
 
   let response
@@ -66,15 +79,12 @@ export async function search(ctx: Ctx) {
   return response
 }
 
-export async function exportRows(ctx: Ctx) {
+export async function exportRows(
+  options: ExportRowsParams
+): Promise<ExportRowsResult> {
+  const { tableId, format, rowIds, columns, query } = options
   const db = context.getAppDB()
-  const table = await db.get(ctx.params.tableId)
-  const rowIds = ctx.request.body.rows
-  let format = ctx.query.format
-  if (typeof format !== "string") {
-    ctx.throw(400, "Format parameter is not valid")
-  }
-  const { columns, query } = ctx.request.body
+  const table = await db.get(tableId)
 
   let result
   if (rowIds) {
@@ -87,7 +97,7 @@ export async function exportRows(ctx: Ctx) {
 
     result = await outputProcessing(table, response)
   } else if (query) {
-    let searchResponse = await search(ctx)
+    let searchResponse = await search({ tableId, query })
     result = searchResponse.rows
   }
 
@@ -108,26 +118,32 @@ export async function exportRows(ctx: Ctx) {
 
   let exportRows = cleanExportRows(rows, schema, format, columns)
   if (format === Format.CSV) {
-    ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
+    return {
+      fileName: "export.csv",
+      content: csv(Object.keys(rows[0]), exportRows),
+    }
   } else if (format === Format.JSON) {
-    ctx.attachment("export.json")
-    return apiFileReturn(json(exportRows))
+    return {
+      fileName: "export.json",
+      content: json(exportRows),
+    }
   } else if (format === Format.JSON_WITH_SCHEMA) {
-    ctx.attachment("export.json")
-    return apiFileReturn(jsonWithSchema(schema, exportRows))
+    return {
+      fileName: "export.json",
+      content: jsonWithSchema(schema, exportRows),
+    }
   } else {
     throw "Format not recognised"
   }
 }
 
-export async function fetch(ctx: Ctx) {
+export async function fetch(tableId: string) {
   const db = context.getAppDB()
 
-  const tableId = ctx.params.tableId
   let table = await db.get(tableId)
   let rows = await getRawTableData(db, tableId)
-  return outputProcessing(table, rows)
+  const result = await outputProcessing(table, rows)
+  return result
 }
 
 async function getRawTableData(db: Database, tableId: string) {
@@ -145,17 +161,17 @@ async function getRawTableData(db: Database, tableId: string) {
   return rows as Row[]
 }
 
-export async function fetchView(ctx: Ctx) {
-  const viewName = decodeURIComponent(ctx.params.viewName)
-
+export async function fetchView(
+  viewName: string,
+  options: { calculation: string; group: string; field: string }
+) {
   // if this is a table view being looked for just transfer to that
   if (viewName.startsWith(DocumentType.TABLE)) {
-    ctx.params.tableId = viewName
-    return fetch(ctx)
+    return fetch(viewName)
   }
 
   const db = context.getAppDB()
-  const { calculation, group, field } = ctx.query
+  const { calculation, group, field } = options
   const viewInfo = await getView(db, viewName)
   let response
   if (env.SELF_HOSTED) {

packages/server/src/sdk/app/rows/utils.ts

@@ -1,3 +1,4 @@
+import { TableSchema } from "@budibase/types"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
@@ -11,9 +12,9 @@ export async function getDatasourceAndQuery(json: any) {
 
 export function cleanExportRows(
   rows: any[],
-  schema: any,
+  schema: TableSchema,
   format: string,
-  columns: string[]
+  columns?: string[]
 ) {
   let cleanRows = [...rows]
 

packages/server/src/sdk/tests/rows/row.spec.ts

@@ -1,6 +1,10 @@
 import { exportRows } from "../../app/rows/search/external"
 import sdk from "../.."
 import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
+import { ExportRowsParams } from "../../app/rows/search"
+import { Format } from "../../../api/controllers/view/exporters"
+import { HTTPError } from "@budibase/backend-core"
+import { Operation } from "@budibase/types"
 
 const mockDatasourcesGet = jest.fn()
 sdk.datasources.get = mockDatasourcesGet
@@ -16,30 +20,21 @@ jest.mock("../../../api/controllers/view/exporters", () => ({
 }))
 jest.mock("../../../utilities/fileSystem")
 
-function getUserCtx() {
-  return {
-    params: {
-      tableId: "datasource__tablename",
-    },
-    query: {
-      format: "csv",
-    },
-    request: {
-      body: {},
-    },
-    throw: jest.fn(() => {
-      throw "Err"
-    }),
-    attachment: jest.fn(),
-  } as any
-}
-
-describe("external row controller", () => {
+describe("external row sdk", () => {
   describe("exportRows", () => {
+    function getExportOptions(): ExportRowsParams {
+      return {
+        tableId: "datasource__tablename",
+        format: Format.CSV,
+        query: {},
+      }
+    }
+
+    const externalRequestCall = jest.fn()
     beforeAll(() => {
       jest
         .spyOn(ExternalRequest.prototype, "run")
-        .mockImplementation(() => Promise.resolve([]))
+        .mockImplementation(externalRequestCall.mockResolvedValue([]))
     })
 
     afterEach(() => {
@@ -47,15 +42,10 @@ describe("external row controller", () => {
     })
 
     it("should throw a 400 if no datasource entities are present", async () => {
-      let userCtx = getUserCtx()
-      try {
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Datasource has not been configured for plus API."
+      const exportOptions = getExportOptions()
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Datasource has not been configured for plus API.", 400)
       )
-      }
     })
 
     it("should handle single quotes from a row ID", async () => {
@@ -66,51 +56,46 @@ describe("external row controller", () => {
           },
         },
       }))
-      let userCtx = getUserCtx()
-      userCtx.request.body = {
-        rows: ["['d001']"],
-      }
+      const exportOptions = getExportOptions()
+      exportOptions.rowIds = ["['d001']"]
 
-      await exportRows(userCtx)
+      await exportRows(exportOptions)
 
-      expect(userCtx.request.body).toEqual({
-        query: {
+      expect(ExternalRequest).toBeCalledTimes(1)
+      expect(ExternalRequest).toBeCalledWith(
+        Operation.READ,
+        exportOptions.tableId,
+        undefined
+      )
+
+      expect(externalRequestCall).toBeCalledTimes(1)
+      expect(externalRequestCall).toBeCalledWith(
+        expect.objectContaining({
+          filters: {
             oneOf: {
               _id: ["d001"],
             },
           },
         })
+      )
     })
 
     it("should throw a 400 if any composite keys are present", async () => {
-      let userCtx = getUserCtx()
-      userCtx.request.body = {
-        rows: ["[123]", "['d001'%2C'10111']"],
-      }
-      try {
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Export data does not support composite keys."
+      const exportOptions = getExportOptions()
+      exportOptions.rowIds = ["[123]", "['d001'%2C'10111']"]
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Export data does not support composite keys.", 400)
       )
-      }
     })
 
     it("should throw a 400 if no table name was found", async () => {
-      let userCtx = getUserCtx()
-      userCtx.params.tableId = "datasource__"
-      userCtx.request.body = {
-        rows: ["[123]"],
-      }
-      try {
-        await exportRows(userCtx)
-      } catch (e) {
-        expect(userCtx.throw).toHaveBeenCalledWith(
-          400,
-          "Could not find table name."
+      const exportOptions = getExportOptions()
+      exportOptions.tableId = "datasource__"
+      exportOptions.rowIds = ["[123]"]
+
+      await expect(exportRows(exportOptions)).rejects.toThrowError(
+        new HTTPError("Could not find table name.", 400)
       )
-      }
     })
   })
 })