From 1b36d8af5196300014847044a8c436fce13a43cf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 19:00:59 +0100 Subject: [PATCH] Getting counting flow working correctly for external datasources. --- packages/backend-core/src/sql/sql.ts | 6 ++-- .../api/controllers/row/ExternalRequest.ts | 28 +++++++++++++------ .../api/controllers/table/ExternalRequest.ts | 2 +- .../src/sdk/app/rows/search/external.ts | 22 ++++++++++----- packages/server/src/sdk/app/rows/sqlAlias.ts | 3 +- packages/server/src/sdk/app/rows/utils.ts | 2 +- packages/types/src/sdk/datasources.ts | 1 + 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index dd350c0da8..1dba4a515e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -599,7 +599,9 @@ class InternalBuilder { aliases: tableAliases, }) // add sorting to pre-query - query = this.addSorting(query, json) + if (!counting) { + query = this.addSorting(query, json) + } const alias = tableAliases?.[tableName] || tableName let preQuery = knex({ [alias]: query, @@ -610,7 +612,7 @@ class InternalBuilder { preQuery = preQuery.select(selectStatement) } // have to add after as well (this breaks MS-SQL) - if (this.client !== SqlClient.MS_SQL) { + if (this.client !== SqlClient.MS_SQL && !counting) { preQuery = this.addSorting(preQuery, json) } // handle joins diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index af27817411..98bd5412d4 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -7,6 +7,7 @@ import { FieldType, FilterType, IncludeRelationship, + isManyToOne, OneToManyRelationshipFieldMetadata, Operation, PaginationJson, @@ -16,22 +17,21 @@ import { SortJson, SortType, Table, - isManyToOne, } from "@budibase/types" import { breakExternalTableId, breakRowIdField, convertRowId, + generateRowIdField, isRowId, isSQL, - generateRowIdField, } from "../../../integrations/utils" import { buildExternalRelationships, buildSqlFieldList, generateIdForRow, - sqlOutputProcessing, isManyToMany, + sqlOutputProcessing, } from "./utils" import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" @@ -61,6 +61,13 @@ export interface RunConfig { includeSqlRelationships?: IncludeRelationship } +export type ExternalRequestReturnType = + T extends Operation.READ + ? Row[] + : T extends Operation.COUNT + ? number + : { row: Row; table: Table } + function buildFilters( id: string | undefined | string[], filters: SearchFilters, @@ -224,9 +231,6 @@ function isEditableColumn(column: FieldSchema) { return !(isExternalAutoColumn || isFormula) } -export type ExternalRequestReturnType = - T extends Operation.READ ? Row[] : { row: Row; table: Table } - export class ExternalRequest { private readonly operation: T private readonly tableId: string @@ -429,7 +433,10 @@ export class ExternalRequest { }) // this is the response from knex if no rows found const rows: Row[] = - !Array.isArray(response) || response?.[0].read ? [] : response + !Array.isArray(response) || + (response.length === 1 && "read" in response[0]) + ? [] + : response const storeTo = isManyToMany(field) ? field.throughFrom || linkPrimaryKey : fieldName @@ -664,10 +671,15 @@ export class ExternalRequest { // aliasing can be disabled fully if desired let response + const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) if (env.SQL_ALIASING_DISABLE) { response = await getDatasourceAndQuery(json) + } else if (this.operation === Operation.COUNT) { + return (await aliasing.countWithAliasing( + json, + makeExternalQuery + )) as ExternalRequestReturnType } else { - const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) response = await aliasing.queryWithAliasing(json, makeExternalQuery) } diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 1e57ea3294..9661e56729 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -33,5 +33,5 @@ export async function makeTableRequest( if (renamed) { json.meta!.renamed = renamed } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json, datasource) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e90ec049d7..366bf46156 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -1,14 +1,14 @@ import { - SortJson, + IncludeRelationship, Operation, PaginationJson, - IncludeRelationship, Row, - SearchFilters, RowSearchParams, + SearchFilters, SearchResponse, - Table, + SortJson, SortOrder, + Table, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -18,7 +18,7 @@ import { } from "../../../../integrations/utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" -import { HTTPError, db } from "@budibase/backend-core" +import { db, HTTPError } from "@budibase/backend-core" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" @@ -75,12 +75,17 @@ export async function search( } try { - let rows = await handleRequest(Operation.READ, tableId, { + const parameters = { filters: query, sort, paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, - }) + } + let rows = await handleRequest(Operation.READ, tableId, parameters) + let totalRows: number | undefined + if (true) { + totalRows = await handleRequest(Operation.COUNT, tableId, parameters) + } let hasNextPage = false // remove the extra row if it's there if (paginate && limit && rows.length > limit) { @@ -103,6 +108,9 @@ export async function search( if (hasNextPage && bookmark != null) { response.bookmark = bookmark + 1 } + if (totalRows != null) { + response.totalRows = totalRows + } return response } catch (err: any) { if (err.message && err.message.includes("does not exist")) { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 52ec2472b2..ab47f98a85 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -249,7 +249,8 @@ export default class AliasTables { json.endpoint.operation = Operation.COUNT let response = await this.queryWithAliasing(json, queryFn) if (response && response.length === 1 && "total" in response[0]) { - return response[0].total + const total = response[0].total + return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bb37fd99f3..da0132c8fa 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -61,7 +61,7 @@ export async function getDatasourceAndQuery( table, } } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json, datasource) } export function cleanExportRows( diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index d6c2bf2bf2..1a9c329153 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -197,6 +197,7 @@ enum DSPlusOperation { export type DatasourcePlusQueryResponse = | Row[] | Record[] + | { total: number }[] | void export interface DatasourcePlus extends IntegrationBase {