diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a48a102349..6c41b71993 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -571,15 +571,10 @@ class InternalBuilder { return query.insert(parsedBody) } - read( - knex: Knex, - json: QueryJson, - limit: number, - opts?: { counting?: boolean } - ): Knex.QueryBuilder { + read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder { let { endpoint, resource, filters, paginate, relationships, tableAliases } = json - const counting = opts?.counting + const counting = endpoint.operation === Operation.COUNT const tableName = endpoint.entityId // select all if not specified @@ -730,6 +725,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { query = builder.create(client, json, opts) break case Operation.READ: + case Operation.COUNT: query = builder.read(client, json, this.limit) break case Operation.UPDATE: @@ -752,20 +748,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return this.convertToNative(query, opts) } - _count(json: QueryJson, opts: QueryOptions = {}) { - const sqlClient = this.getSqlClient() - const config: Knex.Config = { - client: sqlClient, - } - if (sqlClient === SqlClient.SQL_LITE) { - config.useNullAsDefault = true - } - const client = knex(config) - const builder = new InternalBuilder(sqlClient) - const query = builder.read(client, json, this.limit, { counting: true }) - return this.convertToNative(query, opts) - } - async getReturningRow(queryFn: QueryFunction, json: QueryJson) { if (!json.extra || !json.extra.idFilter) { return {} diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index b30c97e289..af27817411 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -39,6 +39,7 @@ import { cloneDeep } from "lodash/fp" import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" import env from "../../../environment" +import { makeExternalQuery } from "../../../integrations/base/query" export interface ManyRelationship { tableId?: string @@ -517,7 +518,7 @@ export class ExternalRequest { // finally cleanup anything that needs to be removed for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) { const table: Table | undefined = this.getTable(tableId) - // if its not the foreign key skip it, nothing to do + // if it's not the foreign key skip it, nothing to do if ( !table || (!isMany && table.primary && table.primary.indexOf(colName) !== -1) @@ -667,7 +668,7 @@ export class ExternalRequest { response = await getDatasourceAndQuery(json) } else { const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) - response = await aliasing.queryWithAliasing(json) + response = await aliasing.queryWithAliasing(json, makeExternalQuery) } const responseRows = Array.isArray(response) ? response : [] diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 371592bece..acef1c6f1e 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -8,8 +8,8 @@ import { getIntegration } from "../index" import sdk from "../../sdk" export async function makeExternalQuery( - datasource: Datasource, - json: QueryJson + json: QueryJson, + datasource?: Datasource ): Promise { const entityId = json.endpoint.entityId, tableName = json.meta.table.name, @@ -22,6 +22,9 @@ export async function makeExternalQuery( ) { throw new Error("Entity ID and table metadata do not align") } + if (!datasource) { + throw new Error("No datasource provided for external query") + } datasource = await sdk.datasources.enrich(datasource) const Integration = await getIntegration(datasource.source) // query is the opinionated function diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 077f971903..c495613856 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -28,7 +28,7 @@ export async function search( table: Table ): Promise> { const { tableId } = options - const { paginate, query, ...params } = options + const { countRows, paginate, query, ...params } = options const { limit } = params let bookmark = (params.bookmark && parseInt(params.bookmark as string)) || undefined @@ -37,10 +37,14 @@ export async function search( } let paginateObj = {} - if (paginate) { + if (paginate && !limit) { + throw new Error("Cannot paginate query without a limit") + } + + if (paginate && limit) { paginateObj = { // add one so we can track if there is another page - limit: limit, + limit: limit + 1, page: bookmark, } } else if (params && limit) { @@ -76,17 +80,10 @@ export async function search( includeSqlRelationships: IncludeRelationship.INCLUDE, }) let hasNextPage = false - if (paginate && rows.length === limit) { - const nextRows = await handleRequest(Operation.READ, tableId, { - filters: query, - sort, - paginate: { - limit: 1, - page: bookmark! * limit + 1, - }, - includeSqlRelationships: IncludeRelationship.INCLUDE, - }) - hasNextPage = nextRows.length > 0 + // remove the extra row if it's there + if (paginate && limit && rows.length > limit) { + rows.pop() + hasNextPage = true } if (options.fields) { diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index e45a7f94a7..bab70134b3 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -12,7 +12,6 @@ import { SortOrder, SortType, SqlClient, - SqlQuery, Table, } from "@budibase/types" import { @@ -114,17 +113,13 @@ async function runSqlQuery( opts?: { countTotalRows?: boolean } ) { const alias = new AliasTables(tables.map(table => table.name)) + if (opts?.countTotalRows) { + json.endpoint.operation = Operation.COUNT + } const processSQLQuery = async (json: QueryJson) => { - let query: SqlQuery | SqlQuery[] - if (opts?.countTotalRows) { - query = builder._count(json, { - disableReturning: true, - }) - } else { - query = builder._query(json, { - disableReturning: true, - }) - } + const query = builder._query(json, { + disableReturning: true, + }) if (Array.isArray(query)) { throw new Error("SQS cannot currently handle multiple queries") @@ -227,14 +222,18 @@ export async function search( ) // check for pagination final row - let nextRow: Row | undefined, rowCount: number | undefined + let nextRow: Row | undefined if (paginate && params.limit && processed.length > params.limit) { - // get the total count of rows - rowCount = await runSqlQuery(request, allTables, { countTotalRows: true }) // remove the extra row that confirmed if there is another row to move to nextRow = processed.pop() } + let rowCount: number | undefined + if (options.countRows) { + // get the total count of rows + rowCount = await runSqlQuery(request, allTables, { countTotalRows: true }) + } + // get the rows let finalRows = await outputProcessing(table, processed, { preserveLinks: true, @@ -255,9 +254,11 @@ export async function search( const hasNextPage = !!nextRow response.hasNextPage = hasNextPage if (hasNextPage) { - response.totalRows = rowCount response.bookmark = bookmark + 1 } + if (rowCount != null) { + response.totalRows = rowCount + } return response } else { return { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 1b470a6a02..52ec2472b2 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -11,7 +11,11 @@ import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" import datasources from "../datasources" -import { makeExternalQuery } from "../../../integrations/base/query" + +type PerformQueryFunction = ( + json: QueryJson, + datasource?: Datasource +) => Promise const WRITE_OPERATIONS: Operation[] = [ Operation.CREATE, @@ -171,7 +175,7 @@ export default class AliasTables { async queryWithAliasing( json: QueryJson, - queryFn?: (json: QueryJson) => Promise + queryFn: PerformQueryFunction ): Promise { const datasourceId = json.endpoint.datasourceId const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL @@ -229,14 +233,7 @@ export default class AliasTables { json.tableAliases = invertedTableAliases } - let response: DatasourcePlusQueryResponse - if (datasource && !isSqs) { - response = await makeExternalQuery(datasource, json) - } else if (queryFn) { - response = await queryFn(json) - } else { - throw new Error("No supplied method to perform aliased query") - } + let response: DatasourcePlusQueryResponse = await queryFn(json, datasource) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { @@ -247,8 +244,9 @@ export default class AliasTables { // handles getting the count out of the query async countWithAliasing( json: QueryJson, - queryFn?: (json: QueryJson) => Promise + queryFn: PerformQueryFunction ): Promise { + json.endpoint.operation = Operation.COUNT let response = await this.queryWithAliasing(json, queryFn) if (response && response.length === 1 && "total" in response[0]) { return response[0].total diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 5d49f01bfc..c120af0628 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -25,6 +25,7 @@ export interface SearchViewRowRequest | "bookmark" | "paginate" | "query" + | "countRows" > {} export interface SearchRowResponse { diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 7f3fc1f391..b0b137034b 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -17,6 +17,7 @@ export interface SearchParams { fields?: string[] indexer?: () => Promise rows?: Row[] + countRows?: boolean } // when searching for rows we want a more extensive search type that requires certain properties