From 7491021ca0e0dfe4d4cee64a47e63d93483a0a48 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 14:35:14 +0100 Subject: [PATCH 01/37] Base implementation of counting (plumbing). --- .../server/src/sdk/app/rows/search/sqs.ts | 46 ++++++++++++++----- packages/server/src/sdk/app/rows/sqlAlias.ts | 21 ++++++++- packages/types/src/sdk/datasources.ts | 1 + yarn.lock | 24 +++++----- 4 files changed, 66 insertions(+), 26 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 87b905a29f..e45a7f94a7 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -12,6 +12,7 @@ import { SortOrder, SortType, SqlClient, + SqlQuery, Table, } from "@budibase/types" import { @@ -101,12 +102,29 @@ function buildTableMap(tables: Table[]) { return tableMap } -async function runSqlQuery(json: QueryJson, tables: Table[]) { +function runSqlQuery(json: QueryJson, tables: Table[]): Promise +function runSqlQuery( + json: QueryJson, + tables: Table[], + opts: { countTotalRows: boolean } +): Promise +async function runSqlQuery( + json: QueryJson, + tables: Table[], + opts?: { countTotalRows?: boolean } +) { const alias = new AliasTables(tables.map(table => table.name)) - return await alias.queryWithAliasing(json, async json => { - const query = builder._query(json, { - disableReturning: true, - }) + const processSQLQuery = async (json: QueryJson) => { + let query: SqlQuery | SqlQuery[] + if (opts?.countTotalRows) { + query = builder._count(json, { + disableReturning: true, + }) + } else { + query = builder._query(json, { + disableReturning: true, + }) + } if (Array.isArray(query)) { throw new Error("SQS cannot currently handle multiple queries") @@ -125,7 +143,12 @@ async function runSqlQuery(json: QueryJson, tables: Table[]) { const db = context.getAppDB() return await db.sql(sql, bindings) - }) + } + if (opts?.countTotalRows) { + return await alias.countWithAliasing(json, processSQLQuery) + } else { + return await alias.queryWithAliasing(json, processSQLQuery) + } } export async function search( @@ -204,8 +227,11 @@ export async function search( ) // check for pagination final row - let nextRow: Row | undefined + let nextRow: Row | undefined, rowCount: number | undefined if (paginate && params.limit && processed.length > params.limit) { + // get the total count of rows + rowCount = await runSqlQuery(request, allTables, { countTotalRows: true }) + // remove the extra row that confirmed if there is another row to move to nextRow = processed.pop() } @@ -226,14 +252,10 @@ export async function search( const response: SearchResponse = { rows: finalRows, } - const prevLimit = request.paginate!.limit - request.paginate = { - limit: 1, - page: bookmark * prevLimit + 1, - } const hasNextPage = !!nextRow response.hasNextPage = hasNextPage if (hasNextPage) { + response.totalRows = rowCount response.bookmark = bookmark + 1 } return response diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index ab4f5d2844..1b470a6a02 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -65,7 +65,7 @@ export default class AliasTables { this.charSeq = new CharSequence() } - isAliasingEnabled(json: QueryJson, datasource: Datasource) { + isAliasingEnabled(json: QueryJson, datasource?: Datasource) { const operation = json.endpoint.operation const fieldLength = json.resource?.fields?.length if ( @@ -75,6 +75,10 @@ export default class AliasTables { ) { return false } + // SQS - doesn't have a datasource + if (!datasource) { + return true + } try { const sqlClient = getSQLClient(datasource) const isWrite = WRITE_OPERATIONS.includes(operation) @@ -173,7 +177,7 @@ export default class AliasTables { const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL let aliasingEnabled: boolean, datasource: Datasource | undefined if (isSqs) { - aliasingEnabled = true + aliasingEnabled = this.isAliasingEnabled(json) } else { datasource = await datasources.get(datasourceId) aliasingEnabled = this.isAliasingEnabled(json, datasource) @@ -239,4 +243,17 @@ export default class AliasTables { return response } } + + // handles getting the count out of the query + async countWithAliasing( + json: QueryJson, + queryFn?: (json: QueryJson) => Promise + ): Promise { + let response = await this.queryWithAliasing(json, queryFn) + if (response && response.length === 1 && "total" in response[0]) { + return response[0].total + } else { + throw new Error("Unable to count rows in query - no count response") + } + } } diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 77e4877dfa..10a697671f 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -8,6 +8,7 @@ export enum Operation { READ = "READ", UPDATE = "UPDATE", DELETE = "DELETE", + COUNT = "COUNT", BULK_CREATE = "BULK_CREATE", CREATE_TABLE = "CREATE_TABLE", UPDATE_TABLE = "UPDATE_TABLE", diff --git a/yarn.lock b/yarn.lock index 426fa2275d..d71dd4da78 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3495,10 +3495,10 @@ dependencies: lodash "^4.17.21" -"@koa/cors@^3.1.0": - version "3.4.3" - resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-3.4.3.tgz#d669ee6e8d6e4f0ec4a7a7b0a17e7a3ed3752ebb" - integrity sha512-WPXQUaAeAMVaLTEFpoq3T2O1C+FstkjJnDQqy95Ck1UdILajsRhu6mhJ8H2f4NFPRBoCNN+qywTJfq/gGki5mw== +"@koa/cors@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@koa/cors/-/cors-5.0.0.tgz#0029b5f057fa0d0ae0e37dd2c89ece315a0daffd" + integrity sha512-x/iUDjcS90W69PryLDIMgFyV21YLTnG9zOpPXS7Bkt2b8AsY3zZsIpOLBkYr9fBcF3HbkKaER5hOBZLfpLgYNw== dependencies: vary "^1.1.2" @@ -5817,10 +5817,10 @@ "@types/koa-compose" "*" "@types/node" "*" -"@types/koa__cors@^3.1.1": - version "3.3.1" - resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-3.3.1.tgz#0ec7543c4c620fd23451bfdd3e21b9a6aadedccd" - integrity sha512-aFGYhTFW7651KhmZZ05VG0QZJre7QxBxDj2LF1lf6GA/wSXEfKVAJxiQQWzRV4ZoMzQIO8vJBXKsUcRuvYK9qw== +"@types/koa__cors@^5.0.0": + version "5.0.0" + resolved "https://registry.yarnpkg.com/@types/koa__cors/-/koa__cors-5.0.0.tgz#74567a045b599266e2cd3940cef96cedecc2ef1f" + integrity sha512-LCk/n25Obq5qlernGOK/2LUwa/2YJb2lxHUkkvYFDOpLXlVI6tKcdfCHRBQnOY4LwH6el5WOLs6PD/a8Uzau6g== dependencies: "@types/koa" "*" @@ -16343,10 +16343,10 @@ node-source-walk@^5.0.0: dependencies: "@babel/parser" "^7.0.0" -nodemailer@6.7.2: - version "6.7.2" - resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.7.2.tgz#44b2ad5f7ed71b7067f7a21c4fedabaec62b85e0" - integrity sha512-Dz7zVwlef4k5R71fdmxwR8Q39fiboGbu3xgswkzGwczUfjp873rVxt1O46+Fh0j1ORnAC6L9+heI8uUpO6DT7Q== +nodemailer@6.9.13: + version "6.9.13" + resolved "https://registry.yarnpkg.com/nodemailer/-/nodemailer-6.9.13.tgz#5b292bf1e92645f4852ca872c56a6ba6c4a3d3d6" + integrity sha512-7o38Yogx6krdoBf3jCAqnIN4oSQFx+fMa0I7dK1D+me9kBxx12D+/33wSb+fhOCtIxvYJ+4x4IMEhmhCKfAiOA== nodemailer@6.9.9: version "6.9.9" From 2c6262844bb719029b864b4d2b2a0ae24f2e915f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 14:35:35 +0100 Subject: [PATCH 02/37] Some work to limiting, changing how limiting works for pagination so that filtering on relationships doesn't cause problems. --- packages/backend-core/src/sql/sql.ts | 112 ++++++++++++++++++--------- 1 file changed, 77 insertions(+), 35 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 61d5849058..a48a102349 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1,10 +1,10 @@ import { Knex, knex } from "knex" import * as dbCore from "../db" import { - isIsoDateString, - isValidFilter, getNativeSql, isExternalTable, + isIsoDateString, + isValidFilter, } from "./utils" import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" @@ -12,21 +12,21 @@ import { BBReferenceFieldMetadata, FieldSchema, FieldType, + INTERNAL_TABLE_SOURCE_ID, JsonFieldMetadata, + JsonTypes, Operation, + prefixed, QueryJson, - SqlQuery, + QueryOptions, RelationshipsJson, SearchFilters, SortDirection, + SqlClient, + SqlQuery, SqlQueryBinding, Table, TableSourceType, - INTERNAL_TABLE_SOURCE_ID, - SqlClient, - QueryOptions, - JsonTypes, - prefixed, } from "@budibase/types" import environment from "../environment" import { helpers } from "@budibase/shared-core" @@ -522,7 +522,7 @@ class InternalBuilder { }) } } - return query.limit(BASE_LIMIT) + return query } knexWithAlias( @@ -571,9 +571,15 @@ class InternalBuilder { return query.insert(parsedBody) } - read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder { + read( + knex: Knex, + json: QueryJson, + limit: number, + opts?: { counting?: boolean } + ): Knex.QueryBuilder { let { endpoint, resource, filters, paginate, relationships, tableAliases } = json + const counting = opts?.counting const tableName = endpoint.entityId // select all if not specified @@ -582,28 +588,16 @@ class InternalBuilder { } let selectStatement: string | (string | Knex.Raw)[] = "*" // handle select - if (resource.fields && resource.fields.length > 0) { + if (!counting && resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided // by the resource builder further up selectStatement = generateSelectStatement(json, knex) } - let foundLimit = limit || BASE_LIMIT - // handle pagination - let foundOffset: number | null = null - if (paginate && paginate.page && paginate.limit) { - // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - foundOffset = offset - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit - } // start building the query let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = query.limit(foundLimit) - if (foundOffset) { - query = query.offset(foundOffset) + // add a base query over all + if (!counting) { + query = query.limit(BASE_LIMIT) } query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, @@ -614,7 +608,12 @@ class InternalBuilder { const alias = tableAliases?.[tableName] || tableName let preQuery = knex({ [alias]: query, - } as any).select(selectStatement) as any + } as any) + if (counting) { + preQuery = preQuery.count("* as total") + } else { + preQuery = preQuery.select(selectStatement) + } // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL) { preQuery = this.addSorting(preQuery, json) @@ -627,6 +626,30 @@ class InternalBuilder { endpoint.schema, tableAliases ) + + let foundLimit = limit || BASE_LIMIT + // handle pagination + let foundOffset: number | null = null + if (paginate && paginate.page && paginate.limit) { + let page = + typeof paginate.page === "string" + ? parseInt(paginate.page) + : paginate.page + page = page <= 1 ? 0 : page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + if (!counting) { + query = query.limit(foundLimit) + } + // add overall pagination + if (!counting && foundOffset) { + query = query.offset(foundOffset) + } + return this.addFilters(query, filters, json.meta.table, { relationship: true, aliases: tableAliases, @@ -671,6 +694,19 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { this.limit = limit } + private convertToNative(query: Knex.QueryBuilder, opts: QueryOptions = {}) { + const sqlClient = this.getSqlClient() + if (opts?.disableBindings) { + return { sql: query.toString() } + } else { + let native = getNativeSql(query) + if (sqlClient === SqlClient.SQL_LITE) { + native = convertBooleans(native) + } + return native + } + } + /** * @param json The JSON query DSL which is to be converted to SQL. * @param opts extra options which are to be passed into the query builder, e.g. disableReturning @@ -713,15 +749,21 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { throw `Operation type is not supported by SQL query builder` } - if (opts?.disableBindings) { - return { sql: query.toString() } - } else { - let native = getNativeSql(query) - if (sqlClient === SqlClient.SQL_LITE) { - native = convertBooleans(native) - } - return native + return this.convertToNative(query, opts) + } + + _count(json: QueryJson, opts: QueryOptions = {}) { + const sqlClient = this.getSqlClient() + const config: Knex.Config = { + client: sqlClient, } + if (sqlClient === SqlClient.SQL_LITE) { + config.useNullAsDefault = true + } + const client = knex(config) + const builder = new InternalBuilder(sqlClient) + const query = builder.read(client, json, this.limit, { counting: true }) + return this.convertToNative(query, opts) } async getReturningRow(queryFn: QueryFunction, json: QueryJson) { From 77556820bf6c05c9d094ab1e982892b11157e53e Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 18:12:36 +0100 Subject: [PATCH 03/37] Bit more work towards row counting, as well as moving external SQL to use row + 1 for working out pagination. --- packages/backend-core/src/sql/sql.ts | 24 ++------------ .../api/controllers/row/ExternalRequest.ts | 5 +-- .../server/src/integrations/base/query.ts | 7 +++-- .../src/sdk/app/rows/search/external.ts | 25 +++++++-------- .../server/src/sdk/app/rows/search/sqs.ts | 31 ++++++++++--------- packages/server/src/sdk/app/rows/sqlAlias.ts | 20 ++++++------ packages/types/src/api/web/app/rows.ts | 1 + packages/types/src/sdk/row.ts | 1 + 8 files changed, 49 insertions(+), 65 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a48a102349..6c41b71993 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -571,15 +571,10 @@ class InternalBuilder { return query.insert(parsedBody) } - read( - knex: Knex, - json: QueryJson, - limit: number, - opts?: { counting?: boolean } - ): Knex.QueryBuilder { + read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder { let { endpoint, resource, filters, paginate, relationships, tableAliases } = json - const counting = opts?.counting + const counting = endpoint.operation === Operation.COUNT const tableName = endpoint.entityId // select all if not specified @@ -730,6 +725,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { query = builder.create(client, json, opts) break case Operation.READ: + case Operation.COUNT: query = builder.read(client, json, this.limit) break case Operation.UPDATE: @@ -752,20 +748,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { return this.convertToNative(query, opts) } - _count(json: QueryJson, opts: QueryOptions = {}) { - const sqlClient = this.getSqlClient() - const config: Knex.Config = { - client: sqlClient, - } - if (sqlClient === SqlClient.SQL_LITE) { - config.useNullAsDefault = true - } - const client = knex(config) - const builder = new InternalBuilder(sqlClient) - const query = builder.read(client, json, this.limit, { counting: true }) - return this.convertToNative(query, opts) - } - async getReturningRow(queryFn: QueryFunction, json: QueryJson) { if (!json.extra || !json.extra.idFilter) { return {} diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index b30c97e289..af27817411 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -39,6 +39,7 @@ import { cloneDeep } from "lodash/fp" import { db as dbCore } from "@budibase/backend-core" import sdk from "../../../sdk" import env from "../../../environment" +import { makeExternalQuery } from "../../../integrations/base/query" export interface ManyRelationship { tableId?: string @@ -517,7 +518,7 @@ export class ExternalRequest { // finally cleanup anything that needs to be removed for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) { const table: Table | undefined = this.getTable(tableId) - // if its not the foreign key skip it, nothing to do + // if it's not the foreign key skip it, nothing to do if ( !table || (!isMany && table.primary && table.primary.indexOf(colName) !== -1) @@ -667,7 +668,7 @@ export class ExternalRequest { response = await getDatasourceAndQuery(json) } else { const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) - response = await aliasing.queryWithAliasing(json) + response = await aliasing.queryWithAliasing(json, makeExternalQuery) } const responseRows = Array.isArray(response) ? response : [] diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 371592bece..acef1c6f1e 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -8,8 +8,8 @@ import { getIntegration } from "../index" import sdk from "../../sdk" export async function makeExternalQuery( - datasource: Datasource, - json: QueryJson + json: QueryJson, + datasource?: Datasource ): Promise { const entityId = json.endpoint.entityId, tableName = json.meta.table.name, @@ -22,6 +22,9 @@ export async function makeExternalQuery( ) { throw new Error("Entity ID and table metadata do not align") } + if (!datasource) { + throw new Error("No datasource provided for external query") + } datasource = await sdk.datasources.enrich(datasource) const Integration = await getIntegration(datasource.source) // query is the opinionated function diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 077f971903..c495613856 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -28,7 +28,7 @@ export async function search( table: Table ): Promise> { const { tableId } = options - const { paginate, query, ...params } = options + const { countRows, paginate, query, ...params } = options const { limit } = params let bookmark = (params.bookmark && parseInt(params.bookmark as string)) || undefined @@ -37,10 +37,14 @@ export async function search( } let paginateObj = {} - if (paginate) { + if (paginate && !limit) { + throw new Error("Cannot paginate query without a limit") + } + + if (paginate && limit) { paginateObj = { // add one so we can track if there is another page - limit: limit, + limit: limit + 1, page: bookmark, } } else if (params && limit) { @@ -76,17 +80,10 @@ export async function search( includeSqlRelationships: IncludeRelationship.INCLUDE, }) let hasNextPage = false - if (paginate && rows.length === limit) { - const nextRows = await handleRequest(Operation.READ, tableId, { - filters: query, - sort, - paginate: { - limit: 1, - page: bookmark! * limit + 1, - }, - includeSqlRelationships: IncludeRelationship.INCLUDE, - }) - hasNextPage = nextRows.length > 0 + // remove the extra row if it's there + if (paginate && limit && rows.length > limit) { + rows.pop() + hasNextPage = true } if (options.fields) { diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index e45a7f94a7..bab70134b3 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -12,7 +12,6 @@ import { SortOrder, SortType, SqlClient, - SqlQuery, Table, } from "@budibase/types" import { @@ -114,17 +113,13 @@ async function runSqlQuery( opts?: { countTotalRows?: boolean } ) { const alias = new AliasTables(tables.map(table => table.name)) + if (opts?.countTotalRows) { + json.endpoint.operation = Operation.COUNT + } const processSQLQuery = async (json: QueryJson) => { - let query: SqlQuery | SqlQuery[] - if (opts?.countTotalRows) { - query = builder._count(json, { - disableReturning: true, - }) - } else { - query = builder._query(json, { - disableReturning: true, - }) - } + const query = builder._query(json, { + disableReturning: true, + }) if (Array.isArray(query)) { throw new Error("SQS cannot currently handle multiple queries") @@ -227,14 +222,18 @@ export async function search( ) // check for pagination final row - let nextRow: Row | undefined, rowCount: number | undefined + let nextRow: Row | undefined if (paginate && params.limit && processed.length > params.limit) { - // get the total count of rows - rowCount = await runSqlQuery(request, allTables, { countTotalRows: true }) // remove the extra row that confirmed if there is another row to move to nextRow = processed.pop() } + let rowCount: number | undefined + if (options.countRows) { + // get the total count of rows + rowCount = await runSqlQuery(request, allTables, { countTotalRows: true }) + } + // get the rows let finalRows = await outputProcessing(table, processed, { preserveLinks: true, @@ -255,9 +254,11 @@ export async function search( const hasNextPage = !!nextRow response.hasNextPage = hasNextPage if (hasNextPage) { - response.totalRows = rowCount response.bookmark = bookmark + 1 } + if (rowCount != null) { + response.totalRows = rowCount + } return response } else { return { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 1b470a6a02..52ec2472b2 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -11,7 +11,11 @@ import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" import datasources from "../datasources" -import { makeExternalQuery } from "../../../integrations/base/query" + +type PerformQueryFunction = ( + json: QueryJson, + datasource?: Datasource +) => Promise const WRITE_OPERATIONS: Operation[] = [ Operation.CREATE, @@ -171,7 +175,7 @@ export default class AliasTables { async queryWithAliasing( json: QueryJson, - queryFn?: (json: QueryJson) => Promise + queryFn: PerformQueryFunction ): Promise { const datasourceId = json.endpoint.datasourceId const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL @@ -229,14 +233,7 @@ export default class AliasTables { json.tableAliases = invertedTableAliases } - let response: DatasourcePlusQueryResponse - if (datasource && !isSqs) { - response = await makeExternalQuery(datasource, json) - } else if (queryFn) { - response = await queryFn(json) - } else { - throw new Error("No supplied method to perform aliased query") - } + let response: DatasourcePlusQueryResponse = await queryFn(json, datasource) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { @@ -247,8 +244,9 @@ export default class AliasTables { // handles getting the count out of the query async countWithAliasing( json: QueryJson, - queryFn?: (json: QueryJson) => Promise + queryFn: PerformQueryFunction ): Promise { + json.endpoint.operation = Operation.COUNT let response = await this.queryWithAliasing(json, queryFn) if (response && response.length === 1 && "total" in response[0]) { return response[0].total diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index 5d49f01bfc..c120af0628 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -25,6 +25,7 @@ export interface SearchViewRowRequest | "bookmark" | "paginate" | "query" + | "countRows" > {} export interface SearchRowResponse { diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 7f3fc1f391..b0b137034b 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -17,6 +17,7 @@ export interface SearchParams { fields?: string[] indexer?: () => Promise rows?: Row[] + countRows?: boolean } // when searching for rows we want a more extensive search type that requires certain properties From 908b77fd9be23ad3f5114467482e692418ee6e15 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 18:27:23 +0100 Subject: [PATCH 04/37] Fixing some issues with using offsets. --- .../server/src/sdk/app/rows/search/external.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index a32a036846..e90ec049d7 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -33,9 +33,9 @@ export async function search( let bookmark = (params.bookmark && parseInt(params.bookmark as string)) || undefined if (paginate && !bookmark) { - bookmark = 1 + bookmark = 0 } - let paginateObj = {} + let paginateObj: PaginationJson | undefined if (paginate && !limit) { throw new Error("Cannot paginate query without a limit") @@ -45,7 +45,9 @@ export async function search( paginateObj = { // add one so we can track if there is another page limit: limit + 1, - page: bookmark, + } + if (bookmark) { + paginateObj.offset = limit * bookmark } } else if (params && limit) { paginateObj = { @@ -97,7 +99,11 @@ export async function search( }) // need wrapper object for bookmarks etc when paginating - return { rows, hasNextPage, bookmark: bookmark && bookmark + 1 } + const response: SearchResponse = { rows, hasNextPage } + if (hasNextPage && bookmark != null) { + response.bookmark = bookmark + 1 + } + return response } catch (err: any) { if (err.message && err.message.includes("does not exist")) { throw new Error( From 1b36d8af5196300014847044a8c436fce13a43cf Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 19:00:59 +0100 Subject: [PATCH 05/37] Getting counting flow working correctly for external datasources. --- packages/backend-core/src/sql/sql.ts | 6 ++-- .../api/controllers/row/ExternalRequest.ts | 28 +++++++++++++------ .../api/controllers/table/ExternalRequest.ts | 2 +- .../src/sdk/app/rows/search/external.ts | 22 ++++++++++----- packages/server/src/sdk/app/rows/sqlAlias.ts | 3 +- packages/server/src/sdk/app/rows/utils.ts | 2 +- packages/types/src/sdk/datasources.ts | 1 + 7 files changed, 44 insertions(+), 20 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index dd350c0da8..1dba4a515e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -599,7 +599,9 @@ class InternalBuilder { aliases: tableAliases, }) // add sorting to pre-query - query = this.addSorting(query, json) + if (!counting) { + query = this.addSorting(query, json) + } const alias = tableAliases?.[tableName] || tableName let preQuery = knex({ [alias]: query, @@ -610,7 +612,7 @@ class InternalBuilder { preQuery = preQuery.select(selectStatement) } // have to add after as well (this breaks MS-SQL) - if (this.client !== SqlClient.MS_SQL) { + if (this.client !== SqlClient.MS_SQL && !counting) { preQuery = this.addSorting(preQuery, json) } // handle joins diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index af27817411..98bd5412d4 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -7,6 +7,7 @@ import { FieldType, FilterType, IncludeRelationship, + isManyToOne, OneToManyRelationshipFieldMetadata, Operation, PaginationJson, @@ -16,22 +17,21 @@ import { SortJson, SortType, Table, - isManyToOne, } from "@budibase/types" import { breakExternalTableId, breakRowIdField, convertRowId, + generateRowIdField, isRowId, isSQL, - generateRowIdField, } from "../../../integrations/utils" import { buildExternalRelationships, buildSqlFieldList, generateIdForRow, - sqlOutputProcessing, isManyToMany, + sqlOutputProcessing, } from "./utils" import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" @@ -61,6 +61,13 @@ export interface RunConfig { includeSqlRelationships?: IncludeRelationship } +export type ExternalRequestReturnType = + T extends Operation.READ + ? Row[] + : T extends Operation.COUNT + ? number + : { row: Row; table: Table } + function buildFilters( id: string | undefined | string[], filters: SearchFilters, @@ -224,9 +231,6 @@ function isEditableColumn(column: FieldSchema) { return !(isExternalAutoColumn || isFormula) } -export type ExternalRequestReturnType = - T extends Operation.READ ? Row[] : { row: Row; table: Table } - export class ExternalRequest { private readonly operation: T private readonly tableId: string @@ -429,7 +433,10 @@ export class ExternalRequest { }) // this is the response from knex if no rows found const rows: Row[] = - !Array.isArray(response) || response?.[0].read ? [] : response + !Array.isArray(response) || + (response.length === 1 && "read" in response[0]) + ? [] + : response const storeTo = isManyToMany(field) ? field.throughFrom || linkPrimaryKey : fieldName @@ -664,10 +671,15 @@ export class ExternalRequest { // aliasing can be disabled fully if desired let response + const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) if (env.SQL_ALIASING_DISABLE) { response = await getDatasourceAndQuery(json) + } else if (this.operation === Operation.COUNT) { + return (await aliasing.countWithAliasing( + json, + makeExternalQuery + )) as ExternalRequestReturnType } else { - const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) response = await aliasing.queryWithAliasing(json, makeExternalQuery) } diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 1e57ea3294..9661e56729 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -33,5 +33,5 @@ export async function makeTableRequest( if (renamed) { json.meta!.renamed = renamed } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json, datasource) } diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index e90ec049d7..366bf46156 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -1,14 +1,14 @@ import { - SortJson, + IncludeRelationship, Operation, PaginationJson, - IncludeRelationship, Row, - SearchFilters, RowSearchParams, + SearchFilters, SearchResponse, - Table, + SortJson, SortOrder, + Table, } from "@budibase/types" import * as exporters from "../../../../api/controllers/view/exporters" import { handleRequest } from "../../../../api/controllers/row/external" @@ -18,7 +18,7 @@ import { } from "../../../../integrations/utils" import { utils } from "@budibase/shared-core" import { ExportRowsParams, ExportRowsResult } from "./types" -import { HTTPError, db } from "@budibase/backend-core" +import { db, HTTPError } from "@budibase/backend-core" import pick from "lodash/pick" import { outputProcessing } from "../../../../utilities/rowProcessor" import sdk from "../../../" @@ -75,12 +75,17 @@ export async function search( } try { - let rows = await handleRequest(Operation.READ, tableId, { + const parameters = { filters: query, sort, paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, - }) + } + let rows = await handleRequest(Operation.READ, tableId, parameters) + let totalRows: number | undefined + if (true) { + totalRows = await handleRequest(Operation.COUNT, tableId, parameters) + } let hasNextPage = false // remove the extra row if it's there if (paginate && limit && rows.length > limit) { @@ -103,6 +108,9 @@ export async function search( if (hasNextPage && bookmark != null) { response.bookmark = bookmark + 1 } + if (totalRows != null) { + response.totalRows = totalRows + } return response } catch (err: any) { if (err.message && err.message.includes("does not exist")) { diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 52ec2472b2..ab47f98a85 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -249,7 +249,8 @@ export default class AliasTables { json.endpoint.operation = Operation.COUNT let response = await this.queryWithAliasing(json, queryFn) if (response && response.length === 1 && "total" in response[0]) { - return response[0].total + const total = response[0].total + return typeof total === "number" ? total : parseInt(total) } else { throw new Error("Unable to count rows in query - no count response") } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bb37fd99f3..da0132c8fa 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -61,7 +61,7 @@ export async function getDatasourceAndQuery( table, } } - return makeExternalQuery(datasource, json) + return makeExternalQuery(json, datasource) } export function cleanExportRows( diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index d6c2bf2bf2..1a9c329153 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -197,6 +197,7 @@ enum DSPlusOperation { export type DatasourcePlusQueryResponse = | Row[] | Record[] + | { total: number }[] | void export interface DatasourcePlus extends IntegrationBase { From f3ca1d0b1e7bee5d40caf72e0ece146bba2a34a5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 19:01:20 +0100 Subject: [PATCH 06/37] Adding countRows parameter to external API for counting. --- packages/server/src/sdk/app/rows/search/external.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 366bf46156..5e2074f56d 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -83,7 +83,7 @@ export async function search( } let rows = await handleRequest(Operation.READ, tableId, parameters) let totalRows: number | undefined - if (true) { + if (countRows) { totalRows = await handleRequest(Operation.COUNT, tableId, parameters) } let hasNextPage = false From cd1e7c0bad0b03f2c5defe7a0cf7f9adcbf1e52b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 19:04:02 +0100 Subject: [PATCH 07/37] Small re-jig make things easier to read. --- .../server/src/api/controllers/row/ExternalRequest.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 98bd5412d4..1679892d03 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -672,15 +672,16 @@ export class ExternalRequest { // aliasing can be disabled fully if desired let response const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) - if (env.SQL_ALIASING_DISABLE) { - response = await getDatasourceAndQuery(json) - } else if (this.operation === Operation.COUNT) { + // if it's a counting operation there will be no more processing, just return the number + if (this.operation === Operation.COUNT) { return (await aliasing.countWithAliasing( json, makeExternalQuery )) as ExternalRequestReturnType } else { - response = await aliasing.queryWithAliasing(json, makeExternalQuery) + response = env.SQL_ALIASING_DISABLE + ? await getDatasourceAndQuery(json) + : await aliasing.queryWithAliasing(json, makeExternalQuery) } const responseRows = Array.isArray(response) ? response : [] From 654a417d66237df845aa3e1c3943b1905cc5a06f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 19:07:50 +0100 Subject: [PATCH 08/37] Type checking. --- packages/server/src/api/controllers/row/utils/utils.ts | 2 +- packages/server/src/api/controllers/row/views.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index c2d62e0204..bf45a85178 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -137,7 +137,7 @@ export async function sqlOutputProcessing( relationships: RelationshipsJson[], opts?: { sqs?: boolean } ): Promise { - if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) { + if (!Array.isArray(rows) || rows.length === 0 || "read" in rows[0]) { return [] } let finalRows: { [key: string]: Row } = {} diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 80aa97d8c0..800cc29ceb 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -60,7 +60,7 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) - const searchOptions: RequiredKeys & + const searchOptions: RequiredKeys> & RequiredKeys> = { tableId: view.tableId, query: enrichedQuery, From c34c219e8fd88c58cc44769391a1adbef9e663cd Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 14 Jun 2024 19:10:52 +0100 Subject: [PATCH 09/37] Tidying up one of the weirder things knex can do. --- .../server/src/api/controllers/row/ExternalRequest.ts | 4 ++-- .../server/src/api/controllers/row/utils/sqlUtils.ts | 10 ++++++++++ packages/server/src/api/controllers/row/utils/utils.ts | 4 ++-- packages/types/src/sdk/datasources.ts | 2 +- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 1679892d03..4e14ce2799 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -30,6 +30,7 @@ import { buildExternalRelationships, buildSqlFieldList, generateIdForRow, + isKnexNoRowReadResponse, isManyToMany, sqlOutputProcessing, } from "./utils" @@ -433,8 +434,7 @@ export class ExternalRequest { }) // this is the response from knex if no rows found const rows: Row[] = - !Array.isArray(response) || - (response.length === 1 && "read" in response[0]) + !Array.isArray(response) || isKnexNoRowReadResponse(response) ? [] : response const storeTo = isManyToMany(field) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 372b8394ff..b236578485 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -1,4 +1,6 @@ import { + DatasourcePlusQueryResponse, + DSPlusOperation, FieldType, ManyToManyRelationshipFieldMetadata, RelationshipFieldMetadata, @@ -192,3 +194,11 @@ export function buildSqlFieldList( } return fields } + +export function isKnexNoRowReadResponse(resp: DatasourcePlusQueryResponse) { + return ( + !Array.isArray(resp) || + resp.length === 0 || + (DSPlusOperation.READ in resp[0] && resp[0].read === true) + ) +} diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index bf45a85178..a607a01f16 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -14,7 +14,7 @@ import { processDates, processFormulas, } from "../../../../utilities/rowProcessor" -import { updateRelationshipColumns } from "./sqlUtils" +import { isKnexNoRowReadResponse, updateRelationshipColumns } from "./sqlUtils" import { basicProcessing, generateIdForRow, @@ -137,7 +137,7 @@ export async function sqlOutputProcessing( relationships: RelationshipsJson[], opts?: { sqs?: boolean } ): Promise { - if (!Array.isArray(rows) || rows.length === 0 || "read" in rows[0]) { + if (isKnexNoRowReadResponse(rows)) { return [] } let finalRows: { [key: string]: Row } = {} diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index 1a9c329153..ba9b1e5f45 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -187,7 +187,7 @@ export interface Schema { } // return these when an operation occurred but we got no response -enum DSPlusOperation { +export enum DSPlusOperation { CREATE = "create", READ = "read", UPDATE = "update", From 73013332aeb8a361692f116ca675c52d4e01d19c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 11:16:21 +0100 Subject: [PATCH 10/37] Adding countRows to search validators. --- packages/server/src/api/controllers/row/views.ts | 3 ++- packages/server/src/api/routes/row.ts | 1 + packages/server/src/api/routes/utils/validators.ts | 1 + yarn.lock | 2 +- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index 800cc29ceb..63ce12f0ab 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -60,7 +60,7 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) - const searchOptions: RequiredKeys> & + const searchOptions: RequiredKeys & RequiredKeys> = { tableId: view.tableId, query: enrichedQuery, @@ -69,6 +69,7 @@ export async function searchView( limit: body.limit, bookmark: body.bookmark, paginate: body.paginate, + countRows: body.countRows, } const result = await sdk.rows.search(searchOptions) diff --git a/packages/server/src/api/routes/row.ts b/packages/server/src/api/routes/row.ts index f1aa39a461..e443b2daeb 100644 --- a/packages/server/src/api/routes/row.ts +++ b/packages/server/src/api/routes/row.ts @@ -86,6 +86,7 @@ router router.post( "/api/v2/views/:viewId/search", + internalSearchValidator(), authorizedResource(PermissionType.VIEW, PermissionLevel.READ, "viewId"), rowController.views.searchView ) diff --git a/packages/server/src/api/routes/utils/validators.ts b/packages/server/src/api/routes/utils/validators.ts index a63b29fe5a..671ce95038 100644 --- a/packages/server/src/api/routes/utils/validators.ts +++ b/packages/server/src/api/routes/utils/validators.ts @@ -109,6 +109,7 @@ export function internalSearchValidator() { sortOrder: OPTIONAL_STRING, sortType: OPTIONAL_STRING, paginate: Joi.boolean(), + countRows: Joi.boolean(), bookmark: Joi.alternatives() .try(OPTIONAL_STRING, OPTIONAL_NUMBER) .optional(), diff --git a/yarn.lock b/yarn.lock index d71dd4da78..7f7def28b3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -21102,7 +21102,7 @@ tar@6.1.15: mkdirp "^1.0.3" yallist "^4.0.0" -tar@^6.1.11, tar@^6.1.2: +tar@6.2.1, tar@^6.1.11, tar@^6.1.2: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" integrity sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A== From 278242fb5575dad88ba7baf29ef7d8a772af83b7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 12:47:59 +0100 Subject: [PATCH 11/37] Moving limits and offsets back into pre-query. --- packages/backend-core/src/sql/sql.ts | 47 ++++++++++++++-------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 1dba4a515e..0f7fedf6c7 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -590,9 +590,28 @@ class InternalBuilder { } // start building the query let query = this.knexWithAlias(knex, endpoint, tableAliases) - // add a base query over all + // handle pagination + let foundOffset: number | null = null + let foundLimit = limit || BASE_LIMIT + if (paginate && paginate.page && paginate.limit) { + // @ts-ignore + const page = paginate.page <= 1 ? 0 : paginate.page - 1 + const offset = page * paginate.limit + foundLimit = paginate.limit + foundOffset = offset + } else if (paginate && paginate.offset && paginate.limit) { + foundLimit = paginate.limit + foundOffset = paginate.offset + } else if (paginate && paginate.limit) { + foundLimit = paginate.limit + } + // always add the found limit, unless counting if (!counting) { - query = query.limit(BASE_LIMIT) + query = query.limit(foundLimit) + } + // add overall pagination + if (!counting && foundOffset) { + query = query.offset(foundOffset) } // add filters to the query (where) query = this.addFilters(query, filters, json.meta.table, { @@ -623,28 +642,10 @@ class InternalBuilder { endpoint.schema, tableAliases ) - // handle pagination - let foundOffset: number | null = null - let foundLimit = limit || BASE_LIMIT - if (paginate && paginate.page && paginate.limit) { - // @ts-ignore - const page = paginate.page <= 1 ? 0 : paginate.page - 1 - const offset = page * paginate.limit - foundLimit = paginate.limit - foundOffset = offset - } else if (paginate && paginate.offset && paginate.limit) { - foundLimit = paginate.limit - foundOffset = paginate.offset - } else if (paginate && paginate.limit) { - foundLimit = paginate.limit - } - // always add the found limit, unless counting + + // add a base query over all if (!counting) { - query = query.limit(foundLimit) - } - // add overall pagination - if (!counting && foundOffset) { - query = query.offset(foundOffset) + query = query.limit(BASE_LIMIT) } return this.addFilters(query, filters, json.meta.table, { From bda83205ee8cd16846a42ca4683693d19375ae8d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 12:48:36 +0100 Subject: [PATCH 12/37] Making sure to measure whether we have paged forward in the query based on raw results. --- packages/server/src/sdk/app/rows/search/sqs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index ccad7d7b55..64dd240f1c 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -219,7 +219,7 @@ export async function search( // check for pagination final row let nextRow: Row | undefined - if (paginate && params.limit && processed.length > params.limit) { + if (paginate && params.limit && rows.length > params.limit) { // remove the extra row that confirmed if there is another row to move to nextRow = processed.pop() } From 5c453707226253aec47d60153113517bb05fb301 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 13:39:10 +0100 Subject: [PATCH 13/37] Fixing an issue with the sort order not being deterministic consistently. --- packages/backend-core/src/sql/sql.ts | 8 +++--- .../scripts/integrations/postgres/init.sql | 27 +++++++++++++++++-- 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 0f7fedf6c7..cd1bd6167c 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -411,8 +411,9 @@ class InternalBuilder { } addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { - let { sort, paginate } = json + let { sort } = json const table = json.meta.table + const mainPrimaryKey = table.primary![0] const tableName = getTableName(table) const aliases = json.tableAliases const aliased = @@ -429,10 +430,9 @@ class InternalBuilder { query = query.orderBy(`${aliased}.${key}`, direction, nulls) } - } else if (this.client === SqlClient.MS_SQL && paginate?.limit) { - // @ts-ignore - query = query.orderBy(`${aliased}.${table?.primary[0]}`) } + // always add sorting by the primary key - make sure result is deterministic + query = query.orderBy(`${aliased}.${mainPrimaryKey}`) return query } diff --git a/packages/server/scripts/integrations/postgres/init.sql b/packages/server/scripts/integrations/postgres/init.sql index b7ce1b7d5b..9624208deb 100644 --- a/packages/server/scripts/integrations/postgres/init.sql +++ b/packages/server/scripts/integrations/postgres/init.sql @@ -54,8 +54,31 @@ INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('Mi INSERT INTO Persons (FirstName, LastName, Address, City, Type, Year) VALUES ('John', 'Smith', '64 Updown Road', 'Dublin', 'programmer', 1996); INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Foo', 'Bar', 'Foo Street', 'Bartown', 'support', 0, 1993); INSERT INTO Persons (FirstName, LastName, Address, City, Type) VALUES ('Jonny', 'Muffin', 'Muffin Street', 'Cork', 'support'); -INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (1, 2, 'assembling', TRUE); -INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) VALUES (2, 1, 'processing', FALSE); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Dave', 'Bar', '2 Foo Street', 'Bartown', 'support', 0, 1993); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('James', 'Bar', '3 Foo Street', 'Bartown', 'support', 0, 1993); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Jenny', 'Bar', '4 Foo Street', 'Bartown', 'support', 0, 1993); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Grace', 'Bar', '5 Foo Street', 'Bartown', 'support', 0, 1993); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Sarah', 'Bar', '6 Foo Street', 'Bartown', 'support', 0, 1993); +INSERT INTO Persons (FirstName, LastName, Address, City, Type, Age, Year) VALUES ('Kelly', 'Bar', '7 Foo Street', 'Bartown', 'support', 0, 1993); + +-- insert a lot of tasks for testing +WITH RECURSIVE generate_series AS ( + SELECT 1 AS n + UNION ALL + SELECT n + 1 FROM generate_series WHERE n < 6000 +), +random_data AS ( + SELECT + n, + (random() * 9 + 1)::int AS ExecutorID, + (random() * 9 + 1)::int AS QaID, + 'assembling' AS TaskName, + (random() < 0.5) AS Completed + FROM generate_series +) +INSERT INTO Tasks (ExecutorID, QaID, TaskName, Completed) +SELECT ExecutorID, QaID, TaskName, Completed +FROM random_data; INSERT INTO Products (ProductName) VALUES ('Computers'); INSERT INTO Products (ProductName) VALUES ('Laptops'); INSERT INTO Products (ProductName) VALUES ('Chairs'); From 0caff1a404c28221b9a6a7dea64201e8b51c2c89 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 13:53:51 +0100 Subject: [PATCH 14/37] Fixing an issue with sorting in SQS. --- packages/server/src/sdk/app/rows/search/sqs.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 64dd240f1c..684e4bd9a6 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -95,6 +95,8 @@ function buildTableMap(tables: Table[]) { // update the table name, should never query by name for SQLite table.originalName = table.name table.name = table._id! + // need a primary for sorting, lookups etc + table.primary = ["_id"] tableMap[table._id!] = table } return tableMap @@ -153,6 +155,10 @@ export async function search( const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) + // make sure we have the mapped/latest table + if (table?._id) { + table = allTablesMap[table?._id] + } if (!table) { throw new Error("Unable to find table") } From e0d8a66fd812c525f067c346286028a10a1357b1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 18:44:17 +0100 Subject: [PATCH 15/37] Adding test cases for counting as well as some small fixes. --- packages/backend-core/src/sql/sql.ts | 13 ++-- .../src/api/routes/tests/search.spec.ts | 76 +++++++++++++++++++ .../server/src/sdk/app/rows/search/sqs.ts | 34 ++++----- 3 files changed, 99 insertions(+), 24 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index cd1bd6167c..1ed8666e21 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -12,21 +12,21 @@ import { BBReferenceFieldMetadata, FieldSchema, FieldType, + INTERNAL_TABLE_SOURCE_ID, JsonFieldMetadata, + JsonTypes, Operation, + prefixed, QueryJson, + QueryOptions, RelationshipsJson, SearchFilters, + SortOrder, SqlClient, SqlQuery, SqlQueryBinding, Table, TableSourceType, - INTERNAL_TABLE_SOURCE_ID, - QueryOptions, - JsonTypes, - prefixed, - SortOrder, } from "@budibase/types" import environment from "../environment" import { helpers } from "@budibase/shared-core" @@ -824,6 +824,9 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { await this.getReturningRow(queryFn, this.checkLookupKeys(id, json)) ) } + if (operation === Operation.COUNT) { + return results + } if (operation !== Operation.READ) { return row } diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index f651908c01..534c08c7c9 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -18,6 +18,7 @@ import { User, Row, RelationshipType, + SearchResponse, } from "@budibase/types" import _ from "lodash" import tk from "timekeeper" @@ -105,6 +106,17 @@ describe.each([ } } + private async performSearchFullResponse(): Promise> { + if (isInMemory) { + return { rows: dataFilters.search(_.cloneDeep(rows), this.query) } + } else { + return config.api.row.search(table._id!, { + ...this.query, + tableId: table._id!, + }) + } + } + // We originally used _.isMatch to compare rows, but found that when // comparing arrays it would return true if the source array was a subset of // the target array. This would sometimes create false matches. This @@ -205,6 +217,34 @@ describe.each([ ) } + // Asserts that the query returns some property values - this cannot be used + // to check row values, however this shouldn't be important for checking properties + async toHaveProperty( + properties: { + key: keyof SearchResponse + value?: any + }[] + ) { + const response = await this.performSearchFullResponse() + for (let property of properties) { + // eslint-disable-next-line jest/no-standalone-expect + expect(response[property.key]).toBeDefined() + if (property.value) { + // eslint-disable-next-line jest/no-standalone-expect + expect(response[property.key]).toEqual(property.value) + } + } + } + + // Asserts that the query doesn't return a property, e.g. pagination parameters. + async toNotHaveProperty(properties: (keyof SearchResponse)[]) { + const response = await this.performSearchFullResponse() + for (let property of properties) { + // eslint-disable-next-line jest/no-standalone-expect + expect(response[property]).toBeUndefined() + } + } + // Asserts that the query returns rows matching the set of rows passed in. // The order of the rows is not important. Extra rows will not cause the // assertion to fail. @@ -1797,4 +1837,40 @@ describe.each([ { two: "foo", other: [{ _id: otherRows[0]._id }] }, ])) }) + + // lucene can't count, and in memory there is no point + !isLucene && + !isInMemory && + describe("row counting", () => { + beforeAll(async () => { + table = await createTable({ + name: { + name: "name", + type: FieldType.STRING, + }, + }) + await createRows([{ name: "a" }, { name: "b" }]) + }) + + it("should be able to count rows when option set", () => + expectSearch({ + countRows: true, + query: { + notEmpty: { + name: true, + }, + }, + }).toHaveProperty([{ key: "totalRows", value: 2 }, { key: "rows" }])) + + it("shouldn't count rows when option is not set", () => { + expectSearch({ + countRows: false, + query: { + notEmpty: { + name: true, + }, + }, + }).toNotHaveProperty(["totalRows"]) + }) + }) }) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 684e4bd9a6..be365c9ed6 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -230,10 +230,12 @@ export async function search( nextRow = processed.pop() } - let rowCount: number | undefined + let totalRows: number | undefined if (options.countRows) { // get the total count of rows - rowCount = await runSqlQuery(request, allTables, { countTotalRows: true }) + totalRows = await runSqlQuery(request, allTables, { + countTotalRows: true, + }) } // get the rows @@ -248,24 +250,18 @@ export async function search( finalRows = finalRows.map((r: any) => pick(r, fields)) } - // check for pagination - if (paginate) { - const response: SearchResponse = { - rows: finalRows, - } - if (nextRow) { - response.hasNextPage = true - response.bookmark = bookmark + 1 - } - if (rowCount != null) { - response.totalRows = rowCount - } - return response - } else { - return { - rows: finalRows, - } + const response: SearchResponse = { + rows: finalRows, } + if (totalRows) { + response.totalRows = totalRows + } + // check for pagination + if (paginate && nextRow) { + response.hasNextPage = true + response.bookmark = bookmark + 1 + } + return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message if (err.status === 404 && msg?.includes(SQLITE_DESIGN_DOC_ID)) { From cf75a8a1f367e585bc44191415bb45070e44b0ab Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 18 Jun 2024 18:49:59 +0100 Subject: [PATCH 16/37] Updating function name. --- packages/server/src/api/controllers/row/ExternalRequest.ts | 4 ++-- packages/server/src/api/controllers/row/utils/sqlUtils.ts | 2 +- packages/server/src/api/controllers/row/utils/utils.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 4e14ce2799..89ad6c617a 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -30,7 +30,7 @@ import { buildExternalRelationships, buildSqlFieldList, generateIdForRow, - isKnexNoRowReadResponse, + isKnexEmptyReadResponse, isManyToMany, sqlOutputProcessing, } from "./utils" @@ -434,7 +434,7 @@ export class ExternalRequest { }) // this is the response from knex if no rows found const rows: Row[] = - !Array.isArray(response) || isKnexNoRowReadResponse(response) + !Array.isArray(response) || isKnexEmptyReadResponse(response) ? [] : response const storeTo = isManyToMany(field) diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index b236578485..6f7bdc7335 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -195,7 +195,7 @@ export function buildSqlFieldList( return fields } -export function isKnexNoRowReadResponse(resp: DatasourcePlusQueryResponse) { +export function isKnexEmptyReadResponse(resp: DatasourcePlusQueryResponse) { return ( !Array.isArray(resp) || resp.length === 0 || diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index a607a01f16..ae34034221 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -14,7 +14,7 @@ import { processDates, processFormulas, } from "../../../../utilities/rowProcessor" -import { isKnexNoRowReadResponse, updateRelationshipColumns } from "./sqlUtils" +import { isKnexEmptyReadResponse, updateRelationshipColumns } from "./sqlUtils" import { basicProcessing, generateIdForRow, @@ -137,7 +137,7 @@ export async function sqlOutputProcessing( relationships: RelationshipsJson[], opts?: { sqs?: boolean } ): Promise { - if (isKnexNoRowReadResponse(rows)) { + if (isKnexEmptyReadResponse(rows)) { return [] } let finalRows: { [key: string]: Row } = {} From 0de94d3535df82413c2f3442731ba24624ec93eb Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 11:52:50 +0100 Subject: [PATCH 17/37] Addressing some PR comments. --- packages/backend-core/src/sql/sql.ts | 5 ++--- packages/server/src/sdk/app/rows/search/sqs.ts | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 1ed8666e21..956842e3ee 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -533,13 +533,12 @@ class InternalBuilder { const tableName = endpoint.entityId const tableAlias = aliases?.[tableName] - const query = knex( + return knex( this.tableNameWithSchema(tableName, { alias: tableAlias, schema: endpoint.schema, }) ) - return query } create(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { @@ -643,7 +642,7 @@ class InternalBuilder { tableAliases ) - // add a base query over all + // add a base limit over the whole query if (!counting) { query = query.limit(BASE_LIMIT) } diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index be365c9ed6..525d414887 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -106,7 +106,7 @@ function runSqlQuery(json: QueryJson, tables: Table[]): Promise function runSqlQuery( json: QueryJson, tables: Table[], - opts: { countTotalRows: boolean } + opts: { countTotalRows: true } ): Promise async function runSqlQuery( json: QueryJson, From abfab054d7872b1415c4269f3051a33b7ca31282 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 12:03:20 +0100 Subject: [PATCH 18/37] Addressing comment about datasource being optional. --- .../src/api/controllers/table/ExternalRequest.ts | 2 +- packages/server/src/integrations/base/query.ts | 4 ++-- packages/server/src/sdk/app/rows/sqlAlias.ts | 10 ++++++---- packages/server/src/sdk/app/rows/utils.ts | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/packages/server/src/api/controllers/table/ExternalRequest.ts b/packages/server/src/api/controllers/table/ExternalRequest.ts index 9661e56729..1e57ea3294 100644 --- a/packages/server/src/api/controllers/table/ExternalRequest.ts +++ b/packages/server/src/api/controllers/table/ExternalRequest.ts @@ -33,5 +33,5 @@ export async function makeTableRequest( if (renamed) { json.meta!.renamed = renamed } - return makeExternalQuery(json, datasource) + return makeExternalQuery(datasource, json) } diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index acef1c6f1e..55886cd20f 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -8,8 +8,8 @@ import { getIntegration } from "../index" import sdk from "../../sdk" export async function makeExternalQuery( - json: QueryJson, - datasource?: Datasource + datasource: Datasource, + json: QueryJson ): Promise { const entityId = json.endpoint.entityId, tableName = json.meta.table.name, diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index ab47f98a85..0eab2f68c8 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -11,10 +11,11 @@ import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core" import { getSQLClient } from "./utils" import { cloneDeep } from "lodash" import datasources from "../datasources" +import { BudibaseInternalDB } from "../../../db/utils" type PerformQueryFunction = ( - json: QueryJson, - datasource?: Datasource + datasource: Datasource, + json: QueryJson ) => Promise const WRITE_OPERATIONS: Operation[] = [ @@ -179,9 +180,10 @@ export default class AliasTables { ): Promise { const datasourceId = json.endpoint.datasourceId const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL - let aliasingEnabled: boolean, datasource: Datasource | undefined + let aliasingEnabled: boolean, datasource: Datasource if (isSqs) { aliasingEnabled = this.isAliasingEnabled(json) + datasource = BudibaseInternalDB } else { datasource = await datasources.get(datasourceId) aliasingEnabled = this.isAliasingEnabled(json, datasource) @@ -233,7 +235,7 @@ export default class AliasTables { json.tableAliases = invertedTableAliases } - let response: DatasourcePlusQueryResponse = await queryFn(json, datasource) + let response: DatasourcePlusQueryResponse = await queryFn(datasource, json) if (Array.isArray(response) && aliasingEnabled) { return this.reverse(response) } else { diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index da0132c8fa..bb37fd99f3 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -61,7 +61,7 @@ export async function getDatasourceAndQuery( table, } } - return makeExternalQuery(json, datasource) + return makeExternalQuery(datasource, json) } export function cleanExportRows( From 0e5de7f16de29e3f165647186a53b06c803ebef5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 13:36:34 +0100 Subject: [PATCH 19/37] Test updates. --- .../src/api/routes/tests/search.spec.ts | 46 ++++++------------- .../src/sdk/app/rows/search/external.ts | 10 ++-- .../server/src/sdk/app/rows/search/sqs.ts | 3 +- 3 files changed, 24 insertions(+), 35 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 534c08c7c9..7b7eb5000f 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -93,20 +93,7 @@ describe.each([ class SearchAssertion { constructor(private readonly query: RowSearchParams) {} - private async performSearch(): Promise { - if (isInMemory) { - return dataFilters.search(_.cloneDeep(rows), this.query) - } else { - return ( - await config.api.row.search(table._id!, { - ...this.query, - tableId: table._id!, - }) - ).rows - } - } - - private async performSearchFullResponse(): Promise> { + private async performSearch(): Promise> { if (isInMemory) { return { rows: dataFilters.search(_.cloneDeep(rows), this.query) } } else { @@ -187,7 +174,7 @@ describe.each([ // different to the one passed in will cause the assertion to fail. Extra // rows returned by the query will also cause the assertion to fail. async toMatchExactly(expectedRows: any[]) { - const foundRows = await this.performSearch() + const { rows: foundRows } = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(expectedRows.length) @@ -203,7 +190,7 @@ describe.each([ // passed in. The order of the rows is not important, but extra rows will // cause the assertion to fail. async toContainExactly(expectedRows: any[]) { - const foundRows = await this.performSearch() + const { rows: foundRows } = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(expectedRows.length) @@ -219,26 +206,23 @@ describe.each([ // Asserts that the query returns some property values - this cannot be used // to check row values, however this shouldn't be important for checking properties - async toHaveProperty( - properties: { - key: keyof SearchResponse - value?: any - }[] - ) { - const response = await this.performSearchFullResponse() - for (let property of properties) { + // typing for this has to be any, Jest doesn't expose types for matchers like expect.any(...) + async toMatch(properties: Record) { + const response = await this.performSearch() + const keys = Object.keys(properties) as Array> + for (let key of keys) { // eslint-disable-next-line jest/no-standalone-expect - expect(response[property.key]).toBeDefined() - if (property.value) { + expect(response[key]).toBeDefined() + if (properties[key]) { // eslint-disable-next-line jest/no-standalone-expect - expect(response[property.key]).toEqual(property.value) + expect(response[key]).toEqual(properties[key]) } } } // Asserts that the query doesn't return a property, e.g. pagination parameters. async toNotHaveProperty(properties: (keyof SearchResponse)[]) { - const response = await this.performSearchFullResponse() + const response = await this.performSearch() for (let property of properties) { // eslint-disable-next-line jest/no-standalone-expect expect(response[property]).toBeUndefined() @@ -249,7 +233,7 @@ describe.each([ // The order of the rows is not important. Extra rows will not cause the // assertion to fail. async toContain(expectedRows: any[]) { - const foundRows = await this.performSearch() + const { rows: foundRows } = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect([...foundRows]).toEqual( @@ -266,7 +250,7 @@ describe.each([ } async toHaveLength(length: number) { - const foundRows = await this.performSearch() + const { rows: foundRows } = await this.performSearch() // eslint-disable-next-line jest/no-standalone-expect expect(foundRows).toHaveLength(length) @@ -1860,7 +1844,7 @@ describe.each([ name: true, }, }, - }).toHaveProperty([{ key: "totalRows", value: 2 }, { key: "rows" }])) + }).toMatch({ totalRows: 2, rows: expect.any(Array) })) it("shouldn't count rows when option is not set", () => { expectSearch({ diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 5e2074f56d..6c2e16dcc4 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -81,11 +81,15 @@ export async function search( paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } - let rows = await handleRequest(Operation.READ, tableId, parameters) - let totalRows: number | undefined + const requests: Promise[] = [] + requests.push(handleRequest(Operation.READ, tableId, parameters)) if (countRows) { - totalRows = await handleRequest(Operation.COUNT, tableId, parameters) + requests.push(handleRequest(Operation.COUNT, tableId, parameters)) } + const responses = await Promise.all(requests) + let rows = responses[0] as Row[] + const totalRows = responses[1] ? (responses[1] as number) : undefined + let hasNextPage = false // remove the extra row if it's there if (paginate && limit && rows.length > limit) { diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 525d414887..a93ae174b0 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -12,6 +12,7 @@ import { SortType, SqlClient, Table, + Datasource, } from "@budibase/types" import { buildInternalRelationships, @@ -117,7 +118,7 @@ async function runSqlQuery( if (opts?.countTotalRows) { json.endpoint.operation = Operation.COUNT } - const processSQLQuery = async (json: QueryJson) => { + const processSQLQuery = async (_: Datasource, json: QueryJson) => { const query = builder._query(json, { disableReturning: true, }) From bc80841554cb666c71b3213c75713ce631fc034a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 13:39:00 +0100 Subject: [PATCH 20/37] Promise.all for both counts (SQS and SQL). --- .../src/sdk/app/rows/search/external.ts | 8 +++---- .../server/src/sdk/app/rows/search/sqs.ts | 22 +++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 6c2e16dcc4..b7e9c7c80a 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -81,12 +81,12 @@ export async function search( paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } - const requests: Promise[] = [] - requests.push(handleRequest(Operation.READ, tableId, parameters)) + const queries: Promise[] = [] + queries.push(handleRequest(Operation.READ, tableId, parameters)) if (countRows) { - requests.push(handleRequest(Operation.COUNT, tableId, parameters)) + queries.push(handleRequest(Operation.COUNT, tableId, parameters)) } - const responses = await Promise.all(requests) + const responses = await Promise.all(queries) let rows = responses[0] as Row[] const totalRows = responses[1] ? (responses[1] as number) : undefined diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index a93ae174b0..5b71b3c6f3 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -213,7 +213,19 @@ export async function search( } try { - const rows = await runSqlQuery(request, allTables) + const queries: Promise[] = [] + queries.push(runSqlQuery(request, allTables)) + if (options.countRows) { + // get the total count of rows + queries.push( + runSqlQuery(request, allTables, { + countTotalRows: true, + }) + ) + } + const responses = await Promise.all(queries) + let rows = responses[0] as Row[] + const totalRows = responses[1] ? (responses[1] as number) : undefined // process from the format of tableId.column to expected format also // make sure JSON columns corrected @@ -231,14 +243,6 @@ export async function search( nextRow = processed.pop() } - let totalRows: number | undefined - if (options.countRows) { - // get the total count of rows - totalRows = await runSqlQuery(request, allTables, { - countTotalRows: true, - }) - } - // get the rows let finalRows = await outputProcessing(table, processed, { preserveLinks: true, From bdbb4c0a668391000d192a958b29e623f2430060 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 13:59:03 +0100 Subject: [PATCH 21/37] Commenting on a bug in knex --- packages/backend-core/src/sql/sql.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 956842e3ee..04f16ea565 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -621,9 +621,13 @@ class InternalBuilder { query = this.addSorting(query, json) } const alias = tableAliases?.[tableName] || tableName - let preQuery = knex({ - [alias]: query, - } as any) + let preQuery: Knex.QueryBuilder = knex({ + // the typescript definition for the knex constructor doesn't support this + // syntax, but it is the only way to alias a pre-query result as part of + // a query - there is an alias dictionary type, but it assumes it can only + // be a table name, not a pre-query + [alias]: query as any, + }) if (counting) { preQuery = preQuery.count("* as total") } else { From aab100b130f54c23a202bdcdf99ed5661ab112ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 14:28:22 +0100 Subject: [PATCH 22/37] Changing how counting is processed. --- .../api/controllers/row/ExternalRequest.ts | 19 +++++++++---------- .../server/src/sdk/app/rows/search/sqs.ts | 6 ++++-- packages/server/src/sdk/app/rows/sqlAlias.ts | 15 --------------- packages/server/src/sdk/app/rows/utils.ts | 11 +++++++++++ 4 files changed, 24 insertions(+), 27 deletions(-) diff --git a/packages/server/src/api/controllers/row/ExternalRequest.ts b/packages/server/src/api/controllers/row/ExternalRequest.ts index 89ad6c617a..619a1e9548 100644 --- a/packages/server/src/api/controllers/row/ExternalRequest.ts +++ b/packages/server/src/api/controllers/row/ExternalRequest.ts @@ -34,7 +34,10 @@ import { isManyToMany, sqlOutputProcessing, } from "./utils" -import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils" +import { + getDatasourceAndQuery, + processRowCountResponse, +} from "../../../sdk/app/rows/utils" import { processObjectSync } from "@budibase/string-templates" import { cloneDeep } from "lodash/fp" import { db as dbCore } from "@budibase/backend-core" @@ -670,18 +673,14 @@ export class ExternalRequest { } // aliasing can be disabled fully if desired - let response const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables)) + let response = env.SQL_ALIASING_DISABLE + ? await getDatasourceAndQuery(json) + : await aliasing.queryWithAliasing(json, makeExternalQuery) + // if it's a counting operation there will be no more processing, just return the number if (this.operation === Operation.COUNT) { - return (await aliasing.countWithAliasing( - json, - makeExternalQuery - )) as ExternalRequestReturnType - } else { - response = env.SQL_ALIASING_DISABLE - ? await getDatasourceAndQuery(json) - : await aliasing.queryWithAliasing(json, makeExternalQuery) + return processRowCountResponse(response) as ExternalRequestReturnType } const responseRows = Array.isArray(response) ? response : [] diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 5b71b3c6f3..5d5fba820d 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -29,6 +29,7 @@ import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" import AliasTables from "../sqlAlias" import { outputProcessing } from "../../../../utilities/rowProcessor" import pick from "lodash/pick" +import { processRowCountResponse } from "../utils" const builder = new sql.Sql(SqlClient.SQL_LITE) @@ -141,10 +142,11 @@ async function runSqlQuery( const db = context.getAppDB() return await db.sql(sql, bindings) } + const response = await alias.queryWithAliasing(json, processSQLQuery) if (opts?.countTotalRows) { - return await alias.countWithAliasing(json, processSQLQuery) + return processRowCountResponse(response) } else { - return await alias.queryWithAliasing(json, processSQLQuery) + return response } } diff --git a/packages/server/src/sdk/app/rows/sqlAlias.ts b/packages/server/src/sdk/app/rows/sqlAlias.ts index 0eab2f68c8..bc8fc56d5e 100644 --- a/packages/server/src/sdk/app/rows/sqlAlias.ts +++ b/packages/server/src/sdk/app/rows/sqlAlias.ts @@ -242,19 +242,4 @@ export default class AliasTables { return response } } - - // handles getting the count out of the query - async countWithAliasing( - json: QueryJson, - queryFn: PerformQueryFunction - ): Promise { - json.endpoint.operation = Operation.COUNT - let response = await this.queryWithAliasing(json, queryFn) - if (response && response.length === 1 && "total" in response[0]) { - const total = response[0].total - return typeof total === "number" ? total : parseInt(total) - } else { - throw new Error("Unable to count rows in query - no count response") - } - } } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index bb37fd99f3..cd1b663f6a 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -50,6 +50,17 @@ export function getSQLClient(datasource: Datasource): SqlClient { throw new Error("Unable to determine client for SQL datasource") } +export function processRowCountResponse( + response: DatasourcePlusQueryResponse +): number { + if (response && response.length === 1 && "total" in response[0]) { + const total = response[0].total + return typeof total === "number" ? total : parseInt(total) + } else { + throw new Error("Unable to count rows in query - no count response") + } +} + export async function getDatasourceAndQuery( json: QueryJson ): Promise { From 1056efdbf6fa6ed91570bb7d3e36b778b295886d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 14:56:06 +0100 Subject: [PATCH 23/37] Changing how counting occurs in SQL layer. --- packages/backend-core/src/sql/sql.ts | 47 +++++++++++++++++++--------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 8f45595988..a2ca837fde 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -595,19 +595,23 @@ class InternalBuilder { return query.upsert(parsedBody) } - read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder { + read( + knex: Knex, + json: QueryJson, + limits?: { base?: number; query?: number } + ): Knex.QueryBuilder { let { endpoint, resource, filters, paginate, relationships, tableAliases } = json - const counting = endpoint.operation === Operation.COUNT const tableName = endpoint.entityId + const counting = endpoint.operation === Operation.COUNT // select all if not specified if (!resource) { resource = { fields: [] } } let selectStatement: string | (string | Knex.Raw)[] = "*" // handle select - if (!counting && resource.fields && resource.fields.length > 0) { + if (resource.fields && resource.fields.length > 0) { // select the resources as the format "table.columnName" - this is what is provided // by the resource builder further up selectStatement = generateSelectStatement(json, knex) @@ -616,7 +620,7 @@ class InternalBuilder { let query = this.knexWithAlias(knex, endpoint, tableAliases) // handle pagination let foundOffset: number | null = null - let foundLimit = limit || BASE_LIMIT + let foundLimit = limits?.query || limits?.base if (paginate && paginate.page && paginate.limit) { // @ts-ignore const page = paginate.page <= 1 ? 0 : paginate.page - 1 @@ -629,12 +633,12 @@ class InternalBuilder { } else if (paginate && paginate.limit) { foundLimit = paginate.limit } - // always add the found limit, unless counting - if (!counting) { + // add the found limit if supplied + if (foundLimit) { query = query.limit(foundLimit) } // add overall pagination - if (!counting && foundOffset) { + if (foundOffset) { query = query.offset(foundOffset) } // add filters to the query (where) @@ -642,9 +646,11 @@ class InternalBuilder { aliases: tableAliases, }) // add sorting to pre-query + // no point in sorting when counting if (!counting) { query = this.addSorting(query, json) } + const alias = tableAliases?.[tableName] || tableName let preQuery: Knex.QueryBuilder = knex({ // the typescript definition for the knex constructor doesn't support this @@ -653,11 +659,7 @@ class InternalBuilder { // be a table name, not a pre-query [alias]: query as any, }) - if (counting) { - preQuery = preQuery.count("* as total") - } else { - preQuery = preQuery.select(selectStatement) - } + preQuery = preQuery.select(selectStatement) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { preQuery = this.addSorting(preQuery, json) @@ -672,8 +674,9 @@ class InternalBuilder { ) // add a base limit over the whole query - if (!counting) { - query = query.limit(BASE_LIMIT) + // if counting we can't set this limit + if (limits?.base) { + query = query.limit(limits.base) } return this.addFilters(query, filters, json.meta.table, { @@ -682,6 +685,15 @@ class InternalBuilder { }) } + count(knex: Knex, json: QueryJson) { + const readQuery = this.read(knex, json) + // have to alias the sub-query, this is a requirement for my-sql and ms-sql + // without this we get an error "Every derived table must have its own alias" + return knex({ + subquery: readQuery as any, + }).count("* as total") + } + update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) @@ -756,8 +768,13 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { query = builder.create(client, json, opts) break case Operation.READ: + query = builder.read(client, json, { + query: this.limit, + base: BASE_LIMIT, + }) + break case Operation.COUNT: - query = builder.read(client, json, this.limit) + query = builder.count(client, json) break case Operation.UPDATE: query = builder.update(client, json, opts) From 2d74927177f94b51316765f93942f185c020d3b8 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 15:08:12 +0100 Subject: [PATCH 24/37] updating how counting disables sorting. --- packages/backend-core/src/sql/sql.ts | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a2ca837fde..a93f1276bb 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -598,13 +598,16 @@ class InternalBuilder { read( knex: Knex, json: QueryJson, - limits?: { base?: number; query?: number } + opts: { + limits?: { base: number; query: number } + disableSorting?: boolean + } = {} ): Knex.QueryBuilder { let { endpoint, resource, filters, paginate, relationships, tableAliases } = json + const { limits, disableSorting } = opts const tableName = endpoint.entityId - const counting = endpoint.operation === Operation.COUNT // select all if not specified if (!resource) { resource = { fields: [] } @@ -647,7 +650,7 @@ class InternalBuilder { }) // add sorting to pre-query // no point in sorting when counting - if (!counting) { + if (!disableSorting) { query = this.addSorting(query, json) } @@ -661,7 +664,7 @@ class InternalBuilder { }) preQuery = preQuery.select(selectStatement) // have to add after as well (this breaks MS-SQL) - if (this.client !== SqlClient.MS_SQL && !counting) { + if (this.client !== SqlClient.MS_SQL && !disableSorting) { preQuery = this.addSorting(preQuery, json) } // handle joins @@ -686,7 +689,9 @@ class InternalBuilder { } count(knex: Knex, json: QueryJson) { - const readQuery = this.read(knex, json) + const readQuery = this.read(knex, json, { + disableSorting: true, + }) // have to alias the sub-query, this is a requirement for my-sql and ms-sql // without this we get an error "Every derived table must have its own alias" return knex({ @@ -769,8 +774,10 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { break case Operation.READ: query = builder.read(client, json, { - query: this.limit, - base: BASE_LIMIT, + limits: { + query: this.limit, + base: BASE_LIMIT, + }, }) break case Operation.COUNT: From a97b24658fdefa0d1e42bb9a54fc6402c1aee21f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 15:08:22 +0100 Subject: [PATCH 25/37] Fixing test case, it didn't provide a primary field. --- packages/server/src/api/routes/tests/queries/generic-sql.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index b060a099d8..e72a091688 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -734,6 +734,7 @@ describe.each( name: entityId, schema: {}, type: "table", + primary: ["id"], sourceId: datasource._id!, sourceType: TableSourceType.EXTERNAL, }, From 2aa911b217119e78aa9b5db0e8a9d6953687f3c7 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 17:10:15 +0100 Subject: [PATCH 26/37] re-jigging things to get counting working properly again. --- packages/backend-core/src/sql/sql.ts | 66 ++++++++++++++-------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index a93f1276bb..4a8ba49d9e 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -114,7 +114,7 @@ function generateSelectStatement( ): (string | Knex.Raw)[] | "*" { const { resource, meta } = json - if (!resource) { + if (!resource || !resource.fields || resource.fields.length === 0) { return "*" } @@ -410,14 +410,32 @@ class InternalBuilder { return query } + addDistinctCount( + query: Knex.QueryBuilder, + json: QueryJson + ): Knex.QueryBuilder { + const table = json.meta.table + const primary = table.primary + const aliases = json.tableAliases + const aliased = + table.name && aliases?.[table.name] ? aliases[table.name] : table.name + if (!primary) { + throw new Error("SQL counting requires primary key to be supplied") + } + return query.countDistinct(`${aliased}.${primary[0]} as total`) + } + addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort } = json const table = json.meta.table - const mainPrimaryKey = table.primary![0] + const primaryKey = table.primary const tableName = getTableName(table) const aliases = json.tableAliases const aliased = tableName && aliases?.[tableName] ? aliases[tableName] : table?.name + if (!Array.isArray(primaryKey)) { + throw new Error("Sorting requires primary key to be specified for table") + } if (sort && Object.keys(sort || {}).length > 0) { for (let [key, value] of Object.entries(sort)) { const direction = @@ -432,7 +450,7 @@ class InternalBuilder { } } // always add sorting by the primary key - make sure result is deterministic - query = query.orderBy(`${aliased}.${mainPrimaryKey}`) + query = query.orderBy(`${aliased}.${primaryKey[0]}`) return query } @@ -600,25 +618,13 @@ class InternalBuilder { json: QueryJson, opts: { limits?: { base: number; query: number } - disableSorting?: boolean } = {} ): Knex.QueryBuilder { - let { endpoint, resource, filters, paginate, relationships, tableAliases } = - json - const { limits, disableSorting } = opts + let { endpoint, filters, paginate, relationships, tableAliases } = json + const { limits } = opts + const counting = endpoint.operation === Operation.COUNT const tableName = endpoint.entityId - // select all if not specified - if (!resource) { - resource = { fields: [] } - } - let selectStatement: string | (string | Knex.Raw)[] = "*" - // handle select - if (resource.fields && resource.fields.length > 0) { - // select the resources as the format "table.columnName" - this is what is provided - // by the resource builder further up - selectStatement = generateSelectStatement(json, knex) - } // start building the query let query = this.knexWithAlias(knex, endpoint, tableAliases) // handle pagination @@ -650,7 +656,7 @@ class InternalBuilder { }) // add sorting to pre-query // no point in sorting when counting - if (!disableSorting) { + if (!counting) { query = this.addSorting(query, json) } @@ -662,9 +668,13 @@ class InternalBuilder { // be a table name, not a pre-query [alias]: query as any, }) - preQuery = preQuery.select(selectStatement) + if (!counting) { + preQuery = preQuery.select(generateSelectStatement(json, knex)) + } else { + preQuery = this.addDistinctCount(preQuery, json) + } // have to add after as well (this breaks MS-SQL) - if (this.client !== SqlClient.MS_SQL && !disableSorting) { + if (this.client !== SqlClient.MS_SQL && !counting) { preQuery = this.addSorting(preQuery, json) } // handle joins @@ -688,17 +698,6 @@ class InternalBuilder { }) } - count(knex: Knex, json: QueryJson) { - const readQuery = this.read(knex, json, { - disableSorting: true, - }) - // have to alias the sub-query, this is a requirement for my-sql and ms-sql - // without this we get an error "Every derived table must have its own alias" - return knex({ - subquery: readQuery as any, - }).count("* as total") - } - update(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) @@ -781,7 +780,8 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { }) break case Operation.COUNT: - query = builder.count(client, json) + // read without any limits to count + query = builder.read(client, json) break case Operation.UPDATE: query = builder.update(client, json, opts) From d121633d8e383f23927b2a55161cfca948a57602 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 17:29:33 +0100 Subject: [PATCH 27/37] Updating queries to be a bit more flexible to updates in the SQL layers. --- .../src/integrations/tests/sqlAlias.spec.ts | 64 ++++++++----------- 1 file changed, 25 insertions(+), 39 deletions(-) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index 67f3d1d05d..0b433896bf 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -57,15 +57,14 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) expect(query).toEqual({ bindings: [relationshipLimit, limit], - sql: multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", + sql: expect.stringContaining( + multiline(`select "a"."year" as "a.year", "a"."firstname" as "a.firstname", "a"."personid" as "a.personid", "a"."address" as "a.address", "a"."age" as "a.age", "a"."type" as "a.type", "a"."city" as "a.city", "a"."lastname" as "a.lastname", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", "b"."completed" as "b.completed", "b"."qaid" as "b.qaid", "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", - "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" - from (select * from "persons" as "a" order by "a"."firstname" asc nulls first limit $1) as "a" - left join "tasks" as "b" on "a"."personid" = "b"."qaid" or "a"."personid" = "b"."executorid" - order by "a"."firstname" asc nulls first limit $2`), + "b"."completed" as "b.completed", "b"."qaid" as "b.qaid"`) + ), }) }) @@ -74,13 +73,10 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) expect(query).toEqual({ bindings: [relationshipLimit, "assembling", limit], - sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", - "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", - "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" - from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" - left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" where COALESCE("b"."taskname" = $2, FALSE) - order by "a"."productname" asc nulls first limit $3`), + sql: expect.stringContaining( + multiline(`where COALESCE("b"."taskname" = $2, FALSE) + order by "a"."productname" asc nulls first, "a"."productid" asc limit $3`) + ), }) }) @@ -89,13 +85,10 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) expect(query).toEqual({ bindings: [relationshipLimit, limit], - sql: multiline(`select "a"."productname" as "a.productname", "a"."productid" as "a.productid", - "b"."executorid" as "b.executorid", "b"."taskname" as "b.taskname", "b"."taskid" as "b.taskid", - "b"."completed" as "b.completed", "b"."qaid" as "b.qaid" - from (select * from "products" as "a" order by "a"."productname" asc nulls first limit $1) as "a" - left join "products_tasks" as "c" on "a"."productid" = "c"."productid" - left join "tasks" as "b" on "b"."taskid" = "c"."taskid" - order by "a"."productname" asc nulls first limit $2`), + sql: expect.stringContaining( + multiline(`left join "products_tasks" as "c" on "a"."productid" = "c"."productid" + left join "tasks" as "b" on "b"."taskid" = "c"."taskid" `) + ), }) }) @@ -106,11 +99,11 @@ describe("Captures of real examples", () => { expect(query).toEqual({ bindings: [...filters, limit, limit], sql: multiline(`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", - "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", - "b"."productname" as "b.productname", "b"."productid" as "b.productid" - from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) limit $3) as "a" - left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" - left join "products" as "b" on "b"."productid" = "c"."productid" limit $4`), + "a"."taskid" as "a.taskid", "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", + "b"."productname" as "b.productname", "b"."productid" as "b.productid" + from (select * from "tasks" as "a" where "a"."taskid" in ($1, $2) order by "a"."taskid" asc limit $3) as "a" + left join "products_tasks" as "c" on "a"."taskid" = "c"."taskid" + left join "products" as "b" on "b"."productid" = "c"."productid" order by "a"."taskid" asc limit $4`), }) }) @@ -132,19 +125,11 @@ describe("Captures of real examples", () => { equalValue, limit, ], - sql: multiline(`select "a"."executorid" as "a.executorid", "a"."taskname" as "a.taskname", "a"."taskid" as "a.taskid", - "a"."completed" as "a.completed", "a"."qaid" as "a.qaid", "b"."productname" as "b.productname", - "b"."productid" as "b.productid", "c"."year" as "c.year", "c"."firstname" as "c.firstname", - "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", - "c"."city" as "c.city", "c"."lastname" as "c.lastname", "c"."year" as "c.year", "c"."firstname" as "c.firstname", - "c"."personid" as "c.personid", "c"."address" as "c.address", "c"."age" as "c.age", "c"."type" as "c.type", - "c"."city" as "c.city", "c"."lastname" as "c.lastname" - from (select * from "tasks" as "a" where COALESCE("a"."completed" != $1, TRUE) - order by "a"."taskname" asc nulls first limit $2) as "a" - left join "products_tasks" as "d" on "a"."taskid" = "d"."taskid" - left join "products" as "b" on "b"."productid" = "d"."productid" - left join "persons" as "c" on "a"."executorid" = "c"."personid" or "a"."qaid" = "c"."personid" - where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE) order by "a"."taskname" asc nulls first limit $6`), + sql: expect.stringContaining( + multiline( + `where "c"."year" between $3 and $4 and COALESCE("b"."productname" = $5, FALSE)` + ) + ), }) }) }) @@ -200,8 +185,9 @@ describe("Captures of real examples", () => { returningQuery = input }, queryJson) expect(returningQuery).toEqual({ - sql: "select * from (select top (@p0) * from [people] where CASE WHEN [people].[name] = @p1 THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p2 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]", - bindings: [1, "Test", 22], + sql: multiline(`select top (@p0) * from (select top (@p1) * from [people] where CASE WHEN [people].[name] = @p2 + THEN 1 ELSE 0 END = 1 and CASE WHEN [people].[age] = @p3 THEN 1 ELSE 0 END = 1 order by [people].[name] asc) as [people]`), + bindings: [5000, 1, "Test", 22], }) }) }) From 58ec7a50b0c49ee1065b0f61d5fbaa409f9d2853 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 17:51:11 +0100 Subject: [PATCH 28/37] Implementing row counting for in-memory, also updating the in memory search function. --- packages/server/src/api/routes/tests/search.spec.ts | 5 ++--- packages/shared-core/src/filters.ts | 13 +++++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 7b7eb5000f..be66253090 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -95,7 +95,7 @@ describe.each([ private async performSearch(): Promise> { if (isInMemory) { - return { rows: dataFilters.search(_.cloneDeep(rows), this.query) } + return dataFilters.search(_.cloneDeep(rows), this.query) } else { return config.api.row.search(table._id!, { ...this.query, @@ -1822,9 +1822,8 @@ describe.each([ ])) }) - // lucene can't count, and in memory there is no point + // lucene can't count the total rows !isLucene && - !isInMemory && describe("row counting", () => { beforeAll(async () => { table = await createTable({ diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 52ab3ed626..bd75406e26 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -12,6 +12,7 @@ import { SortOrder, RowSearchParams, EmptyFilterOption, + SearchResponse, } from "@budibase/types" import dayjs from "dayjs" import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants" @@ -262,15 +263,23 @@ export const buildQuery = (filter: SearchFilter[]) => { return query } -export const search = (docs: Record[], query: RowSearchParams) => { +export const search = ( + docs: Record[], + query: RowSearchParams +): SearchResponse> => { let result = runQuery(docs, query.query) if (query.sort) { result = sort(result, query.sort, query.sortOrder || SortOrder.ASCENDING) } + let totalRows = result.length if (query.limit) { result = limit(result, query.limit.toString()) } - return result + const response: SearchResponse> = { rows: result } + if (query.countRows) { + response.totalRows = totalRows + } + return response } /** From 67c00c9e4cc1d31cc0fa932bea64c27f808ea19b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 18:46:48 +0100 Subject: [PATCH 29/37] Addressing PR comments. --- packages/backend-core/src/sql/sql.ts | 34 +++++++++---------- .../src/sdk/app/rows/search/external.ts | 6 +++- .../server/src/sdk/app/rows/search/sqs.ts | 8 +++-- 3 files changed, 28 insertions(+), 20 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 4a8ba49d9e..72ff8d4578 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -642,23 +642,24 @@ class InternalBuilder { } else if (paginate && paginate.limit) { foundLimit = paginate.limit } - // add the found limit if supplied - if (foundLimit) { - query = query.limit(foundLimit) - } - // add overall pagination - if (foundOffset) { - query = query.offset(foundOffset) + // counting should not sort, limit or offset + if (!counting) { + // add the found limit if supplied + if (foundLimit != null) { + query = query.limit(foundLimit) + } + // add overall pagination + if (foundOffset != null) { + query = query.offset(foundOffset) + } + // add sorting to pre-query + // no point in sorting when counting + query = this.addSorting(query, json) } // add filters to the query (where) query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) - // add sorting to pre-query - // no point in sorting when counting - if (!counting) { - query = this.addSorting(query, json) - } const alias = tableAliases?.[tableName] || tableName let preQuery: Knex.QueryBuilder = knex({ @@ -668,11 +669,10 @@ class InternalBuilder { // be a table name, not a pre-query [alias]: query as any, }) - if (!counting) { - preQuery = preQuery.select(generateSelectStatement(json, knex)) - } else { - preQuery = this.addDistinctCount(preQuery, json) - } + // if counting, use distinct count, else select + preQuery = !counting + ? preQuery.select(generateSelectStatement(json, knex)) + : this.addDistinctCount(preQuery, json) // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { preQuery = this.addSorting(preQuery, json) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index b7e9c7c80a..9fc3487f62 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -88,7 +88,8 @@ export async function search( } const responses = await Promise.all(queries) let rows = responses[0] as Row[] - const totalRows = responses[1] ? (responses[1] as number) : undefined + const totalRows = + responses.length > 1 ? (responses[1] as number) : undefined let hasNextPage = false // remove the extra row if it's there @@ -115,6 +116,9 @@ export async function search( if (totalRows != null) { response.totalRows = totalRows } + if (paginate && !hasNextPage) { + response.hasNextPage = false + } return response } catch (err: any) { if (err.message && err.message.includes("does not exist")) { diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 5d5fba820d..0ba2adb570 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -227,7 +227,8 @@ export async function search( } const responses = await Promise.all(queries) let rows = responses[0] as Row[] - const totalRows = responses[1] ? (responses[1] as number) : undefined + const totalRows = + responses.length > 1 ? (responses[1] as number) : undefined // process from the format of tableId.column to expected format also // make sure JSON columns corrected @@ -260,7 +261,7 @@ export async function search( const response: SearchResponse = { rows: finalRows, } - if (totalRows) { + if (totalRows != null) { response.totalRows = totalRows } // check for pagination @@ -268,6 +269,9 @@ export async function search( response.hasNextPage = true response.bookmark = bookmark + 1 } + if (paginate && !nextRow) { + response.hasNextPage = false + } return response } catch (err: any) { const msg = typeof err === "string" ? err : err.message From 86d9de5a2dd5511313005038f82b7bfde4b70eac Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 19 Jun 2024 18:57:37 +0100 Subject: [PATCH 30/37] Fixing view test cases, adding SQS to it and correcting the default sort order. --- .../src/api/routes/tests/viewV2.spec.ts | 36 ++++++++++++++----- .../server/src/sdk/app/rows/search/sqs.ts | 7 ++-- 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 06921037dd..99ff4f8db7 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -7,6 +7,7 @@ import { INTERNAL_TABLE_SOURCE_ID, PermissionLevel, QuotaUsageType, + Row, SaveTableRequest, SearchFilterOperator, SortOrder, @@ -17,6 +18,7 @@ import { UpdateViewRequest, ViewUIFieldMetadata, ViewV2, + SearchResponse, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -25,17 +27,21 @@ import { quotas } from "@budibase/pro" import { db, roles } from "@budibase/backend-core" describe.each([ - ["internal", undefined], + ["lucene", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], -])("/v2/views (%s)", (_, dsProvider) => { +])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() - const isInternal = !dsProvider + const isSqs = name === "sqs" + const isLucene = name === "lucene" + const isInternal = isSqs || isLucene let table: Table let datasource: Datasource + let envCleanup: (() => void) | undefined function saveTableRequest( ...overrides: Partial>[] @@ -82,6 +88,9 @@ describe.each([ } beforeAll(async () => { + if (isSqs) { + envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + } await config.init() if (dsProvider) { @@ -94,6 +103,9 @@ describe.each([ afterAll(async () => { setup.afterAll() + if (envCleanup) { + envCleanup() + } }) beforeEach(() => { @@ -1252,12 +1264,13 @@ describe.each([ paginate: true, limit: 4, query: {}, + countRows: true, }) expect(page1).toEqual({ rows: expect.arrayContaining(rows.slice(0, 4)), - totalRows: isInternal ? 10 : undefined, hasNextPage: true, bookmark: expect.anything(), + totalRows: 10, }) const page2 = await config.api.viewV2.search(view.id, { @@ -1265,12 +1278,13 @@ describe.each([ limit: 4, bookmark: page1.bookmark, query: {}, + countRows: true, }) expect(page2).toEqual({ rows: expect.arrayContaining(rows.slice(4, 8)), - totalRows: isInternal ? 10 : undefined, hasNextPage: true, bookmark: expect.anything(), + totalRows: 10, }) const page3 = await config.api.viewV2.search(view.id, { @@ -1278,13 +1292,17 @@ describe.each([ limit: 4, bookmark: page2.bookmark, query: {}, + countRows: true, }) - expect(page3).toEqual({ + const expectation: SearchResponse = { rows: expect.arrayContaining(rows.slice(8)), - totalRows: isInternal ? 10 : undefined, hasNextPage: false, - bookmark: expect.anything(), - }) + totalRows: 10, + } + if (isLucene) { + expectation.bookmark = expect.anything() + } + expect(page3).toEqual(expectation) }) const sortTestOptions: [ diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 0ba2adb570..bb1d62affc 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -154,7 +154,7 @@ export async function search( options: RowSearchParams, table: Table ): Promise> { - const { paginate, query, ...params } = options + let { paginate, query, ...params } = options const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) @@ -196,7 +196,7 @@ export async function search( sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING request.sort = { [sortField.name]: { - direction: params.sortOrder || SortOrder.DESCENDING, + direction: params.sortOrder || SortOrder.ASCENDING, type: sortType as SortType, }, } @@ -207,7 +207,8 @@ export async function search( } const bookmark: number = (params.bookmark as number) || 0 - if (paginate && params.limit) { + if (params.limit) { + paginate = true request.paginate = { limit: params.limit + 1, offset: bookmark * params.limit, From 580e36f3015e54aafef1bc904da23dde4eb1d86b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 10:46:52 +0100 Subject: [PATCH 31/37] Updating test case. --- .../server/src/integrations/tests/sql.spec.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index cad1b346c0..b595508093 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -142,7 +142,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson({ schema: "production" })) expect(query).toEqual({ bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" limit $2`, + sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "production"."brands" order by "test"."id" asc limit $1) as "brands" left join "production"."products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, }) }) @@ -150,7 +150,7 @@ describe("SQL query builder", () => { const query = sql._query(generateRelationshipJson()) expect(query).toEqual({ bindings: [500, 5000], - sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" limit $2`, + sql: `select "brands"."brand_id" as "brands.brand_id", "brands"."brand_name" as "brands.brand_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name", "products"."brand_id" as "products.brand_id" from (select * from "brands" order by "test"."id" asc limit $1) as "brands" left join "products" as "products" on "brands"."brand_id" = "products"."brand_id" order by "test"."id" asc limit $2`, }) }) @@ -160,7 +160,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [500, 5000], - sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" limit $2`, + sql: `select "stores"."store_id" as "stores.store_id", "stores"."store_name" as "stores.store_name", "products"."product_id" as "products.product_id", "products"."product_name" as "products.product_name" from (select * from "production"."stores" order by "test"."id" asc limit $1) as "stores" left join "production"."stocks" as "stocks" on "stores"."store_id" = "stocks"."store_id" left join "production"."products" as "products" on "products"."product_id" = "stocks"."product_id" order by "test"."id" asc limit $2`, }) }) @@ -175,8 +175,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: ["john%", limit], - sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`, + bindings: ["john%", limit, 5000], + sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -190,8 +190,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit], - sql: `select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4)) where rownum <= :5) "test"`, + bindings: ["%20%", "%25%", `%"john"%`, `%"mary"%`, limit, 5000], + sql: `select * from (select * from (select * from (select * from "test" where (COALESCE(LOWER("test"."age"), '') LIKE :1 AND COALESCE(LOWER("test"."age"), '') LIKE :2) and (COALESCE(LOWER("test"."name"), '') LIKE :3 AND COALESCE(LOWER("test"."name"), '') LIKE :4) order by "test"."id" asc) where rownum <= :5) "test" order by "test"."id" asc) where rownum <= :6`, }) query = new Sql(SqlClient.ORACLE, limit)._query( @@ -204,8 +204,8 @@ describe("SQL query builder", () => { }) ) expect(query).toEqual({ - bindings: [`%jo%`, limit], - sql: `select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1) where rownum <= :2) "test"`, + bindings: [`%jo%`, limit, 5000], + sql: `select * from (select * from (select * from (select * from "test" where LOWER("test"."name") LIKE :1 order by "test"."id" asc) where rownum <= :2) "test" order by "test"."id" asc) where rownum <= :3`, }) }) }) From 33453646758ddba07a6d2b6a53f24a70bbb5055c Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 14:36:08 +0100 Subject: [PATCH 32/37] Updating test case - not exactly sure what it was testing before, but now it definitely confirms paginated results are stable. --- .../src/api/routes/tests/search.spec.ts | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index be66253090..c5c18997e0 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -1548,19 +1548,32 @@ describe.each([ // be stable or pagination will break. We don't want the user to need // to specify an order for pagination to work. it("is stable without a sort specified", async () => { - let { rows } = await config.api.row.search(table._id!, { - tableId: table._id!, - query: {}, - }) + let { rows: fullRowList } = await config.api.row.search( + table._id!, + { + tableId: table._id!, + query: {}, + } + ) - for (let i = 0; i < 10; i++) { + // repeat the search many times to check the first row is always the same + let bookmark: string | number | undefined, + hasNextPage: boolean | undefined = true, + rowCount: number = 0 + do { const response = await config.api.row.search(table._id!, { tableId: table._id!, limit: 1, + paginate: true, query: {}, + bookmark, }) - expect(response.rows).toEqual(rows) - } + bookmark = response.bookmark + hasNextPage = response.hasNextPage + expect(response.rows.length).toEqual(1) + const foundRow = response.rows[0] + expect(foundRow).toEqual(fullRowList[rowCount++]) + } while (hasNextPage) }) }) From b6b05e08b182fefc36c2ff3da533c61fe930a47d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 15:52:56 +0100 Subject: [PATCH 33/37] Removing SQS from view test to check. --- .../src/api/routes/tests/viewV2.spec.ts | 1 - yarn.lock | 34 ++++++++----------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 99ff4f8db7..4efd20e66b 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -28,7 +28,6 @@ import { db, roles } from "@budibase/backend-core" describe.each([ ["lucene", undefined], - ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], diff --git a/yarn.lock b/yarn.lock index 3606f068b1..9914c334df 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10296,7 +10296,7 @@ engine.io-parser@~5.0.3: resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.0.6.tgz#7811244af173e157295dec9b2718dfe42a64ef45" integrity sha512-tjuoZDMAdEhVnSFleYPCtdL2GXwVTGtNjoeJd9IhIG3C1xs9uwxqRNEu5WpnDZCaozwVlK/nuQhpodhXSIMaxw== -engine.io@~6.4.1: +engine.io@~6.4.2: version "6.4.2" resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.4.2.tgz#ffeaf68f69b1364b0286badddf15ff633476473f" integrity sha512-FKn/3oMiJjrOEOeUub2WCox6JhxBXq/Zn3fZOMCBxKnNYtsdKjxhl7yR3fZhM9PV+rdE75SU5SYMc+2PGzo+Tg== @@ -20160,17 +20160,25 @@ socket.io-parser@~4.2.1: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" -socket.io@4.6.1: - version "4.6.1" - resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.1.tgz#62ec117e5fce0692fa50498da9347cfb52c3bc70" - integrity sha512-KMcaAi4l/8+xEjkRICl6ak8ySoxsYG+gG6/XfRCPJPQ/haCRIJBTL4wIl8YCsmtaBovcAXGLOShyVWQ/FG8GZA== +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + +socket.io@4.6.2: + version "4.6.2" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.6.2.tgz#d597db077d4df9cbbdfaa7a9ed8ccc3d49439786" + integrity sha512-Vp+lSks5k0dewYTfwgPT9UeGGd+ht7sCpB7p0e83VgO4X/AHYWhXITMrNk/pg8syY2bpx23ptClCQuHhqi2BgQ== dependencies: accepts "~1.3.4" base64id "~2.0.0" debug "~4.3.2" - engine.io "~6.4.1" + engine.io "~6.4.2" socket.io-adapter "~2.5.2" - socket.io-parser "~4.2.1" + socket.io-parser "~4.2.4" socks-proxy-agent@^7.0.0: version "7.0.0" @@ -21102,18 +21110,6 @@ tar@6.1.11: mkdirp "^1.0.3" yallist "^4.0.0" -tar@6.1.15: - version "6.1.15" - resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.15.tgz#c9738b0b98845a3b344d334b8fa3041aaba53a69" - integrity sha512-/zKt9UyngnxIT/EAGYuxaMYgOIJiP81ab9ZfkILq4oNLPFX50qyYmu7jRj9qeXoxmJHjGlbH0+cm2uy1WCs10A== - dependencies: - chownr "^2.0.0" - fs-minipass "^2.0.0" - minipass "^5.0.0" - minizlib "^2.1.1" - mkdirp "^1.0.3" - yallist "^4.0.0" - tar@6.2.1, tar@^6.1.11, tar@^6.1.2: version "6.2.1" resolved "https://registry.yarnpkg.com/tar/-/tar-6.2.1.tgz#717549c541bc3c2af15751bea94b1dd068d4b03a" From 66ef0cb79afe0c28662bebd62dad36f9a3a3e6ff Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 16:05:03 +0100 Subject: [PATCH 34/37] Adding back SQS - wasn't causing a problem. --- packages/server/src/api/routes/tests/viewV2.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 4efd20e66b..99ff4f8db7 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -28,6 +28,7 @@ import { db, roles } from "@budibase/backend-core" describe.each([ ["lucene", undefined], + ["sqs", undefined], [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], From 295961edb15c1811060bb86972aeb789a49cdf5b Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 16:18:32 +0100 Subject: [PATCH 35/37] Attempting without promise.all in external. --- packages/server/src/sdk/app/rows/search/external.ts | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index 9fc3487f62..bfb1ccc442 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -81,15 +81,11 @@ export async function search( paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } - const queries: Promise[] = [] - queries.push(handleRequest(Operation.READ, tableId, parameters)) + let rows = await handleRequest(Operation.READ, tableId, parameters) + let totalRows: number | undefined if (countRows) { - queries.push(handleRequest(Operation.COUNT, tableId, parameters)) + totalRows = await handleRequest(Operation.COUNT, tableId, parameters) } - const responses = await Promise.all(queries) - let rows = responses[0] as Row[] - const totalRows = - responses.length > 1 ? (responses[1] as number) : undefined let hasNextPage = false // remove the extra row if it's there From df56371ab6e589ad9310f99ea792ede2368ef8e1 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 16:36:18 +0100 Subject: [PATCH 36/37] Reverting change to promises. --- packages/server/src/sdk/app/rows/search/external.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/external.ts b/packages/server/src/sdk/app/rows/search/external.ts index bfb1ccc442..9fc3487f62 100644 --- a/packages/server/src/sdk/app/rows/search/external.ts +++ b/packages/server/src/sdk/app/rows/search/external.ts @@ -81,11 +81,15 @@ export async function search( paginate: paginateObj as PaginationJson, includeSqlRelationships: IncludeRelationship.INCLUDE, } - let rows = await handleRequest(Operation.READ, tableId, parameters) - let totalRows: number | undefined + const queries: Promise[] = [] + queries.push(handleRequest(Operation.READ, tableId, parameters)) if (countRows) { - totalRows = await handleRequest(Operation.COUNT, tableId, parameters) + queries.push(handleRequest(Operation.COUNT, tableId, parameters)) } + const responses = await Promise.all(queries) + let rows = responses[0] as Row[] + const totalRows = + responses.length > 1 ? (responses[1] as number) : undefined let hasNextPage = false // remove the extra row if it's there From 86bae92ada951adb57f417628a3486d0219f353a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Thu, 20 Jun 2024 17:13:42 +0100 Subject: [PATCH 37/37] Refactoring search test to make it easier to find promises which aren't handled. --- .../src/api/routes/tests/search.spec.ts | 997 ++++++++++-------- 1 file changed, 584 insertions(+), 413 deletions(-) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index c5c18997e0..cff966ab89 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -274,55 +274,63 @@ describe.each([ }) describe("equal", () => { - it("successfully finds true row", () => - expectQuery({ equal: { isTrue: true } }).toMatchExactly([ + it("successfully finds true row", async () => { + await expectQuery({ equal: { isTrue: true } }).toMatchExactly([ { isTrue: true }, - ])) + ]) + }) - it("successfully finds false row", () => - expectQuery({ equal: { isTrue: false } }).toMatchExactly([ + it("successfully finds false row", async () => { + await expectQuery({ equal: { isTrue: false } }).toMatchExactly([ { isTrue: false }, - ])) + ]) + }) }) describe("notEqual", () => { - it("successfully finds false row", () => - expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ + it("successfully finds false row", async () => { + await expectQuery({ notEqual: { isTrue: true } }).toContainExactly([ { isTrue: false }, - ])) + ]) + }) - it("successfully finds true row", () => - expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ + it("successfully finds true row", async () => { + await expectQuery({ notEqual: { isTrue: false } }).toContainExactly([ { isTrue: true }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds true row", () => - expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ + it("successfully finds true row", async () => { + await expectQuery({ oneOf: { isTrue: [true] } }).toContainExactly([ { isTrue: true }, - ])) + ]) + }) - it("successfully finds false row", () => - expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ + it("successfully finds false row", async () => { + await expectQuery({ oneOf: { isTrue: [false] } }).toContainExactly([ { isTrue: false }, - ])) + ]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "isTrue", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ isTrue: false }, { isTrue: true }])) + }).toMatchExactly([{ isTrue: false }, { isTrue: true }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "isTrue", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ isTrue: true }, { isTrue: false }])) + }).toMatchExactly([{ isTrue: true }, { isTrue: false }]) + }) }) }) @@ -676,191 +684,230 @@ describe.each([ }) describe("misc", () => { - it("should return all if no query is passed", () => - expectSearch({} as RowSearchParams).toContainExactly([ + it("should return all if no query is passed", async () => { + await expectSearch({} as RowSearchParams).toContainExactly([ { name: "foo" }, { name: "bar" }, - ])) + ]) + }) - it("should return all if empty query is passed", () => - expectQuery({}).toContainExactly([{ name: "foo" }, { name: "bar" }])) + it("should return all if empty query is passed", async () => { + await expectQuery({}).toContainExactly([ + { name: "foo" }, + { name: "bar" }, + ]) + }) - it("should return all if onEmptyFilter is RETURN_ALL", () => - expectQuery({ + it("should return all if onEmptyFilter is RETURN_ALL", async () => { + await expectQuery({ onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - it("should return nothing if onEmptyFilter is RETURN_NONE", () => - expectQuery({ + it("should return nothing if onEmptyFilter is RETURN_NONE", async () => { + await expectQuery({ onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toFindNothing()) + }).toFindNothing() + }) - it("should respect limit", () => - expectSearch({ limit: 1, paginate: true, query: {} }).toHaveLength(1)) + it("should respect limit", async () => { + await expectSearch({ + limit: 1, + paginate: true, + query: {}, + }).toHaveLength(1) + }) }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { name: "foo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { name: "foo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { name: "none" } }).toFindNothing() + }) - it("works as an or condition", () => - expectQuery({ + it("works as an or condition", async () => { + await expectQuery({ allOr: true, equal: { name: "foo" }, oneOf: { name: ["bar"] }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) - it("can have multiple values for same column", () => - expectQuery({ + it("can have multiple values for same column", async () => { + await expectQuery({ allOr: true, equal: { "1:name": "foo", "2:name": "bar" }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { name: "foo" } }).toContainExactly([ { name: "bar" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { name: "bar" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { name: ["foo"] } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { name: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { name: ["none"] } }).toFindNothing() + }) }) describe("fuzzy", () => { - it("successfully finds a row", () => - expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ fuzzy: { name: "oo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ fuzzy: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ fuzzy: { name: "none" } }).toFindNothing() + }) }) describe("string", () => { - it("successfully finds a row", () => - expectQuery({ string: { name: "fo" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ string: { name: "fo" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ string: { name: "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ string: { name: "none" } }).toFindNothing() + }) - it("is case-insensitive", () => - expectQuery({ string: { name: "FO" } }).toContainExactly([ + it("is case-insensitive", async () => { + await expectQuery({ string: { name: "FO" } }).toContainExactly([ { name: "foo" }, - ])) + ]) + }) }) describe("range", () => { - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { name: { low: "a", high: "z" } }, - }).toContainExactly([{ name: "bar" }, { name: "foo" }])) + }).toContainExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { name: { low: "a", high: "c" } }, - }).toContainExactly([{ name: "bar" }])) + }).toContainExactly([{ name: "bar" }]) + }) - it("successfully finds a row with a low bound", () => - expectQuery({ + it("successfully finds a row with a low bound", async () => { + await expectQuery({ range: { name: { low: "f", high: "z" } }, - }).toContainExactly([{ name: "foo" }])) + }).toContainExactly([{ name: "foo" }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { name: { low: "g", high: "h" } }, - }).toFindNothing()) + }).toFindNothing() + }) !isLucene && - it("ignores low if it's an empty object", () => - expectQuery({ + it("ignores low if it's an empty object", async () => { + await expectQuery({ // @ts-ignore range: { name: { low: {}, high: "z" } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) !isLucene && - it("ignores high if it's an empty object", () => - expectQuery({ + it("ignores high if it's an empty object", async () => { + await expectQuery({ // @ts-ignore range: { name: { low: "a", high: {} } }, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("empty", () => { - it("finds no empty rows", () => - expectQuery({ empty: { name: null } }).toFindNothing()) + it("finds no empty rows", async () => { + await expectQuery({ empty: { name: null } }).toFindNothing() + }) - it("should not be affected by when filter empty behaviour", () => - expectQuery({ + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ empty: { name: null }, onEmptyFilter: EmptyFilterOption.RETURN_ALL, - }).toFindNothing()) + }).toFindNothing() + }) }) describe("notEmpty", () => { - it("finds all non-empty rows", () => - expectQuery({ notEmpty: { name: null } }).toContainExactly([ + it("finds all non-empty rows", async () => { + await expectQuery({ notEmpty: { name: null } }).toContainExactly([ { name: "foo" }, { name: "bar" }, - ])) + ]) + }) - it("should not be affected by when filter empty behaviour", () => - expectQuery({ + it("should not be affected by when filter empty behaviour", async () => { + await expectQuery({ notEmpty: { name: null }, onEmptyFilter: EmptyFilterOption.RETURN_NONE, - }).toContainExactly([{ name: "foo" }, { name: "bar" }])) + }).toContainExactly([{ name: "foo" }, { name: "bar" }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "name", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "name", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ name: "bar" }, { name: "foo" }])) + }).toMatchExactly([{ name: "bar" }, { name: "foo" }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "name", sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ name: "foo" }, { name: "bar" }])) + }).toMatchExactly([{ name: "foo" }, { name: "bar" }]) + }) }) }) }) @@ -874,97 +921,119 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ equal: { age: 1 } }).toContainExactly([{ age: 1 }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { age: 2 } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { age: 2 } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { age: 1 } }).toContainExactly([{ age: 10 }])) + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { age: 1 } }).toContainExactly([ + { age: 10 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { age: 10 } }).toContainExactly([{ age: 1 }])) + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { age: 10 } }).toContainExactly([ + { age: 1 }, + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { age: [1] } }).toContainExactly([{ age: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { age: [1] } }).toContainExactly([ + { age: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { age: [2] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { age: [2] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { age: { low: 1, high: 5 } }, - }).toContainExactly([{ age: 1 }])) + }).toContainExactly([{ age: 1 }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { age: { low: 1, high: 10 } }, - }).toContainExactly([{ age: 1 }, { age: 10 }])) + }).toContainExactly([{ age: 1 }, { age: 10 }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { age: { low: 5, high: 10 } }, - }).toContainExactly([{ age: 10 }])) + }).toContainExactly([{ age: 10 }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { age: { low: 5, high: 9 } }, - }).toFindNothing()) + }).toFindNothing() + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { age: { low: 5 } }, - }).toContainExactly([{ age: 10 }])) + }).toContainExactly([{ age: 10 }]) + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { age: { high: 5 } }, - }).toContainExactly([{ age: 1 }])) + }).toContainExactly([{ age: 1 }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "age", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }])) + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "age", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }])) + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) describe("sortType NUMBER", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "age", sortType: SortType.NUMBER, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ age: 1 }, { age: 10 }])) + }).toMatchExactly([{ age: 1 }, { age: 10 }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "age", sortType: SortType.NUMBER, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ age: 10 }, { age: 1 }])) + }).toMatchExactly([{ age: 10 }, { age: 1 }]) + }) }) }) @@ -984,104 +1053,120 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { dob: JAN_1ST } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { dob: JAN_2ND } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { dob: JAN_1ST } }).toContainExactly([ { dob: JAN_10TH }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { dob: JAN_10TH } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { dob: [JAN_1ST] } }).toContainExactly([ { dob: JAN_1ST }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { dob: [JAN_2ND] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { dob: { low: JAN_1ST, high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }])) + }).toContainExactly([{ dob: JAN_1ST }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { dob: { low: JAN_1ST, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_10TH } }, - }).toContainExactly([{ dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_10TH }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_9TH } }, - }).toFindNothing()) + }).toFindNothing() + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { dob: { low: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_10TH }])) + }).toContainExactly([{ dob: JAN_10TH }]) + }) // We never implemented half-open ranges in Lucene. !isLucene && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { dob: { high: JAN_5TH } }, - }).toContainExactly([{ dob: JAN_1ST }])) + }).toContainExactly([{ dob: JAN_1ST }]) + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "dob", sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "dob", sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }])) + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.ASCENDING, - }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }])) + }).toMatchExactly([{ dob: JAN_1ST }, { dob: JAN_10TH }]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "dob", sortType: SortType.STRING, sortOrder: SortOrder.DESCENDING, - }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }])) + }).toMatchExactly([{ dob: JAN_10TH }, { dob: JAN_1ST }]) + }) }) }) }) @@ -1115,72 +1200,85 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { time: T_1000 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { time: T_1000 } }).toContainExactly([ { time: "10:00:00" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { time: UNEXISTING_TIME } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + equal: { time: UNEXISTING_TIME }, + }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { time: T_1000 } }).toContainExactly([ { timeid: NULL_TIME__ID }, { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, { time: "00:00:00" }, - ])) + ]) + }) - it("return all when requesting non-existing", () => - expectQuery({ notEqual: { time: UNEXISTING_TIME } }).toContainExactly( - [ - { timeid: NULL_TIME__ID }, - { time: "10:00:00" }, - { time: "10:45:00" }, - { time: "12:00:00" }, - { time: "15:30:00" }, - { time: "00:00:00" }, - ] - )) + it("return all when requesting non-existing", async () => { + await expectQuery({ + notEqual: { time: UNEXISTING_TIME }, + }).toContainExactly([ + { timeid: NULL_TIME__ID }, + { time: "10:00:00" }, + { time: "10:45:00" }, + { time: "12:00:00" }, + { time: "15:30:00" }, + { time: "00:00:00" }, + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { time: [T_1000] } }).toContainExactly([ { time: "10:00:00" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { time: [UNEXISTING_TIME] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + oneOf: { time: [UNEXISTING_TIME] }, + }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { time: { low: T_1045, high: T_1045 } }, - }).toContainExactly([{ time: "10:45:00" }])) + }).toContainExactly([{ time: "10:45:00" }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { time: { low: T_1045, high: T_1530 } }, }).toContainExactly([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { time: { low: UNEXISTING_TIME, high: UNEXISTING_TIME } }, - }).toFindNothing()) + }).toFindNothing() + }) }) describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "time", sortOrder: SortOrder.ASCENDING, @@ -1191,10 +1289,11 @@ describe.each([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "time", sortOrder: SortOrder.DESCENDING, @@ -1205,11 +1304,12 @@ describe.each([ { time: "10:00:00" }, { time: "00:00:00" }, { timeid: NULL_TIME__ID }, - ])) + ]) + }) describe("sortType STRING", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "time", sortType: SortType.STRING, @@ -1221,10 +1321,11 @@ describe.each([ { time: "10:45:00" }, { time: "12:00:00" }, { time: "15:30:00" }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "time", sortType: SortType.STRING, @@ -1236,7 +1337,8 @@ describe.each([ { time: "10:00:00" }, { time: "00:00:00" }, { timeid: NULL_TIME__ID }, - ])) + ]) + }) }) }) }) @@ -1254,66 +1356,78 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => - expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ contains: { numbers: ["one"] } }).toContainExactly([ { numbers: ["one", "two"] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ contains: { numbers: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { numbers: ["none"] } }).toFindNothing() + }) - it("fails to find row containing all", () => - expectQuery({ + it("fails to find row containing all", async () => { + await expectQuery({ contains: { numbers: ["one", "two", "three"] }, - }).toFindNothing()) + }).toFindNothing() + }) - it("finds all with empty list", () => - expectQuery({ contains: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ contains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) describe("notContains", () => { - it("successfully finds a row", () => - expectQuery({ notContains: { numbers: ["one"] } }).toContainExactly([ - { numbers: ["three"] }, - ])) + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { numbers: ["one"] }, + }).toContainExactly([{ numbers: ["three"] }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ + it("fails to find nonexistent row", async () => { + await expectQuery({ notContains: { numbers: ["one", "two", "three"] }, }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) // Not sure if this is correct behaviour but changing it would be a // breaking change. - it("finds all with empty list", () => - expectQuery({ notContains: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ notContains: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) describe("containsAny", () => { - it("successfully finds rows", () => - expectQuery({ + it("successfully finds rows", async () => { + await expectQuery({ containsAny: { numbers: ["one", "two", "three"] }, }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ containsAny: { numbers: ["none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { numbers: ["none"] }, + }).toFindNothing() + }) - it("finds all with empty list", () => - expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ + it("finds all with empty list", async () => { + await expectQuery({ containsAny: { numbers: [] } }).toContainExactly([ { numbers: ["one", "two"] }, { numbers: ["three"] }, - ])) + ]) + }) }) }) @@ -1332,48 +1446,56 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { num: SMALL } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { num: SMALL } }).toContainExactly([ { num: SMALL }, - ])) + ]) + }) - it("successfully finds a big value", () => - expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }])) + it("successfully finds a big value", async () => { + await expectQuery({ equal: { num: BIG } }).toContainExactly([ + { num: BIG }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { num: "2" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { num: "2" } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ { num: MEDIUM }, { num: BIG }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { num: 10 } }).toContainExactly([ { num: SMALL }, { num: MEDIUM }, { num: BIG }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ { num: SMALL }, - ])) + ]) + }) - it("successfully finds all rows", () => - expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([ - { num: SMALL }, - { num: MEDIUM }, - { num: BIG }, - ])) + it("successfully finds all rows", async () => { + await expectQuery({ + oneOf: { num: [SMALL, MEDIUM, BIG] }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { num: [2] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { num: [2] } }).toFindNothing() + }) }) // Range searches against bigints don't seem to work at all in Lucene, and I @@ -1381,35 +1503,41 @@ describe.each([ // we've decided not to spend time on it. !isLucene && describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { num: { low: SMALL, high: "5" } }, - }).toContainExactly([{ num: SMALL }])) + }).toContainExactly([{ num: SMALL }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { num: { low: SMALL, high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { num: { low: MEDIUM, high: BIG } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { num: { low: "5", high: "5" } }, - }).toFindNothing()) + }).toFindNothing() + }) - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { num: { low: MEDIUM } }, - }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + }).toContainExactly([{ num: MEDIUM }, { num: BIG }]) + }) - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { num: { high: MEDIUM } }, - }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }]) + }) }) }) @@ -1428,16 +1556,20 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { auto: 1 } }).toContainExactly([{ auto: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ equal: { auto: 1 } }).toContainExactly([ + { auto: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { auto: 0 } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { auto: 0 } }).toFindNothing() + }) }) describe("not equal", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { auto: 1 } }).toContainExactly([ { auto: 2 }, { auto: 3 }, { auto: 4 }, @@ -1447,10 +1579,11 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { auto: 0 } }).toContainExactly([ { auto: 1 }, { auto: 2 }, { auto: 3 }, @@ -1461,55 +1594,66 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { auto: [1] } }).toContainExactly([{ auto: 1 }])) + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { auto: [1] } }).toContainExactly([ + { auto: 1 }, + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { auto: [0] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { auto: [0] } }).toFindNothing() + }) }) describe("range", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ range: { auto: { low: 1, high: 1 } }, - }).toContainExactly([{ auto: 1 }])) + }).toContainExactly([{ auto: 1 }]) + }) - it("successfully finds multiple rows", () => - expectQuery({ + it("successfully finds multiple rows", async () => { + await expectQuery({ range: { auto: { low: 1, high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }])) + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) - it("successfully finds a row with a high bound", () => - expectQuery({ + it("successfully finds a row with a high bound", async () => { + await expectQuery({ range: { auto: { low: 2, high: 2 } }, - }).toContainExactly([{ auto: 2 }])) + }).toContainExactly([{ auto: 2 }]) + }) - it("successfully finds no rows", () => - expectQuery({ + it("successfully finds no rows", async () => { + await expectQuery({ range: { auto: { low: 0, high: 0 } }, - }).toFindNothing()) + }).toFindNothing() + }) isSqs && - it("can search using just a low value", () => - expectQuery({ + it("can search using just a low value", async () => { + await expectQuery({ range: { auto: { low: 9 } }, - }).toContainExactly([{ auto: 9 }, { auto: 10 }])) + }).toContainExactly([{ auto: 9 }, { auto: 10 }]) + }) isSqs && - it("can search using just a high value", () => - expectQuery({ + it("can search using just a high value", async () => { + await expectQuery({ range: { auto: { high: 2 } }, - }).toContainExactly([{ auto: 1 }, { auto: 2 }])) + }).toContainExactly([{ auto: 1 }, { auto: 2 }]) + }) }) isSqs && describe("sort", () => { - it("sorts ascending", () => - expectSearch({ + it("sorts ascending", async () => { + await expectSearch({ query: {}, sort: "auto", sortOrder: SortOrder.ASCENDING, @@ -1524,10 +1668,11 @@ describe.each([ { auto: 8 }, { auto: 9 }, { auto: 10 }, - ])) + ]) + }) - it("sorts descending", () => - expectSearch({ + it("sorts descending", async () => { + await expectSearch({ query: {}, sort: "auto", sortOrder: SortOrder.DESCENDING, @@ -1542,7 +1687,8 @@ describe.each([ { auto: 3 }, { auto: 2 }, { auto: 1 }, - ])) + ]) + }) // This is important for pagination. The order of results must always // be stable or pagination will break. We don't want the user to need @@ -1615,13 +1761,15 @@ describe.each([ await createRows([{ "1:name": "bar" }, { "1:name": "foo" }]) }) - it("successfully finds a row", () => - expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { "1:1:name": "bar" } }).toContainExactly([ { "1:name": "bar" }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { "1:1:name": "none" } }).toFindNothing() + }) }) describe("user", () => { @@ -1648,51 +1796,59 @@ describe.each([ }) describe("equal", () => { - it("successfully finds a row", () => - expectQuery({ equal: { user: user1._id } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ equal: { user: user1._id } }).toContainExactly([ { user: { _id: user1._id } }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ equal: { user: "us_none" } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { user: "us_none" } }).toFindNothing() + }) }) describe("notEqual", () => { - it("successfully finds a row", () => - expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ notEqual: { user: user1._id } }).toContainExactly([ { user: { _id: user2._id } }, {}, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ notEqual: { user: "us_none" } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, {}, - ])) + ]) + }) }) describe("oneOf", () => { - it("successfully finds a row", () => - expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ oneOf: { user: [user1._id] } }).toContainExactly([ { user: { _id: user1._id } }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ oneOf: { user: ["us_none"] } }).toFindNothing() + }) }) describe("empty", () => { - it("finds empty rows", () => - expectQuery({ empty: { user: null } }).toContainExactly([{}])) + it("finds empty rows", async () => { + await expectQuery({ empty: { user: null } }).toContainExactly([{}]) + }) }) describe("notEmpty", () => { - it("finds non-empty rows", () => - expectQuery({ notEmpty: { user: null } }).toContainExactly([ + it("finds non-empty rows", async () => { + await expectQuery({ notEmpty: { user: null } }).toContainExactly([ { user: { _id: user1._id } }, { user: { _id: user2._id } }, - ])) + ]) + }) }) }) @@ -1726,58 +1882,71 @@ describe.each([ }) describe("contains", () => { - it("successfully finds a row", () => - expectQuery({ contains: { users: [user1._id] } }).toContainExactly([ + it("successfully finds a row", async () => { + await expectQuery({ + contains: { users: [user1._id] }, + }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ contains: { users: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ contains: { users: ["us_none"] } }).toFindNothing() + }) }) describe("notContains", () => { - it("successfully finds a row", () => - expectQuery({ notContains: { users: [user1._id] } }).toContainExactly([ - { users: [{ _id: user2._id }] }, - {}, - ])) + it("successfully finds a row", async () => { + await expectQuery({ + notContains: { users: [user1._id] }, + }).toContainExactly([{ users: [{ _id: user2._id }] }, {}]) + }) - it("fails to find nonexistent row", () => - expectQuery({ notContains: { users: ["us_none"] } }).toContainExactly([ + it("fails to find nonexistent row", async () => { + await expectQuery({ + notContains: { users: ["us_none"] }, + }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, {}, - ])) + ]) + }) }) describe("containsAny", () => { - it("successfully finds rows", () => - expectQuery({ + it("successfully finds rows", async () => { + await expectQuery({ containsAny: { users: [user1._id, user2._id] }, }).toContainExactly([ { users: [{ _id: user1._id }] }, { users: [{ _id: user2._id }] }, { users: [{ _id: user1._id }, { _id: user2._id }] }, - ])) + ]) + }) - it("fails to find nonexistent row", () => - expectQuery({ containsAny: { users: ["us_none"] } }).toFindNothing()) + it("fails to find nonexistent row", async () => { + await expectQuery({ + containsAny: { users: ["us_none"] }, + }).toFindNothing() + }) }) describe("multi-column equals", () => { - it("successfully finds a row", () => - expectQuery({ + it("successfully finds a row", async () => { + await expectQuery({ equal: { number: 1 }, contains: { users: [user1._id] }, - }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }])) + }).toContainExactly([{ users: [{ _id: user1._id }], number: 1 }]) + }) - it("fails to find nonexistent row", () => - expectQuery({ + it("fails to find nonexistent row", async () => { + await expectQuery({ equal: { number: 2 }, contains: { users: [user1._id] }, - }).toFindNothing()) + }).toFindNothing() + }) }) }) @@ -1827,12 +1996,13 @@ describe.each([ rows = await config.api.row.fetch(table._id!) }) - it("can search through relations", () => - expectQuery({ + it("can search through relations", async () => { + await expectQuery({ equal: { [`${otherTable.name}.one`]: "foo" }, }).toContainExactly([ { two: "foo", other: [{ _id: otherRows[0]._id }] }, - ])) + ]) + }) }) // lucene can't count the total rows @@ -1848,18 +2018,19 @@ describe.each([ await createRows([{ name: "a" }, { name: "b" }]) }) - it("should be able to count rows when option set", () => - expectSearch({ + it("should be able to count rows when option set", async () => { + await expectSearch({ countRows: true, query: { notEmpty: { name: true, }, }, - }).toMatch({ totalRows: 2, rows: expect.any(Array) })) + }).toMatch({ totalRows: 2, rows: expect.any(Array) }) + }) - it("shouldn't count rows when option is not set", () => { - expectSearch({ + it("shouldn't count rows when option is not set", async () => { + await expectSearch({ countRows: false, query: { notEmpty: {