From cb99007a429d0b9803dbac5f79e382feba128213 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 28 May 2024 18:30:06 +0100 Subject: [PATCH 1/6] Fix for issue with templated apps - some template apps are *very* old and therefore don't necessarily abide by the rules that are generally followed for new apps, this is something we should resolve, but for now adding an option that allows searching for only a specific document type. --- packages/backend-core/src/sql/sql.ts | 25 ++++++++++++++++--- .../server/src/sdk/app/rows/search/sqs.ts | 2 ++ packages/types/src/sdk/search.ts | 3 ++- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 2d01c6a7ee..63103a0571 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -1,6 +1,11 @@ import { Knex, knex } from "knex" import * as dbCore from "../db" -import { isIsoDateString, isValidFilter, getNativeSql } from "./utils" +import { + isIsoDateString, + isValidFilter, + getNativeSql, + isExternalTable, +} from "./utils" import { SqlStatements } from "./sqlStatements" import SqlTableQueryBuilder from "./sqlTable" import { @@ -21,6 +26,7 @@ import { SqlClient, QueryOptions, JsonTypes, + prefixed, } from "@budibase/types" import environment from "../environment" import { helpers } from "@budibase/shared-core" @@ -556,8 +562,15 @@ class InternalBuilder { } read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder { - let { endpoint, resource, filters, paginate, relationships, tableAliases } = - json + let { + endpoint, + resource, + filters, + paginate, + relationships, + tableAliases, + meta, + } = json const tableName = endpoint.entityId // select all if not specified @@ -592,6 +605,12 @@ class InternalBuilder { query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) + + // when searching internal tables make sure long looking for rows + if (meta.documentType && !isExternalTable(meta.table)) { + query.andWhereLike("_id", `${prefixed(meta.documentType)}%`) + } + // add sorting to pre-query query = this.addSorting(query, json) const alias = tableAliases?.[tableName] || tableName diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 5083f9b79f..a21b3366ed 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -1,4 +1,5 @@ import { + DocumentType, FieldType, Operation, QueryJson, @@ -153,6 +154,7 @@ export async function search( meta: { table, tables: allTablesMap, + documentType: DocumentType.ROW, }, resource: { fields: buildInternalFieldList(table, allTables), diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 62c5054c7a..a4221280bf 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -1,5 +1,5 @@ import { Operation, SortDirection } from "./datasources" -import { Row, Table } from "../documents" +import { Row, Table, DocumentType } from "../documents" import { SortType } from "../api" import { Knex } from "knex" @@ -117,6 +117,7 @@ export interface QueryJson { table: Table tables?: Record renamed?: RenameColumn + documentType?: DocumentType } extra?: { idFilter?: SearchFilters From f89f469e01e47efe1825d83178e470ec934d19e5 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 28 May 2024 18:31:19 +0100 Subject: [PATCH 2/6] Comment to explain what the new documentType meta option does. --- packages/types/src/sdk/search.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index a4221280bf..322afa81b7 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -117,6 +117,9 @@ export interface QueryJson { table: Table tables?: Record renamed?: RenameColumn + // specific to SQS/SQLite search on internal tables this can be used + // to make sure the documents returned are always filtered down to a + // specific document type (such as just rows) documentType?: DocumentType } extra?: { From 83fa6f952ac500c5d68e2875414284fe2fcf7925 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 28 May 2024 18:34:08 +0100 Subject: [PATCH 3/6] re-jigging, making this part of filters rather than meta. --- packages/backend-core/src/sql/sql.ts | 22 ++++++++-------------- packages/types/src/sdk/search.ts | 10 +++++----- 2 files changed, 13 insertions(+), 19 deletions(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index 63103a0571..b85f109376 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -397,6 +397,12 @@ class InternalBuilder { contains(filters.containsAny, true) } + // when searching internal tables make sure long looking for rows + if (filters.documentType && !isExternalTable(table)) { + // has to be its own option, must always be AND onto the search + query.andWhereLike("_id", `${prefixed(filters.documentType)}%`) + } + return query } @@ -562,15 +568,8 @@ class InternalBuilder { } read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder { - let { - endpoint, - resource, - filters, - paginate, - relationships, - tableAliases, - meta, - } = json + let { endpoint, resource, filters, paginate, relationships, tableAliases } = + json const tableName = endpoint.entityId // select all if not specified @@ -606,11 +605,6 @@ class InternalBuilder { aliases: tableAliases, }) - // when searching internal tables make sure long looking for rows - if (meta.documentType && !isExternalTable(meta.table)) { - query.andWhereLike("_id", `${prefixed(meta.documentType)}%`) - } - // add sorting to pre-query query = this.addSorting(query, json) const alias = tableAliases?.[tableName] || tableName diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 322afa81b7..dc8ea09bcc 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -62,11 +62,15 @@ export interface SearchFilters { [SearchFilterOperator.CONTAINS_ANY]?: { [key: string]: any[] } + // specific to SQS/SQLite search on internal tables this can be used + // to make sure the documents returned are always filtered down to a + // specific document type (such as just rows) + documentType?: DocumentType } export type SearchFilterKey = keyof Omit< SearchFilters, - "allOr" | "onEmptyFilter" | "fuzzyOr" + "allOr" | "onEmptyFilter" | "fuzzyOr" | "documentType" > export type SearchQueryFields = Omit @@ -117,10 +121,6 @@ export interface QueryJson { table: Table tables?: Record renamed?: RenameColumn - // specific to SQS/SQLite search on internal tables this can be used - // to make sure the documents returned are always filtered down to a - // specific document type (such as just rows) - documentType?: DocumentType } extra?: { idFilter?: SearchFilters From 394fa2536c484321bd6e5d6fdbfab9e591d9078f Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 29 May 2024 11:02:27 +0100 Subject: [PATCH 4/6] Build fix. --- packages/server/src/sdk/app/rows/search/sqs.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index a21b3366ed..87b905a29f 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -149,12 +149,14 @@ export async function search( entityId: table._id!, operation: Operation.READ, }, - filters: cleanupFilters(query, allTables), + filters: { + ...cleanupFilters(query, allTables), + documentType: DocumentType.ROW, + }, table, meta: { table, tables: allTablesMap, - documentType: DocumentType.ROW, }, resource: { fields: buildInternalFieldList(table, allTables), From 6b7a4f07cba93ba6f065309f9876e78abc87818d Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 29 May 2024 11:56:52 +0100 Subject: [PATCH 5/6] Quick fix for ambiguous column names _id. --- packages/backend-core/src/sql/sql.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index b85f109376..61d5849058 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -399,8 +399,12 @@ class InternalBuilder { // when searching internal tables make sure long looking for rows if (filters.documentType && !isExternalTable(table)) { + const tableRef = opts?.aliases?.[table._id!] || table._id // has to be its own option, must always be AND onto the search - query.andWhereLike("_id", `${prefixed(filters.documentType)}%`) + query.andWhereLike( + `${tableRef}._id`, + `${prefixed(filters.documentType)}%` + ) } return query From 786abde888ed3f6af4052856febeb45b0f321cb2 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Wed, 29 May 2024 16:31:55 +0100 Subject: [PATCH 6/6] Reducing concurrency of type checking (for CI). --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 41726f5cb4..08176fae90 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "build:oss": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --ignore @budibase/account-portal --ignore @budibase/account-portal-server --ignore @budibase/account-portal-ui", "build:account-portal": "NODE_OPTIONS=--max-old-space-size=1500 lerna run build --stream --scope @budibase/account-portal --scope @budibase/account-portal-server --scope @budibase/account-portal-ui", "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput", - "check:types": "lerna run check:types", + "check:types": "lerna run --concurrency 2 check:types", "build:sdk": "lerna run --stream build:sdk", "deps:circular": "madge packages/server/dist/index.js packages/worker/src/index.ts packages/backend-core/dist/src/index.js packages/cli/src/index.js --circular", "release": "lerna publish from-package --yes --force-publish --no-git-tag-version --no-push --no-git-reset",