diff --git a/.vscode/settings.json b/.vscode/settings.json index e22d5a8866..0723219a8b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,8 @@ }, "[svelte]": { "editor.defaultFormatter": "svelte.svelte-vscode" + }, + "[handlebars]": { + "editor.formatOnSave": false } } diff --git a/lerna.json b/lerna.json index a2be7be7b4..2ce2b69508 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.23.6", + "version": "2.23.9", "npmClient": "yarn", "packages": [ "packages/*", diff --git a/nx.json b/nx.json index 618395ec90..8ba8798946 100644 --- a/nx.json +++ b/nx.json @@ -9,10 +9,7 @@ }, "targetDefaults": { "build": { - "inputs": [ - "{workspaceRoot}/scripts/build.js", - "{workspaceRoot}/lerna.json" - ] + "inputs": ["{workspaceRoot}/scripts/*", "{workspaceRoot}/lerna.json"] } } } diff --git a/package.json b/package.json index e520b7c2cf..e60a086e17 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "dev:camunda": "./scripts/deploy-camunda.sh", "dev:all": "yarn run kill-all && lerna run --stream dev", "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built", - "dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0", + "dev:docker": "./scripts/devDocker.sh", "test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream", "lint:eslint": "eslint packages --max-warnings=0", "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"", diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index 6165a68e57..f77c6385ba 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -50,6 +50,8 @@ type CreateAdminUserOpts = { hashPassword?: boolean requirePassword?: boolean skipPasswordValidation?: boolean + firstName?: string + lastName?: string } type FeatureFns = { isSSOEnforced: FeatureFn; isAppBuildersEnabled: FeatureFn } @@ -517,6 +519,8 @@ export class UserDB { global: true, }, tenantId, + firstName: opts?.firstName, + lastName: opts?.lastName, } if (opts?.ssoId) { user.ssoId = opts.ssoId diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index ff9830c92a..d271462f3e 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -34,15 +34,15 @@ import { RowUtils } from "@budibase/frontend-core" import ServerBindingPanel from "components/common/bindings/ServerBindingPanel.svelte" - const AUTO_TYPE = FIELDS.AUTO.type - const FORMULA_TYPE = FIELDS.FORMULA.type - const LINK_TYPE = FIELDS.LINK.type - const STRING_TYPE = FIELDS.STRING.type - const NUMBER_TYPE = FIELDS.NUMBER.type - const JSON_TYPE = FIELDS.JSON.type - const DATE_TYPE = FIELDS.DATETIME.type - const USER_TYPE = FIELDS.USER.subtype - const USERS_TYPE = FIELDS.USERS.subtype + const AUTO_TYPE = FieldType.AUTO + const FORMULA_TYPE = FieldType.FORMULA + const LINK_TYPE = FieldType.LINK + const STRING_TYPE = FieldType.STRING + const NUMBER_TYPE = FieldType.NUMBER + const JSON_TYPE = FieldType.JSON + const DATE_TYPE = FieldType.DATETIME + const USER_TYPE = FieldSubtype.USER + const USERS_TYPE = FieldSubtype.USERS const dispatch = createEventDispatcher() const PROHIBITED_COLUMN_NAMES = ["type", "_id", "_rev", "tableId"] @@ -626,7 +626,7 @@ /> - {:else if editableColumn.type === FieldType.LINK} + {:else if editableColumn.type === FieldType.LINK && !editableColumn.autocolumn} { ) { continue } - let tableId: string | undefined, + let relatedTableId: string | undefined, lookupField: string | undefined, fieldName: string | undefined if (isManyToMany(field)) { - tableId = field.through + relatedTableId = field.through lookupField = primaryKey fieldName = field.throughTo || primaryKey } else if (isManyToOne(field)) { - tableId = field.tableId + relatedTableId = field.tableId lookupField = field.foreignKey fieldName = field.fieldName } - if (!tableId || !lookupField || !fieldName) { + if (!relatedTableId || !lookupField || !fieldName) { throw new Error( "Unable to lookup relationships - undefined column properties." ) } - const { tableName: relatedTableName } = breakExternalTableId(tableId) + const { tableName: relatedTableName } = + breakExternalTableId(relatedTableId) // @ts-ignore const linkPrimaryKey = this.tables[relatedTableName].primary[0] if (!lookupField || !row[lookupField]) { continue } + const endpoint = getEndpoint(relatedTableId, Operation.READ) + const relatedTable = this.tables[endpoint.entityId] + if (!relatedTable) { + throw new Error("unable to find related table") + } const response = await getDatasourceAndQuery({ - endpoint: getEndpoint(tableId, Operation.READ), + endpoint: endpoint, filters: { equal: { [fieldName]: row[lookupField], }, }, meta: { - table, + table: relatedTable, }, }) // this is the response from knex if no rows found @@ -414,7 +420,11 @@ export class ExternalRequest { const storeTo = isManyToMany(field) ? field.throughFrom || linkPrimaryKey : fieldName - related[storeTo] = { rows, isMany: isManyToMany(field), tableId } + related[storeTo] = { + rows, + isMany: isManyToMany(field), + tableId: relatedTableId, + } } return related } @@ -437,7 +447,6 @@ export class ExternalRequest { // if we're creating (in a through table) need to wipe the existing ones first const promises = [] const related = await this.lookupRelations(mainTableId, row) - const table = this.getTable(mainTableId)! for (let relationship of relationships) { const { key, tableId, isUpdate, id, ...rest } = relationship const body: { [key: string]: any } = processObjectSync(rest, row, {}) @@ -484,7 +493,7 @@ export class ExternalRequest { body, filters: buildFilters(id, {}, linkTable), meta: { - table, + table: linkTable, }, }) ) diff --git a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts index 7790f909e7..e8a38dcfaa 100644 --- a/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts +++ b/packages/server/src/api/routes/tests/queries/generic-sql.spec.ts @@ -4,6 +4,7 @@ import { Query, QueryPreview, SourceName, + TableSourceType, } from "@budibase/types" import * as setup from "../utilities" import { @@ -740,12 +741,25 @@ describe.each( }) describe("query through datasource", () => { - it("should be able to query a pg datasource", async () => { + it("should be able to query the datasource", async () => { + const entityId = "test_table" + await config.api.datasource.update({ + ...datasource, + entities: { + [entityId]: { + name: entityId, + schema: {}, + type: "table", + sourceId: datasource._id!, + sourceType: TableSourceType.EXTERNAL, + }, + }, + }) const res = await config.api.datasource.query({ endpoint: { datasourceId: datasource._id!, operation: Operation.READ, - entityId: "test_table", + entityId, }, resource: { fields: ["id", "name"], diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts index 5b71ec9044..698ea0c10b 100644 --- a/packages/server/src/api/routes/tests/search.spec.ts +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -26,6 +26,7 @@ describe.each([ [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], ])("/api/:sourceId/search (%s)", (name, dsProvider) => { const isSqs = name === "internal-sqs" + const isInternal = name === "internal" const config = setup.getConfig() let envCleanup: (() => void) | undefined @@ -336,6 +337,20 @@ describe.each([ expectQuery({ range: { age: { low: 5, high: 9 } }, }).toFindNothing()) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a low value", () => + expectQuery({ + range: { age: { low: 5 } }, + }).toContainExactly([{ age: 10 }])) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a high value", () => + expectQuery({ + range: { age: { high: 5 } }, + }).toContainExactly([{ age: 1 }])) }) describe("sort", () => { @@ -440,6 +455,20 @@ describe.each([ expectQuery({ range: { dob: { low: JAN_5TH, high: JAN_9TH } }, }).toFindNothing()) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a low value", () => + expectQuery({ + range: { dob: { low: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_10TH }])) + + // We never implemented half-open ranges in Lucene. + !isInternal && + it("can search using just a high value", () => + expectQuery({ + range: { dob: { high: JAN_5TH } }, + }).toContainExactly([{ dob: JAN_1ST }])) }) describe("sort", () => { @@ -550,4 +579,100 @@ describe.each([ ])) }) }) + + describe("bigints", () => { + const SMALL = "1" + const MEDIUM = "10000000" + + // Our bigints are int64s in most datasources. + const BIG = "9223372036854775807" + + beforeAll(async () => { + await createTable({ + num: { name: "num", type: FieldType.BIGINT }, + }) + await createRows([{ num: SMALL }, { num: MEDIUM }, { num: BIG }]) + }) + + describe("equal", () => { + it("successfully finds a row", () => + expectQuery({ equal: { num: SMALL } }).toContainExactly([ + { num: SMALL }, + ])) + + it("successfully finds a big value", () => + expectQuery({ equal: { num: BIG } }).toContainExactly([{ num: BIG }])) + + it("fails to find nonexistent row", () => + expectQuery({ equal: { num: "2" } }).toFindNothing()) + }) + + describe("notEqual", () => { + it("successfully finds a row", () => + expectQuery({ notEqual: { num: SMALL } }).toContainExactly([ + { num: MEDIUM }, + { num: BIG }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ notEqual: { num: 10 } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, + ])) + }) + + describe("oneOf", () => { + it("successfully finds a row", () => + expectQuery({ oneOf: { num: [SMALL] } }).toContainExactly([ + { num: SMALL }, + ])) + + it("successfully finds all rows", () => + expectQuery({ oneOf: { num: [SMALL, MEDIUM, BIG] } }).toContainExactly([ + { num: SMALL }, + { num: MEDIUM }, + { num: BIG }, + ])) + + it("fails to find nonexistent row", () => + expectQuery({ oneOf: { num: [2] } }).toFindNothing()) + }) + + // Range searches against bigints don't seem to work at all in Lucene, and I + // couldn't figure out why. Given that we're replacing Lucene with SQS, + // we've decided not to spend time on it. + !isInternal && + describe("range", () => { + it("successfully finds a row", () => + expectQuery({ + range: { num: { low: SMALL, high: "5" } }, + }).toContainExactly([{ num: SMALL }])) + + it("successfully finds multiple rows", () => + expectQuery({ + range: { num: { low: SMALL, high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + + it("successfully finds a row with a high bound", () => + expectQuery({ + range: { num: { low: MEDIUM, high: BIG } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + + it("successfully finds no rows", () => + expectQuery({ + range: { num: { low: "5", high: "5" } }, + }).toFindNothing()) + + it("can search using just a low value", () => + expectQuery({ + range: { num: { low: MEDIUM } }, + }).toContainExactly([{ num: MEDIUM }, { num: BIG }])) + + it("can search using just a high value", () => + expectQuery({ + range: { num: { high: MEDIUM } }, + }).toContainExactly([{ num: SMALL }, { num: MEDIUM }])) + }) + }) }) diff --git a/packages/server/src/integrations/base/query.ts b/packages/server/src/integrations/base/query.ts index 03e6028e32..371592bece 100644 --- a/packages/server/src/integrations/base/query.ts +++ b/packages/server/src/integrations/base/query.ts @@ -2,6 +2,7 @@ import { QueryJson, Datasource, DatasourcePlusQueryResponse, + RowOperations, } from "@budibase/types" import { getIntegration } from "../index" import sdk from "../../sdk" @@ -10,6 +11,17 @@ export async function makeExternalQuery( datasource: Datasource, json: QueryJson ): Promise { + const entityId = json.endpoint.entityId, + tableName = json.meta.table.name, + tableId = json.meta.table._id + // case found during testing - make sure this doesn't happen again + if ( + RowOperations.includes(json.endpoint.operation) && + entityId !== tableId && + entityId !== tableName + ) { + throw new Error("Entity ID and table metadata do not align") + } datasource = await sdk.datasources.enrich(datasource) const Integration = await getIntegration(datasource.source) // query is the opinionated function diff --git a/packages/server/src/integrations/base/sql.ts b/packages/server/src/integrations/base/sql.ts index 59684422e7..28b7eb910b 100644 --- a/packages/server/src/integrations/base/sql.ts +++ b/packages/server/src/integrations/base/sql.ts @@ -6,6 +6,7 @@ import { SqlClient, isValidFilter, getNativeSql, + SqlStatements, } from "../utils" import SqlTableQueryBuilder from "./sqlTable" import { @@ -160,9 +161,19 @@ class InternalBuilder { addFilters( query: Knex.QueryBuilder, filters: SearchFilters | undefined, - tableName: string, + table: Table, opts: { aliases?: Record; relationship?: boolean } ): Knex.QueryBuilder { + if (!filters) { + return query + } + filters = parseFilters(filters) + // if all or specified in filters, then everything is an or + const allOr = filters.allOr + const sqlStatements = new SqlStatements(this.client, table, { allOr }) + const tableName = + this.client === SqlClient.SQL_LITE ? table._id! : table.name + function getTableAlias(name: string) { const alias = opts.aliases?.[name] return alias || name @@ -258,12 +269,6 @@ class InternalBuilder { } } - if (!filters) { - return query - } - filters = parseFilters(filters) - // if all or specified in filters, then everything is an or - const allOr = filters.allOr if (filters.oneOf) { iterate(filters.oneOf, (key, array) => { const fnc = allOr ? "orWhereIn" : "whereIn" @@ -306,17 +311,11 @@ class InternalBuilder { const lowValid = isValidFilter(value.low), highValid = isValidFilter(value.high) if (lowValid && highValid) { - // Use a between operator if we have 2 valid range values - const fnc = allOr ? "orWhereBetween" : "whereBetween" - query = query[fnc](key, [value.low, value.high]) + query = sqlStatements.between(query, key, value.low, value.high) } else if (lowValid) { - // Use just a single greater than operator if we only have a low - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, ">", value.low) + query = sqlStatements.lte(query, key, value.low) } else if (highValid) { - // Use just a single less than operator if we only have a high - const fnc = allOr ? "orWhere" : "where" - query = query[fnc](key, "<", value.high) + query = sqlStatements.gte(query, key, value.high) } }) } @@ -359,7 +358,7 @@ class InternalBuilder { addSorting(query: Knex.QueryBuilder, json: QueryJson): Knex.QueryBuilder { let { sort, paginate } = json - const table = json.meta?.table + const table = json.meta.table const tableName = getTableName(table) const aliases = json.tableAliases const aliased = @@ -473,14 +472,13 @@ class InternalBuilder { ): Knex.QueryBuilder { const tableName = endpoint.entityId const tableAlias = aliases?.[tableName] - let table: string | Record = tableName - if (tableAlias) { - table = { [tableAlias]: tableName } - } - let query = knex(table) - if (endpoint.schema) { - query = query.withSchema(endpoint.schema) - } + + const query = knex( + this.tableNameWithSchema(tableName, { + alias: tableAlias, + schema: endpoint.schema, + }) + ) return query } @@ -547,7 +545,7 @@ class InternalBuilder { if (foundOffset) { query = query.offset(foundOffset) } - query = this.addFilters(query, filters, tableName, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // add sorting to pre-query @@ -568,7 +566,7 @@ class InternalBuilder { endpoint.schema, tableAliases ) - return this.addFilters(query, filters, tableName, { + return this.addFilters(query, filters, json.meta.table, { relationship: true, aliases: tableAliases, }) @@ -578,7 +576,7 @@ class InternalBuilder { const { endpoint, body, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) const parsedBody = parseBody(body) - query = this.addFilters(query, filters, endpoint.entityId, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // mysql can't use returning @@ -592,7 +590,7 @@ class InternalBuilder { delete(knex: Knex, json: QueryJson, opts: QueryOptions): Knex.QueryBuilder { const { endpoint, filters, tableAliases } = json let query = this.knexWithAlias(knex, endpoint, tableAliases) - query = this.addFilters(query, filters, endpoint.entityId, { + query = this.addFilters(query, filters, json.meta.table, { aliases: tableAliases, }) // mysql can't use returning @@ -684,7 +682,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder { // when creating if an ID has been inserted need to make sure // the id filter is enriched with it before trying to retrieve the row checkLookupKeys(id: any, json: QueryJson) { - if (!id || !json.meta?.table || !json.meta.table.primary) { + if (!id || !json.meta.table || !json.meta.table.primary) { return json } const primaryKey = json.meta.table.primary?.[0] diff --git a/packages/server/src/integrations/tests/sql.spec.ts b/packages/server/src/integrations/tests/sql.spec.ts index 4ee544cc5e..5de9cc4fbc 100644 --- a/packages/server/src/integrations/tests/sql.spec.ts +++ b/packages/server/src/integrations/tests/sql.spec.ts @@ -1,11 +1,11 @@ import { SqlClient } from "../utils" import Sql from "../base/sql" import { + FieldType, Operation, QueryJson, - TableSourceType, Table, - FieldType, + TableSourceType, } from "@budibase/types" const TABLE_NAME = "test" @@ -13,7 +13,12 @@ const TABLE: Table = { type: "table", sourceType: TableSourceType.EXTERNAL, sourceId: "SOURCE_ID", - schema: {}, + schema: { + id: { + name: "id", + type: FieldType.NUMBER, + }, + }, name: TABLE_NAME, primary: ["id"], } @@ -73,7 +78,7 @@ function generateUpdateJson({ meta?: any }): QueryJson { if (!meta.table) { - meta.table = table + meta.table = TABLE } return { endpoint: endpoint(table, "UPDATE"), @@ -158,6 +163,9 @@ function generateManyRelationshipJson(config: { schema?: string } = {}) { }, ], extra: { idFilter: {} }, + meta: { + table: TABLE, + }, } } @@ -341,7 +349,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" > $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" >= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -360,7 +368,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: [date, limit], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" < $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."property" <= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -594,7 +602,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["2000-01-01 00:00:00", 500], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" > $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" >= $1 limit $2) as "${TABLE_NAME}"`, }) }) @@ -613,7 +621,7 @@ describe("SQL query builder", () => { ) expect(query).toEqual({ bindings: ["2010-01-01 00:00:00", 500], - sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" < $1 limit $2) as "${TABLE_NAME}"`, + sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."dob" <= $1 limit $2) as "${TABLE_NAME}"`, }) }) diff --git a/packages/server/src/integrations/tests/sqlAlias.spec.ts b/packages/server/src/integrations/tests/sqlAlias.spec.ts index f4edab8dad..fda2a091fa 100644 --- a/packages/server/src/integrations/tests/sqlAlias.spec.ts +++ b/packages/server/src/integrations/tests/sqlAlias.spec.ts @@ -117,7 +117,8 @@ describe("Captures of real examples", () => { let query = new Sql(SqlClient.POSTGRES, limit)._query(queryJson) const filters = queryJson.filters const notEqualsValue = Object.values(filters?.notEqual!)[0] - const rangeValue = Object.values(filters?.range!)[0] + const rangeValue: { high?: string | number; low?: string | number } = + Object.values(filters?.range!)[0] const equalValue = Object.values(filters?.equal!)[0] expect(query).toEqual({ diff --git a/packages/server/src/integrations/utils/index.ts b/packages/server/src/integrations/utils/index.ts new file mode 100644 index 0000000000..a9c2019ba2 --- /dev/null +++ b/packages/server/src/integrations/utils/index.ts @@ -0,0 +1,2 @@ +export * from "./utils" +export { SqlStatements } from "./sqlStatements" diff --git a/packages/server/src/integrations/utils/sqlStatements.ts b/packages/server/src/integrations/utils/sqlStatements.ts new file mode 100644 index 0000000000..7a5482830b --- /dev/null +++ b/packages/server/src/integrations/utils/sqlStatements.ts @@ -0,0 +1,80 @@ +import { FieldType, Table, FieldSchema } from "@budibase/types" +import { SqlClient } from "./utils" +import { Knex } from "knex" + +export class SqlStatements { + client: string + table: Table + allOr: boolean | undefined + constructor( + client: string, + table: Table, + { allOr }: { allOr?: boolean } = {} + ) { + this.client = client + this.table = table + this.allOr = allOr + } + + getField(key: string): FieldSchema | undefined { + const fieldName = key.split(".")[1] + return this.table.schema[fieldName] + } + + between( + query: Knex.QueryBuilder, + key: string, + low: number | string, + high: number | string + ) { + // Use a between operator if we have 2 valid range values + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw( + `CAST(${key} AS INTEGER) BETWEEN CAST(? AS INTEGER) AND CAST(? AS INTEGER)`, + [low, high] + ) + } else { + const fnc = this.allOr ? "orWhereBetween" : "whereBetween" + query = query[fnc](key, [low, high]) + } + return query + } + + lte(query: Knex.QueryBuilder, key: string, low: number | string) { + // Use just a single greater than operator if we only have a low + const field = this.getField(key) + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) >= CAST(? AS INTEGER)`, [ + low, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, ">=", low) + } + return query + } + + gte(query: Knex.QueryBuilder, key: string, high: number | string) { + const field = this.getField(key) + // Use just a single less than operator if we only have a high + if ( + field?.type === FieldType.BIGINT && + this.client === SqlClient.SQL_LITE + ) { + query = query.whereRaw(`CAST(${key} AS INTEGER) <= CAST(? AS INTEGER)`, [ + high, + ]) + } else { + const fnc = this.allOr ? "orWhere" : "where" + query = query[fnc](key, "<=", high) + } + return query + } +} diff --git a/packages/server/src/integrations/utils.ts b/packages/server/src/integrations/utils/utils.ts similarity index 98% rename from packages/server/src/integrations/utils.ts rename to packages/server/src/integrations/utils/utils.ts index cc832d8062..cc75f0444d 100644 --- a/packages/server/src/integrations/utils.ts +++ b/packages/server/src/integrations/utils/utils.ts @@ -5,10 +5,10 @@ import { FieldType, TableSourceType, } from "@budibase/types" -import { DocumentType, SEPARATOR } from "../db/utils" -import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants" +import { DocumentType, SEPARATOR } from "../../db/utils" +import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../../constants" import { SWITCHABLE_TYPES, helpers } from "@budibase/shared-core" -import env from "../environment" +import env from "../../environment" import { Knex } from "knex" const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}` diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 36576c58a3..10cc5aa6c6 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -200,6 +200,6 @@ export async function search( } } catch (err: any) { const msg = typeof err === "string" ? err : err.message - throw new Error(`Unable to search by SQL - ${msg}`) + throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/sdk/app/rows/utils.ts b/packages/server/src/sdk/app/rows/utils.ts index d307b17947..a9df4f89cd 100644 --- a/packages/server/src/sdk/app/rows/utils.ts +++ b/packages/server/src/sdk/app/rows/utils.ts @@ -52,6 +52,12 @@ export async function getDatasourceAndQuery( ): Promise { const datasourceId = json.endpoint.datasourceId const datasource = await sdk.datasources.get(datasourceId) + const table = datasource.entities?.[json.endpoint.entityId] + if (!json.meta && table) { + json.meta = { + table, + } + } return makeExternalQuery(datasource, json) } diff --git a/packages/server/src/sdk/app/tables/external/index.ts b/packages/server/src/sdk/app/tables/external/index.ts index f3f2f070be..2a78600cfc 100644 --- a/packages/server/src/sdk/app/tables/external/index.ts +++ b/packages/server/src/sdk/app/tables/external/index.ts @@ -51,6 +51,12 @@ export async function save( !oldTable && (tableToSave.primary == null || tableToSave.primary.length === 0) ) { + if (tableToSave.schema.id) { + throw new Error( + "External tables with no `primary` column set will define an `id` column, but we found an `id` column in the supplied schema. Either set a `primary` column or remove the `id` column." + ) + } + tableToSave.primary = ["id"] tableToSave.schema.id = { type: FieldType.NUMBER, diff --git a/packages/server/src/sdk/app/tables/internal/sqs.ts b/packages/server/src/sdk/app/tables/internal/sqs.ts index 99240c28d4..0726c94962 100644 --- a/packages/server/src/sdk/app/tables/internal/sqs.ts +++ b/packages/server/src/sdk/app/tables/internal/sqs.ts @@ -42,7 +42,7 @@ const FieldTypeMap: Record = { [FieldType.ATTACHMENT_SINGLE]: SQLiteType.BLOB, [FieldType.ARRAY]: SQLiteType.BLOB, [FieldType.LINK]: SQLiteType.BLOB, - [FieldType.BIGINT]: SQLiteType.REAL, + [FieldType.BIGINT]: SQLiteType.TEXT, // TODO: consider the difference between multi-user and single user types (subtyping) [FieldType.BB_REFERENCE]: SQLiteType.TEXT, } diff --git a/packages/server/src/tests/utilities/api/datasource.ts b/packages/server/src/tests/utilities/api/datasource.ts index 0362a25940..6ac624f0db 100644 --- a/packages/server/src/tests/utilities/api/datasource.ts +++ b/packages/server/src/tests/utilities/api/datasource.ts @@ -61,7 +61,7 @@ export class DatasourceAPI extends TestAPI { } query = async ( - query: Omit, + query: Omit & Partial>, expectations?: Expectations ) => { return await this._post(`/api/datasources/query`, { diff --git a/packages/shared-core/src/filters.ts b/packages/shared-core/src/filters.ts index 6010f064bf..0554e0c1e4 100644 --- a/packages/shared-core/src/filters.ts +++ b/packages/shared-core/src/filters.ts @@ -218,14 +218,16 @@ export const buildLuceneQuery = (filter: SearchFilter[]) => { high: type === "number" ? maxint : "9999-00-00T00:00:00.000Z", } } - if ((operator as any) === "rangeLow" && value != null && value !== "") { - query.range[field].low = value - } else if ( - (operator as any) === "rangeHigh" && - value != null && - value !== "" - ) { - query.range[field].high = value + if (operator === "rangeLow" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + low: value, + } + } else if (operator === "rangeHigh" && value != null && value !== "") { + query.range[field] = { + ...query.range[field], + high: value, + } } } else if (query[queryOperator] && operator !== "onEmptyFilter") { if (type === "boolean") { diff --git a/packages/types/src/api/web/user.ts b/packages/types/src/api/web/user.ts index 10630c272c..75f00760bf 100644 --- a/packages/types/src/api/web/user.ts +++ b/packages/types/src/api/web/user.ts @@ -66,6 +66,8 @@ export interface CreateAdminUserRequest { password?: string tenantId: string ssoId?: string + familyName?: string + givenName?: string } export interface AddSSoUserRequest { diff --git a/packages/types/src/documents/global/user.ts b/packages/types/src/documents/global/user.ts index a4e6b613c6..9c7dc80e49 100644 --- a/packages/types/src/documents/global/user.ts +++ b/packages/types/src/documents/global/user.ts @@ -22,6 +22,13 @@ export interface UserSSO { providerType: SSOProviderType oauth2?: OAuth2 thirdPartyProfile?: SSOProfileJson + profile?: { + displayName?: string + name?: { + givenName?: string + familyName?: string + } + } } export type SSOUser = User & UserSSO diff --git a/packages/types/src/sdk/datasources.ts b/packages/types/src/sdk/datasources.ts index e1a012d81e..77e4877dfa 100644 --- a/packages/types/src/sdk/datasources.ts +++ b/packages/types/src/sdk/datasources.ts @@ -14,6 +14,14 @@ export enum Operation { DELETE_TABLE = "DELETE_TABLE", } +export const RowOperations = [ + Operation.CREATE, + Operation.READ, + Operation.UPDATE, + Operation.DELETE, + Operation.BULK_CREATE, +] + export enum SortDirection { ASCENDING = "ASCENDING", DESCENDING = "DESCENDING", diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 88c70d56bd..40f411f02a 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -27,10 +27,13 @@ export interface SearchFilters { [key: string]: string } [SearchFilterOperator.RANGE]?: { - [key: string]: { - high: number | string - low: number | string - } + [key: string]: + | { + high: number | string + low: number | string + } + | { high: number | string } + | { low: number | string } } [SearchFilterOperator.EQUAL]?: { [key: string]: any diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts index 4c1af90d38..b610ecce1a 100644 --- a/packages/worker/src/api/controllers/global/users.ts +++ b/packages/worker/src/api/controllers/global/users.ts @@ -116,7 +116,8 @@ const parseBooleanParam = (param: any) => { export const adminUser = async ( ctx: Ctx ) => { - const { email, password, tenantId, ssoId } = ctx.request.body + const { email, password, tenantId, ssoId, givenName, familyName } = + ctx.request.body if (await platform.tenants.exists(tenantId)) { ctx.throw(403, "Organisation already exists.") @@ -151,6 +152,8 @@ export const adminUser = async ( ssoId, hashPassword, requirePassword, + firstName: givenName, + lastName: familyName, }) // events diff --git a/packages/worker/src/api/routes/global/users.ts b/packages/worker/src/api/routes/global/users.ts index e7c77678fc..b40c491830 100644 --- a/packages/worker/src/api/routes/global/users.ts +++ b/packages/worker/src/api/routes/global/users.ts @@ -16,6 +16,8 @@ function buildAdminInitValidation() { password: OPTIONAL_STRING, tenantId: Joi.string().required(), ssoId: Joi.string(), + familyName: OPTIONAL_STRING, + givenName: OPTIONAL_STRING, }) .required() .unknown(false) diff --git a/scripts/devDocker.sh b/scripts/devDocker.sh new file mode 100755 index 0000000000..5e01e5813a --- /dev/null +++ b/scripts/devDocker.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Check if the pro submodule is loaded +if [ ! -d "./packages/pro/src" ]; then + echo "[ERROR] Submodule is not loaded. This is only allowed with loaded submodules." + exit 1 +fi + +yarn build --scope @budibase/server --scope @budibase/worker +docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0 + +