diff --git a/packages/backend-core/src/sql/sql.ts b/packages/backend-core/src/sql/sql.ts index e6738d4b36..b2fec5fe57 100644 --- a/packages/backend-core/src/sql/sql.ts +++ b/packages/backend-core/src/sql/sql.ts @@ -10,10 +10,12 @@ import { } from "./utils" import SqlTableQueryBuilder from "./sqlTable" import { + Aggregation, AnySearchFilter, ArrayOperator, BasicOperator, BBReferenceFieldMetadata, + CalculationType, FieldSchema, FieldType, INTERNAL_TABLE_SOURCE_ID, @@ -789,6 +791,38 @@ class InternalBuilder { return query.countDistinct(`${aliased}.${primary[0]} as total`) } + addAggregations( + query: Knex.QueryBuilder, + aggregations: Aggregation[] + ): Knex.QueryBuilder { + const fields = this.query.resource?.fields || [] + if (fields.length > 0) { + query = query.groupBy(fields.map(field => `${this.table.name}.${field}`)) + } + for (const aggregation of aggregations) { + const op = aggregation.calculationType + const field = `${this.table.name}.${aggregation.field} as ${aggregation.name}` + switch (op) { + case CalculationType.COUNT: + query = query.count(field) + break + case CalculationType.SUM: + query = query.sum(field) + break + case CalculationType.AVG: + query = query.avg(field) + break + case CalculationType.MIN: + query = query.min(field) + break + case CalculationType.MAX: + query = query.max(field) + break + } + } + return query + } + addSorting(query: Knex.QueryBuilder): Knex.QueryBuilder { let { sort } = this.query const primaryKey = this.table.primary @@ -1172,10 +1206,18 @@ class InternalBuilder { } } - // if counting, use distinct count, else select - query = !counting - ? query.select(this.generateSelectStatement()) - : this.addDistinctCount(query) + const aggregations = this.query.resource?.aggregations || [] + if (counting) { + query = this.addDistinctCount(query) + } else if (aggregations.length > 0) { + query = query.select( + this.knex.raw("ROW_NUMBER() OVER (ORDER BY (SELECT 0)) as _id") + ) + query = this.addAggregations(query, aggregations) + } else { + query = query.select(this.generateSelectStatement()) + } + // have to add after as well (this breaks MS-SQL) if (this.client !== SqlClient.MS_SQL && !counting) { query = this.addSorting(query) diff --git a/packages/server/src/api/controllers/row/views.ts b/packages/server/src/api/controllers/row/views.ts index d2541dfa25..ba48bc5664 100644 --- a/packages/server/src/api/controllers/row/views.ts +++ b/packages/server/src/api/controllers/row/views.ts @@ -7,8 +7,9 @@ import { RowSearchParams, SearchFilterKey, LogicalOperator, + Aggregation, } from "@budibase/types" -import { dataFilters } from "@budibase/shared-core" +import { dataFilters, helpers } from "@budibase/shared-core" import sdk from "../../../sdk" import { db, context, features } from "@budibase/backend-core" import { enrichSearchContext } from "./utils" @@ -27,7 +28,7 @@ export async function searchView( ctx.throw(400, `This method only supports viewsV2`) } - const viewFields = Object.entries(view.schema || {}) + const viewFields = Object.entries(helpers.views.basicFields(view)) .filter(([_, value]) => value.visible) .map(([key]) => key) const { body } = ctx.request @@ -74,6 +75,14 @@ export async function searchView( user: sdk.users.getUserContextBindings(ctx.user), }) + const aggregations: Aggregation[] = Object.entries( + helpers.views.calculationFields(view) + ).map(([name, { field, calculationType }]) => ({ + name, + calculationType, + field, + })) + const searchOptions: RequiredKeys & RequiredKeys> = { tableId: view.tableId, @@ -84,6 +93,7 @@ export async function searchView( bookmark: body.bookmark, paginate: body.paginate, countRows: body.countRows, + aggregations, } const result = await sdk.rows.search(searchOptions) diff --git a/packages/server/src/api/controllers/view/viewsV2.ts b/packages/server/src/api/controllers/view/viewsV2.ts index 4208772fa6..18306ba245 100644 --- a/packages/server/src/api/controllers/view/viewsV2.ts +++ b/packages/server/src/api/controllers/view/viewsV2.ts @@ -8,8 +8,45 @@ import { ViewResponse, ViewResponseEnriched, ViewV2, + BasicViewUIFieldMetadata, + ViewCalculationFieldMetadata, } from "@budibase/types" import { builderSocket, gridSocket } from "../../../websockets" +import { helpers } from "@budibase/shared-core" + +function stripUnknownFields( + field: ViewUIFieldMetadata +): RequiredKeys { + if (helpers.views.isCalculationField(field)) { + const strippedField: RequiredKeys = { + order: field.order, + width: field.width, + visible: field.visible, + readonly: field.readonly, + icon: field.icon, + calculationType: field.calculationType, + field: field.field, + } + return strippedField + } else { + const strippedField: RequiredKeys = { + order: field.order, + width: field.width, + visible: field.visible, + readonly: field.readonly, + icon: field.icon, + } + return strippedField + } +} + +function stripUndefinedFields(obj: Record): void { + Object.keys(obj) + .filter(key => obj[key] === undefined) + .forEach(key => { + delete obj[key] + }) +} async function parseSchema(view: CreateViewRequest) { if (!view.schema) { @@ -18,18 +55,8 @@ async function parseSchema(view: CreateViewRequest) { const finalViewSchema = view.schema && Object.entries(view.schema).reduce((p, [fieldName, schemaValue]) => { - const fieldSchema: RequiredKeys = { - order: schemaValue.order, - width: schemaValue.width, - visible: schemaValue.visible, - readonly: schemaValue.readonly, - icon: schemaValue.icon, - } - Object.entries(fieldSchema) - .filter(([, val]) => val === undefined) - .forEach(([key]) => { - delete fieldSchema[key as keyof ViewUIFieldMetadata] - }) + const fieldSchema = stripUnknownFields(schemaValue) + stripUndefinedFields(fieldSchema) p[fieldName] = fieldSchema return p }, {} as Record>) diff --git a/packages/server/src/api/routes/tests/viewV2.spec.ts b/packages/server/src/api/routes/tests/viewV2.spec.ts index 356f01dee0..72f55c16d2 100644 --- a/packages/server/src/api/routes/tests/viewV2.spec.ts +++ b/packages/server/src/api/routes/tests/viewV2.spec.ts @@ -18,6 +18,7 @@ import { ViewV2, SearchResponse, BasicOperator, + CalculationType, } from "@budibase/types" import { generator, mocks } from "@budibase/backend-core/tests" import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" @@ -32,13 +33,13 @@ import { import sdk from "../../../sdk" describe.each([ - ["lucene", undefined], + // ["lucene", undefined], ["sqs", undefined], - [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], - [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], - [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], - [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], - [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], + // [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + // [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + // [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + // [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], + // [DatabaseName.ORACLE, getDatasource(DatabaseName.ORACLE)], ])("/v2/views (%s)", (name, dsProvider) => { const config = setup.getConfig() const isSqs = name === "sqs" @@ -1978,6 +1979,64 @@ describe.each([ }) ) }) + + describe("calculations", () => { + let table: Table + let rows: Row[] + + beforeAll(async () => { + table = await config.api.table.save( + saveTableRequest({ + schema: { + quantity: { + type: FieldType.NUMBER, + name: "quantity", + }, + price: { + type: FieldType.NUMBER, + name: "price", + }, + }, + }) + ) + + rows = await Promise.all( + Array.from({ length: 10 }, () => + config.api.row.save(table._id!, { + quantity: generator.natural({ min: 1, max: 10 }), + price: generator.natural({ min: 1, max: 10 }), + }) + ) + ) + }) + + it.only("should be able to search by calculations", async () => { + const view = await config.api.viewV2.create({ + tableId: table._id!, + name: generator.guid(), + schema: { + "Quantity Sum": { + visible: true, + calculationType: CalculationType.SUM, + field: "quantity", + }, + }, + }) + + const response = await config.api.viewV2.search(view.id, { + query: {}, + }) + + expect(response.rows).toHaveLength(1) + expect(response.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + "Quantity Sum": rows.reduce((acc, r) => acc + r.quantity, 0), + }), + ]) + ) + }) + }) }) describe("permissions", () => { diff --git a/packages/server/src/sdk/app/rows/search.ts b/packages/server/src/sdk/app/rows/search.ts index 3d5de2e6cb..95cbc919a1 100644 --- a/packages/server/src/sdk/app/rows/search.ts +++ b/packages/server/src/sdk/app/rows/search.ts @@ -46,6 +46,9 @@ export async function search( paginate: options.paginate, fields: options.fields, countRows: options.countRows, + aggregations: options.aggregations + ?.map(a => `${a.field}:${a.calculationType}`) + .join(", "), }) const isExternalTable = isExternalTableID(options.tableId) diff --git a/packages/server/src/sdk/app/rows/search/internal/sqs.ts b/packages/server/src/sdk/app/rows/search/internal/sqs.ts index fb140e3c14..93c6cab2ea 100644 --- a/packages/server/src/sdk/app/rows/search/internal/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/internal/sqs.ts @@ -48,6 +48,7 @@ import { } from "@budibase/shared-core" import { isSearchingByRowID } from "../utils" import tracer from "dd-trace" +import { cloneDeep } from "lodash" const builder = new sql.Sql(SqlClient.SQL_LITE) const SQLITE_COLUMN_LIMIT = 2000 @@ -285,7 +286,7 @@ export async function search( table: Table, opts?: { retrying?: boolean } ): Promise> { - let { paginate, query, ...params } = options + let { paginate, query, ...params } = cloneDeep(options) const allTables = await sdk.tables.getAllInternalTables() const allTablesMap = buildTableMap(allTables) @@ -303,6 +304,21 @@ export async function search( ...cleanupFilters(query, table, allTables), documentType: DocumentType.ROW, } + + let fields = options.fields + if (fields === undefined) { + fields = buildInternalFieldList(table, allTables, { relationships }) + } else { + fields = fields.map(f => mapToUserColumn(f)) + } + + if (options.aggregations) { + options.aggregations = options.aggregations.map(a => { + a.field = mapToUserColumn(a.field) + return a + }) + } + const request: QueryJson = { endpoint: { // not important, we query ourselves @@ -317,9 +333,7 @@ export async function search( tables: allTablesMap, columnPrefix: USER_COLUMN_PREFIX, }, - resource: { - fields: buildInternalFieldList(table, allTables, { relationships }), - }, + resource: { fields, aggregations: options.aggregations }, relationships, } @@ -426,6 +440,7 @@ export async function search( if (err.status === 400 && msg?.match(MISSING_COLUMN_REGEX)) { return { rows: [] } } + throw err throw new Error(`Unable to search by SQL - ${msg}`, { cause: err }) } } diff --git a/packages/server/src/sdk/app/views/index.ts b/packages/server/src/sdk/app/views/index.ts index ed624c2b5c..01c2d3d2fa 100644 --- a/packages/server/src/sdk/app/views/index.ts +++ b/packages/server/src/sdk/app/views/index.ts @@ -1,8 +1,9 @@ import { + BasicViewUIFieldMetadata, RenameColumn, + Table, TableSchema, View, - ViewUIFieldMetadata, ViewV2, ViewV2Enriched, } from "@budibase/types" @@ -38,31 +39,84 @@ export async function getEnriched(viewId: string): Promise { return pickApi(tableId).getEnriched(viewId) } +async function guardCalculationViewSchema( + table: Table, + view: Omit +) { + const calculationFields = helpers.views.calculationFields(view) + for (const calculationFieldName of Object.keys(calculationFields)) { + const schema = calculationFields[calculationFieldName] + const targetSchema = table.schema[schema.field] + if (!targetSchema) { + throw new HTTPError( + `Calculation field "${calculationFieldName}" references field "${schema.field}" which does not exist in the table schema`, + 400 + ) + } + + if (!helpers.schema.isNumeric(targetSchema)) { + throw new HTTPError( + `Calculation field "${calculationFieldName}" references field "${schema.field}" which is not a numeric field`, + 400 + ) + } + } + + const groupByFields = helpers.views.basicFields(view) + for (const groupByFieldName of Object.keys(groupByFields)) { + const targetSchema = table.schema[groupByFieldName] + if (!targetSchema) { + throw new HTTPError( + `Group by field "${groupByFieldName}" does not exist in the table schema`, + 400 + ) + } + } +} + async function guardViewSchema( tableId: string, view: Omit ) { - const viewSchema = view.schema || {} const table = await sdk.tables.getTable(tableId) + if (helpers.views.isCalculationView(view)) { + await guardCalculationViewSchema(table, view) + } + + await checkReadonlyFields(table, view) + checkRequiredFields(table, view) + checkDisplayField(view) +} + +async function checkReadonlyFields( + table: Table, + view: Omit +) { + const viewSchema = view.schema || {} for (const field of Object.keys(viewSchema)) { - const tableSchemaField = table.schema[field] - if (!tableSchemaField) { + const viewFieldSchema = viewSchema[field] + if (helpers.views.isCalculationField(viewFieldSchema)) { + continue + } + + const tableFieldSchema = table.schema[field] + if (!tableFieldSchema) { throw new HTTPError( `Field "${field}" is not valid for the requested table`, 400 ) } - if (viewSchema[field].readonly) { + if (viewFieldSchema.readonly) { if ( !(await features.isViewReadonlyColumnsEnabled()) && - !(tableSchemaField as ViewUIFieldMetadata).readonly + !(tableFieldSchema as BasicViewUIFieldMetadata).readonly ) { throw new HTTPError(`Readonly fields are not enabled`, 400) } - if (!viewSchema[field].visible) { + if (!viewFieldSchema.visible) { throw new HTTPError( `Field "${field}" must be visible if you want to make it readonly`, 400 @@ -70,18 +124,33 @@ async function guardViewSchema( } } } +} - const existingView = - table?.views && (table.views[view.name] as ViewV2 | undefined) +function checkDisplayField(view: Omit) { + if (view.primaryDisplay) { + const viewSchemaField = view.schema?.[view.primaryDisplay] + if (!viewSchemaField?.visible) { + throw new HTTPError( + `You can't hide "${view.primaryDisplay}" because it is the display column.`, + 400 + ) + } + } +} + +function checkRequiredFields( + table: Table, + view: Omit +) { + const existingView = table.views?.[view.name] as ViewV2 | undefined for (const field of Object.values(table.schema)) { if (!helpers.schema.isRequired(field.constraints)) { continue } - const viewSchemaField = viewSchema[field.name] - const existingViewSchema = - existingView?.schema && existingView.schema[field.name] + const viewSchemaField = view.schema?.[field.name] + const existingViewSchema = existingView?.schema?.[field.name] if (!viewSchemaField && !existingViewSchema?.visible) { // Supporting existing configs with required columns but hidden in views continue @@ -94,24 +163,16 @@ async function guardViewSchema( ) } - if (viewSchemaField.readonly) { + if ( + helpers.views.isBasicViewField(viewSchemaField) && + viewSchemaField.readonly + ) { throw new HTTPError( `You can't make "${field.name}" readonly because it is a required field.`, 400 ) } } - - if (view.primaryDisplay) { - const viewSchemaField = viewSchema[view.primaryDisplay] - - if (!viewSchemaField?.visible) { - throw new HTTPError( - `You can't hide "${view.primaryDisplay}" because it is the display column.`, - 400 - ) - } - } } export async function create( diff --git a/packages/shared-core/src/helpers/index.ts b/packages/shared-core/src/helpers/index.ts index c33ff2ab1e..503f71e4eb 100644 --- a/packages/shared-core/src/helpers/index.ts +++ b/packages/shared-core/src/helpers/index.ts @@ -2,3 +2,4 @@ export * from "./helpers" export * from "./integrations" export * as cron from "./cron" export * as schema from "./schema" +export * as views from "./views" diff --git a/packages/shared-core/src/helpers/schema.ts b/packages/shared-core/src/helpers/schema.ts index d0035cc305..c9200fee18 100644 --- a/packages/shared-core/src/helpers/schema.ts +++ b/packages/shared-core/src/helpers/schema.ts @@ -45,3 +45,7 @@ export function decodeNonAscii(str: string): string { String.fromCharCode(parseInt(p1, 16)) ) } + +export function isNumeric(field: FieldSchema) { + return field.type === FieldType.NUMBER || field.type === FieldType.BIGINT +} diff --git a/packages/shared-core/src/helpers/views.ts b/packages/shared-core/src/helpers/views.ts new file mode 100644 index 0000000000..0364ccff41 --- /dev/null +++ b/packages/shared-core/src/helpers/views.ts @@ -0,0 +1,39 @@ +import { + BasicViewUIFieldMetadata, + ViewCalculationFieldMetadata, + ViewUIFieldMetadata, + ViewV2, +} from "@budibase/types" +import { pickBy } from "lodash" + +export function isCalculationField( + field: ViewUIFieldMetadata +): field is ViewCalculationFieldMetadata { + return "calculationType" in field +} + +export function isBasicViewField( + field: ViewUIFieldMetadata +): field is BasicViewUIFieldMetadata { + return !isCalculationField(field) +} + +type UnsavedViewV2 = Omit + +export function isCalculationView(view: UnsavedViewV2) { + return Object.values(view.schema || {}).some(isCalculationField) +} + +export function calculationFields(view: UnsavedViewV2) { + if (!isCalculationView(view)) { + throw new Error("View is not a calculation view") + } + return pickBy(view.schema || {}, isCalculationField) +} + +export function basicFields(view: UnsavedViewV2) { + if (!isCalculationView(view)) { + throw new Error("View is not a calculation view") + } + return pickBy(view.schema || {}, field => !isCalculationField(field)) +} diff --git a/packages/types/src/api/web/app/rows.ts b/packages/types/src/api/web/app/rows.ts index ce6f6f672d..5d49ac1812 100644 --- a/packages/types/src/api/web/app/rows.ts +++ b/packages/types/src/api/web/app/rows.ts @@ -26,6 +26,7 @@ export interface SearchViewRowRequest | "paginate" | "query" | "countRows" + | "aggregations" > {} export interface SearchRowResponse { diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 24dad0bcca..539c1e0d23 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -33,10 +33,19 @@ export interface View { groupBy?: string } -export type ViewUIFieldMetadata = UIFieldMetadata & { +export interface BasicViewUIFieldMetadata extends UIFieldMetadata { readonly?: boolean } +export interface ViewCalculationFieldMetadata extends BasicViewUIFieldMetadata { + calculationType: CalculationType + field: string +} + +export type ViewUIFieldMetadata = + | BasicViewUIFieldMetadata + | ViewCalculationFieldMetadata + export enum CalculationType { SUM = "sum", AVG = "avg", @@ -45,11 +54,6 @@ export enum CalculationType { MAX = "max", } -export type ViewCalculationFieldMetadata = ViewUIFieldMetadata & { - calculationType: CalculationType - field: string -} - export interface ViewV2 { version: 2 id: string @@ -62,7 +66,7 @@ export interface ViewV2 { order?: SortOrder type?: SortType } - schema?: Record + schema?: Record } export type ViewSchema = ViewCountOrSumSchema | ViewStatisticsSchema diff --git a/packages/types/src/sdk/row.ts b/packages/types/src/sdk/row.ts index 6850359cc3..8ee0338731 100644 --- a/packages/types/src/sdk/row.ts +++ b/packages/types/src/sdk/row.ts @@ -1,8 +1,14 @@ import { SortOrder, SortType } from "../api" import { SearchFilters } from "./search" -import { Row } from "../documents" +import { CalculationType, Row } from "../documents" import { WithRequired } from "../shared" +export interface Aggregation { + name: string + calculationType: CalculationType + field: string +} + export interface SearchParams { tableId?: string query?: SearchFilters @@ -18,6 +24,7 @@ export interface SearchParams { indexer?: () => Promise rows?: Row[] countRows?: boolean + aggregations?: Aggregation[] } // when searching for rows we want a more extensive search type that requires certain properties diff --git a/packages/types/src/sdk/search.ts b/packages/types/src/sdk/search.ts index 6feea40766..ba3c388480 100644 --- a/packages/types/src/sdk/search.ts +++ b/packages/types/src/sdk/search.ts @@ -2,6 +2,7 @@ import { Operation } from "./datasources" import { Row, Table, DocumentType } from "../documents" import { SortOrder, SortType } from "../api" import { Knex } from "knex" +import { Aggregation } from "./row" export enum BasicOperator { EQUAL = "equal", @@ -143,6 +144,7 @@ export interface QueryJson { } resource?: { fields: string[] + aggregations?: Aggregation[] } filters?: SearchFilters sort?: SortJson diff --git a/packages/types/src/shared/typeUtils.ts b/packages/types/src/shared/typeUtils.ts index c7ecebed0a..dbb3fc2553 100644 --- a/packages/types/src/shared/typeUtils.ts +++ b/packages/types/src/shared/typeUtils.ts @@ -4,6 +4,29 @@ export type DeepPartial = { export type ISO8601 = string +/** + * RequiredKeys make it such that you _must_ assign a value to every key in the + * type. It differs subtly from Required in that it doesn't change the type + * of the fields, you can specify undefined as a value and that's fine. + * + * Example: + * + * ```ts + * interface Foo { + * bar: string + * baz?: string + * } + * + * type FooRequiredKeys = RequiredKeys + * type FooRequired = Required + * + * const a: FooRequiredKeys = { bar: "hello", baz: undefined } + * const b: FooRequired = { bar: "hello", baz: undefined } + * ``` + * + * In this code, a passes type checking whereas b does not. This is because + * Required makes baz non-optional. + */ export type RequiredKeys = { [K in keyof Required]: T[K] }