Merge remote-tracking branch 'origin/develop' into feature/frequency-histogram

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.8.28-alpha.1",
+  "version": "2.8.29-alpha.1",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -51,9 +51,9 @@
     "kill-builder": "kill-port 3000",
     "kill-server": "kill-port 4001 4002",
     "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && yarn nx run-many --target=dev:builder",
-    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && yarn nx run-many --target=dev:builder --exclude=@budibase/backend-core,@budibase/server,@budibase/worker",
-    "dev:server": "yarn run kill-server &&  yarn nx run-many --target=dev:builder --projects=@budibase/worker,@budibase/server",
+    "dev": "yarn run kill-all && lerna run --stream --parallel dev:builder",
+    "dev:noserver": "yarn run kill-builder && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
     "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
     "dev:docker": "yarn build:docker:pre && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
     "test": "lerna run --stream test --stream",

packages/server/src/api/controllers/row/external.ts

@@ -8,26 +8,13 @@ import {
   Datasource,
   IncludeRelationship,
   Operation,
+  PatchRowRequest,
+  PatchRowResponse,
   Row,
   Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
-import * as utils from "./utils"
-
-async function getRow(
-  tableId: string,
-  rowId: string,
-  opts?: { relationships?: boolean }
-) {
-  const response = (await handleRequest(Operation.READ, tableId, {
-    id: breakRowIdField(rowId),
-    includeSqlRelationships: opts?.relationships
-      ? IncludeRelationship.INCLUDE
-      : IncludeRelationship.EXCLUDE,
-  })) as Row[]
-  return response ? response[0] : response
-}
 
 export async function handleRequest(
   operation: Operation,
@@ -55,14 +42,12 @@ export async function handleRequest(
   )
 }
 
-export async function patch(ctx: UserCtx) {
-  const inputs = ctx.request.body
+export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const tableId = ctx.params.tableId
-  const id = inputs._id
-  // don't save the ID to db
-  delete inputs._id
-  const validateResult = await utils.validate({
-    row: inputs,
+  const { id, ...rowData } = ctx.request.body
+
+  const validateResult = await sdk.rows.utils.validate({
+    row: rowData,
     tableId,
   })
   if (!validateResult.valid) {
@@ -70,9 +55,11 @@ export async function patch(ctx: UserCtx) {
   }
   const response = await handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(id),
-    row: inputs,
+    row: rowData,
+  })
+  const row = await sdk.rows.external.getRow(tableId, id, {
+    relationships: true,
   })
-  const row = await getRow(tableId, id, { relationships: true })
   const table = await sdk.tables.getTable(tableId)
   return {
     ...response,
@@ -84,7 +71,7 @@ export async function patch(ctx: UserCtx) {
 export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
-  const validateResult = await utils.validate({
+  const validateResult = await sdk.rows.utils.validate({
     row: inputs,
     tableId,
   })
@@ -97,7 +84,9 @@ export async function save(ctx: UserCtx) {
   const responseRow = response as { row: Row }
   const rowId = responseRow.row._id
   if (rowId) {
-    const row = await getRow(tableId, rowId, { relationships: true })
+    const row = await sdk.rows.external.getRow(tableId, rowId, {
+      relationships: true,
+    })
     return {
       ...response,
       row,
@@ -110,7 +99,7 @@ export async function save(ctx: UserCtx) {
 export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
-  return getRow(tableId, id)
+  return sdk.rows.external.getRow(tableId, id)
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/api/controllers/row/index.ts

@@ -9,6 +9,8 @@ import {
   DeleteRow,
   DeleteRows,
   Row,
+  PatchRowRequest,
+  PatchRowResponse,
   SearchResponse,
   SortOrder,
   SortType,
@@ -29,7 +31,9 @@ function pickApi(tableId: any) {
   return internal
 }
 
-export async function patch(ctx: any): Promise<any> {
+export async function patch(
+  ctx: UserCtx<PatchRowRequest, PatchRowResponse>
+): Promise<any> {
   const appId = ctx.appId
   const tableId = utils.getTableId(ctx)
   const body = ctx.request.body
@@ -38,7 +42,7 @@ export async function patch(ctx: any): Promise<any> {
     return save(ctx)
   }
   try {
-    const { row, table } = await quotas.addQuery<any>(
+    const { row, table } = await quotas.addQuery(
       () => pickApi(tableId).patch(ctx),
       {
         datasourceId: tableId,
@@ -53,7 +57,7 @@ export async function patch(ctx: any): Promise<any> {
     ctx.message = `${table.name} updated successfully.`
     ctx.body = row
     gridSocket?.emitRowUpdate(ctx, row)
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
@@ -78,6 +82,7 @@ export const save = async (ctx: any) => {
   ctx.body = row || squashed
   gridSocket?.emitRowUpdate(ctx, row || squashed)
 }
+
 export async function fetchView(ctx: any) {
   const tableId = utils.getTableId(ctx)
   const viewName = decodeURIComponent(ctx.params.viewName)
@@ -267,7 +272,7 @@ export async function searchView(ctx: Ctx<void, SearchResponse>) {
     undefined
 
   ctx.status = 200
-  ctx.body = await quotas.addQuery(
+  const result = await quotas.addQuery(
     () =>
       sdk.rows.search({
         tableId: view.tableId,
@@ -279,6 +284,9 @@ export async function searchView(ctx: Ctx<void, SearchResponse>) {
       datasourceId: view.tableId,
     }
   )
+
+  result.rows.forEach(r => (r._viewId = view.id))
+  ctx.body = result
 }
 
 export async function validate(ctx: Ctx) {
@@ -287,7 +295,7 @@ export async function validate(ctx: Ctx) {
   if (isExternalTable(tableId)) {
     ctx.body = { valid: true }
   } else {
-    ctx.body = await utils.validate({
+    ctx.body = await sdk.rows.utils.validate({
       row: ctx.request.body,
       tableId,
     })

packages/server/src/api/controllers/row/internal.ts

@@ -15,19 +15,26 @@ import * as utils from "./utils"
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
+import {
+  UserCtx,
+  LinkDocumentValue,
+  Row,
+  Table,
+  PatchRowRequest,
+  PatchRowResponse,
+} from "@budibase/types"
 import sdk from "../../../sdk"
 
-export async function patch(ctx: UserCtx) {
+export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
   const inputs = ctx.request.body
   const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
+  const dbTable = await sdk.tables.getTable(tableId)
   try {
-    let dbTable = await sdk.tables.getTable(tableId)
     oldRow = await outputProcessing(
       dbTable,
-      await utils.findRow(ctx, tableId, inputs._id)
+      await utils.findRow(ctx, tableId, inputs._id!)
     )
   } catch (err) {
     if (isUserTable) {
@@ -40,7 +47,7 @@ export async function patch(ctx: UserCtx) {
       throw "Row does not exist"
     }
   }
-  let dbTable = await sdk.tables.getTable(tableId)
+
   // need to build up full patch fields before coerce
   let combinedRow: any = cloneDeep(oldRow)
   for (let key of Object.keys(inputs)) {
@@ -53,7 +60,7 @@ export async function patch(ctx: UserCtx) {
 
   // this returns the table and row incase they have been updated
   let { table, row } = inputProcessing(ctx.user, tableClone, combinedRow)
-  const validateResult = await utils.validate({
+  const validateResult = await sdk.rows.utils.validate({
     row,
     table,
   })
@@ -74,7 +81,7 @@ export async function patch(ctx: UserCtx) {
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
-    ctx.request.body = row
+    ctx.request.body = row as any
     await userController.updateMetadata(ctx)
     return { row: ctx.body as Row, table }
   }
@@ -102,7 +109,7 @@ export async function save(ctx: UserCtx) {
 
   let { table, row } = inputProcessing(ctx.user, tableClone, inputs)
 
-  const validateResult = await utils.validate({
+  const validateResult = await sdk.rows.utils.validate({
     row,
     table,
   })

packages/server/src/api/controllers/user.ts

@@ -1,6 +1,5 @@
-import { generateUserMetadataID, generateUserFlagID } from "../../db/utils"
+import { generateUserFlagID } from "../../db/utils"
 import { InternalTables } from "../../db/utils"
-import { getGlobalUsers } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
 import { context } from "@budibase/backend-core"
 import { Ctx, UserCtx } from "@budibase/types"

packages/server/src/api/routes/tests/row.spec.ts

@@ -5,7 +5,7 @@ tk.freeze(timestamp)
 import { outputProcessing } from "../../../utilities/rowProcessor"
 import * as setup from "./utilities"
 const { basicRow } = setup.structures
-import { context, db, tenancy } from "@budibase/backend-core"
+import { context, tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import {
   QuotaUsageType,
@@ -16,6 +16,7 @@ import {
   FieldType,
   SortType,
   SortOrder,
+  PatchRowRequest,
 } from "@budibase/types"
 import {
   expectAnyInternalColsAttributes,
@@ -399,17 +400,12 @@ describe("/rows", () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      const res = await request
-        .patch(`/api/${table._id}/rows`)
-        .send({
-          _id: existing._id,
-          _rev: existing._rev,
-          tableId: table._id,
-          name: "Updated Name",
-        })
-        .set(config.defaultHeaders())
-        .expect("Content-Type", /json/)
-        .expect(200)
+      const res = await config.api.row.patch(table._id!, {
+        _id: existing._id!,
+        _rev: existing._rev!,
+        tableId: table._id!,
+        name: "Updated Name",
+      })
 
       expect((res as any).res.statusMessage).toEqual(
         `${table.name} updated successfully.`
@@ -430,16 +426,16 @@ describe("/rows", () => {
       const rowUsage = await getRowUsage()
       const queryUsage = await getQueryUsage()
 
-      await request
-        .patch(`/api/${table._id}/rows`)
-        .send({
-          _id: existing._id,
-          _rev: existing._rev,
-          tableId: table._id,
+      await config.api.row.patch(
+        table._id!,
+        {
+          _id: existing._id!,
+          _rev: existing._rev!,
+          tableId: table._id!,
           name: 1,
-        })
-        .set(config.defaultHeaders())
-        .expect(400)
+        },
+        { expectStatus: 400 }
+      )
 
       await assertRowUsage(rowUsage)
       await assertQueryUsage(queryUsage)
@@ -986,16 +982,17 @@ describe("/rows", () => {
         )
       }
 
-      const createViewResponse = await config.api.viewV2.create({
+      const view = await config.api.viewV2.create({
         columns: { name: { visible: true } },
       })
-      const response = await config.api.viewV2.search(createViewResponse.id)
+      const response = await config.api.viewV2.search(view.id)
 
       expect(response.body.rows).toHaveLength(10)
       expect(response.body.rows).toEqual(
         expect.arrayContaining(
           rows.map(r => ({
             ...expectAnyInternalColsAttributes,
+            _viewId: view.id,
             name: r.name,
           }))
         )

packages/server/src/sdk/app/rows/external.ts

@@ -0,0 +1,17 @@
+import { IncludeRelationship, Operation, Row } from "@budibase/types"
+import { handleRequest } from "../../../api/controllers/row/external"
+import { breakRowIdField } from "../../../integrations/utils"
+
+export async function getRow(
+  tableId: string,
+  rowId: string,
+  opts?: { relationships?: boolean }
+) {
+  const response = (await handleRequest(Operation.READ, tableId, {
+    id: breakRowIdField(rowId),
+    includeSqlRelationships: opts?.relationships
+      ? IncludeRelationship.INCLUDE
+      : IncludeRelationship.EXCLUDE,
+  })) as Row[]
+  return response ? response[0] : response
+}

packages/server/src/sdk/app/rows/index.ts

@@ -2,10 +2,12 @@ import * as attachments from "./attachments"
 import * as rows from "./rows"
 import * as search from "./search"
 import * as utils from "./utils"
+import * as external from "./external"
 
 export default {
   ...attachments,
   ...rows,
   ...search,
-  utils: utils,
+  utils,
+  external,
 }

packages/server/src/sdk/app/rows/search/internal.ts

@@ -147,8 +147,8 @@ export async function exportRows(
 export async function fetch(tableId: string) {
   const db = context.getAppDB()
 
-  let table = await sdk.tables.getTable(tableId)
-  let rows = await getRawTableData(db, tableId)
+  const table = await sdk.tables.getTable(tableId)
+  const rows = await getRawTableData(db, tableId)
   const result = await outputProcessing(table, rows)
   return result
 }
@@ -171,7 +171,7 @@ async function getRawTableData(db: Database, tableId: string) {
 export async function fetchView(
   viewName: string,
   options: { calculation: string; group: string; field: string }
-) {
+): Promise<Row[]> {
   // if this is a table view being looked for just transfer to that
   if (viewName.startsWith(DocumentType.TABLE)) {
     return fetch(viewName)
@@ -197,7 +197,7 @@ export async function fetchView(
     )
   }
 
-  let rows
+  let rows: Row[] = []
   if (!calculation) {
     response.rows = response.rows.map(row => row.doc)
     let table: Table

packages/server/src/sdk/app/rows/utils.ts

@@ -1,4 +1,6 @@
-import { TableSchema } from "@budibase/types"
+import cloneDeep from "lodash/cloneDeep"
+import validateJs from "validate.js"
+import { FieldType, Row, Table, TableSchema } from "@budibase/types"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Format } from "../../../api/controllers/view/exporters"
@@ -46,3 +48,90 @@ export function cleanExportRows(
 
   return cleanRows
 }
+
+function isForeignKey(key: string, table: Table) {
+  const relationships = Object.values(table.schema).filter(
+    column => column.type === FieldType.LINK
+  )
+  return relationships.some(relationship => relationship.foreignKey === key)
+}
+
+export async function validate({
+  tableId,
+  row,
+  table,
+}: {
+  tableId?: string
+  row: Row
+  table?: Table
+}): Promise<{
+  valid: boolean
+  errors: Record<string, any>
+}> {
+  let fetchedTable: Table
+  if (!table) {
+    fetchedTable = await sdk.tables.getTable(tableId)
+  } else {
+    fetchedTable = table
+  }
+  const errors: Record<string, any> = {}
+  for (let fieldName of Object.keys(fetchedTable.schema)) {
+    const column = fetchedTable.schema[fieldName]
+    const constraints = cloneDeep(column.constraints)
+    const type = column.type
+    // foreign keys are likely to be enriched
+    if (isForeignKey(fieldName, fetchedTable)) {
+      continue
+    }
+    // formulas shouldn't validated, data will be deleted anyway
+    if (type === FieldTypes.FORMULA || column.autocolumn) {
+      continue
+    }
+    // special case for options, need to always allow unselected (empty)
+    if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
+      constraints.inclusion.push(null as any, "")
+    }
+    let res
+
+    // Validate.js doesn't seem to handle array
+    if (type === FieldTypes.ARRAY && row[fieldName]) {
+      if (row[fieldName].length) {
+        if (!Array.isArray(row[fieldName])) {
+          row[fieldName] = row[fieldName].split(",")
+        }
+        row[fieldName].map((val: any) => {
+          if (
+            !constraints?.inclusion?.includes(val) &&
+            constraints?.inclusion?.length !== 0
+          ) {
+            errors[fieldName] = "Field not in list"
+          }
+        })
+      } else if (constraints?.presence && row[fieldName].length === 0) {
+        // non required MultiSelect creates an empty array, which should not throw errors
+        errors[fieldName] = [`${fieldName} is required`]
+      }
+    } else if (
+      (type === FieldTypes.ATTACHMENT || type === FieldTypes.JSON) &&
+      typeof row[fieldName] === "string"
+    ) {
+      // this should only happen if there is an error
+      try {
+        const json = JSON.parse(row[fieldName])
+        if (type === FieldTypes.ATTACHMENT) {
+          if (Array.isArray(json)) {
+            row[fieldName] = json
+          } else {
+            errors[fieldName] = [`Must be an array`]
+          }
+        }
+      } catch (err) {
+        errors[fieldName] = [`Contains invalid JSON`]
+      }
+    } else {
+      res = validateJs.single(row[fieldName], constraints)
+    }
+    if (res) errors[fieldName] = res
+  }
+  return { valid: Object.keys(errors).length === 0, errors }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -1,13 +1,16 @@
 import TestConfiguration from "../TestConfiguration"
+import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
 
 export default class API {
   table: TableAPI
   viewV2: ViewV2API
+  row: RowAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
     this.viewV2 = new ViewV2API(config)
+    this.row = new RowAPI(config)
   }
 }

packages/server/src/tests/utilities/api/row.ts

@@ -0,0 +1,22 @@
+import { PatchRowRequest } from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class RowAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  patch = async (
+    tableId: string,
+    row: PatchRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    return this.request
+      .patch(`/api/${tableId}/rows`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
+}

packages/server/src/utilities/rowProcessor/index.ts

@@ -186,18 +186,21 @@ export function inputProcessing(
  * @param {object} opts used to set some options for the output, such as disabling relationship squashing.
  * @returns {object[]|object} the enriched rows will be returned.
  */
-export async function outputProcessing(
+export async function outputProcessing<T extends Row[] | Row>(
   table: Table,
-  rows: Row[] | Row,
+  rows: T,
   opts = { squash: true }
-) {
+): Promise<T> {
+  let safeRows: Row[]
   let wasArray = true
   if (!(rows instanceof Array)) {
-    rows = [rows]
+    safeRows = [rows]
     wasArray = false
+  } else {
+    safeRows = rows
   }
   // attach any linked row information
-  let enriched = await linkRows.attachFullLinkedDocs(table, rows as Row[])
+  let enriched = await linkRows.attachFullLinkedDocs(table, safeRows)
 
   // process formulas
   enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
@@ -221,7 +224,7 @@ export async function outputProcessing(
       enriched
     )) as Row[]
   }
-  return wasArray ? enriched : enriched[0]
+  return (wasArray ? enriched : enriched[0]) as T
 }
 
 /**

packages/types/src/api/web/app/rows.ts

@@ -1,3 +1,13 @@
+import { Row } from "../../../documents"
+
+export interface PatchRowRequest extends Row {
+  _id: string
+  _rev: string
+  tableId: string
+}
+
+export interface PatchRowResponse extends Row {}
+
 export interface SearchResponse {
   rows: any[]
 }

packages/types/src/documents/app/row.ts

@@ -30,5 +30,6 @@ export interface RowAttachment {
 export interface Row extends Document {
   type?: string
   tableId?: string
+  _viewId?: string
   [key: string]: any
 }