Merge branch 'develop' of github.com:Budibase/budibase into fix/ci-disk-space

.github/workflows/stale_bot.yml

@@ -2,7 +2,7 @@ name: Close stale issues and PRs # https://github.com/actions/stale
 on:
   workflow_dispatch:
   schedule:
-    - cron: '30 1 * * *' # 1:30 every morning
+    - cron: '*/30 * * * *' # Every 30 mins
 
 jobs:
   stale:

.prettierignore

@@ -9,4 +9,5 @@ packages/backend-core/coverage
 packages/server/client
 packages/server/src/definitions/openapi.ts
 packages/builder/.routify
-packages/sdk/sdk
+packages/sdk/sdk
+packages/pro/coverage

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.10.8",
+  "version": "2.10.9-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -1,4 +1,5 @@
 import {
+  AutoReason,
   Datasource,
   FieldSchema,
   FieldType,
@@ -24,7 +25,7 @@ import {
   isSQL,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
-import { FieldTypes } from "../../../constants"
+import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
 import { processDates, processFormulas } from "../../../utilities/rowProcessor"
@@ -259,6 +260,15 @@ function isOneSide(field: FieldSchema) {
   )
 }
 
+function isEditableColumn(column: FieldSchema) {
+  const isExternalAutoColumn =
+    column.autocolumn &&
+    column.autoReason !== AutoReason.FOREIGN_KEY &&
+    column.subtype !== AutoFieldSubTypes.AUTO_ID
+  const isFormula = column.type === FieldTypes.FORMULA
+  return !(isExternalAutoColumn || isFormula)
+}
+
 export class ExternalRequest {
   private operation: Operation
   private tableId: string
@@ -295,11 +305,7 @@ export class ExternalRequest {
       manyRelationships: ManyRelationship[] = []
     for (let [key, field] of Object.entries(table.schema)) {
       // if set already, or not set just skip it
-      if (
-        row[key] == null ||
-        newRow[key] ||
-        !sdk.tables.isEditableColumn(field)
-      ) {
+      if (row[key] == null || newRow[key] || !isEditableColumn(field)) {
         continue
       }
       // if its an empty string then it means return the column to null (if possible)

packages/server/src/api/controllers/row/external.ts

@@ -18,7 +18,8 @@ import {
 import sdk from "../../../sdk"
 import * as utils from "./utils"
 import { dataFilters } from "@budibase/shared-core"
-import { removeEmptyFilters } from "./utils"
+import { inputProcessing } from "../../../utilities/rowProcessor"
+import { cloneDeep, isEqual } from "lodash"
 
 export async function handleRequest(
   operation: Operation,
@@ -77,10 +78,24 @@ export async function save(ctx: UserCtx) {
   if (!validateResult.valid) {
     throw { validation: validateResult.errors }
   }
+
+  const table = await sdk.tables.getTable(tableId)
+  const { table: updatedTable, row } = inputProcessing(
+    ctx.user,
+    cloneDeep(table),
+    inputs
+  )
+
   const response = await handleRequest(Operation.CREATE, tableId, {
-    row: inputs,
+    row,
   })
+
   const responseRow = response as { row: Row }
+
+  if (!isEqual(table, updatedTable)) {
+    await sdk.tables.saveTable(updatedTable)
+  }
+
   const rowId = responseRow.row._id
   if (rowId) {
     const row = await sdk.rows.external.getRow(tableId, rowId, {
@@ -95,10 +110,18 @@ export async function save(ctx: UserCtx) {
   }
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const id = ctx.params.rowId
   const tableId = utils.getTableId(ctx)
-  return sdk.rows.external.getRow(tableId, id)
+  const row = await sdk.rows.external.getRow(tableId, id, {
+    relationships: true,
+  })
+
+  if (!row) {
+    ctx.throw(404)
+  }
+
+  return row
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -108,7 +131,7 @@ export async function destroy(ctx: UserCtx) {
     id: breakRowIdField(_id),
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })) as { row: Row }
-  return { response: { ok: true }, row }
+  return { response: { ok: true, id: _id }, row }
 }
 
 export async function bulkDestroy(ctx: UserCtx) {

packages/server/src/api/controllers/row/index.ts

@@ -14,6 +14,10 @@ import {
   SearchRowResponse,
   SearchRowRequest,
   SearchParams,
+  GetRowResponse,
+  ValidateResponse,
+  ExportRowsRequest,
+  ExportRowsResponse,
 } from "@budibase/types"
 import * as utils from "./utils"
 import { gridSocket } from "../../../websockets"
@@ -111,7 +115,7 @@ export async function fetch(ctx: any) {
   })
 }
 
-export async function find(ctx: any) {
+export async function find(ctx: UserCtx<void, GetRowResponse>) {
   const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
     datasourceId: tableId,
@@ -214,11 +218,11 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
   })
 }
 
-export async function validate(ctx: Ctx) {
+export async function validate(ctx: Ctx<Row, ValidateResponse>) {
   const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
   if (isExternalTable(tableId)) {
-    ctx.body = { valid: true }
+    ctx.body = { valid: true, errors: {} }
   } else {
     ctx.body = await sdk.rows.utils.validate({
       row: ctx.request.body,
@@ -237,7 +241,9 @@ export async function fetchEnrichedRow(ctx: any) {
   )
 }
 
-export const exportRows = async (ctx: any) => {
+export const exportRows = async (
+  ctx: Ctx<ExportRowsRequest, ExportRowsResponse>
+) => {
   const tableId = utils.getTableId(ctx)
 
   const format = ctx.query.format

packages/server/src/api/controllers/row/internal.ts

@@ -131,7 +131,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function find(ctx: UserCtx) {
+export async function find(ctx: UserCtx): Promise<Row> {
   const tableId = utils.getTableId(ctx),
     rowId = ctx.params.rowId
   const table = await sdk.tables.getTable(tableId)

packages/server/src/api/controllers/row/utils.ts

@@ -34,7 +34,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
-  let row
+  let row: Row
   // TODO remove special user case in future
   if (tableId === InternalTables.USER_METADATA) {
     ctx.params = {

packages/server/src/api/controllers/table/index.ts

@@ -78,9 +78,9 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
   ctx.eventEmitter &&
-    ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
+    ctx.eventEmitter.emitTable(`table:save`, appId, { ...savedTable })
   ctx.body = savedTable
-  builderSocket?.emitTableUpdate(ctx, savedTable)
+  builderSocket?.emitTableUpdate(ctx, { ...savedTable })
 }
 
 export async function destroy(ctx: UserCtx) {

packages/server/src/api/routes/tests/table.spec.ts

@@ -1,6 +1,6 @@
 import { generator } from "@budibase/backend-core/tests"
 import { events, context } from "@budibase/backend-core"
-import { FieldType, Table } from "@budibase/types"
+import { FieldType, Table, ViewCalculation } from "@budibase/types"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import * as setup from "./utilities"
 const { basicTable } = setup.structures
@@ -90,8 +90,10 @@ describe("/tables", () => {
       await config.createLegacyView({
         name: "TestView",
         field: "Price",
-        calculation: "stats",
-        tableId: testTable._id,
+        calculation: ViewCalculation.STATISTICS,
+        tableId: testTable._id!,
+        schema: {},
+        filters: [],
       })
 
       const testRow = await request
@@ -254,7 +256,7 @@ describe("/tables", () => {
       }))
 
       await config.api.viewV2.create({ tableId })
-      await config.createLegacyView({ tableId, name: generator.guid() })
+      await config.createLegacyView()
 
       const res = await config.api.table.fetch()
 
@@ -366,7 +368,7 @@ describe("/tables", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.message).toEqual(`Table ${testTable._id} deleted.`)
-      const dependentTable = await config.getTable(linkedTable._id)
+      const dependentTable = await config.api.table.get(linkedTable._id!)
       expect(dependentTable.schema.TestTable).not.toBeDefined()
     })
 

packages/server/src/api/routes/tests/viewV2.spec.ts

@@ -6,6 +6,7 @@ import {
   SortOrder,
   SortType,
   Table,
+  UIFieldMetadata,
   UpdateViewRequest,
   ViewV2,
 } from "@budibase/types"
@@ -418,9 +419,12 @@ describe.each([
       const res = await config.api.viewV2.create(newView)
       const view = await config.api.viewV2.get(res.id)
       expect(view!.schema?.Price).toBeUndefined()
-      const updatedTable = await config.getTable(table._id!)
-      const viewSchema = updatedTable.views[view!.name!].schema
-      expect(viewSchema.Price.visible).toEqual(false)
+      const updatedTable = await config.api.table.get(table._id!)
+      const viewSchema = updatedTable.views![view!.name!].schema as Record<
+        string,
+        UIFieldMetadata
+      >
+      expect(viewSchema.Price?.visible).toEqual(false)
     })
   })
 })

packages/server/src/db/tests/linkController.spec.js

@@ -6,7 +6,7 @@ const { RelationshipType } = require("../../constants")
 const { cloneDeep } = require("lodash/fp")
 
 describe("test the link controller", () => {
-  let config = new TestConfig(false)
+  let config = new TestConfig()
   let table1, table2, appId
 
   beforeAll(async () => {

packages/server/src/db/tests/linkTests.spec.js

@@ -4,7 +4,7 @@ const linkUtils = require("../linkedRows/linkUtils")
 const { context } = require("@budibase/backend-core")
 
 describe("test link functionality", () => {
-  const config = new TestConfig(false)
+  const config = new TestConfig()
   let appId
 
   describe("getLinkedTable", () => {

packages/server/src/integration-test/postgres.spec.ts

@@ -12,18 +12,15 @@ import {
   FieldType,
   RelationshipType,
   Row,
-  SourceName,
   Table,
 } from "@budibase/types"
 import _ from "lodash"
 import { generator } from "@budibase/backend-core/tests"
 import { utils } from "@budibase/backend-core"
-import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+import { databaseTestProviders } from "../integrations/tests/utils"
 
 const config = setup.getConfig()!
 
-jest.setTimeout(30000)
-
 jest.unmock("pg")
 jest.mock("../websockets")
 
@@ -35,62 +32,18 @@ describe("postgres integrations", () => {
     manyToOneRelationshipInfo: ForeignTableInfo,
     manyToManyRelationshipInfo: ForeignTableInfo
 
-  let host: string
-  let port: number
-  const containers: StartedTestContainer[] = []
-
   beforeAll(async () => {
-    const containerPostgres = await new GenericContainer("postgres")
-      .withExposedPorts(5432)
-      .withEnv("POSTGRES_PASSWORD", "password")
-      .withWaitStrategy(
-        Wait.forLogMessage(
-          "PostgreSQL init process complete; ready for start up."
-        )
-      )
-      .start()
-
-    host = containerPostgres.getContainerIpAddress()
-    port = containerPostgres.getMappedPort(5432)
-
     await config.init()
     const apiKey = await config.generateApiKey()
 
-    containers.push(containerPostgres)
-
     makeRequest = generateMakeRequest(apiKey, true)
-  })
 
-  afterAll(async () => {
-    for (let container of containers) {
-      await container.stop()
-    }
+    postgresDatasource = await config.api.datasource.create(
+      await databaseTestProviders.postgres.getDsConfig()
+    )
   })
 
-  function pgDatasourceConfig() {
-    return {
-      datasource: {
-        type: "datasource",
-        source: SourceName.POSTGRES,
-        plus: true,
-        config: {
-          host,
-          port,
-          database: "postgres",
-          user: "postgres",
-          password: "password",
-          schema: "public",
-          ssl: false,
-          rejectUnauthorized: false,
-          ca: false,
-        },
-      },
-    }
-  }
-
   beforeEach(async () => {
-    postgresDatasource = await config.createDatasource(pgDatasourceConfig())
-
     async function createAuxTable(prefix: string) {
       return await config.createTable({
         name: `${prefix}_${generator.word({ length: 6 })}`,
@@ -226,25 +179,6 @@ describe("postgres integrations", () => {
     let { rowData } = opts as any
     let foreignRows: ForeignRowsInfo[] = []
 
-    async function createForeignRow(tableInfo: ForeignTableInfo) {
-      const foreignKey = `fk_${tableInfo.table.name}_${tableInfo.fieldName}`
-
-      const foreignRow = await config.createRow({
-        tableId: tableInfo.table._id,
-        title: generator.name(),
-      })
-
-      rowData = {
-        ...rowData,
-        [foreignKey]: foreignRow.id,
-      }
-      foreignRows.push({
-        row: foreignRow,
-
-        relationshipType: tableInfo.relationshipType,
-      })
-    }
-
     if (opts?.createForeignRows?.createOneToMany) {
       const foreignKey = `fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`
 
@@ -322,6 +256,14 @@ describe("postgres integrations", () => {
     })
   }
 
+  const createRandomTableWithRows = async () => {
+    const tableId = (await createDefaultPgTable())._id!
+    return await config.api.row.save(tableId, {
+      tableId,
+      title: generator.name(),
+    })
+  }
+
   async function populatePrimaryRows(
     count: number,
     opts?: {
@@ -357,9 +299,9 @@ describe("postgres integrations", () => {
       config: {
         ca: false,
         database: "postgres",
-        host,
+        host: postgresDatasource.config!.host,
         password: "--secret-value--",
-        port,
+        port: postgresDatasource.config!.port,
         rejectUnauthorized: false,
         schema: "public",
         ssl: false,
@@ -401,12 +343,16 @@ describe("postgres integrations", () => {
 
       it("multiple rows can be persisted", async () => {
         const numberOfRows = 10
-        const newRows = Array(numberOfRows).fill(generateRandomPrimaryRowData())
+        const newRows: Row[] = Array(numberOfRows).fill(
+          generateRandomPrimaryRowData()
+        )
 
-        for (const newRow of newRows) {
-          const res = await createRow(primaryPostgresTable._id, newRow)
-          expect(res.status).toBe(200)
-        }
+        await Promise.all(
+          newRows.map(async newRow => {
+            const res = await createRow(primaryPostgresTable._id, newRow)
+            expect(res.status).toBe(200)
+          })
+        )
 
         const persistedRows = await config.getRows(primaryPostgresTable._id!)
         expect(persistedRows).toHaveLength(numberOfRows)
@@ -567,7 +513,7 @@ describe("postgres integrations", () => {
           foreignRows = createdRow.foreignRows
         })
 
-        it("only one to many foreign keys are retrieved", async () => {
+        it("only one to primary keys are retrieved", async () => {
           const res = await getRow(primaryPostgresTable._id, row.id)
 
           expect(res.status).toBe(200)
@@ -575,6 +521,12 @@ describe("postgres integrations", () => {
           const one2ManyForeignRows = foreignRows.filter(
             x => x.relationshipType === RelationshipType.ONE_TO_MANY
           )
+          const many2OneForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_ONE
+          )
+          const many2ManyForeignRows = foreignRows.filter(
+            x => x.relationshipType === RelationshipType.MANY_TO_MANY
+          )
           expect(one2ManyForeignRows).toHaveLength(1)
 
           expect(res.body).toEqual({
@@ -585,9 +537,25 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               one2ManyForeignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              one2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2OneForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              many2ManyForeignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -616,9 +584,13 @@ describe("postgres integrations", () => {
             _rev: expect.any(String),
             [`fk_${oneToManyRelationshipInfo.table.name}_${oneToManyRelationshipInfo.fieldName}`]:
               foreignRows[0].row.id,
+            [oneToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -645,9 +617,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToOneRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
 
@@ -674,9 +650,13 @@ describe("postgres integrations", () => {
             tableId: row.tableId,
             _id: expect.any(String),
             _rev: expect.any(String),
+            [manyToManyRelationshipInfo.fieldName]: expect.arrayContaining(
+              foreignRows.map(r => ({
+                _id: r.row._id,
+                primaryDisplay: r.row.title,
+              }))
+            ),
           })
-
-          expect(res.body[oneToManyRelationshipInfo.fieldName]).toBeUndefined()
         })
       })
     })
@@ -730,12 +710,6 @@ describe("postgres integrations", () => {
       describe("given than multiple tables have multiple rows", () => {
         const rowsCount = 6
         beforeEach(async () => {
-          const createRandomTableWithRows = async () =>
-            await config.createRow({
-              tableId: (await createDefaultPgTable())._id,
-              title: generator.name(),
-            })
-
           await createRandomTableWithRows()
           await createRandomTableWithRows()
 
@@ -1023,12 +997,6 @@ describe("postgres integrations", () => {
       const rowsCount = 6
 
       beforeEach(async () => {
-        const createRandomTableWithRows = async () =>
-          await config.createRow({
-            tableId: (await createDefaultPgTable())._id,
-            title: generator.name(),
-          })
-
         await createRandomTableWithRows()
         await populatePrimaryRows(rowsCount)
         await createRandomTableWithRows()
@@ -1046,24 +1014,25 @@ describe("postgres integrations", () => {
 
   describe("POST /api/datasources/verify", () => {
     it("should be able to verify the connection", async () => {
-      const config = pgDatasourceConfig()
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const response = await config.api.datasource.verify({
+        datasource: await databaseTestProviders.postgres.getDsConfig(),
+      })
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(true)
     })
 
     it("should state an invalid datasource cannot connect", async () => {
-      const config = pgDatasourceConfig()
-      config.datasource.config.password = "wrongpassword"
-      const response = await makeRequest(
-        "post",
-        "/api/datasources/verify",
-        config
-      )
+      const dbConfig = await databaseTestProviders.postgres.getDsConfig()
+      const response = await config.api.datasource.verify({
+        datasource: {
+          ...dbConfig,
+          config: {
+            ...dbConfig.config,
+            password: "wrongpassword",
+          },
+        },
+      })
+
       expect(response.status).toBe(200)
       expect(response.body.connected).toBe(false)
       expect(response.body.error).toBeDefined()

packages/server/src/integrations/tests/utils/index.ts

@@ -0,0 +1,14 @@
+jest.unmock("pg")
+
+import { Datasource } from "@budibase/types"
+import * as pg from "./postgres"
+
+jest.setTimeout(30000)
+
+export interface DatabasePlusTestProvider {
+  getDsConfig(): Promise<Datasource>
+}
+
+export const databaseTestProviders = {
+  postgres: pg,
+}

packages/server/src/integrations/tests/utils/postgres.ts

@@ -0,0 +1,38 @@
+import { Datasource, SourceName } from "@budibase/types"
+import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
+
+let container: StartedTestContainer | undefined
+
+export async function getDsConfig(): Promise<Datasource> {
+  if (!container) {
+    container = await new GenericContainer("postgres")
+      .withExposedPorts(5432)
+      .withEnv("POSTGRES_PASSWORD", "password")
+      .withWaitStrategy(
+        Wait.forLogMessage(
+          "PostgreSQL init process complete; ready for start up."
+        )
+      )
+      .start()
+  }
+
+  const host = container.getContainerIpAddress()
+  const port = container.getMappedPort(5432)
+
+  return {
+    type: "datasource_plus",
+    source: SourceName.POSTGRES,
+    plus: true,
+    config: {
+      host,
+      port,
+      database: "postgres",
+      user: "postgres",
+      password: "password",
+      schema: "public",
+      ssl: false,
+      rejectUnauthorized: false,
+      ca: false,
+    },
+  }
+}

packages/server/src/migrations/tests/index.spec.ts

@@ -11,6 +11,7 @@ import { MIGRATIONS } from "../"
 import * as helpers from "./helpers"
 
 import tk from "timekeeper"
+import { View } from "@budibase/types"
 const timestamp = new Date().toISOString()
 tk.freeze(timestamp)
 
@@ -52,7 +53,9 @@ describe("migrations", () => {
         await config.createTable()
         await config.createLegacyView()
         await config.createTable()
-        await config.createLegacyView(structures.view(config.table!._id!))
+        await config.createLegacyView(
+          structures.view(config.table!._id!) as View
+        )
         await config.createScreen()
         await config.createScreen()
 

packages/server/src/sdk/app/rows/search.ts

@@ -30,7 +30,7 @@ export interface ExportRowsParams {
   format: Format
   rowIds?: string[]
   columns?: string[]
-  query: SearchFilters
+  query?: SearchFilters
 }
 
 export interface ExportRowsResult {

packages/server/src/sdk/app/tables/index.ts

@@ -12,7 +12,7 @@ import {
   TableViewsResponse,
 } from "@budibase/types"
 import datasources from "../datasources"
-import { isEditableColumn, populateExternalTableSchemas } from "./validation"
+import { populateExternalTableSchemas } from "./validation"
 import sdk from "../../../sdk"
 
 async function getAllInternalTables(db?: Database): Promise<Table[]> {
@@ -73,12 +73,23 @@ function enrichViewSchemas(table: Table): TableResponse {
   }
 }
 
+async function saveTable(table: Table) {
+  const db = context.getAppDB()
+  if (isExternalTable(table._id!)) {
+    const datasource = await sdk.datasources.get(table.sourceId!)
+    datasource.entities![table.name] = table
+    await db.put(datasource)
+  } else {
+    await db.put(table)
+  }
+}
+
 export default {
   getAllInternalTables,
   getAllExternalTables,
   getExternalTable,
   getTable,
   populateExternalTableSchemas,
-  isEditableColumn,
   enrichViewSchemas,
+  saveTable,
 }

packages/server/src/sdk/app/tables/validation.ts

@@ -55,13 +55,6 @@ function checkForeignKeysAreAutoColumns(datasource: Datasource) {
   return datasource
 }
 
-export function isEditableColumn(column: FieldSchema) {
-  const isAutoColumn =
-    column.autocolumn && column.autoReason !== AutoReason.FOREIGN_KEY
-  const isFormula = column.type === FieldTypes.FORMULA
-  return !(isAutoColumn || isFormula)
-}
-
 export function populateExternalTableSchemas(datasource: Datasource) {
   return checkForeignKeysAreAutoColumns(datasource)
 }

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -50,6 +50,11 @@ import {
   SearchFilters,
   UserRoles,
   Automation,
+  View,
+  FieldType,
+  RelationshipType,
+  ViewV2,
+  CreateViewRequest,
 } from "@budibase/types"
 
 import API from "./api"
@@ -75,9 +80,8 @@ class TestConfiguration {
   globalUserId: any
   userMetadataId: any
   table?: Table
-  linkedTable: any
   automation: any
-  datasource: any
+  datasource?: Datasource
   tenantId?: string
   defaultUserValues: DefaultUserValues
   api: API
@@ -527,7 +531,7 @@ class TestConfiguration {
   // TABLE
 
   async updateTable(
-    config?: any,
+    config?: Table,
     { skipReassigning } = { skipReassigning: false }
   ): Promise<Table> {
     config = config || basicTable()
@@ -542,33 +546,50 @@ class TestConfiguration {
     if (config != null && config._id) {
       delete config._id
     }
+    config = config || basicTable()
+    if (this.datasource && !config.sourceId) {
+      config.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        config.type = "external"
+      }
+    }
+
     return this.updateTable(config, options)
   }
 
   async getTable(tableId?: string) {
-    tableId = tableId || this.table?._id
+    tableId = tableId || this.table!._id!
     return this._req(null, { tableId }, controllers.table.find)
   }
 
-  async createLinkedTable(relationshipType?: string, links: any = ["link"]) {
+  async createLinkedTable(
+    relationshipType = RelationshipType.ONE_TO_MANY,
+    links: any = ["link"],
+    config?: Table
+  ) {
     if (!this.table) {
       throw "Must have created a table first."
     }
-    const tableConfig: any = basicTable()
+    const tableConfig = config || basicTable()
     tableConfig.primaryDisplay = "name"
     for (let link of links) {
       tableConfig.schema[link] = {
-        type: "link",
+        type: FieldType.LINK,
         fieldName: link,
         tableId: this.table._id,
         name: link,
-      }
-      if (relationshipType) {
-        tableConfig.schema[link].relationshipType = relationshipType
+        relationshipType,
       }
     }
+
+    if (this.datasource && !tableConfig.sourceId) {
+      tableConfig.sourceId = this.datasource._id
+      if (this.datasource.plus) {
+        tableConfig.type = "external"
+      }
+    }
+
     const linkedTable = await this.createTable(tableConfig)
-    this.linkedTable = linkedTable
     return linkedTable
   }
 
@@ -621,17 +642,36 @@ class TestConfiguration {
 
   // VIEW
 
-  async createLegacyView(config?: any) {
-    if (!this.table) {
+  async createLegacyView(config?: View) {
+    if (!this.table && !config) {
       throw "Test requires table to be configured."
     }
     const view = config || {
-      tableId: this.table._id,
-      name: "ViewTest",
+      tableId: this.table!._id,
+      name: generator.guid(),
     }
     return this._req(view, null, controllers.view.v1.save)
   }
 
+  async createView(
+    config?: Omit<CreateViewRequest, "tableId" | "name"> & {
+      name?: string
+      tableId?: string
+    }
+  ) {
+    if (!this.table && !config?.tableId) {
+      throw "Test requires table to be configured."
+    }
+
+    const view: CreateViewRequest = {
+      ...config,
+      tableId: config?.tableId || this.table!._id!,
+      name: config?.name || generator.word(),
+    }
+
+    return await this.api.viewV2.create(view)
+  }
+
   // AUTOMATION
 
   async createAutomation(config?: any) {
@@ -677,17 +717,17 @@ class TestConfiguration {
     config = config || basicDatasource()
     const response = await this._req(config, null, controllers.datasource.save)
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
-  async updateDatasource(datasource: any) {
+  async updateDatasource(datasource: Datasource): Promise<Datasource> {
     const response = await this._req(
       datasource,
       { datasourceId: datasource._id },
       controllers.datasource.update
     )
     this.datasource = response.datasource
-    return this.datasource
+    return this.datasource!
   }
 
   async restDatasource(cfg?: any) {
@@ -771,7 +811,7 @@ class TestConfiguration {
     if (!this.datasource && !config) {
       throw "No datasource created for query."
     }
-    config = config || basicQuery(this.datasource._id)
+    config = config || basicQuery(this.datasource!._id!)
     return this._req(config, null, controllers.query.save)
   }
 

packages/server/src/tests/utilities/api/datasource.ts

@@ -0,0 +1,57 @@
+import {
+  CreateDatasourceRequest,
+  Datasource,
+  VerifyDatasourceRequest,
+  VerifyDatasourceResponse,
+} from "@budibase/types"
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class DatasourceAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  create = async (
+    config: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const body: CreateDatasourceRequest = {
+      datasource: config,
+      tablesFilter: [],
+    }
+    const result = await this.request
+      .post(`/api/datasources`)
+      .send(body)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+
+  update = async (
+    datasource: Datasource,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Datasource> => {
+    const result = await this.request
+      .put(`/api/datasources/${datasource._id}`)
+      .send(datasource)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result.body.datasource as Datasource
+  }
+
+  verify = async (
+    data: VerifyDatasourceRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const result = await this.request
+      .post(`/api/datasources/verify`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return result
+  }
+}

packages/server/src/tests/utilities/api/index.ts

@@ -3,17 +3,23 @@ import { PermissionAPI } from "./permission"
 import { RowAPI } from "./row"
 import { TableAPI } from "./table"
 import { ViewV2API } from "./viewV2"
+import { DatasourceAPI } from "./datasource"
+import { LegacyViewAPI } from "./legacyView"
 
 export default class API {
   table: TableAPI
+  legacyView: LegacyViewAPI
   viewV2: ViewV2API
   row: RowAPI
   permission: PermissionAPI
+  datasource: DatasourceAPI
 
   constructor(config: TestConfiguration) {
     this.table = new TableAPI(config)
+    this.legacyView = new LegacyViewAPI(config)
     this.viewV2 = new ViewV2API(config)
     this.row = new RowAPI(config)
     this.permission = new PermissionAPI(config)
+    this.datasource = new DatasourceAPI(config)
   }
 }

packages/server/src/tests/utilities/api/legacyView.ts

@@ -0,0 +1,16 @@
+import TestConfiguration from "../TestConfiguration"
+import { TestAPI } from "./base"
+
+export class LegacyViewAPI extends TestAPI {
+  constructor(config: TestConfiguration) {
+    super(config)
+  }
+
+  get = async (id: string, { expectStatus } = { expectStatus: 200 }) => {
+    return await this.request
+      .get(`/api/views/${id}`)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+  }
+}

packages/server/src/tests/utilities/api/row.ts

@@ -1,4 +1,10 @@
-import { PatchRowRequest, SaveRowRequest, Row } from "@budibase/types"
+import {
+  PatchRowRequest,
+  SaveRowRequest,
+  Row,
+  ValidateResponse,
+  ExportRowsRequest,
+} from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -22,6 +28,21 @@ export class RowAPI extends TestAPI {
     return request
   }
 
+  getEnriched = async (
+    sourceId: string,
+    rowId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .get(`/api/${sourceId}/${rowId}/enrich`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+    if (expectStatus !== 404) {
+      request.expect("Content-Type", /json/)
+    }
+    return request
+  }
+
   save = async (
     sourceId: string,
     row: SaveRowRequest,
@@ -36,6 +57,20 @@ export class RowAPI extends TestAPI {
     return resp.body as Row
   }
 
+  validate = async (
+    sourceId: string,
+    row: SaveRowRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<ValidateResponse> => {
+    const resp = await this.request
+      .post(`/api/${sourceId}/rows/validate`)
+      .send(row)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return resp.body as ValidateResponse
+  }
+
   patch = async (
     sourceId: string,
     row: PatchRowRequest,
@@ -51,14 +86,40 @@ export class RowAPI extends TestAPI {
 
   delete = async (
     sourceId: string,
-    rows: Row[],
+    rows: Row | string | (Row | string)[],
     { expectStatus } = { expectStatus: 200 }
   ) => {
     return this.request
       .delete(`/api/${sourceId}/rows`)
-      .send({ rows })
+      .send(Array.isArray(rows) ? { rows } : rows)
       .set(this.config.defaultHeaders())
       .expect("Content-Type", /json/)
       .expect(expectStatus)
   }
+
+  fetch = async (
+    sourceId: string,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<Row[]> => {
+    const request = this.request
+      .get(`/api/${sourceId}/rows`)
+      .set(this.config.defaultHeaders())
+      .expect(expectStatus)
+
+    return (await request).body
+  }
+
+  exportRows = async (
+    tableId: string,
+    body: ExportRowsRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ) => {
+    const request = this.request
+      .post(`/api/${tableId}/rows/exportRows?format=json`)
+      .set(this.config.defaultHeaders())
+      .send(body)
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return request
+  }
 }

packages/server/src/tests/utilities/api/table.ts

@@ -1,4 +1,4 @@
-import { Table } from "@budibase/types"
+import { SaveTableRequest, SaveTableResponse, Table } from "@budibase/types"
 import TestConfiguration from "../TestConfiguration"
 import { TestAPI } from "./base"
 
@@ -7,6 +7,19 @@ export class TableAPI extends TestAPI {
     super(config)
   }
 
+  create = async (
+    data: SaveTableRequest,
+    { expectStatus } = { expectStatus: 200 }
+  ): Promise<SaveTableResponse> => {
+    const res = await this.request
+      .post(`/api/tables`)
+      .send(data)
+      .set(this.config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(expectStatus)
+    return res.body
+  }
+
   fetch = async (
     { expectStatus } = { expectStatus: 200 }
   ): Promise<Table[]> => {

packages/server/src/tests/utilities/api/viewV2.ts

@@ -23,8 +23,8 @@ export class ViewV2API extends TestAPI {
     if (!tableId && !this.config.table) {
       throw "Test requires table to be configured."
     }
-    const table = this.config.table
-    tableId = table!._id!
+
+    tableId = tableId || this.config.table!._id!
     const view = {
       tableId,
       name: generator.guid(),

packages/types/src/api/web/app/row.ts

@@ -1,5 +1,7 @@
 import { Row } from "../../../documents/app/row"
 
+export interface GetRowResponse extends Row {}
+
 export interface DeleteRows {
   rows: (Row | string)[]
 }
@@ -9,3 +11,8 @@ export interface DeleteRow {
 }
 
 export type DeleteRowRequest = DeleteRows | DeleteRow
+
+export interface ValidateResponse {
+  valid: boolean
+  errors: Record<string, any>
+}

packages/types/src/api/web/app/rows.ts

@@ -1,5 +1,6 @@
-import { SearchParams } from "../../../sdk"
+import { SearchFilters, SearchParams } from "../../../sdk"
 import { Row } from "../../../documents"
+import { ReadStream } from "fs"
 
 export interface SaveRowRequest extends Row {}
 
@@ -28,3 +29,11 @@ export interface SearchViewRowRequest
 export interface SearchRowResponse {
   rows: any[]
 }
+
+export interface ExportRowsRequest {
+  rows: string[]
+  columns?: string[]
+  query?: SearchFilters
+}
+
+export type ExportRowsResponse = ReadStream