Merge branch 'master' into feat/pick-relationship-fields

packages/pro

@@ -1 +1 @@
-Subproject commit 14c89a5b20ee4de07723063458a2437b0dd4f2c9
+Subproject commit 51a83b791a7a11b1d51c1fdb91f2ac246298279e

packages/server/src/api/controllers/row/external.ts

@@ -38,7 +38,7 @@ export async function handleRequest<T extends Operation>(
 }
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
 
   const { _id, ...rowData } = ctx.request.body
   const table = await sdk.tables.getTable(tableId)
@@ -93,7 +93,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 }
 
 export async function destroy(ctx: UserCtx) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const _id = ctx.request.body._id
   const { row } = await handleRequest(Operation.DELETE, tableId, {
     id: breakRowIdField(_id),
@@ -104,7 +104,7 @@ export async function destroy(ctx: UserCtx) {
 
 export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   let promises: Promise<{ row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
@@ -123,7 +123,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const { datasourceId, tableName } = breakExternalTableId(tableId)
   const datasource: Datasource = await sdk.datasources.get(datasourceId)
   if (!datasource || !datasource.entities) {

packages/server/src/api/controllers/row/index.ts

@@ -47,7 +47,7 @@ export async function patch(
   ctx: UserCtx<PatchRowRequest, PatchRowResponse>
 ): Promise<any> {
   const appId = ctx.appId
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const body = ctx.request.body
 
   // if it doesn't have an _id then its save
@@ -72,7 +72,7 @@ export async function patch(
 
 export const save = async (ctx: UserCtx<Row, Row>) => {
   const appId = ctx.appId
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const body = ctx.request.body
 
   // user metadata doesn't exist yet - don't allow creation
@@ -97,13 +97,12 @@ export const save = async (ctx: UserCtx<Row, Row>) => {
   gridSocket?.emitRowUpdate(ctx, row || squashed)
 }
 
-export async function fetchView(ctx: any) {
+export async function fetchLegacyView(ctx: any) {
-  const tableId = utils.getTableId(ctx)
   const viewName = decodeURIComponent(ctx.params.viewName)
 
   const { calculation, group, field } = ctx.query
 
-  ctx.body = await sdk.rows.fetchView(tableId, viewName, {
+  ctx.body = await sdk.rows.fetchLegacyView(viewName, {
     calculation,
     group: calculation ? group : null,
     field,
@@ -111,12 +110,12 @@ export async function fetchView(ctx: any) {
 }
 
 export async function fetch(ctx: any) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   ctx.body = await sdk.rows.fetch(tableId)
 }
 
 export async function find(ctx: UserCtx<void, GetRowResponse>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const rowId = ctx.params.rowId
 
   ctx.body = await sdk.rows.find(tableId, rowId)
@@ -132,7 +131,7 @@ function isDeleteRow(input: any): input is DeleteRow {
 
 async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
   let request = ctx.request.body as DeleteRows
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
 
   const processedRows = request.rows.map(row => {
     let processedRow: Row = typeof row == "string" ? { _id: row } : row
@@ -148,7 +147,7 @@ async function processDeleteRowsRequest(ctx: UserCtx<DeleteRowRequest>) {
 }
 
 async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const appId = ctx.appId
 
   let deleteRequest = ctx.request.body as DeleteRows
@@ -170,7 +169,7 @@ async function deleteRows(ctx: UserCtx<DeleteRowRequest>) {
 
 async function deleteRow(ctx: UserCtx<DeleteRowRequest>) {
   const appId = ctx.appId
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
 
   const resp = await pickApi(tableId).destroy(ctx)
   if (!tableId.includes("datasource_plus")) {
@@ -204,7 +203,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
 }
 
 export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
 
   await context.ensureSnippetContext(true)
 
@@ -226,7 +225,7 @@ export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
 }
 
 export async function validate(ctx: Ctx<Row, ValidateResponse>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   // external tables are hard to validate currently
   if (isExternalTableID(tableId)) {
     ctx.body = { valid: true, errors: {} }
@@ -239,14 +238,14 @@ export async function validate(ctx: Ctx<Row, ValidateResponse>) {
 }
 
 export async function fetchEnrichedRow(ctx: UserCtx<void, Row>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx)
 }
 
 export const exportRows = async (
   ctx: Ctx<ExportRowsRequest, ExportRowsResponse>
 ) => {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
 
   const format = ctx.query.format
 
@@ -279,7 +278,7 @@ export const exportRows = async (
 export async function downloadAttachment(ctx: UserCtx) {
   const { columnName } = ctx.params
 
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const rowId = ctx.params.rowId
   const row = await sdk.rows.find(tableId, rowId)
 

packages/server/src/api/controllers/row/internal.ts

@@ -16,7 +16,6 @@ import {
   PatchRowRequest,
   PatchRowResponse,
   Row,
-  Table,
   UserCtx,
 } from "@budibase/types"
 import sdk from "../../../sdk"
@@ -24,7 +23,7 @@ import { getLinkedTableIDs } from "../../../db/linkedRows/linkUtils"
 import { flatten } from "lodash"
 
 export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const inputs = ctx.request.body
   const isUserTable = tableId === InternalTables.USER_METADATA
   let oldRow
@@ -98,7 +97,7 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
 
 export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const { _id } = ctx.request.body
   let row = await db.get<Row>(_id)
   let _rev = ctx.request.body._rev || row._rev
@@ -137,7 +136,7 @@ export async function destroy(ctx: UserCtx) {
 }
 
 export async function bulkDestroy(ctx: UserCtx) {
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const table = await sdk.tables.getTable(tableId)
   let { rows } = ctx.request.body
 
@@ -179,7 +178,7 @@ export async function bulkDestroy(ctx: UserCtx) {
 export async function fetchEnrichedRow(ctx: UserCtx) {
   const fieldName = ctx.request.query.field as string | undefined
   const db = context.getAppDB()
-  const tableId = utils.getTableId(ctx)
+  const { tableId } = utils.getSourceId(ctx)
   const rowId = ctx.params.rowId as string
   // need table to work out where links go in row, as well as the link docs
   const [table, links] = await Promise.all([
@@ -197,7 +196,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   )
 
   // get the linked tables
-  const linkTableIds = getLinkedTableIDs(table as Table)
+  const linkTableIds = getLinkedTableIDs(table.schema)
   const linkTables = await sdk.tables.getTables(linkTableIds)
 
   // perform output processing

packages/server/src/api/controllers/row/utils/utils.ts

@@ -1,4 +1,4 @@
-import { InternalTables } from "../../../../db/utils"
+import * as utils from "../../../../db/utils"
 
 import { context } from "@budibase/backend-core"
 import {
@@ -67,7 +67,7 @@ export async function findRow(tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row: Row
   // TODO remove special user case in future
-  if (tableId === InternalTables.USER_METADATA) {
+  if (tableId === utils.InternalTables.USER_METADATA) {
     row = await getFullUser(rowId)
   } else {
     row = await db.get(rowId)
@@ -78,22 +78,25 @@ export async function findRow(tableId: string, rowId: string) {
   return row
 }
 
-export function getTableId(ctx: Ctx): string {
+export function getSourceId(ctx: Ctx): { tableId: string; viewId?: string } {
   // top priority, use the URL first
   if (ctx.params?.sourceId) {
-    return ctx.params.sourceId
+    const { sourceId } = ctx.params
+    if (utils.isViewID(sourceId)) {
+      return {
+        tableId: utils.extractViewInfoFromID(sourceId).tableId,
+        viewId: sourceId,
+      }
+    }
+    return { tableId: ctx.params.sourceId }
   }
   // now check for old way of specifying table ID
   if (ctx.params?.tableId) {
-    return ctx.params.tableId
+    return { tableId: ctx.params.tableId }
   }
   // check body for a table ID
   if (ctx.request.body?.tableId) {
-    return ctx.request.body.tableId
+    return { tableId: ctx.request.body.tableId }
-  }
-  // now check if a specific view name
-  if (ctx.params?.viewName) {
-    return ctx.params.viewName
   }
   throw new Error("Unable to find table ID in request")
 }
@@ -198,7 +201,7 @@ export async function sqlOutputProcessing(
 }
 
 export function isUserMetadataTable(tableId: string) {
-  return tableId === InternalTables.USER_METADATA
+  return tableId === utils.InternalTables.USER_METADATA
 }
 
 export async function enrichArrayContext(

packages/server/src/api/controllers/view/views.ts

@@ -2,7 +2,7 @@ import viewTemplate from "./viewBuilder"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
-import { fetchView } from "../row"
+import { fetchLegacyView } from "../row"
 import { context, events } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 import {
@@ -170,7 +170,7 @@ export async function exportView(ctx: Ctx) {
     ctx.params.viewName = viewName
   }
 
-  await fetchView(ctx)
+  await fetchLegacyView(ctx)
   let rows = ctx.body as Row[]
 
   let schema: TableSchema = view && view.meta && view.meta.schema

packages/server/src/api/routes/tests/row.spec.ts

@@ -96,12 +96,7 @@ describe.each([
   let envCleanup: (() => void) | undefined
 
   beforeAll(async () => {
-    await withCoreEnv(
+    await withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, () => config.init())
-      {
-        SQS_SEARCH_ENABLE: "true",
-      },
-      () => config.init()
-    )
     if (isSqs) {
       envCleanup = setCoreEnv({
         SQS_SEARCH_ENABLE: "true",

packages/server/src/api/routes/view.ts

@@ -46,7 +46,7 @@ router
       permissions.PermissionType.TABLE,
       permissions.PermissionLevel.READ
     ),
-    rowController.fetchView
+    rowController.fetchLegacyView
   )
   .get("/api/views", authorized(permissions.BUILDER), viewController.v1.fetch)
   .delete(

packages/server/src/db/linkedRows/index.ts

@@ -19,6 +19,7 @@ import {
   LinkDocumentValue,
   Row,
   Table,
+  TableSchema,
 } from "@budibase/types"
 import sdk from "../../sdk"
 
@@ -47,8 +48,8 @@ export const EventType = {
   TABLE_DELETE: "table:delete",
 }
 
-function clearRelationshipFields(table: Table, rows: Row[]) {
+function clearRelationshipFields(schema: TableSchema, rows: Row[]) {
-  for (let [key, field] of Object.entries(table.schema)) {
+  for (let [key, field] of Object.entries(schema)) {
     if (field.type === FieldType.LINK) {
       rows = rows.map(row => {
         delete row[key]
@@ -159,11 +160,11 @@ export async function updateLinks(args: {
  * @return returns the rows with all of the enriched relationships on it.
  */
 export async function attachFullLinkedDocs(
-  table: Table,
+  schema: TableSchema,
   rows: Row[],
   opts?: { fromRow?: Row }
 ) {
-  const linkedTableIds = getLinkedTableIDs(table)
+  const linkedTableIds = getLinkedTableIDs(schema)
   if (linkedTableIds.length === 0) {
     return rows
   }
@@ -183,7 +184,7 @@ export async function attachFullLinkedDocs(
   }
   const linkedTables = response[1] as Table[]
   // clear any existing links that could be dupe'd
-  rows = clearRelationshipFields(table, rows)
+  rows = clearRelationshipFields(schema, rows)
   // now get the docs and combine into the rows
   let linked: Row[] = []
   if (linksWithoutFromRow.length > 0) {
@@ -202,7 +203,7 @@ export async function attachFullLinkedDocs(
       }
       if (linkedRow) {
         const linkedTableId =
-          linkedRow.tableId || getRelatedTableForField(table, link.fieldName)
+          linkedRow.tableId || getRelatedTableForField(schema, link.fieldName)
         const linkedTable = linkedTables.find(
           table => table._id === linkedTableId
         )
@@ -268,7 +269,8 @@ export async function squashLinksToPrimaryDisplay(
       }
       const newLinks = []
       for (const link of row[column]) {
-        const linkTblId = link.tableId || getRelatedTableForField(table, column)
+        const linkTblId =
+          link.tableId || getRelatedTableForField(table.schema, column)
         const linkedTable = await getLinkedTable(linkTblId!, linkedTables)
         const obj: any = { _id: link._id }
         obj.primaryDisplay = getPrimaryDisplayValue(link, linkedTable)

packages/server/src/db/linkedRows/linkUtils.ts

@@ -7,6 +7,7 @@ import {
   LinkDocument,
   LinkDocumentValue,
   Table,
+  TableSchema,
 } from "@budibase/types"
 import sdk from "../../sdk"
 
@@ -121,8 +122,8 @@ export function getUniqueByProp(array: any[], prop: string) {
   return filteredArray
 }
 
-export function getLinkedTableIDs(table: Table): string[] {
+export function getLinkedTableIDs(schema: TableSchema): string[] {
-  return Object.values(table.schema)
+  return Object.values(schema)
     .filter(isRelationshipColumn)
     .map(column => column.tableId)
 }
@@ -139,13 +140,16 @@ export async function getLinkedTable(id: string, tables: Table[]) {
   return linkedTable
 }
 
-export function getRelatedTableForField(table: Table, fieldName: string) {
+export function getRelatedTableForField(
+  schema: TableSchema,
+  fieldName: string
+) {
   // look to see if its on the table, straight in the schema
-  const field = table.schema[fieldName]
+  const field = schema[fieldName]
   if (field?.type === FieldType.LINK) {
     return field.tableId
   }
-  for (let column of Object.values(table.schema)) {
+  for (let column of Object.values(schema)) {
     if (column.type === FieldType.LINK && column.fieldName === fieldName) {
       return column.tableId
     }

packages/server/src/db/tests/linkTests.spec.ts

@@ -34,7 +34,7 @@ describe("test link functionality", () => {
   })
 
   describe("getRelatedTableForField", () => {
-    let link = basicTable()
+    const link = basicTable()
     link.schema.link = {
       name: "link",
       relationshipType: RelationshipType.ONE_TO_MANY,
@@ -44,11 +44,13 @@ describe("test link functionality", () => {
     }
 
     it("should get the field from the table directly", () => {
-      expect(linkUtils.getRelatedTableForField(link, "link")).toBe("tableID")
+      expect(linkUtils.getRelatedTableForField(link.schema, "link")).toBe(
+        "tableID"
+      )
     })
 
     it("should get the field from the link", () => {
-      expect(linkUtils.getRelatedTableForField(link, "otherLink")).toBe(
+      expect(linkUtils.getRelatedTableForField(link.schema, "otherLink")).toBe(
         "tableID"
       )
     })

packages/server/src/middleware/trimViewRowInfo.ts

@@ -1,33 +1,23 @@
 import { Ctx, Row } from "@budibase/types"
-import * as utils from "../db/utils"
+
 import sdk from "../sdk"
 import { Next } from "koa"
-import { getTableId } from "../api/controllers/row/utils"
+import { getSourceId } from "../api/controllers/row/utils"
 
 export default async (ctx: Ctx<Row>, next: Next) => {
   const { body } = ctx.request
-  let { _viewId: viewId } = body
+  const viewId = getSourceId(ctx).viewId ?? body._viewId
-
-  const possibleViewId = getTableId(ctx)
-  if (utils.isViewID(possibleViewId)) {
-    viewId = possibleViewId
-  }
 
   // nothing to do, it is not a view (just a table ID)
   if (!viewId) {
     return next()
   }
 
-  const { tableId } = utils.extractViewInfoFromID(viewId)
-
   // don't need to trim delete requests
   if (ctx?.method?.toLowerCase() !== "delete") {
     await trimViewFields(ctx.request.body, viewId)
   }
 
-  ctx.params.sourceId = tableId
-  ctx.params.viewId = viewId
-
   return next()
 }
 

packages/server/src/sdk/app/rows/search.ts

@@ -121,10 +121,9 @@ export async function fetchRaw(tableId: string): Promise<Row[]> {
   return pickApi(tableId).fetchRaw(tableId)
 }
 
-export async function fetchView(
+export async function fetchLegacyView(
-  tableId: string,
   viewName: string,
   params: ViewParams
 ): Promise<Row[]> {
-  return pickApi(tableId).fetchView(viewName, params)
+  return internal.fetchLegacyView(viewName, params)
 }

packages/server/src/sdk/app/rows/search/external.ts

@@ -272,11 +272,3 @@ export async function fetchRaw(tableId: string): Promise<Row[]> {
   })
   return response.rows
 }
-
-export async function fetchView(viewName: string) {
-  // there are no views in external datasources, shouldn't ever be called
-  // for now just fetch
-  const split = viewName.split("all_")
-  const tableId = split[1] ? split[1] : split[0]
-  return fetch(tableId)
-}

packages/server/src/sdk/app/rows/search/internal/internal.ts

@@ -145,7 +145,7 @@ export async function fetchRaw(tableId: string): Promise<Row[]> {
   return rows as Row[]
 }
 
-export async function fetchView(
+export async function fetchLegacyView(
   viewName: string,
   options: { calculation: string; group: string; field: string }
 ): Promise<Row[]> {

packages/server/src/utilities/rowProcessor/index.ts

@@ -263,7 +263,7 @@ export async function outputProcessing<T extends Row[] | Row>(
   }
   // attach any linked row information
   let enriched = !opts.preserveLinks
-    ? await linkRows.attachFullLinkedDocs(table, safeRows, {
+    ? await linkRows.attachFullLinkedDocs(table.schema, safeRows, {
         fromRow: opts?.fromRow,
       })
     : safeRows

packages/server/src/websockets/grid.ts

@@ -4,7 +4,7 @@ import { BaseSocket } from "./websocket"
 import { auth, permissions } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
-import { getTableId } from "../api/controllers/row/utils"
+import { getSourceId } from "../api/controllers/row/utils"
 import { Row, Table, View, ViewV2 } from "@budibase/types"
 import { Socket } from "socket.io"
 import { GridSocketEvent } from "@budibase/shared-core"
@@ -80,7 +80,7 @@ export default class GridSocket extends BaseSocket {
   }
 
   emitRowUpdate(ctx: any, row: Row) {
-    const resourceId = ctx.params?.viewId || getTableId(ctx)
+    const resourceId = ctx.params?.viewId || getSourceId(ctx)
     const room = `${ctx.appId}-${resourceId}`
     this.emitToRoom(ctx, room, GridSocketEvent.RowChange, {
       id: row._id,
@@ -89,7 +89,7 @@ export default class GridSocket extends BaseSocket {
   }
 
   emitRowDeletion(ctx: any, row: Row) {
-    const resourceId = ctx.params?.viewId || getTableId(ctx)
+    const resourceId = ctx.params?.viewId || getSourceId(ctx)
     const room = `${ctx.appId}-${resourceId}`
     this.emitToRoom(ctx, room, GridSocketEvent.RowChange, {
       id: row._id,