Merge branch 'master' of github.com:Budibase/budibase into develop

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.7.25-alpha.10",
+  "version": "2.7.25",
   "npmClient": "yarn",
   "useNx": true,
   "packages": [
@@ -31,4 +31,4 @@
       "loadEnvFiles": false
     }
   }
-}
+}

packages/server/src/integrations/googlesheets.ts

@@ -13,7 +13,7 @@ import {
   Row,
   SearchFilters,
   SortJson,
-  Table,
+  ExternalTable,
   TableRequest,
 } from "@budibase/types"
 import { OAuth2Client } from "google-auth-library"
@@ -139,7 +139,7 @@ const SCHEMA: Integration = {
 class GoogleSheetsIntegration implements DatasourcePlus {
   private readonly config: GoogleSheetsConfig
   private client: GoogleSpreadsheet
-  public tables: Record<string, Table> = {}
+  public tables: Record<string, ExternalTable> = {}
   public schemaErrors: Record<string, string> = {}
 
   constructor(config: GoogleSheetsConfig) {
@@ -253,12 +253,18 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     return sheets.map(s => s.title)
   }
 
-  getTableSchema(title: string, headerValues: string[], id?: string) {
+  getTableSchema(
+    title: string,
+    headerValues: string[],
+    datasourceId: string,
+    id?: string
+  ) {
     // base table
-    const table: Table = {
+    const table: ExternalTable = {
       name: title,
       primary: [GOOGLE_SHEETS_PRIMARY_KEY],
       schema: {},
+      sourceId: datasourceId,
     }
     if (id) {
       table._id = id
@@ -273,20 +279,28 @@ class GoogleSheetsIntegration implements DatasourcePlus {
     return table
   }
 
-  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+  async buildSchema(
+    datasourceId: string,
+    entities: Record<string, ExternalTable>
+  ) {
+    // not fully configured yet
+    if (!this.config.auth) {
+      return
+    }
     await this.connect()
     const sheets = this.client.sheetsByIndex
-    const tables: Record<string, Table> = {}
+    const tables: Record<string, ExternalTable> = {}
     await utils.parallelForeach(
       sheets,
       async sheet => {
         // must fetch rows to determine schema
-        await sheet.getRows({ limit: 0, offset: 0 })
+        await sheet.getRows()
 
         const id = buildExternalTableId(datasourceId, sheet.title)
         tables[sheet.title] = this.getTableSchema(
           sheet.title,
           sheet.headerValues,
+          datasourceId,
           id
         )
       },

packages/server/src/integrations/microsoftSqlServer.ts

@@ -2,7 +2,7 @@ import {
   DatasourceFieldType,
   Integration,
   Operation,
-  Table,
+  ExternalTable,
   TableSchema,
   QueryJson,
   QueryType,
@@ -98,7 +98,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
   private index: number = 0
   private readonly pool: any
   private client: any
-  public tables: Record<string, Table> = {}
+  public tables: Record<string, ExternalTable> = {}
   public schemaErrors: Record<string, string> = {}
 
   MASTER_TABLES = [
@@ -221,7 +221,10 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    * @param {*} datasourceId - datasourceId to fetch
    * @param entities - the tables that are to be built
    */
-  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+  async buildSchema(
+    datasourceId: string,
+    entities: Record<string, ExternalTable>
+  ) {
     await this.connect()
     let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
     if (tableInfo == null || !Array.isArray(tableInfo)) {
@@ -234,7 +237,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       .map((record: any) => record.TABLE_NAME)
       .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
 
-    const tables: Record<string, Table> = {}
+    const tables: Record<string, ExternalTable> = {}
     for (let tableName of tableNames) {
       // get the column definition (type)
       const definition = await this.runSQL(this.getDefinitionSQL(tableName))
@@ -277,6 +280,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       }
       tables[tableName] = {
         _id: buildExternalTableId(datasourceId, tableName),
+        sourceId: datasourceId,
         primary: primaryKeys,
         name: tableName,
         schema,

packages/server/src/integrations/mysql.ts

@@ -4,7 +4,7 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  Table,
+  ExternalTable,
   TableSchema,
   DatasourcePlus,
   DatasourceFeature,
@@ -124,7 +124,7 @@ export function bindingTypeCoerce(bindings: any[]) {
 class MySQLIntegration extends Sql implements DatasourcePlus {
   private config: MySQLConfig
   private client?: mysql.Connection
-  public tables: Record<string, Table> = {}
+  public tables: Record<string, ExternalTable> = {}
   public schemaErrors: Record<string, string> = {}
 
   constructor(config: MySQLConfig) {
@@ -221,8 +221,11 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
     }
   }
 
-  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    const tables: { [key: string]: Table } = {}
+  async buildSchema(
+    datasourceId: string,
+    entities: Record<string, ExternalTable>
+  ) {
+    const tables: { [key: string]: ExternalTable } = {}
     await this.connect()
 
     try {
@@ -260,6 +263,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
         if (!tables[tableName]) {
           tables[tableName] = {
             _id: buildExternalTableId(datasourceId, tableName),
+            sourceId: datasourceId,
             primary: primaryKeys,
             name: tableName,
             schema,

packages/server/src/integrations/oracle.ts

@@ -5,7 +5,7 @@ import {
   QueryJson,
   QueryType,
   SqlQuery,
-  Table,
+  ExternalTable,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
@@ -108,7 +108,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   private readonly config: OracleConfig
   private index: number = 1
 
-  public tables: Record<string, Table> = {}
+  public tables: Record<string, ExternalTable> = {}
   public schemaErrors: Record<string, string> = {}
 
   private readonly COLUMNS_SQL = `
@@ -262,13 +262,16 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    * @param {*} datasourceId - datasourceId to fetch
    * @param entities - the tables that are to be built
    */
-  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+  async buildSchema(
+    datasourceId: string,
+    entities: Record<string, ExternalTable>
+  ) {
     const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
       sql: this.COLUMNS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
 
-    const tables: { [key: string]: Table } = {}
+    const tables: { [key: string]: ExternalTable } = {}
 
     // iterate each table
     Object.values(oracleTables).forEach(oracleTable => {
@@ -279,6 +282,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
           primary: [],
           name: oracleTable.name,
           schema: {},
+          sourceId: datasourceId,
         }
         tables[oracleTable.name] = table
       }

packages/server/src/integrations/postgres.ts

@@ -5,7 +5,7 @@ import {
   QueryType,
   QueryJson,
   SqlQuery,
-  Table,
+  ExternalTable,
   DatasourcePlus,
   DatasourceFeature,
   ConnectionInfo,
@@ -143,7 +143,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
   private readonly config: PostgresConfig
   private index: number = 1
   private open: boolean
-  public tables: Record<string, Table> = {}
+  public tables: Record<string, ExternalTable> = {}
   public schemaErrors: Record<string, string> = {}
 
   COLUMNS_SQL!: string
@@ -261,7 +261,10 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    * @param {*} datasourceId - datasourceId to fetch
    * @param entities - the tables that are to be built
    */
-  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
+  async buildSchema(
+    datasourceId: string,
+    entities: Record<string, ExternalTable>
+  ) {
     let tableKeys: { [key: string]: string[] } = {}
     await this.openConnection()
     try {
@@ -287,7 +290,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
       const columnsResponse: { rows: PostgresColumn[] } =
         await this.client.query(this.COLUMNS_SQL)
 
-      const tables: { [key: string]: Table } = {}
+      const tables: { [key: string]: ExternalTable } = {}
 
       for (let column of columnsResponse.rows) {
         const tableName: string = column.table_name
@@ -300,6 +303,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
             primary: tableKeys[tableName] || [],
             name: tableName,
             schema: {},
+            sourceId: datasourceId,
           }
         }
 

packages/server/src/utilities/global.ts

@@ -9,7 +9,7 @@ import {
 import env from "../environment"
 import { groups } from "@budibase/pro"
 import { UserCtx, ContextUser, User, UserGroup } from "@budibase/types"
-import { global } from "yargs"
+import { cloneDeep } from "lodash"
 
 export function updateAppRole(
   user: ContextUser,
@@ -65,16 +65,20 @@ export async function processUser(
   user: ContextUser,
   opts: { appId?: string; groups?: UserGroup[] } = {}
 ) {
-  if (user) {
-    delete user.password
+  let clonedUser = cloneDeep(user)
+  if (clonedUser) {
+    delete clonedUser.password
   }
   const appId = opts.appId || context.getAppId()
-  user = updateAppRole(user, { appId })
-  if (!user.roleId && user?.userGroups?.length) {
-    user = await checkGroupRoles(user, { appId, groups: opts?.groups })
+  clonedUser = updateAppRole(clonedUser, { appId })
+  if (!clonedUser.roleId && clonedUser?.userGroups?.length) {
+    clonedUser = await checkGroupRoles(clonedUser, {
+      appId,
+      groups: opts?.groups,
+    })
   }
 
-  return user
+  return clonedUser
 }
 
 export async function getCachedSelf(ctx: UserCtx, appId: string) {

packages/types/src/documents/app/table.ts

@@ -82,6 +82,10 @@ export interface Table extends Document {
   rowHeight?: number
 }
 
+export interface ExternalTable extends Table {
+  sourceId: string
+}
+
 export interface TableRequest extends Table {
   _rename?: RenameColumn
   created?: boolean