Merge remote-tracking branch 'origin/develop' into feature/user-onboarding-overlays

.github/ISSUE_TEMPLATE/bug_report.md

@@ -6,7 +6,7 @@ labels: bug
 assignees: ''
 
 ---
-## Checklist
+**Checklist**
 - [ ] I have searched budibase discussions and github issues to check if my issue already exists
 
 **Hosting**

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/backend-core",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase backend core libraries used in server and worker",
   "main": "dist/src/index.js",
   "types": "dist/src/index.d.ts",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@budibase/nano": "10.1.1",
-    "@budibase/types": "2.2.12-alpha.16",
+    "@budibase/types": "2.2.12-alpha.32",
     "@shopify/jest-koa-mocks": "5.0.1",
     "@techpass/passport-openidconnect": "0.3.2",
     "aws-cloudfront-sign": "2.2.0",

packages/backend-core/src/events/publishers/rows.ts

@@ -3,7 +3,6 @@ import {
   Event,
   RowsImportedEvent,
   RowsCreatedEvent,
-  RowImportFormat,
   Table,
 } from "@budibase/types"
 
@@ -16,14 +15,9 @@ const created = async (count: number, timestamp?: string | number) => {
   await publishEvent(Event.ROWS_CREATED, properties, timestamp)
 }
 
-const imported = async (
+const imported = async (table: Table, count: number) => {
-  table: Table,
-  format: RowImportFormat,
-  count: number
-) => {
   const properties: RowsImportedEvent = {
     tableId: table._id as string,
-    format,
     count,
   }
   await publishEvent(Event.ROWS_IMPORTED, properties)

packages/backend-core/src/events/publishers/table.ts

@@ -2,7 +2,6 @@ import { publishEvent } from "../events"
 import {
   Event,
   TableExportFormat,
-  TableImportFormat,
   Table,
   TableCreatedEvent,
   TableUpdatedEvent,
@@ -40,10 +39,9 @@ async function exported(table: Table, format: TableExportFormat) {
   await publishEvent(Event.TABLE_EXPORTED, properties)
 }
 
-async function imported(table: Table, format: TableImportFormat) {
+async function imported(table: Table) {
   const properties: TableImportedEvent = {
     tableId: table._id as string,
-    format,
   }
   await publishEvent(Event.TABLE_IMPORTED, properties)
 }

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",
@@ -38,7 +38,7 @@
   ],
   "dependencies": {
     "@adobe/spectrum-css-workflow-icons": "1.2.1",
-    "@budibase/string-templates": "2.2.12-alpha.16",
+    "@budibase/string-templates": "2.2.12-alpha.32",
     "@spectrum-css/actionbutton": "1.0.1",
     "@spectrum-css/actiongroup": "1.0.1",
     "@spectrum-css/avatar": "3.0.2",

packages/bbui/src/Form/Core/Picker.svelte

@@ -45,7 +45,9 @@
     getOptionLabel
   )
 
-  const onClick = () => {
+  const onClick = e => {
+    e.preventDefault()
+    e.stopPropagation()
     dispatch("click")
     if (readonly) {
       return
@@ -88,7 +90,6 @@
   class:is-open={open}
   aria-haspopup="listbox"
   on:click={onClick}
-  use:clickOutside={() => (open = false)}
   bind:this={button}
 >
   {#if fieldIcon}
@@ -130,14 +131,25 @@
 <Popover
   anchor={button}
   align="left"
+<<<<<<< HEAD
   portalTarget={document.documentElement}
+=======
+>>>>>>> origin/develop
   bind:this={popover}
   {open}
   on:close={() => (open = false)}
   useAnchorWidth={!autoWidth}
   maxWidth={autoWidth ? 400 : null}
 >
+<<<<<<< HEAD
   <div class="popover-content" class:auto-width={autoWidth}>
+=======
+  <div
+    class="popover-content"
+    class:auto-width={autoWidth}
+    use:clickOutside={() => (open = false)}
+  >
+>>>>>>> origin/develop
     {#if autocomplete}
       <Search
         value={searchTerm}

packages/bbui/src/Popover/Popover.svelte

@@ -5,6 +5,8 @@
   import positionDropdown from "../Actions/position_dropdown"
   import clickOutside from "../Actions/click_outside"
   import { fly } from "svelte/transition"
+  import { getContext } from "svelte"
+  import Context from "../context"
 
   const dispatch = createEventDispatcher()
 
@@ -25,6 +27,7 @@
   $: tooltipClasses = showTip
     ? `spectrum-Popover--withTip spectrum-Popover--${direction}`
     : ""
+  $: target = portalTarget || getContext(Context.PopoverRoot) || ".spectrum"
 
   export const show = () => {
     dispatch("open")
@@ -62,7 +65,7 @@
 </script>
 
 {#if open}
-  <Portal target={portalTarget}>
+  <Portal {target}>
     <div
       tabindex="0"
       use:positionDropdown={{

packages/bbui/src/context.js

@@ -1,3 +1,4 @@
 export default {
   Modal: "bbui-modal",
+  PopoverRoot: "bbui-popover-root",
 }

packages/builder/cypress.json

@@ -11,9 +11,5 @@
     "WORKER_PORT": "4200",
     "JWT_SECRET": "test",
     "HOST_IP": ""
-  },
-  "retries": {
-    "runMode": 1,
-    "openMode": 0
   }
 }

packages/builder/cypress/integration/adminAndManagement/authentication.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../../support/filterTests"
 // const interact = require("../support/interact")
 
 filterTests(["smoke", "all"], () => {
-  context("Auth Configuration", () => {
+  xcontext("Auth Configuration", () => {
     before(() => {
       cy.login()
     })
@@ -21,7 +21,7 @@ filterTests(["smoke", "all"], () => {
       cy.get("[data-cy=oidc-active]").should('not.be.checked')
 
       cy.intercept("POST", "/api/global/configs").as("updateAuth")
-      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({ force: true })
       cy.wait("@updateAuth")
       cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
 
@@ -45,7 +45,7 @@ filterTests(["smoke", "all"], () => {
       cy.get("button[data-cy=oidc-save]").should("not.be.disabled");
 
       cy.intercept("POST", "/api/global/configs").as("updateAuth")
-      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({ force: true })
       cy.wait("@updateAuth")
       cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
 
@@ -85,11 +85,11 @@ filterTests(["smoke", "all"], () => {
       cy.get(".auth-form input.spectrum-Textfield-input").type("Another ")
       cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 6)
       cy.get(".spectrum-Tags-item").contains("Another")
-      
+
       cy.get("button[data-cy=oidc-save]").should("not.be.disabled");
 
       cy.intercept("POST", "/api/global/configs").as("updateAuth")
-      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({ force: true })
       cy.wait("@updateAuth")
       cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
 
@@ -123,7 +123,7 @@ filterTests(["smoke", "all"], () => {
       cy.get("button[data-cy=oidc-save]").should("not.be.disabled");
 
       cy.intercept("POST", "/api/global/configs").as("updateAuth")
-      cy.get("button[data-cy=oidc-save]").contains("Save").click({force: true})
+      cy.get("button[data-cy=oidc-save]").contains("Save").click({ force: true })
       cy.wait("@updateAuth")
       cy.get("@updateAuth").its("response.statusCode").should("eq", 200)
 
@@ -144,7 +144,7 @@ filterTests(["smoke", "all"], () => {
 
       cy.get("div.content").scrollTo("bottom")
 
-      cy.get("[data-cy=restore-oidc-default-scopes]").click({force: true})
+      cy.get("[data-cy=restore-oidc-default-scopes]").click({ force: true })
 
       cy.get(".spectrum-Tags").find(".spectrum-Tags-item").its("length").should("eq", 4)
 

packages/builder/cypress/integration/adminAndManagement/userSettings.spec.js

@@ -3,107 +3,112 @@ const interact = require('../../support/interact')
 
 filterTests(["smoke", "all"], () => {
   context("User Settings Menu", () => {
-    
+
     before(() => {
       cy.login()
     })
-    
+
     it("should update user information via user settings menu", () => {
-        const fname = "test"
+      const fname = "test"
-        const lname = "user"
+      const lname = "user"
 
-        cy.visit(`${Cypress.config().baseUrl}/builder`)
+      cy.visit(`${Cypress.config().baseUrl}/builder`)
-        cy.updateUserInformation(fname, lname)
+      cy.updateUserInformation(fname, lname)
 
-        // Go to user info and confirm name update
+      // Go to user info and confirm name update
-        cy.contains("Users").click()
+      cy.contains("Users").click()
-        cy.contains("test@test.com").click()
+      cy.contains("test@test.com").click()
 
-        cy.get(interact.FIELD, { timeout: 1000 }).eq(1).within(() => {
+      cy.get(interact.FIELD, { timeout: 1000 }).eq(1).within(() => {
-            cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).should('have.value', fname)
+        cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).should('have.value', fname)
-          })
+      })
-        cy.get(interact.FIELD).eq(2).within(() => {
+      cy.get(interact.FIELD).eq(2).within(() => {
-            cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).should('have.value', lname)
+        cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).should('have.value', lname)
-          })
+      })
     })
 
-    it("should allow copying of the users API key", () => {
+    xit("should allow copying of the users API key", () => {
-        cy.get(".user-dropdown .avatar > .icon", { timeout: 2000 }).click({ force: true })
+      cy.get(".user-dropdown .avatar > .icon", { timeout: 2000 }).click({ force: true })
-        cy.get(interact.SPECTRUM_MENU_ITEM).contains("View API key").click({ force: true })
+      cy.get(interact.SPECTRUM_MENU_ITEM).contains("View API key").click({ force: true })
-        cy.get(interact.SPECTRUM_DIALOG_CONTENT).within(() => {
+      cy.get(interact.SPECTRUM_DIALOG_CONTENT).within(() => {
-            cy.get(interact.SPECTRUM_ICON).click({force: true})
+        cy.get(interact.SPECTRUM_ICON).click({ force: true })
-        })
+      })
-        // There may be timing issues with this on the smoke build
+      // There may be timing issues with this on the smoke build
-        cy.wait(500)
+      cy.wait(500)
-        cy.get(".spectrum-Toast-content")
+      cy.get(".spectrum-Toast-content")
         .contains("URL copied to clipboard")
         .should("be.visible")
     })
 
     it("should allow API key regeneration", () => {
-        // Get initial API key value
+      cy.get(".user-dropdown .icon", { timeout: 2000 }).click({ force: true })
-        cy.get(interact.SPECTRUM_DIALOG_CONTENT)
+      cy.get(interact.SPECTRUM_MENU_ITEM).contains("View API key").click({ force: true })
+      cy.get(interact.SPECTRUM_DIALOG_CONTENT).within(() => {
+        cy.get(interact.SPECTRUM_ICON).click({ force: true })
+      })
+      // Get initial API key value
+      cy.get(interact.SPECTRUM_DIALOG_CONTENT)
         .find(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').as('keyOne')
 
-        // Click re-generate key button
+      // Click re-generate key button
-        cy.get("button").contains("Re-generate key").click({ force: true })
+      cy.get("button").contains("Regenerate key").click({ force: true })
 
-        // Verify API key was changed
+      // Verify API key was changed
-        cy.get(interact.SPECTRUM_DIALOG_CONTENT).within(() => {
+      cy.get(interact.SPECTRUM_DIALOG_CONTENT).within(() => {
-            cy.get('@keyOne').then((keyOne) => {
+        cy.get('@keyOne').then((keyOne) => {
-                cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').should('not.eq', keyOne)
+          cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).invoke('val').should('not.eq', keyOne)
-            })
         })
-        cy.closeModal()
+      })
+      cy.closeModal()
     })
 
     it("should update password", () => {
-        // Access Update password modal
+      // Access Update password modal
-        cy.get(".user-dropdown .avatar > .icon", { timeout: 2000 }).click({ force: true })
+      cy.get(".user-dropdown .icon", { timeout: 2000 }).click({ force: true })
-        cy.get(interact.SPECTRUM_MENU_ITEM).contains("Update password").click({ force: true })
+      cy.get(interact.SPECTRUM_MENU_ITEM).contains("Update password").click({ force: true })
 
-        // Enter new password and update
+      // Enter new password and update
-        cy.get(interact.SPECTRUM_DIALOG_GRID).within(() => {
+      cy.get(interact.SPECTRUM_DIALOG_GRID).within(() => {
-            for (let i = 0; i < 2; i++) {
+        for (let i = 0; i < 2; i++) {
-                // password set to 'newpwd'
+          // password set to 'newpwd'
-                cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).eq(i).type("newpwd")
+          cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).eq(i).type("newpwd")
-              }
+        }
-            cy.get("button").contains("Update password").click({ force: true })
+        cy.get("button").contains("Update password").click({ force: true })
-        })
+      })
 
-        // Logout & in with new password
+      // Logout & in with new password
-        //cy.logOut()
+      //cy.logOut()
-        cy.login("test@test.com", "newpwd")
+      cy.login("test@test.com", "newpwd")
     })
 
-    it("should open and close developer mode", () => {
+    xit("should open and close developer mode", () => {
-        cy.get(".user-dropdown .avatar > .icon", { timeout: 2000 }).click({ force: true })
+      cy.get(".user-dropdown .icon", { timeout: 2000 }).click({ force: true })
-        
-        // Close developer mode & verify
-        cy.get(interact.SPECTRUM_MENU_ITEM).contains("Close developer mode").click({ force: true })
-        cy.get(interact.SPECTRUM_SIDENAV).should('not.exist') // No config sections
-        cy.get(interact.CREATE_APP_BUTTON).should('not.exist') // No create app button
-        cy.get(".app").should('not.exist') // At least one app should be available
 
-        // Open developer mode & verify
+      // Close developer mode & verify
-        cy.get(".avatar > .icon").click({ force: true })
+      cy.get(interact.SPECTRUM_MENU_ITEM).contains("Close developer mode").click({ force: true })
-        cy.get(interact.SPECTRUM_MENU_ITEM).contains("Open developer mode").click({ force: true })
+      cy.get(interact.SPECTRUM_SIDENAV).should('not.exist') // No config sections
-        cy.get(interact.SPECTRUM_SIDENAV).should('exist') // config sections available
+      cy.get(interact.CREATE_APP_BUTTON).should('not.exist') // No create app button
-        cy.get(interact.CREATE_APP_BUTTON).should('exist') // create app button available
+      cy.get(".app").should('not.exist') // At least one app should be available
+
+      // Open developer mode & verify
+      cy.get(".avatar > .icon").click({ force: true })
+      cy.get(interact.SPECTRUM_MENU_ITEM).contains("Open developer mode").click({ force: true })
+      cy.get(".app-table").should('exist') // config sections available
+      cy.get(interact.CREATE_APP_BUTTON).should('exist') // create app button available
     })
 
     after(() => {
-        // Change password back to original value
+      // Change password back to original value
-        cy.get(".user-dropdown .avatar > .icon", { timeout: 2000 }).click({ force: true })
+      cy.get(".user-dropdown .icon", { timeout: 2000 }).click({ force: true })
-        cy.get(interact.SPECTRUM_MENU_ITEM).contains("Update password").click({ force: true })
+      cy.get(interact.SPECTRUM_MENU_ITEM).contains("Update password").click({ force: true })
-        cy.get(interact.SPECTRUM_DIALOG_GRID).within(() => {
+      cy.get(interact.SPECTRUM_DIALOG_GRID).within(() => {
-            for (let i = 0; i < 2; i++) {
+        for (let i = 0; i < 2; i++) {
-                cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).eq(i).type("test")
+          cy.get(interact.SPECTRUM_TEXTFIELD_INPUT).eq(i).type("test")
-              }
+        }
-            cy.get("button").contains("Update password").click({ force: true })
+        cy.get("button").contains("Update password").click({ force: true })
-        })
-        // Remove users name
-        cy.updateUserInformation()
       })
+      // Remove users name
+      cy.updateUserInformation()
+    })
   })
 })

packages/builder/cypress/integration/appOverview.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 import clientPackage from "@budibase/client/package.json"
 
 filterTests(["all"], () => {
-  context("Application Overview screen", () => {
+  xcontext("Application Overview screen", () => {
     before(() => {
       cy.login()
       cy.deleteAllApps()

packages/builder/cypress/integration/createApp.spec.js

@@ -14,15 +14,15 @@ filterTests(['smoke', 'all'], () => {
         cy.visit(`${Cypress.config().baseUrl}/builder/portal/apps/create`, { timeout: 5000 }) //added /portal/apps/create
         cy.wait(1000)
         cy.get(interact.CREATE_APP_BUTTON, { timeout: 10000 }).contains('Start from scratch').should("exist")
-        
+
         cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist")
         cy.get(interact.TEMPLATE_CATEGORY).should("exist")
-        
+
         cy.get(interact.APP_TABLE).should("not.exist")
       })
     }
 
-    it("should provide filterable templates", () => {
+    xit("should provide filterable templates", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
       cy.wait(500)
 
@@ -30,16 +30,16 @@ filterTests(['smoke', 'all'], () => {
         .its("body")
         .then(val => {
           if (val.length > 0) {
-            cy.get(interact.SPECTRUM_BUTTON).contains("Templates").click({force: true})
+            cy.get(interact.SPECTRUM_BUTTON).contains("View Templates").click({ force: true })
           }
         })
 
       cy.get(interact.TEMPLATE_CATEGORY_FILTER).should("exist")
       cy.get(interact.TEMPLATE_CATEGORY).should("exist")
-      
+
       cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).its('length').should('be.gt', 1)
       cy.get(interact.TEMPLATE_CATEGORY_FILTER_ACTIONBUTTON).its('length').should('be.gt', 2)
-      
+
       cy.get(interact.TEMPLATE_CATEGORY_FILTER_ACTIONBUTTON).eq(1).click()
       cy.get(interact.TEMPLATE_CATEGORY_ACTIONGROUP).should('have.length', 1)
 
@@ -104,14 +104,14 @@ filterTests(['smoke', 'all'], () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 5000 })
 
       cy.updateUserInformation("Ted", "Userman")
-      
+
       cy.createApp("", false)
       cy.applicationInAppTable("Teds app")
       cy.deleteApp("Teds app")
 
       // Accomodate names that end in 'S'
       cy.updateUserInformation("Chris", "Userman")
-      
+
       cy.createApp("", false)
       cy.applicationInAppTable("Chris app")
       cy.deleteApp("Chris app")
@@ -123,35 +123,49 @@ filterTests(['smoke', 'all'], () => {
       const exportedApp = 'cypress/fixtures/exported-app.txt'
 
       cy.importApp(exportedApp, "")
-
       cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 2000 })
-
       cy.applicationInAppTable("My app")
-      
+      cy.get(".app-table .name").eq(0).click()
-      cy.get(".appTable .name").eq(0).click()
+      cy.closeModal()
-
+      cy.get(`[aria-label="ShowMenu"]`).click()
-      cy.deleteApp("My app")
+      cy.get(".spectrum-Menu").within(() => {
+        cy.contains("Overview").click()
+      })
+      cy.get(".app-overview-actions-icon").within(() => {
+        cy.get(".spectrum-Icon").click({ force: true })
+      })
+      cy.get(".spectrum-Menu").contains("Delete").click({ force: true })
+      cy.get(".spectrum-Dialog-grid").within(() => {
+        cy.get("input").type("My app")
+      })
+      cy.get(".spectrum-Button--warning").click()
     })
 
     it("should create an application from an export, using the users first name as the default app name", () => {
       const exportedApp = 'cypress/fixtures/exported-app.txt'
 
       cy.updateUserInformation("Ted", "Userman")
-
       cy.importApp(exportedApp, "")
-
       cy.visit(`${Cypress.config().baseUrl}/builder`)
-
       cy.applicationInAppTable("Teds app")
-      
+      cy.get(".app-table .name").eq(0).click()
-      cy.get(".appTable .name").eq(0).click()
+      cy.closeModal()
-
+      cy.get(`[aria-label="ShowMenu"]`).click()
-      cy.deleteApp("Teds app")
+      cy.get(".spectrum-Menu").within(() => {
-
+        cy.contains("Overview").click()
+      })
+      cy.get(".app-overview-actions-icon").within(() => {
+        cy.get(".spectrum-Icon").click({ force: true })
+      })
+      cy.get(".spectrum-Menu").contains("Delete").click({ force: true })
+      cy.get(".spectrum-Dialog-grid").within(() => {
+        cy.get("input").type("Teds app")
+      })
+      cy.get(".spectrum-Button--warning").click()
       cy.updateUserInformation("", "")
     })
 
-    it("should generate the first application from a template", () => {
+    xit("should generate the first application from a template", () => {
       cy.visit(`${Cypress.config().baseUrl}/builder`)
       cy.wait(500)
 
@@ -172,28 +186,28 @@ filterTests(['smoke', 'all'], () => {
         const card = cy.get('.template-card').eq(0).should("exist");
         const cardOverlay = card.get('.template-thumbnail-action-overlay').should("exist")
         cardOverlay.invoke("show")
-        cardOverlay.get("button").contains("Use template").should("exist").click({force: true})
+        cardOverlay.get("button").contains("Use template").should("exist").click({ force: true })
       })
 
       // CMD Create app from theme card
       cy.get(".spectrum-Modal").should('be.visible')
-      
+
       const templateName = cy.get(".spectrum-Modal .template-thumbnail-text")
       templateName.invoke('text')
-      .then(templateNameText => {
+        .then(templateNameText => {
-        const templateNameParsed = "/"+templateNameText.toLowerCase().replace(/\s+/g, "-")
+          const templateNameParsed = "/" + templateNameText.toLowerCase().replace(/\s+/g, "-")
-        cy.get(interact.SPECTRUM_MODAL_INPUT).eq(0).should("have.value", templateNameText)
+          cy.get(interact.SPECTRUM_MODAL_INPUT).eq(0).should("have.value", templateNameText)
-        cy.get(interact.SPECTRUM_MODAL_INPUT).eq(1).should("have.value", templateNameParsed)
+          cy.get(interact.SPECTRUM_MODAL_INPUT).eq(1).should("have.value", templateNameParsed)
 
-        cy.get(".spectrum-Modal .spectrum-ButtonGroup").contains("Create app").click()
+          cy.get(".spectrum-Modal .spectrum-ButtonGroup").contains("Create app").click()
-        cy.wait(5000)
+          cy.wait(5000)
-        
-        cy.visit(`${Cypress.config().baseUrl}/builder`)
-        cy.wait(2000)
 
-        cy.applicationInAppTable(templateNameText)
+          cy.visit(`${Cypress.config().baseUrl}/builder`)
-        cy.deleteApp(templateNameText) 
+          cy.wait(2000)
-      });
+
+          cy.applicationInAppTable(templateNameText)
+          cy.deleteApp(templateNameText)
+        });
 
     })
 
@@ -217,5 +231,5 @@ filterTests(['smoke', 'all'], () => {
       cy.deleteApp(secondAppName)
     })
 
-  })  
+  })
 })

packages/builder/cypress/integration/createScreen.spec.js

@@ -2,7 +2,7 @@ import filterTests from "../support/filterTests"
 const interact = require('../support/interact')
 
 filterTests(["smoke", "all"], () => {
-  context("Screen Tests", () => {
+  xcontext("Screen Tests", () => {
     before(() => {
       cy.login()
       cy.createTestApp()
@@ -25,7 +25,7 @@ filterTests(["smoke", "all"], () => {
 
     it.skip("should delete all screens then create first screen via button", () => {
       cy.deleteAllScreens()
-      
+
       cy.contains("Create first screen").click()
       cy.get(interact.BODY, { timeout: 2000 }).should('contain', '/home')
     })
@@ -33,7 +33,7 @@ filterTests(["smoke", "all"], () => {
     it("Should create and filter screens by access level", () => {
       const accessLevels = ["Basic", "Admin", "Public", "Power"]
 
-      for (const access of accessLevels){
+      for (const access of accessLevels) {
         // Create screen with specified access level
         cy.createScreen(access, access)
         // Filter by access level and confirm screen visible
@@ -46,9 +46,9 @@ filterTests(["smoke", "all"], () => {
       // Filter by All screens - Confirm all screens visible
       cy.filterScreensAccessLevel("All screens")
       cy.get(interact.BODY).should('contain', accessLevels[0])
-      .and('contain', accessLevels[1])
+        .and('contain', accessLevels[1])
-      .and('contain', accessLevels[2])
+        .and('contain', accessLevels[2])
-      .and('contain', accessLevels[3])
+        .and('contain', accessLevels[3])
     })
   })
 })

packages/builder/cypress/support/commands.js

@@ -101,7 +101,7 @@ Cypress.Commands.add("deleteUser", email => {
 })
 
 Cypress.Commands.add("updateUserInformation", (firstName, lastName) => {
-  cy.get(".user-dropdown .avatar > .icon", { timeout: 2000 }).click({
+  cy.get(".user-dropdown .icon", { timeout: 2000 }).click({
     force: true,
   })
 
@@ -132,7 +132,7 @@ Cypress.Commands.add("updateUserInformation", (firstName, lastName) => {
         .blur()
     }
     cy.get(".confirm-wrap").within(() => {
-      cy.get("button").contains("Update information").click({ force: true })
+      cy.get("button").contains("Save").click({ force: true })
     })
     cy.get(".spectrum-Dialog-grid").should("not.exist")
   })
@@ -222,9 +222,12 @@ Cypress.Commands.add("deleteApp", name => {
           // Go to app overview
           const appIdParsed = appId.split("_").pop()
           const actionEleId = `[data-cy=row_actions_${appIdParsed}]`
-          cy.get(actionEleId).within(() => {
+          cy.get(actionEleId).click()
-            cy.contains("Manage").click({ force: true })
+          cy.get(`[aria-label="ShowMenu"]`).click()
+          cy.get(".spectrum-Menu").within(() => {
+            cy.contains("Overview").click()
           })
+
           cy.wait(500)
 
           // Unpublish first if needed
@@ -400,7 +403,7 @@ Cypress.Commands.add("searchForApplication", appName => {
         return
       } else {
         // Searches for the app
-        cy.get(".filter").then(() => {
+        cy.get(".spectrum-Search").then(() => {
           cy.get(".spectrum-Textfield").within(() => {
             cy.get("input").eq(0).clear({ force: true })
             cy.get("input").eq(0).type(appName, { force: true })
@@ -413,7 +416,7 @@ Cypress.Commands.add("searchForApplication", appName => {
 // Assumes there are no others
 Cypress.Commands.add("applicationInAppTable", appName => {
   cy.visit(`${Cypress.config().baseUrl}/builder`, { timeout: 30000 })
-  cy.get(".appTable", { timeout: 30000 }).within(() => {
+  cy.get(".app-table", { timeout: 30000 }).within(() => {
     cy.get(".title").contains(appName).should("exist")
   })
 })

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "license": "GPL-3.0",
   "private": true,
   "scripts": {
@@ -71,10 +71,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.16",
+    "@budibase/bbui": "2.2.12-alpha.32",
-    "@budibase/client": "2.2.12-alpha.16",
+    "@budibase/client": "2.2.12-alpha.32",
-    "@budibase/frontend-core": "2.2.12-alpha.16",
+    "@budibase/frontend-core": "2.2.12-alpha.32",
-    "@budibase/string-templates": "2.2.12-alpha.16",
+    "@budibase/string-templates": "2.2.12-alpha.32",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/builderStore/dataBinding.js

@@ -378,6 +378,7 @@ const getProviderContextBindings = (asset, dataProviders) => {
           providerId,
           // Table ID is used by JSON fields to know what table the field is in
           tableId: table?._id,
+          component: component._component,
           category: component._instanceName,
           icon: def.icon,
           display: {

packages/builder/src/components/backend/DataTable/modals/ExportModal.svelte

@@ -12,6 +12,10 @@
       name: "JSON",
       key: "json",
     },
+    {
+      name: "JSON with Schema",
+      key: "jsonWithSchema",
+    },
   ]
 
   export let view
@@ -24,7 +28,7 @@
         viewName: view,
         format: exportFormat,
       })
-      download(data, `export.${exportFormat}`)
+      download(data, `export.${exportFormat === "csv" ? "csv" : "json"}`)
     } catch (error) {
       notifications.error(`Unable to export ${exportFormat.toUpperCase()} data`)
     }

packages/builder/src/components/backend/DataTable/modals/ImportModal.svelte

@@ -6,22 +6,22 @@
     Body,
     Layout,
   } from "@budibase/bbui"
-  import TableDataImport from "../../TableNavigator/TableDataImport.svelte"
+  import TableDataImport from "../../TableNavigator/ExistingTableDataImport.svelte"
   import { API } from "api"
   import { createEventDispatcher } from "svelte"
 
   const dispatch = createEventDispatcher()
 
   export let tableId
-  let dataImport
+  let rows = []
-
+  let allValid = false
-  $: valid = dataImport?.csvString != null && dataImport?.valid
+  let displayColumn = null
 
   async function importData() {
     try {
       await API.importTableData({
         tableId,
-        data: dataImport,
+        rows,
       })
       notifications.success("Rows successfully imported")
     } catch (error) {
@@ -37,14 +37,14 @@
   title="Import Data"
   confirmText="Import"
   onConfirm={importData}
-  disabled={!valid}
+  disabled={!allValid}
 >
   <Body size="S">
-    Import rows to an existing table from a CSV. Only columns from the CSV which
+    Import rows to an existing table from a CSV or JSON file. Only columns from
-    exist in the table will be imported.
+    the file which exist in the table will be imported.
   </Body>
   <Layout gap="XS" noPadding>
-    <Label grey extraSmall>CSV to import</Label>
+    <Label grey extraSmall>CSV or JSON file to import</Label>
-    <TableDataImport bind:dataImport bind:existingTableId={tableId} />
+    <TableDataImport {tableId} bind:rows bind:allValid bind:displayColumn />
   </Layout>
 </ModalContent>

packages/builder/src/components/backend/Datasources/CreateEditRelationship.svelte

@@ -10,7 +10,6 @@
   } from "@budibase/bbui"
   import { tables } from "stores/backend"
   import { Helpers } from "@budibase/bbui"
-  import { writable } from "svelte/store"
 
   export let save
   export let datasource
@@ -18,41 +17,95 @@
   export let fromRelationship = {}
   export let toRelationship = {}
   export let close
-  export let selectedFromTable
 
-  let originalFromName = fromRelationship.name,
+  const colNotSet = "Please specify a column name"
-    originalToName = toRelationship.name
+  const relationshipTypes = [
-  let fromTable, toTable, through, linkTable, tableOptions
+    {
-  let isManyToMany, isManyToOne, relationshipTypes
+      label: "One to Many",
-  let errors, valid
+      value: RelationshipTypes.MANY_TO_ONE,
-  let currentTables = {}
+    },
+    {
+      label: "Many to Many",
+      value: RelationshipTypes.MANY_TO_MANY,
+    },
+  ]
 
-  if (fromRelationship && !fromRelationship.relationshipType) {
+  let originalFromColumnName = toRelationship.name,
-    fromRelationship.relationshipType = RelationshipTypes.MANY_TO_ONE
+    originalToColumnName = fromRelationship.name
-  }
+  let originalFromTable = plusTables.find(
+    table => table._id === toRelationship?.tableId
+  )
+  let originalToTable = plusTables.find(
+    table => table._id === fromRelationship?.tableId
+  )
 
-  if (toRelationship && selectedFromTable) {
+  let tableOptions
-    toRelationship.tableId = selectedFromTable._id
+  let errors = {}
-  }
+  let hasClickedSave = !!fromRelationship.relationshipType
+  let fromPrimary,
+    fromForeign,
+    fromTable,
+    toTable,
+    throughTable,
+    fromColumn,
+    toColumn
+  let fromId, toId, throughId, throughToKey, throughFromKey
+  let isManyToMany, isManyToOne, relationshipType
 
-  function inSchema(table, prop, ogName) {
+  $: {
-    if (!table || !prop || prop === ogName) {
+    if (!fromPrimary) {
-      return false
+      fromPrimary = fromRelationship.foreignKey
+      fromForeign = toRelationship.foreignKey
+    }
+    if (!fromColumn && !errors.fromColumn) {
+      fromColumn = toRelationship.name
+    }
+    if (!toColumn && !errors.toColumn) {
+      toColumn = fromRelationship.name
+    }
+    if (!fromId) {
+      fromId = toRelationship.tableId
+    }
+    if (!toId) {
+      toId = fromRelationship.tableId
+    }
+    if (!throughId) {
+      throughId = fromRelationship.through
+      throughFromKey = fromRelationship.throughFrom
+      throughToKey = fromRelationship.throughTo
+    }
+    if (!relationshipType) {
+      relationshipType = fromRelationship.relationshipType
     }
-    const keys = Object.keys(table.schema).map(key => key.toLowerCase())
-    return keys.indexOf(prop.toLowerCase()) !== -1
   }
 
-  const touched = writable({})
+  $: tableOptions = plusTables.map(table => ({
+    label: table.name,
+    value: table._id,
+  }))
+  $: valid = getErrorCount(errors) === 0 || !hasClickedSave
 
-  function invalidThroughTable({ through, throughTo, throughFrom }) {
+  $: isManyToMany = relationshipType === RelationshipTypes.MANY_TO_MANY
+  $: isManyToOne = relationshipType === RelationshipTypes.MANY_TO_ONE
+  $: fromTable = plusTables.find(table => table._id === fromId)
+  $: toTable = plusTables.find(table => table._id === toId)
+  $: throughTable = plusTables.find(table => table._id === throughId)
+
+  $: toRelationship.relationshipType = fromRelationship?.relationshipType
+
+  const getErrorCount = errors =>
+    Object.entries(errors)
+      .filter(entry => !!entry[1])
+      .map(entry => entry[0]).length
+
+  function invalidThroughTable() {
     // need to know the foreign key columns to check error
-    if (!through || !throughTo || !throughFrom) {
+    if (!throughId || !throughToKey || !throughFromKey) {
       return false
     }
-    const throughTable = plusTables.find(tbl => tbl._id === through)
+    const throughTbl = plusTables.find(tbl => tbl._id === throughId)
-    const otherColumns = Object.values(throughTable.schema).filter(
+    const otherColumns = Object.values(throughTbl.schema).filter(
-      col => col.name !== throughFrom && col.name !== throughTo
+      col => col.name !== throughFromKey && col.name !== throughToKey
     )
     for (let col of otherColumns) {
       if (col.constraints?.presence && !col.autocolumn) {
@@ -62,142 +115,134 @@
     return false
   }
 
-  function checkForErrors(fromRelate, toRelate) {
+  function validate() {
-    const isMany =
+    const isMany = relationshipType === RelationshipTypes.MANY_TO_MANY
-      fromRelate.relationshipType === RelationshipTypes.MANY_TO_MANY
     const tableNotSet = "Please specify a table"
+    const foreignKeyNotSet = "Please pick a foreign key"
     const errObj = {}
-    if ($touched.from && !fromTable) {
+    if (!relationshipType) {
-      errObj.from = tableNotSet
+      errObj.relationshipType = "Please specify a relationship type"
     }
-    if ($touched.to && !toTable) {
+    if (!fromTable) {
-      errObj.to = tableNotSet
+      errObj.fromTable = tableNotSet
     }
-    if ($touched.through && isMany && !fromRelate.through) {
+    if (!toTable) {
-      errObj.through = tableNotSet
+      errObj.toTable = tableNotSet
     }
-    if ($touched.through && invalidThroughTable(fromRelate)) {
+    if (isMany && !throughTable) {
-      errObj.through =
+      errObj.throughTable = tableNotSet
-        "Ensure all columns in table are nullable or auto generated"
     }
-    if ($touched.foreign && !isMany && !fromRelate.fieldName) {
+    if (isMany && !throughFromKey) {
-      errObj.foreign = "Please pick the foreign key"
+      errObj.throughFromKey = foreignKeyNotSet
     }
-    const colNotSet = "Please specify a column name"
+    if (isMany && !throughToKey) {
-    if ($touched.fromCol && !fromRelate.name) {
+      errObj.throughToKey = foreignKeyNotSet
-      errObj.fromCol = colNotSet
     }
-    if ($touched.toCol && !toRelate.name) {
+    if (invalidThroughTable()) {
-      errObj.toCol = colNotSet
+      errObj.throughTable =
+        "Ensure non-key columns are nullable or auto-generated"
     }
-    if ($touched.primary && !fromPrimary) {
+    if (!isMany && !fromForeign) {
-      errObj.primary = "Please pick the primary key"
+      errObj.fromForeign = foreignKeyNotSet
     }
+    if (!fromColumn) {
+      errObj.fromColumn = colNotSet
+    }
+    if (!toColumn) {
+      errObj.toColumn = colNotSet
+    }
+    if (!isMany && !fromPrimary) {
+      errObj.fromPrimary = "Please pick the primary key"
+    }
+
     // currently don't support relationships back onto the table itself, needs to relate out
     const tableError = "From/to/through tables must be different"
-    if (fromTable && (fromTable === toTable || fromTable === through)) {
+    if (fromTable && (fromTable === toTable || fromTable === throughTable)) {
-      errObj.from = tableError
+      errObj.fromTable = tableError
     }
-    if (toTable && (toTable === fromTable || toTable === through)) {
+    if (toTable && (toTable === fromTable || toTable === throughTable)) {
-      errObj.to = tableError
+      errObj.toTable = tableError
     }
-    if (through && (through === fromTable || through === toTable)) {
+    if (
-      errObj.through = tableError
+      throughTable &&
+      (throughTable === fromTable || throughTable === toTable)
+    ) {
+      errObj.throughTable = tableError
     }
     const colError = "Column name cannot be an existing column"
-    if (inSchema(fromTable, fromRelate.name, originalFromName)) {
+    if (isColumnNameBeingUsed(toTable, fromColumn, originalFromColumnName)) {
-      errObj.fromCol = colError
+      errObj.fromColumn = colError
     }
-    if (inSchema(toTable, toRelate.name, originalToName)) {
+    if (isColumnNameBeingUsed(fromTable, toColumn, originalToColumnName)) {
-      errObj.toCol = colError
+      errObj.toColumn = colError
     }
 
     let fromType, toType
-    if (fromPrimary && fromRelate.fieldName) {
+    if (fromPrimary && fromForeign) {
       fromType = fromTable?.schema[fromPrimary]?.type
-      toType = toTable?.schema[fromRelate.fieldName]?.type
+      toType = toTable?.schema[fromForeign]?.type
     }
     if (fromType && toType && fromType !== toType) {
-      errObj.foreign =
+      errObj.fromForeign =
         "Column type of the foreign key must match the primary key"
     }
+
     errors = errObj
+    return getErrorCount(errors) === 0
   }
 
-  let fromPrimary
+  function isColumnNameBeingUsed(table, columnName, originalName) {
-  $: {
+    if (!table || !columnName || columnName === originalName) {
-    if (!fromPrimary && fromTable) {
+      return false
-      fromPrimary = fromTable.primary[0]
-    }
-  }
-  $: isManyToMany =
-    fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_MANY
-  $: isManyToOne =
-    fromRelationship?.relationshipType === RelationshipTypes.MANY_TO_ONE
-  $: tableOptions = plusTables.map(table => ({
-    label: table.name,
-    value: table._id,
-  }))
-  $: fromTable = plusTables.find(table => table._id === toRelationship?.tableId)
-  $: toTable = plusTables.find(table => table._id === fromRelationship?.tableId)
-  $: through = plusTables.find(table => table._id === fromRelationship?.through)
-  $: checkForErrors(fromRelationship, toRelationship)
-  $: valid =
-    Object.keys(errors).length === 0 && Object.keys($touched).length !== 0
-  $: linkTable = through || toTable
-  $: relationshipTypes = [
-    {
-      label: "Many",
-      value: RelationshipTypes.MANY_TO_MANY,
-    },
-    {
-      label: "One",
-      value: RelationshipTypes.MANY_TO_ONE,
-    },
-  ]
-  $: updateRelationshipType(fromRelationship?.relationshipType)
-  $: tableChanged(fromTable, toTable)
-
-  function updateRelationshipType(fromType) {
-    if (fromType === RelationshipTypes.MANY_TO_MANY) {
-      toRelationship.relationshipType = RelationshipTypes.MANY_TO_MANY
-    } else {
-      toRelationship.relationshipType = RelationshipTypes.MANY_TO_ONE
     }
+    const keys = Object.keys(table.schema).map(key => key.toLowerCase())
+    return keys.indexOf(columnName.toLowerCase()) !== -1
   }
 
   function buildRelationships() {
-    // if any to many only need to check from
-    const manyToMany =
-      fromRelationship.relationshipType === RelationshipTypes.MANY_TO_MANY
-    // main is simply used to know this is the side the user configured it from
     const id = Helpers.uuid()
-    if (!manyToMany) {
+    //Map temporary variables
-      delete fromRelationship.through
-      delete toRelationship.through
-    }
     let relateFrom = {
       ...fromRelationship,
+      tableId: toId,
+      name: toColumn,
+      relationshipType,
+      fieldName: fromForeign,
+      through: throughId,
+      throughFrom: throughFromKey,
+      throughTo: throughToKey,
       type: "link",
       main: true,
       _id: id,
     }
-    let relateTo = {
+    let relateTo = (toRelationship = {
       ...toRelationship,
+      tableId: fromId,
+      name: fromColumn,
+      through: throughId,
       type: "link",
       _id: id,
+    })
+
+    // if any to many only need to check from
+    const manyToMany =
+      relateFrom.relationshipType === RelationshipTypes.MANY_TO_MANY
+
+    if (!manyToMany) {
+      delete relateFrom.through
+      delete relateTo.through
     }
 
     // [0] is because we don't support composite keys for relationships right now
     if (manyToMany) {
       relateFrom = {
         ...relateFrom,
-        through: through._id,
+        through: throughTable._id,
         fieldName: toTable.primary[0],
       }
       relateTo = {
         ...relateTo,
-        through: through._id,
+        through: throughTable._id,
         fieldName: fromTable.primary[0],
         throughFrom: relateFrom.throughTo,
         throughTo: relateFrom.throughFrom,
@@ -226,9 +271,27 @@
     toRelationship = relateTo
   }
 
-  // save the relationship on to the datasource
+  function removeExistingRelationship() {
+    if (originalFromTable && originalFromColumnName) {
+      delete datasource.entities[originalFromTable.name].schema[
+        originalToColumnName
+      ]
+    }
+    if (originalToTable && originalToColumnName) {
+      delete datasource.entities[originalToTable.name].schema[
+        originalFromColumnName
+      ]
+    }
+  }
+
   async function saveRelationship() {
+    hasClickedSave = true
+    if (!validate()) {
+      return false
+    }
     buildRelationships()
+    removeExistingRelationship()
+
     // source of relationship
     datasource.entities[fromTable.name].schema[fromRelationship.name] =
       fromRelationship
@@ -236,43 +299,14 @@
     datasource.entities[toTable.name].schema[toRelationship.name] =
       toRelationship
 
-    // If relationship has been renamed
-    if (originalFromName !== fromRelationship.name) {
-      delete datasource.entities[fromTable.name].schema[originalFromName]
-    }
-    if (originalToName !== toRelationship.name) {
-      delete datasource.entities[toTable.name].schema[originalToName]
-    }
-
-    // store the original names so it won't cause an error
-    originalToName = toRelationship.name
-    originalFromName = fromRelationship.name
     await save()
   }
-
   async function deleteRelationship() {
-    delete datasource.entities[fromTable.name].schema[fromRelationship.name]
+    removeExistingRelationship()
-    delete datasource.entities[toTable.name].schema[toRelationship.name]
     await save()
     await tables.fetch()
     close()
   }
-
-  function tableChanged(fromTbl, toTbl) {
-    if (
-      (currentTables?.from?._id === fromTbl?._id &&
-        currentTables?.to?._id === toTbl?._id) ||
-      originalFromName ||
-      originalToName
-    ) {
-      return
-    }
-    fromRelationship.name = toTbl?.name || ""
-    errors.fromCol = ""
-    toRelationship.name = fromTbl?.name || ""
-    errors.toCol = ""
-    currentTables = { from: fromTbl, to: toTbl }
-  }
 </script>
 
 <ModalContent
@@ -284,7 +318,9 @@
   <Select
     label="Relationship type"
     options={relationshipTypes}
-    bind:value={fromRelationship.relationshipType}
+    bind:value={relationshipType}
+    bind:error={errors.relationshipType}
+    on:change={() => (errors.relationshipType = null)}
   />
   <div class="headings">
     <Detail>Tables</Detail>
@@ -292,60 +328,74 @@
   <Select
     label="Select from table"
     options={tableOptions}
-    disabled={!!selectedFromTable}
+    bind:value={fromId}
-    on:change={() => ($touched.from = true)}
+    bind:error={errors.fromTable}
-    bind:error={errors.from}
+    on:change={e => {
-    bind:value={toRelationship.tableId}
+      fromColumn = tableOptions.find(opt => opt.value === e.detail)?.label || ""
+      errors.fromTable = null
+      errors.fromColumn = null
+    }}
   />
   {#if isManyToOne && fromTable}
     <Select
-      label={`Primary Key (${fromTable?.name})`}
+      label={`Primary Key (${fromTable.name})`}
-      options={Object.keys(fromTable?.schema)}
+      options={Object.keys(fromTable.schema)}
-      on:change={() => ($touched.primary = true)}
-      bind:error={errors.primary}
       bind:value={fromPrimary}
+      bind:error={errors.fromPrimary}
+      on:change={() => (errors.fromPrimary = null)}
     />
   {/if}
   <Select
     label={"Select to table"}
     options={tableOptions}
-    on:change={() => ($touched.to = true)}
+    bind:value={toId}
-    bind:error={errors.to}
+    bind:error={errors.toTable}
-    bind:value={fromRelationship.tableId}
+    on:change={e => {
+      toColumn = tableOptions.find(opt => opt.value === e.detail)?.label || ""
+      errors.toTable = null
+      errors.toColumn = null
+    }}
   />
   {#if isManyToMany}
     <Select
       label={"Through"}
       options={tableOptions}
-      on:change={() => ($touched.through = true)}
+      bind:value={throughId}
-      bind:error={errors.through}
+      bind:error={errors.throughTable}
-      bind:value={fromRelationship.through}
     />
-    {#if fromTable && toTable && through}
+    {#if fromTable && toTable && throughTable}
       <Select
         label={`Foreign Key (${fromTable?.name})`}
-        options={Object.keys(through?.schema)}
+        options={Object.keys(throughTable?.schema)}
-        on:change={() => ($touched.fromForeign = true)}
+        bind:value={throughToKey}
-        bind:error={errors.fromForeign}
+        bind:error={errors.throughToKey}
-        bind:value={fromRelationship.throughTo}
+        on:change={e => {
+          if (throughFromKey === e.detail) {
+            throughFromKey = null
+          }
+          errors.throughToKey = null
+        }}
       />
       <Select
         label={`Foreign Key (${toTable?.name})`}
-        options={Object.keys(through?.schema)}
+        options={Object.keys(throughTable?.schema)}
-        on:change={() => ($touched.toForeign = true)}
+        bind:value={throughFromKey}
-        bind:error={errors.toForeign}
+        bind:error={errors.throughFromKey}
-        bind:value={fromRelationship.throughFrom}
+        on:change={e => {
+          if (throughToKey === e.detail) {
+            throughToKey = null
+          }
+          errors.throughFromKey = null
+        }}
       />
     {/if}
   {:else if isManyToOne && toTable}
     <Select
       label={`Foreign Key (${toTable?.name})`}
-      options={Object.keys(toTable?.schema).filter(
+      options={Object.keys(toTable?.schema)}
-        field => toTable?.primary.indexOf(field) === -1
+      bind:value={fromForeign}
-      )}
+      bind:error={errors.fromForeign}
-      on:change={() => ($touched.foreign = true)}
+      on:change={() => (errors.fromForeign = null)}
-      bind:error={errors.foreign}
-      bind:value={fromRelationship.fieldName}
     />
   {/if}
   <div class="headings">
@@ -356,19 +406,21 @@
     provide a name for these columns.
   </Body>
   <Input
-    on:blur={() => ($touched.fromCol = true)}
-    bind:error={errors.fromCol}
     label="From table column"
-    bind:value={fromRelationship.name}
+    bind:value={fromColumn}
+    bind:error={errors.fromColumn}
+    on:change={e => {
+      errors.fromColumn = e.detail?.length > 0 ? null : colNotSet
+    }}
   />
   <Input
-    on:blur={() => ($touched.toCol = true)}
-    bind:error={errors.toCol}
     label="To table column"
-    bind:value={toRelationship.name}
+    bind:value={toColumn}
+    bind:error={errors.toColumn}
+    on:change={e => (errors.toColumn = e.detail?.length > 0 ? null : colNotSet)}
   />
   <div slot="footer">
-    {#if originalFromName != null}
+    {#if originalFromColumnName != null}
       <Button warning text on:click={deleteRelationship}>Delete</Button>
     {/if}
   </div>

packages/builder/src/components/backend/TableNavigator/ExistingTableDataImport.svelte

@@ -0,0 +1,251 @@
+<script>
+  import { Select } from "@budibase/bbui"
+  import { FIELDS } from "constants/backend"
+  import { API } from "api"
+  import { parseFile } from "./utils"
+
+  let error = null
+  let fileName = null
+  let fileType = null
+
+  let loading = false
+  let validation = {}
+  let validateHash = ""
+  let schema = null
+  let invalidColumns = []
+
+  export let tableId = null
+  export let rows = []
+  export let allValid = false
+
+  const typeOptions = [
+    {
+      label: "Text",
+      value: FIELDS.STRING.type,
+    },
+    {
+      label: "Number",
+      value: FIELDS.NUMBER.type,
+    },
+    {
+      label: "Date",
+      value: FIELDS.DATETIME.type,
+    },
+    {
+      label: "Options",
+      value: FIELDS.OPTIONS.type,
+    },
+    {
+      label: "Multi-select",
+      value: FIELDS.ARRAY.type,
+    },
+    {
+      label: "Barcode/QR",
+      value: FIELDS.BARCODEQR.type,
+    },
+    {
+      label: "Long Form Text",
+      value: FIELDS.LONGFORM.type,
+    },
+  ]
+
+  $: {
+    schema = fetchSchema(tableId)
+  }
+
+  async function fetchSchema(tableId) {
+    try {
+      const definition = await API.fetchTableDefinition(tableId)
+      schema = definition.schema
+    } catch (e) {
+      error = e
+    }
+  }
+
+  async function handleFile(e) {
+    loading = true
+    error = null
+    validation = {}
+
+    try {
+      const response = await parseFile(e)
+      rows = response.rows
+      fileName = response.fileName
+      fileType = response.fileType
+    } catch (e) {
+      loading = false
+      error = e
+    }
+  }
+
+  async function validate(rows) {
+    loading = true
+    error = null
+    validation = {}
+    allValid = false
+
+    try {
+      if (rows.length > 0) {
+        const response = await API.validateExistingTableImport({
+          rows,
+          tableId,
+        })
+
+        validation = response.schemaValidation
+        invalidColumns = response.invalidColumns
+        allValid = response.allValid
+      }
+    } catch (e) {
+      error = e.message
+    }
+
+    loading = false
+  }
+
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows)
+
+    if (newValidateHash !== validateHash) {
+      validate(rows)
+    }
+
+    validateHash = newValidateHash
+  }
+</script>
+
+<div class="dropzone">
+  <input
+    disabled={!schema || loading}
+    id="file-upload"
+    accept="text/csv,application/json"
+    type="file"
+    on:change={handleFile}
+  />
+  <label for="file-upload" class:uploaded={rows.length > 0}>
+    {#if loading}
+      loading...
+    {:else if error}
+      error: {error}
+    {:else if fileName}
+      {fileName}
+    {:else}
+      Upload
+    {/if}
+  </label>
+</div>
+{#if fileName && Object.keys(validation).length === 0}
+  <p>No valid fields, try another file</p>
+{:else if rows.length > 0 && !error}
+  <div class="schema-fields">
+    {#each Object.keys(validation) as name}
+      <div class="field">
+        <span>{name}</span>
+        <Select
+          value={schema[name]?.type}
+          options={typeOptions}
+          placeholder={null}
+          getOptionLabel={option => option.label}
+          getOptionValue={option => option.value}
+          disabled
+        />
+        <span
+          class={loading || validation[name]
+            ? "fieldStatusSuccess"
+            : "fieldStatusFailure"}
+        >
+          {validation[name] ? "Success" : "Failure"}
+        </span>
+      </div>
+    {/each}
+  </div>
+  {#if invalidColumns.length > 0}
+    <p class="spectrum-FieldLabel spectrum-FieldLabel--sizeM">
+      The following columns are present in the data you wish to import, but do
+      not match the schema of this table and will be ignored.
+    </p>
+    <ul class="ignoredList">
+      {#each invalidColumns as column}
+        <li>{column}</li>
+      {/each}
+    </ul>
+  {/if}
+{/if}
+
+<style>
+  .dropzone {
+    text-align: center;
+    display: flex;
+    align-items: center;
+    flex-direction: column;
+    border-radius: 10px;
+    transition: all 0.3s;
+  }
+
+  input {
+    display: none;
+  }
+
+  label {
+    font-family: var(--font-sans);
+    cursor: pointer;
+    font-weight: 600;
+    box-sizing: border-box;
+    overflow: hidden;
+    border-radius: var(--border-radius-s);
+    color: var(--ink);
+    padding: var(--spacing-m) var(--spacing-l);
+    transition: all 0.2s ease 0s;
+    display: inline-flex;
+    text-rendering: optimizeLegibility;
+    min-width: auto;
+    outline: none;
+    font-feature-settings: "case" 1, "rlig" 1, "calt" 0;
+    -webkit-box-align: center;
+    user-select: none;
+    flex-shrink: 0;
+    align-items: center;
+    justify-content: center;
+    width: 100%;
+    background-color: var(--grey-2);
+    font-size: var(--font-size-xs);
+    line-height: normal;
+    border: var(--border-transparent);
+  }
+
+  .uploaded {
+    color: var(--blue);
+  }
+
+  .schema-fields {
+    margin-top: var(--spacing-xl);
+  }
+
+  .field {
+    display: grid;
+    grid-template-columns: 2fr 2fr 1fr auto;
+    margin-top: var(--spacing-m);
+    align-items: center;
+    grid-gap: var(--spacing-m);
+    font-size: var(--spectrum-global-dimension-font-size-75);
+  }
+
+  .fieldStatusSuccess {
+    color: var(--green);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .fieldStatusFailure {
+    color: var(--red);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .ignoredList {
+    margin: 0;
+    padding: 0;
+    list-style: none;
+    font-size: var(--spectrum-global-dimension-font-size-75);
+  }
+</style>

packages/builder/src/components/backend/TableNavigator/TableDataImport.svelte

@@ -1,107 +1,21 @@
 <script>
-  import { Select, InlineAlert, notifications } from "@budibase/bbui"
+  import { Select } from "@budibase/bbui"
   import { FIELDS } from "constants/backend"
   import { API } from "api"
+  import { parseFile } from "./utils"
 
-  const BYTES_IN_MB = 1000000
+  let error = null
-  const FILE_SIZE_LIMIT = BYTES_IN_MB * 5
+  let fileName = null
+  let fileType = null
 
-  export let files = []
+  let loading = false
-  export let dataImport = {
+  let validation = {}
-    valid: true,
+  let validateHash = ""
-    schema: {},
-  }
-  export let existingTableId
 
-  let csvString = undefined
+  export let rows = []
-  let primaryDisplay = undefined
+  export let schema = {}
-  let schema = {}
+  export let allValid = true
-  let fields = []
+  export let displayColumn = null
-  let hasValidated = false
-
-  $: valid =
-    !schema ||
-    (fields.every(column => schema[column].success) &&
-      (!hasValidated || Object.keys(schema).length > 0))
-  $: dataImport = {
-    valid,
-    schema: buildTableSchema(schema),
-    csvString,
-    primaryDisplay,
-  }
-  $: noFieldsError = existingTableId
-    ? "No columns in CSV match existing table schema"
-    : "Could not find any columns to import"
-
-  function buildTableSchema(schema) {
-    const tableSchema = {}
-    for (let key in schema) {
-      const type = schema[key].type
-
-      if (type === "omit") continue
-
-      tableSchema[key] = {
-        name: key,
-        type,
-        constraints: FIELDS[type.toUpperCase()].constraints,
-      }
-    }
-    return tableSchema
-  }
-
-  async function validateCSV() {
-    try {
-      const parseResult = await API.validateTableCSV({
-        csvString,
-        schema: schema || {},
-        tableId: existingTableId,
-      })
-      schema = parseResult?.schema
-      fields = Object.keys(schema || {}).filter(
-        key => schema[key].type !== "omit"
-      )
-
-      // Check primary display is valid
-      if (!primaryDisplay || fields.indexOf(primaryDisplay) === -1) {
-        primaryDisplay = fields[0]
-      }
-
-      hasValidated = true
-    } catch (error) {
-      notifications.error("CSV Invalid, please try another CSV file")
-    }
-  }
-
-  async function handleFile(evt) {
-    const fileArray = Array.from(evt.target.files)
-    if (fileArray.some(file => file.size >= FILE_SIZE_LIMIT)) {
-      notifications.error(
-        `Files cannot exceed ${
-          FILE_SIZE_LIMIT / BYTES_IN_MB
-        }MB. Please try again with smaller files.`
-      )
-      return
-    }
-
-    // Read CSV as plain text to upload alongside schema
-    let reader = new FileReader()
-    reader.addEventListener("load", function (e) {
-      csvString = e.target.result
-      files = fileArray
-      validateCSV()
-    })
-    reader.readAsText(fileArray[0])
-  }
-
-  async function omitColumn(columnName) {
-    schema[columnName].type = "omit"
-    await validateCSV()
-  }
-
-  const handleTypeChange = column => evt => {
-    schema[column].type = evt.detail
-    validateCSV()
-  }
 
   const typeOptions = [
     {
@@ -133,54 +47,114 @@
       value: FIELDS.LONGFORM.type,
     },
   ]
+
+  async function handleFile(e) {
+    loading = true
+    error = null
+    validation = {}
+
+    try {
+      const response = await parseFile(e)
+      rows = response.rows
+      schema = response.schema
+      fileName = response.fileName
+      fileType = response.fileType
+    } catch (e) {
+      loading = false
+      error = e
+    }
+  }
+
+  async function validate(rows, schema) {
+    loading = true
+    error = null
+    validation = {}
+    allValid = false
+
+    try {
+      if (rows.length > 0) {
+        const response = await API.validateNewTableImport({ rows, schema })
+        validation = response.schemaValidation
+        allValid = response.allValid
+      }
+    } catch (e) {
+      error = e.message
+    }
+
+    loading = false
+  }
+
+  $: {
+    // binding in consumer is causing double renders here
+    const newValidateHash = JSON.stringify(rows) + JSON.stringify(schema)
+
+    if (newValidateHash !== validateHash) {
+      validate(rows, schema)
+    }
+
+    validateHash = newValidateHash
+  }
 </script>
 
 <div class="dropzone">
-  <input id="file-upload" accept=".csv" type="file" on:change={handleFile} />
+  <input
-  <label for="file-upload" class:uploaded={files[0]}>
+    disabled={loading}
-    {#if files[0]}{files[0].name}{:else}Upload{/if}
+    id="file-upload"
+    accept="text/csv,application/json"
+    type="file"
+    on:change={handleFile}
+  />
+  <label for="file-upload" class:uploaded={rows.length > 0}>
+    {#if loading}
+      loading...
+    {:else if error}
+      error: {error}
+    {:else if fileName}
+      {fileName}
+    {:else}
+      Upload
+    {/if}
   </label>
 </div>
-{#if fields.length}
+{#if rows.length > 0 && !error}
   <div class="schema-fields">
-    {#each fields as columnName}
+    {#each Object.values(schema) as column}
       <div class="field">
-        <span>{columnName}</span>
+        <span>{column.name}</span>
         <Select
-          bind:value={schema[columnName].type}
+          bind:value={column.type}
-          on:change={handleTypeChange(columnName)}
+          on:change={e => (column.type = e.detail)}
           options={typeOptions}
           placeholder={null}
           getOptionLabel={option => option.label}
           getOptionValue={option => option.value}
-          disabled={!!existingTableId}
+          disabled={loading}
         />
-        <span class="field-status" class:error={!schema[columnName].success}>
+        <span
-          {schema[columnName].success ? "Success" : "Failure"}
+          class={loading || validation[column.name]
+            ? "fieldStatusSuccess"
+            : "fieldStatusFailure"}
+        >
+          {validation[column.name] ? "Success" : "Failure"}
         </span>
         <i
-          class="omit-button ri-close-circle-fill"
+          class={`omit-button ri-close-circle-fill ${
-          on:click={() => omitColumn(columnName)}
+            loading ? "omit-button-disabled" : ""
+          }`}
+          on:click={() => {
+            delete schema[column.name]
+            schema = schema
+          }}
         />
       </div>
     {/each}
   </div>
-  {#if !existingTableId}
+  <div class="display-column">
-    <div class="display-column">
+    <Select
-      <Select
+      label="Display Column"
-        label="Display Column"
+      bind:value={displayColumn}
-        bind:value={primaryDisplay}
+      options={Object.keys(schema)}
-        options={fields}
+      sort
-        sort
-      />
-    </div>
-  {/if}
-{:else if hasValidated}
-  <div>
-    <InlineAlert
-      header="Invalid CSV"
-      bind:message={noFieldsError}
-      type="error"
     />
   </div>
 {/if}
@@ -195,28 +169,10 @@
     transition: all 0.3s;
   }
 
-  .field-status {
+  input {
-    color: var(--green);
-    justify-self: center;
-    font-weight: 600;
-  }
-
-  .error {
-    color: var(--red);
-  }
-
-  .uploaded {
-    color: var(--blue);
-  }
-
-  input[type="file"] {
     display: none;
   }
 
-  .schema-fields {
-    margin-top: var(--spacing-xl);
-  }
-
   label {
     cursor: pointer;
     font-weight: 600;
@@ -243,11 +199,12 @@
     border: var(--border-transparent);
   }
 
-  .omit-button {
+  .uploaded {
-    font-size: 1.2em;
+    color: var(--blue);
-    color: var(--grey-7);
+  }
-    cursor: pointer;
+
-    justify-self: flex-end;
+  .schema-fields {
+    margin-top: var(--spacing-xl);
   }
 
   .field {
@@ -259,6 +216,30 @@
     font-size: var(--spectrum-global-dimension-font-size-75);
   }
 
+  .fieldStatusSuccess {
+    color: var(--green);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .fieldStatusFailure {
+    color: var(--red);
+    justify-self: center;
+    font-weight: 600;
+  }
+
+  .omit-button {
+    font-size: 1.2em;
+    color: var(--grey-7);
+    cursor: pointer;
+    justify-self: flex-end;
+  }
+
+  .omit-button-disabled {
+    pointer-events: none;
+    opacity: 70%;
+  }
+
   .display-column {
     margin-top: var(--spacing-xl);
   }

packages/builder/src/components/backend/TableNavigator/modals/CreateTableModal.svelte

@@ -29,18 +29,27 @@
     : BUDIBASE_INTERNAL_DB_ID
 
   export let name
-  let dataImport
   let error = ""
   let autoColumns = getAutoColumnInformation()
+  let schema = {}
+  let rows = []
+  let allValid = true
+  let displayColumn = null
 
-  function addAutoColumns(tableName, schema) {
+  function getAutoColumns() {
-    for (let [subtype, col] of Object.entries(autoColumns)) {
+    const selectedAutoColumns = {}
-      if (!col.enabled) {
+
-        continue
+    Object.entries(autoColumns).forEach(([subtype, column]) => {
+      if (column.enabled) {
+        selectedAutoColumns[column.name] = buildAutoColumn(
+          name,
+          column.name,
+          subtype
+        )
       }
-      schema[col.name] = buildAutoColumn(tableName, col.name, subtype)
+    })
-    }
+
-    return schema
+    return selectedAutoColumns
   }
 
   function checkValid(evt) {
@@ -55,15 +64,15 @@
   async function saveTable() {
     let newTable = {
       name,
-      schema: addAutoColumns(name, dataImport.schema || {}),
+      schema: { ...schema, ...getAutoColumns() },
-      dataImport,
+      rows,
       type: "internal",
       sourceId: targetDatasourceId,
     }
 
     // Only set primary display if defined
-    if (dataImport.primaryDisplay && dataImport.primaryDisplay.length) {
+    if (displayColumn && displayColumn.length) {
-      newTable.primaryDisplay = dataImport.primaryDisplay
+      newTable.primaryDisplay = displayColumn
     }
 
     // Create table
@@ -90,7 +99,7 @@
   title="Create Table"
   confirmText="Create"
   onConfirm={saveTable}
-  disabled={error || !name || (dataImport && !dataImport.valid)}
+  disabled={error || !name || (rows.length && !allValid)}
 >
   <Input
     data-cy="table-name-input"
@@ -117,8 +126,10 @@
   </div>
   <div>
     <Layout gap="XS" noPadding>
-      <Label grey extraSmall>Create Table from CSV (Optional)</Label>
+      <Label grey extraSmall
-      <TableDataImport bind:dataImport />
+        >Create a Table from a CSV or JSON file (Optional)</Label
+      >
+      <TableDataImport bind:rows bind:schema bind:allValid bind:displayColumn />
     </Layout>
   </div>
 </ModalContent>

packages/builder/src/components/backend/TableNavigator/utils.js

@@ -0,0 +1,71 @@
+import { API } from "api"
+import { FIELDS } from "constants/backend"
+
+const BYTES_IN_MB = 1000000
+const FILE_SIZE_LIMIT = BYTES_IN_MB * 5
+
+const getDefaultSchema = rows => {
+  const newSchema = {}
+
+  rows.forEach(row => {
+    Object.keys(row).forEach(column => {
+      newSchema[column] = {
+        name: column,
+        type: "string",
+        constraints: FIELDS["STRING"].constraints,
+      }
+    })
+  })
+
+  return newSchema
+}
+
+export const parseFile = e => {
+  return new Promise((resolve, reject) => {
+    const file = Array.from(e.target.files)[0]
+
+    if (file.size >= FILE_SIZE_LIMIT) {
+      reject("file too large")
+      return
+    }
+
+    let reader = new FileReader()
+
+    const resolveRows = (rows, schema = null) => {
+      resolve({
+        rows,
+        schema: schema ?? getDefaultSchema(rows),
+        fileName: file.name,
+        fileType: file.type,
+      })
+    }
+
+    reader.addEventListener("load", function (e) {
+      const fileData = e.target.result
+
+      if (file.type === "text/csv") {
+        API.csvToJson(fileData)
+          .then(rows => {
+            resolveRows(rows)
+          })
+          .catch(() => {
+            reject("can't convert csv to json")
+          })
+      } else if (file.type === "application/json") {
+        const parsedFileData = JSON.parse(fileData)
+
+        if (Array.isArray(parsedFileData)) {
+          resolveRows(parsedFileData)
+        } else if (typeof parsedFileData === "object") {
+          resolveRows(parsedFileData.rows, parsedFileData.schema)
+        } else {
+          reject("invalid json format")
+        }
+      } else {
+        reject("invalid file type")
+      }
+    })
+
+    reader.readAsText(file)
+  })
+}

packages/builder/src/components/design/settings/controls/DataSourceSelect.svelte

@@ -70,7 +70,10 @@
       type: "provider",
     }))
   $: links = bindings
+    // Get only link bindings
     .filter(x => x.fieldSchema?.type === "link")
+    // Filter out bindings provided by forms
+    .filter(x => !x.component?.endsWith("/form"))
     .map(binding => {
       const { providerId, readableBinding, fieldSchema } = binding || {}
       const { name, tableId } = fieldSchema || {}

packages/builder/src/pages/builder/app/[application]/design/[screenId]/components/[componentId]/new/_components/NewComponentPanel.svelte

@@ -176,7 +176,6 @@
   const addComponent = async component => {
     try {
       await store.actions.components.create(component)
-      $goto("../")
     } catch (error) {
       notifications.error(error || "Error creating component")
     }

packages/builder/src/pages/builder/portal/_components/UpgradeButton.svelte

@@ -7,6 +7,7 @@
 {#if $admin.cloud && $auth?.user?.accountPortalAccess}
   <Button
     cta
+    size="S"
     on:click
     on:click={() => {
       $goto($admin.accountPortalUrl + "/portal/upgrade")

packages/builder/src/pages/builder/portal/_components/UserDropdown.svelte

@@ -26,9 +26,6 @@
     <Avatar size="M" initials={$auth.initials} url={$auth.user.pictureUrl} />
     <Icon size="XL" name="ChevronDown" />
   </div>
-  <MenuItem icon="Moon" on:click={() => themeModal.show()} dataCy="theme">
-    Theme
-  </MenuItem>
   <MenuItem
     icon="UserEdit"
     on:click={() => profileModal.show()}
@@ -36,6 +33,9 @@
   >
     My profile
   </MenuItem>
+  <MenuItem icon="Moon" on:click={() => themeModal.show()} dataCy="theme">
+    Theme
+  </MenuItem>
   <MenuItem icon="LockClosed" on:click={() => updatePasswordModal.show()}>
     Update password
   </MenuItem>

packages/builder/src/pages/builder/portal/settings/auth/index.svelte

@@ -172,6 +172,8 @@
       delete element.createdAt
       delete element.updatedAt
 
+      const { activated } = element.config
+
       if (element.type === ConfigTypes.OIDC) {
         // Add a UUID here so each config is distinguishable when it arrives at the login page
         for (let config of element.config.configs) {
@@ -181,30 +183,26 @@
           // Callback urls shouldn't be included
           delete config.callbackURL
         }
-        if (partialOidc) {
+        if ((partialOidc || activated) && !oidcComplete) {
-          if (!oidcComplete) {
+          notifications.error(
-            notifications.error(
+            `Please fill in all required ${ConfigTypes.OIDC} fields`
-              `Please fill in all required ${ConfigTypes.OIDC} fields`
+          )
-            )
+        } else if (oidcComplete || !activated) {
-          } else {
+          calls.push(API.saveConfig(element))
-            calls.push(API.saveConfig(element))
+          // Turn the save button grey when clicked
-            // Turn the save button grey when clicked
+          oidcSaveButtonDisabled = true
-            oidcSaveButtonDisabled = true
+          originalOidcDoc = cloneDeep(providers.oidc)
-            originalOidcDoc = cloneDeep(providers.oidc)
-          }
         }
       }
       if (element.type === ConfigTypes.Google) {
-        if (partialGoogle) {
+        if ((partialGoogle || activated) && !googleComplete) {
-          if (!googleComplete) {
+          notifications.error(
-            notifications.error(
+            `Please fill in all required ${ConfigTypes.Google} fields`
-              `Please fill in all required ${ConfigTypes.Google} fields`
+          )
-            )
+        } else if (googleComplete || !activated) {
-          } else {
+          calls.push(API.saveConfig(element))
-            calls.push(API.saveConfig(element))
+          googleSaveButtonDisabled = true
-            googleSaveButtonDisabled = true
+          originalGoogleDoc = cloneDeep(providers.google)
-            originalGoogleDoc = cloneDeep(providers.google)
-          }
         }
       }
     })

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {
@@ -26,9 +26,9 @@
     "outputPath": "build"
   },
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.16",
+    "@budibase/backend-core": "2.2.12-alpha.32",
-    "@budibase/string-templates": "2.2.12-alpha.16",
+    "@budibase/string-templates": "2.2.12-alpha.32",
-    "@budibase/types": "2.2.12-alpha.16",
+    "@budibase/types": "2.2.12-alpha.32",
     "axios": "0.21.2",
     "chalk": "4.1.0",
     "cli-progress": "3.11.2",

packages/client/manifest.json

@@ -4019,7 +4019,8 @@
       {
         "type": "filter",
         "label": "Filtering",
-        "key": "filter"
+        "key": "filter",
+        "nested": true
       },
       {
         "type": "field",
@@ -4535,7 +4536,8 @@
           {
             "type": "filter",
             "label": "Filtering",
-            "key": "filter"
+            "key": "filter",
+            "nested": true
           },
           {
             "type": "searchfield",
@@ -4665,7 +4667,8 @@
       {
         "type": "filter",
         "label": "Filtering",
-        "key": "filter"
+        "key": "filter",
+        "nested": true
       },
       {
         "type": "field/sortable",
@@ -4831,7 +4834,8 @@
       {
         "type": "filter",
         "label": "Filtering",
-        "key": "filter"
+        "key": "filter",
+        "nested": true
       },
       {
         "type": "field/sortable",

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.16",
+    "@budibase/bbui": "2.2.12-alpha.32",
-    "@budibase/frontend-core": "2.2.12-alpha.16",
+    "@budibase/frontend-core": "2.2.12-alpha.32",
-    "@budibase/string-templates": "2.2.12-alpha.16",
+    "@budibase/string-templates": "2.2.12-alpha.32",
     "@spectrum-css/button": "^3.0.3",
     "@spectrum-css/card": "^3.0.3",
     "@spectrum-css/divider": "^1.0.3",

packages/client/src/components/Block.svelte

@@ -38,7 +38,7 @@
         ...$component.styles?.normal,
       },
       custom:
-        definition._styles?.custom || "" + $component.styles?.custom || "",
+        (definition._styles?.custom || "") + ($component.styles?.custom || ""),
     }
 
     // Create component tree

packages/client/src/components/CustomThemeWrapper.svelte

@@ -1,5 +1,9 @@
 <script>
   import { themeStore } from "stores"
+  import { setContext } from "svelte"
+  import { Context } from "@budibase/bbui"
+
+  setContext(Context.PopoverRoot, "#theme-root")
 </script>
 
 <div style={$themeStore.customThemeCss} id="theme-root">

packages/client/src/components/app/blocks/CardsBlock.svelte

@@ -36,9 +36,12 @@
   let dataProviderId
   let repeaterId
   let schema
+  let enrichedSearchColumns
 
   $: fetchSchema(dataSource)
-  $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema)
+  $: enrichSearchColumns(searchColumns, schema).then(
+    val => (enrichedSearchColumns = val)
+  )
   $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId)
   $: cardWidth = cardHorizontal ? 420 : 300
   $: fullCardURL = buildFullCardUrl(

packages/client/src/components/app/blocks/TableBlock.svelte

@@ -36,9 +36,12 @@
   let newRowSidePanelId
   let schema
   let primaryDisplay
+  let enrichedSearchColumns
 
   $: fetchSchema(dataSource)
-  $: enrichedSearchColumns = enrichSearchColumns(searchColumns, schema)
+  $: enrichSearchColumns(searchColumns, schema).then(
+    val => (enrichedSearchColumns = val)
+  )
   $: enrichedFilter = enrichFilter(filter, enrichedSearchColumns, formId)
   $: editTitle = getEditTitle(detailsFormBlockId, primaryDisplay)
   $: normalFields = getNormalFields(schema)

packages/client/src/components/app/forms/Form.svelte

@@ -24,6 +24,11 @@
   let table
 
   $: fetchSchema(dataSource)
+  $: schemaKey = generateSchemaKey(schema)
+  $: initialValues = getInitialValues(actionType, dataSource, $context)
+  $: resetKey = Helpers.hashString(
+    schemaKey + JSON.stringify(initialValues) + disabled
+  )
 
   // Returns the closes data context which isn't a built in context
   const getInitialValues = (type, dataSource, context) => {
@@ -57,13 +62,17 @@
     schema = res || {}
   }
 
-  $: initialValues = getInitialValues(actionType, dataSource, $context)
+  // Generates a predictable string that uniquely identifies a schema. We can't
-  $: resetKey = Helpers.hashString(
+  // simply stringify the whole schema as there are array fields which have
-    !!schema +
+  // random order.
-      JSON.stringify(initialValues) +
+  const generateSchemaKey = schema => {
-      JSON.stringify(dataSource) +
+    if (!schema) {
-      disabled
+      return null
-  )
+    }
+    const fields = Object.keys(schema)
+    fields.sort()
+    return fields.map(field => `${field}:${schema[field].type}`).join("-")
+  }
 </script>
 
 {#key resetKey}

packages/client/src/utils/blocks.js

@@ -1,4 +1,5 @@
 import { makePropSafe as safe } from "@budibase/string-templates"
+import { API } from "../api/index.js"
 
 // Map of data types to component types for search fields inside blocks
 const schemaComponentMap = {
@@ -15,10 +16,28 @@ const schemaComponentMap = {
  * @param searchColumns the search columns to use
  * @param schema the datasource schema
  */
-export const enrichSearchColumns = (searchColumns, schema) => {
+export const enrichSearchColumns = async (searchColumns, schema) => {
+  if (!searchColumns?.length || !schema) {
+    return []
+  }
   let enrichedColumns = []
-  searchColumns?.forEach(column => {
+  for (let column of searchColumns) {
-    const schemaType = schema?.[column]?.type
+    let schemaType = schema[column]?.type
+
+    // Check if this is a field in another related table. The only way we can
+    // check this is checking for a "." inside the column, then checking if we
+    // have a link field named the same as that field prefix.
+    if (column.includes(".")) {
+      const split = column.split(".")
+      const sourceField = split[0]
+      const linkField = split.slice(1).join(".")
+      const linkSchema = schema[sourceField]
+      if (linkSchema?.type === "link") {
+        const linkedDef = await API.fetchTableDefinition(linkSchema.tableId)
+        schemaType = linkedDef?.schema?.[linkField]?.type
+      }
+    }
+
     const componentType = schemaComponentMap[schemaType]
     if (componentType) {
       enrichedColumns.push({
@@ -27,7 +46,7 @@ export const enrichSearchColumns = (searchColumns, schema) => {
         type: schemaType,
       })
     }
-  })
+  }
   return enrichedColumns.slice(0, 5)
 }
 
@@ -57,12 +76,14 @@ export const enrichFilter = (filter, columns, formId) => {
         value: `{{ ${binding} }}`,
       })
       const format = "YYYY-MM-DDTHH:mm:ss.SSSZ"
+      let hbs = `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`
+      hbs = `{{#if ${binding} }}${hbs}{{/if}}`
       enrichedFilter.push({
         field: column.name,
         type: column.type,
         operator: "rangeHigh",
         valueType: "Binding",
-        value: `{{ date (add (date ${binding} "x") 86399999) "${format}" }}`,
+        value: hbs,
       })
     }
 

packages/frontend-core/package.json

@@ -1,12 +1,12 @@
 {
   "name": "@budibase/frontend-core",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase frontend core libraries used in builder and client",
   "author": "Budibase",
   "license": "MPL-2.0",
   "svelte": "src/index.js",
   "dependencies": {
-    "@budibase/bbui": "2.2.12-alpha.16",
+    "@budibase/bbui": "2.2.12-alpha.32",
     "lodash": "^4.17.21",
     "svelte": "^3.46.2"
   }

packages/frontend-core/src/api/tables.js

@@ -64,32 +64,22 @@ export const buildTableEndpoints = API => ({
    * @param tableId the table ID to import to
    * @param data the data import object
    */
-  importTableData: async ({ tableId, data }) => {
+  importTableData: async ({ tableId, rows }) => {
     return await API.post({
       url: `/api/tables/${tableId}/import`,
       body: {
-        dataImport: data,
+        rows,
       },
     })
   },
-
+  csvToJson: async csvString => {
-  /**
-   * Validates a candidate CSV to be imported for a certain table.
-   * @param tableId the table ID to import to
-   * @param csvString the CSV contents as a string
-   * @param schema the proposed schema
-   */
-  validateTableCSV: async ({ tableId, csvString, schema }) => {
     return await API.post({
-      url: "/api/tables/csv/validate",
+      url: "/api/convert/csvToJson",
       body: {
         csvString,
-        schema,
-        tableId,
       },
     })
   },
-
   /**
    * Gets a list o tables.
    */
@@ -120,4 +110,22 @@ export const buildTableEndpoints = API => ({
       url: `/api/tables/${tableId}/${tableRev}`,
     })
   },
+  validateNewTableImport: async ({ rows, schema }) => {
+    return await API.post({
+      url: "/api/tables/validateNewTableImport",
+      body: {
+        rows,
+        schema,
+      },
+    })
+  },
+  validateExistingTableImport: async ({ rows, tableId }) => {
+    return await API.post({
+      url: "/api/tables/validateExistingTableImport",
+      body: {
+        rows,
+        tableId,
+      },
+    })
+  },
 })

packages/sdk/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/sdk",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase Public API SDK",
   "author": "Budibase",
   "license": "MPL-2.0",

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.16",
+    "@budibase/backend-core": "2.2.12-alpha.32",
-    "@budibase/client": "2.2.12-alpha.16",
+    "@budibase/client": "2.2.12-alpha.32",
-    "@budibase/pro": "2.2.12-alpha.16",
+    "@budibase/pro": "2.2.12-alpha.32",
-    "@budibase/string-templates": "2.2.12-alpha.16",
+    "@budibase/string-templates": "2.2.12-alpha.32",
-    "@budibase/types": "2.2.12-alpha.16",
+    "@budibase/types": "2.2.12-alpha.32",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -77,7 +77,7 @@
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
     "jsonschema": "1.4.0",
-    "knex": "0.95.15",
+    "knex": "2.4.0",
     "koa": "2.13.4",
     "koa-body": "4.2.0",
     "koa-compress": "4.0.1",

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -681,6 +681,12 @@ export class ExternalRequest {
       config,
       table
     )
+    //if the sort column is a formula, remove it
+    for (let sortColumn of Object.keys(sort || {})) {
+      if (table.schema[sortColumn]?.type === "formula") {
+        delete sort?.[sortColumn]
+      }
+    }
     filters = buildFilters(id, filters || {}, table)
     const relationships = this.buildRelationships(table)
     // clean up row on ingress using schema

packages/server/src/api/controllers/row/internal.ts

@@ -27,7 +27,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import * as exporters from "../view/exporters"
+import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Ctx,
@@ -412,14 +412,15 @@ export async function exportRows(ctx: Ctx) {
     rows = result
   }
 
-  let headers = Object.keys(rows[0])
+  if (format === Format.CSV) {
-  // @ts-ignore
+    ctx.attachment("export.csv")
-  const exporter = exporters[format]
+    return apiFileReturn(csv(Object.keys(rows[0]), rows))
-  const filename = `export.${format}`
+  } else if (format === Format.JSON) {
-
+    ctx.attachment("export.json")
-  // send down the file
+    return apiFileReturn(json(rows))
-  ctx.attachment(filename)
+  } else {
-  return apiFileReturn(exporter(headers, rows))
+    throw "Format not recognised"
+  }
 }
 
 export async function fetchEnrichedRow(ctx: Ctx) {

packages/server/src/api/controllers/table/external.ts

@@ -10,9 +10,9 @@ import {
 } from "./utils"
 import { FieldTypes, RelationshipTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
-import * as csvParser from "../../../utilities/csvParser"
 import { handleRequest } from "../row/external"
 import { events, context } from "@budibase/backend-core"
+import { parse, isRows, isSchema } from "../../../utilities/schema"
 import {
   Datasource,
   Table,
@@ -197,7 +197,7 @@ export async function save(ctx: BBContext) {
   const table: TableRequest = ctx.request.body
   const renamed = table?._rename
   // can't do this right now
-  delete table.dataImport
+  delete table.rows
   const datasourceId = getDatasourceId(ctx.request.body)!
   // table doesn't exist already, note that it is created
   if (!table._id) {
@@ -338,17 +338,17 @@ export async function destroy(ctx: BBContext) {
 
 export async function bulkImport(ctx: BBContext) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { dataImport } = ctx.request.body
+  const { rows }: { rows: unknown } = ctx.request.body
-  if (!dataImport || !dataImport.schema || !dataImport.csvString) {
+  const schema: unknown = table.schema
+
+  if (!rows || !isRows(rows) || !isSchema(schema)) {
     ctx.throw(400, "Provided data import information is invalid.")
   }
-  const rows = await csvParser.transform({
+
-    ...dataImport,
+  const parsedRows = await parse(rows, schema)
-    existingTable: table,
-  })
   await handleRequest(Operation.BULK_CREATE, table._id!, {
-    rows,
+    rows: parsedRows,
   })
-  await events.rows.imported(table, "csv", rows.length)
+  await events.rows.imported(table, parsedRows.length)
   return table
 }

packages/server/src/api/controllers/table/index.ts

@@ -1,11 +1,16 @@
 import * as internal from "./internal"
 import * as external from "./external"
-import * as csvParser from "../../../utilities/csvParser"
+import {
+  validate as validateSchema,
+  isSchema,
+  isRows,
+} from "../../../utilities/schema"
 import { isExternalTable, isSQL } from "../../../integrations/utils"
 import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
 import { Table, BBContext } from "@budibase/types"
 import sdk from "../../../sdk"
+import csv from "csvtojson"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -56,16 +61,16 @@ export async function find(ctx: BBContext) {
 export async function save(ctx: BBContext) {
   const appId = ctx.appId
   const table = ctx.request.body
-  const importFormat =
+  const isImport = table.rows
-    table.dataImport && table.dataImport.csvString ? "csv" : undefined
+
   const savedTable = await pickApi({ table }).save(ctx)
   if (!table._id) {
     await events.table.created(savedTable)
   } else {
     await events.table.updated(savedTable)
   }
-  if (importFormat) {
+  if (isImport) {
-    await events.table.imported(savedTable, importFormat)
+    await events.table.imported(savedTable)
   }
   ctx.status = 200
   ctx.message = `Table ${table.name} saved successfully.`
@@ -96,19 +101,43 @@ export async function bulkImport(ctx: BBContext) {
   ctx.body = { message: `Bulk rows created.` }
 }
 
-export async function validateCSVSchema(ctx: BBContext) {
+export async function csvToJson(ctx: BBContext) {
-  // tableId being specified means its an import to an existing table
+  const { csvString } = ctx.request.body
-  const { csvString, schema = {}, tableId } = ctx.request.body
+
-  let existingTable
+  const result = await csv().fromString(csvString)
-  if (tableId) {
+
-    existingTable = await sdk.tables.getTable(tableId)
+  ctx.status = 200
-  }
+  ctx.body = result
-  let result: Record<string, any> | undefined = await csvParser.parse(
+}
-    csvString,
+
-    schema
+export async function validateNewTableImport(ctx: BBContext) {
-  )
+  const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
-  if (existingTable) {
+
-    result = csvParser.updateSchema({ schema: result, existingTable })
+  if (isRows(rows) && isSchema(schema)) {
-  }
+    ctx.status = 200
-  ctx.body = { schema: result }
+    ctx.body = validateSchema(rows, schema)
+  } else {
+    ctx.status = 422
+  }
+}
+
+export async function validateExistingTableImport(ctx: BBContext) {
+  const { rows, tableId }: { rows: unknown; tableId: unknown } =
+    ctx.request.body
+
+  let schema = null
+  if (tableId) {
+    const table = await sdk.tables.getTable(tableId)
+    schema = table.schema
+  } else {
+    ctx.status = 422
+    return
+  }
+
+  if (tableId && isRows(rows) && isSchema(schema)) {
+    ctx.status = 200
+    ctx.body = validateSchema(rows, schema)
+  } else {
+    ctx.status = 422
+  }
 }

packages/server/src/api/controllers/table/internal.ts

@@ -35,7 +35,7 @@ function checkAutoColumns(table: Table, oldTable: Table) {
 
 export async function save(ctx: any) {
   const db = context.getAppDB()
-  const { dataImport, ...rest } = ctx.request.body
+  const { rows, ...rest } = ctx.request.body
   let tableToSave = {
     type: "table",
     _id: generateTableID(),
@@ -61,7 +61,7 @@ export async function save(ctx: any) {
   const tableSaveFunctions = new TableSaveFunctions({
     user: ctx.user,
     oldTable,
-    dataImport,
+    importRows: rows,
   })
   tableToSave = await tableSaveFunctions.before(tableToSave)
 
@@ -185,7 +185,7 @@ export async function destroy(ctx: any) {
 
 export async function bulkImport(ctx: any) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
-  const { dataImport } = ctx.request.body
+  const { rows } = ctx.request.body
-  await handleDataImport(ctx.user, table, dataImport)
+  await handleDataImport(ctx.user, table, rows)
   return table
 }

packages/server/src/api/controllers/table/utils.ts

@@ -1,4 +1,4 @@
-import { transform } from "../../../utilities/csvParser"
+import { parse, isSchema, isRows } from "../../../utilities/schema"
 import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
 import { isEqual } from "lodash"
 import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
@@ -128,24 +128,23 @@ export function importToRows(data: any, table: any, user: any = {}) {
   return finalData
 }
 
-export async function handleDataImport(user: any, table: any, dataImport: any) {
+export async function handleDataImport(user: any, table: any, rows: any) {
-  if (!dataImport || !dataImport.csvString) {
+  const schema: unknown = table.schema
+
+  if (!rows || !isRows(rows) || !isSchema(schema)) {
     return table
   }
 
   const db = context.getAppDB()
-  // Populate the table with rows imported from CSV in a bulk update
+  const data = parse(rows, schema)
-  const data = await transform({
-    ...dataImport,
-    existingTable: table,
-  })
 
   let finalData: any = importToRows(data, table, user)
 
   await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
     tableId: table._id,
   })
-  await events.rows.imported(table, "csv", finalData.length)
+
+  await events.rows.imported(table, finalData.length)
   return table
 }
 
@@ -210,14 +209,14 @@ class TableSaveFunctions {
   db: any
   user: any
   oldTable: any
-  dataImport: any
+  importRows: any
   rows: any
 
-  constructor({ user, oldTable, dataImport }: any) {
+  constructor({ user, oldTable, importRows }: any) {
     this.db = context.getAppDB()
     this.user = user
     this.oldTable = oldTable
-    this.dataImport = dataImport
+    this.importRows = importRows
     // any rows that need updated
     this.rows = []
   }
@@ -241,7 +240,7 @@ class TableSaveFunctions {
   // after saving
   async after(table: any) {
     table = await handleSearchIndexes(table)
-    table = await handleDataImport(this.user, table, this.dataImport)
+    table = await handleDataImport(this.user, table, this.importRows)
     return table
   }
 

packages/server/src/api/controllers/user.ts

@@ -4,19 +4,21 @@ import { getGlobalUsers, getRawGlobalUser } from "../../utilities/global"
 import { getFullUser } from "../../utilities/users"
 import {
   context,
-  constants,
   roles as rolesCore,
   db as dbCore,
 } from "@budibase/backend-core"
-import { BBContext, User } from "@budibase/types"
+import { BBContext, Ctx, SyncUserRequest, User } from "@budibase/types"
 import sdk from "../../sdk"
 
-export async function syncUser(ctx: BBContext) {
+export async function syncUser(ctx: Ctx<SyncUserRequest>) {
   let deleting = false,
     user: User | any
   const userId = ctx.params.id
+
+  const previousUser = ctx.request.body?.previousUser
+
   try {
-    user = await getRawGlobalUser(userId)
+    user = (await getRawGlobalUser(userId)) as User
   } catch (err: any) {
     if (err && err.status === 404) {
       user = {}
@@ -25,6 +27,11 @@ export async function syncUser(ctx: BBContext) {
       throw err
     }
   }
+
+  let previousApps = previousUser
+    ? Object.keys(previousUser.roles).map(appId => appId)
+    : []
+
   const roles = deleting ? {} : user.roles
   // remove props which aren't useful to metadata
   delete user.password
@@ -40,8 +47,9 @@ export async function syncUser(ctx: BBContext) {
       .filter(entry => entry[1] !== rolesCore.BUILTIN_ROLE_IDS.PUBLIC)
       .map(([appId]) => appId)
   }
-  for (let prodAppId of prodAppIds) {
+  for (let prodAppId of new Set([...prodAppIds, ...previousApps])) {
     const roleId = roles[prodAppId]
+    const deleteFromApp = !roleId
     const devAppId = dbCore.getDevelopmentAppID(prodAppId)
     for (let appId of [prodAppId, devAppId]) {
       if (!(await dbCore.dbExists(appId))) {
@@ -54,24 +62,24 @@ export async function syncUser(ctx: BBContext) {
         try {
           metadata = await db.get(metadataId)
         } catch (err) {
-          if (deleting) {
+          if (deleteFromApp) {
             return
           }
           metadata = {
             tableId: InternalTables.USER_METADATA,
           }
         }
+
+        if (deleteFromApp) {
+          await db.remove(metadata)
+          return
+        }
+
         // assign the roleId for the metadata doc
         if (roleId) {
           metadata.roleId = roleId
         }
-        let combined = !deleting
+        let combined = sdk.users.combineMetadataAndUser(user, metadata)
-          ? sdk.users.combineMetadataAndUser(user, metadata)
-          : {
-              ...metadata,
-              status: constants.UserStatus.INACTIVE,
-              metadata: rolesCore.BUILTIN_ROLE_IDS.PUBLIC,
-            }
         // if its null then there was no updates required
         if (combined) {
           await db.put(combined)

packages/server/src/api/controllers/view/exporters.ts

@@ -1,4 +1,4 @@
-import { Row } from "@budibase/types"
+import { Row, TableSchema } from "@budibase/types"
 
 export function csv(headers: string[], rows: Row[]) {
   let csv = headers.map(key => `"${key}"`).join(",")
@@ -18,11 +18,26 @@ export function csv(headers: string[], rows: Row[]) {
   return csv
 }
 
-export function json(headers: string[], rows: Row[]) {
+export function json(rows: Row[]) {
   return JSON.stringify(rows, undefined, 2)
 }
 
-export const ExportFormats = {
+export function jsonWithSchema(schema: TableSchema, rows: Row[]) {
-  CSV: "csv",
+  const newSchema: TableSchema = {}
-  JSON: "json",
+  Object.values(schema).forEach(column => {
+    if (!column.autocolumn) {
+      newSchema[column.name] = column
+    }
+  })
+  return JSON.stringify({ schema: newSchema, rows }, undefined, 2)
+}
+
+export enum Format {
+  CSV = "csv",
+  JSON = "json",
+  JSON_WITH_SCHEMA = "jsonWithSchema",
+}
+
+export function isFormat(format: any): format is Format {
+  return Object.values(Format).includes(format as Format)
 }

packages/server/src/api/controllers/view/index.ts

@@ -1,6 +1,6 @@
 import viewTemplate from "./viewBuilder"
 import { apiFileReturn } from "../../../utilities/fileSystem"
-import * as exporters from "./exporters"
+import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
 import { FieldTypes } from "../../../constants"
@@ -127,9 +127,13 @@ export async function exportView(ctx: BBContext) {
   const viewName = decodeURIComponent(ctx.query.view as string)
   const view = await getView(viewName)
 
-  const format = ctx.query.format as string
+  const format = ctx.query.format as unknown
-  if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
+
-    ctx.throw(400, "Format must be specified, either csv or json")
+  if (!isFormat(format)) {
+    ctx.throw(
+      400,
+      "Format must be specified, either csv, json or jsonWithSchema"
+    )
   }
 
   if (view) {
@@ -171,7 +175,7 @@ export async function exportView(ctx: BBContext) {
   })
 
   // make sure no "undefined" entries appear in the CSV
-  if (format === exporters.ExportFormats.CSV) {
+  if (format === Format.CSV) {
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
       for (let row of rows) {
@@ -182,13 +186,18 @@ export async function exportView(ctx: BBContext) {
     }
   }
 
-  // Export part
+  if (format === Format.CSV) {
-  let headers = Object.keys(schema)
+    ctx.attachment(`${viewName}.csv`)
-  const exporter = format === "csv" ? exporters.csv : exporters.json
+    ctx.body = apiFileReturn(csv(Object.keys(schema), rows))
-  const filename = `${viewName}.${format}`
+  } else if (format === Format.JSON) {
-  // send down the file
+    ctx.attachment(`${viewName}.json`)
-  ctx.attachment(filename)
+    ctx.body = apiFileReturn(json(rows))
-  ctx.body = apiFileReturn(exporter(headers, rows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment(`${viewName}.json`)
+    ctx.body = apiFileReturn(jsonWithSchema(schema, rows))
+  } else {
+    throw "Format not recognised"
+  }
 
   if (viewName.startsWith(DocumentType.TABLE)) {
     await events.table.exported(table, format as TableExportFormat)

packages/server/src/api/routes/table.ts

@@ -67,10 +67,7 @@ router
    * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
    * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
    * the rows in the table. This functionality is only available for internal tables.
-   * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the
+   * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
-   * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call
-   * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating
-   * or for external tables.
    *
    * @apiParamExample {json} Example:
    * {
@@ -99,15 +96,7 @@ router
    *     "old": "columnName",
    *     "updated": "newColumnName",
    *   },
-   *   "dataImport": {
+   *   "rows": []
-   *     "csvString": "column\nvalue",
-   *     "primaryDisplay": "column",
-   *     "schema": {
-   *       "column": {
-   *         "type": "string"
-   *       }
-   *     }
-   *   }
    * }
    *
    * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
@@ -121,30 +110,20 @@ router
     tableValidator(),
     tableController.save
   )
-  /**
-   * @api {post} /api/tables/csv/validate Validate a CSV for a table
-   * @apiName Validate a CSV for a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and
-   * converted into a Budibase schema; this endpoint does this.
-   *
-   * @apiParam (Body) {string} csvString The CSV which is to be validated as a string.
-   * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the
-   * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can
-   * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g.
-   * using a number instead of strings.
-   * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and
-   * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When
-   * importing a CSV to an existing table only fields that are present on the table can be imported.
-   *
-   * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for
-   * the CSV - this will be in the same format used for table schema.s
-   */
   .post(
-    "/api/tables/csv/validate",
+    "/api/convert/csvToJson",
     authorized(BUILDER),
-    tableController.validateCSVSchema
+    tableController.csvToJson
+  )
+  .post(
+    "/api/tables/validateNewTableImport",
+    authorized(BUILDER),
+    tableController.validateNewTableImport
+  )
+  .post(
+    "/api/tables/validateExistingTableImport",
+    authorized(BUILDER),
+    tableController.validateExistingTableImport
   )
   /**
    * @api {post} /api/tables/:tableId/:revId Delete a table
@@ -177,9 +156,7 @@ router
    *
    * @apiParam {string} tableId The ID of the table which the data should be imported to.
    *
-   * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with
+   * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
-   * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be
-   * turned into rows.
    *
    * @apiSuccess {string} message A message stating that the data was imported successfully.
    */

packages/server/src/api/routes/tests/misc.spec.js

@@ -42,7 +42,7 @@ describe("run misc tests", () => {
   })
 
   describe("test table utilities", () => {
-    it("should be able to import a CSV", async () => {
+    it("should be able to import data", async () => {
       return config.doInContext(null, async () => {
         const table = await config.createTable({
           name: "table",
@@ -75,17 +75,11 @@ describe("run misc tests", () => {
             },
           },
         })
-        const dataImport = {
+
-          csvString: "a,b,c,d\n1,2,3,4",
-          schema: {},
-        }
-        for (let col of ["a", "b", "c", "d"]) {
-          dataImport.schema[col] = { type: "string" }
-        }
         await tableUtils.handleDataImport(
           { userId: "test" },
           table,
-          dataImport
+          [{ a: '1', b: '2', c: '3', d: '4'}]
         )
         const rows = await config.getRows()
         expect(rows[0].a).toEqual("1")
@@ -94,4 +88,4 @@ describe("run misc tests", () => {
       })
     })
   })
-})
+})

packages/server/src/api/routes/tests/table.spec.js

@@ -43,21 +43,18 @@ describe("/tables", () => {
       expect(events.table.created).toBeCalledWith(res.body)
     })
 
-    it("creates a table via data import CSV", async () => {
+    it("creates a table via data import", async () => {
       const table = basicTable()
-      table.dataImport = {
+      table.rows = [{ name: 'test-name', description: 'test-desc' }]
-        csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-      }
-      table.dataImport.schema = table.schema
 
       const res = await createTable(table)
 
       expect(events.table.created).toBeCalledTimes(1)
       expect(events.table.created).toBeCalledWith(res.body)
       expect(events.table.imported).toBeCalledTimes(1)
-      expect(events.table.imported).toBeCalledWith(res.body, "csv")
+      expect(events.table.imported).toBeCalledWith(res.body)
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(res.body, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(res.body, 1)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -155,11 +152,10 @@ describe("/tables", () => {
     it("imports rows successfully", async () => {
       const table = await config.createTable()
       const importRequest = {
-        dataImport: {
+        schema: table.schema,
-          csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
+        rows: [{ name: 'test-name', description: 'test-desc' }]
-          schema: table.schema
-        }
       }
+
       jest.clearAllMocks()
 
       await request
@@ -171,7 +167,7 @@ describe("/tables", () => {
 
       expect(events.table.created).not.toHaveBeenCalled()
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(table, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(table, 1)
     })
   })
 
@@ -206,24 +202,6 @@ describe("/tables", () => {
     })
   })
 
-  describe("validate csv", () => {
-    it("should be able to validate a CSV layout", async () => {
-      const res = await request
-        .post(`/api/tables/csv/validate`)
-        .send({
-          csvString: "a,b,c,d\n1,2,3,4"
-        })
-        .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
-        .expect(200)
-      expect(res.body.schema).toBeDefined()
-      expect(res.body.schema.a).toEqual({
-        type: "string",
-        success: true,
-      })
-    })
-  })
-
   describe("indexing", () => {
     it("should be able to create a table with indexes", async () => {
       await context.doInAppContext(appId, async () => {

packages/server/src/api/routes/tests/user.spec.js

@@ -171,9 +171,28 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.message).toEqual('User synced.')
     })
+
+
+    it("should sync the user when a previous user is specified", async () => {
+      const app1 = await config.createApp('App 1')
+      const app2 = await config.createApp('App 2')
+
+      let user = await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        true,
+        { [app1.appId]: 'ADMIN' })
+      let res = await request
+        .post(`/api/users/metadata/sync/${user._id}`)
+        .set(config.defaultHeaders())
+        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
+        .expect(200)
+        .expect("Content-Type", /json/)
+
+      expect(res.body.message).toEqual('User synced.')
+    })
   })
-
-
-
-
 })

packages/server/src/api/routes/utils/validators.ts

@@ -18,7 +18,7 @@ export function tableValidator() {
     schema: Joi.object().required(),
     name: Joi.string().required(),
     views: Joi.object(),
-    dataImport: Joi.object(),
+    rows: Joi.array(),
   }).unknown(true))
 }
 

packages/server/src/middleware/currentapp.ts

@@ -25,6 +25,7 @@ export default async (ctx: BBContext, next: any) => {
   if (!appCookie && !requestAppId) {
     return next()
   }
+
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
@@ -51,7 +52,7 @@ export default async (ctx: BBContext, next: any) => {
 
   let appId: string | undefined,
     roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
-  if (!ctx.user) {
+  if (!ctx.user?._id) {
     // not logged in, try to set a cookie for public apps
     appId = requestAppId
   } else if (requestAppId != null) {
@@ -96,7 +97,7 @@ export default async (ctx: BBContext, next: any) => {
     // need to judge this only based on the request app ID,
     if (
       env.MULTI_TENANCY &&
-      ctx.user &&
+      ctx.user?._id &&
       requestAppId &&
       !tenancy.isUserInAppTenant(requestAppId, ctx.user)
     ) {

packages/server/src/utilities/csvParser.ts

@@ -1,161 +0,0 @@
-import { FieldSchema, Table } from "@budibase/types"
-import csv from "csvtojson"
-import { FieldTypes } from "../constants"
-
-type CsvParseOpts = {
-  schema?: { [key: string]: any }
-  existingTable: Table
-  csvString?: string
-}
-
-const VALIDATORS: any = {
-  [FieldTypes.STRING]: () => true,
-  [FieldTypes.OPTIONS]: () => true,
-  [FieldTypes.BARCODEQR]: () => true,
-  [FieldTypes.NUMBER]: (attribute?: string) => {
-    // allow not to be present
-    if (!attribute) {
-      return true
-    }
-    return !isNaN(Number(attribute))
-  },
-  [FieldTypes.DATETIME]: (attribute?: string) => {
-    // allow not to be present
-    if (!attribute) {
-      return true
-    }
-    return !isNaN(new Date(attribute).getTime())
-  },
-}
-
-const PARSERS: any = {
-  [FieldTypes.NUMBER]: (attribute?: string) => {
-    if (!attribute) {
-      return attribute
-    }
-    return Number(attribute)
-  },
-  [FieldTypes.DATETIME]: (attribute?: string) => {
-    if (!attribute) {
-      return attribute
-    }
-    return new Date(attribute).toISOString()
-  },
-}
-
-export function parse(csvString: string, parsers: any): Record<string, any> {
-  const result = csv().fromString(csvString)
-
-  const schema: Record<string, any> = {}
-
-  return new Promise((resolve, reject) => {
-    result.on("header", headers => {
-      for (let header of headers) {
-        schema[header] = {
-          type: parsers[header] ? parsers[header].type : "string",
-          success: true,
-        }
-      }
-    })
-    result.subscribe(row => {
-      // For each CSV row parse all the columns that need parsed
-      for (let key of Object.keys(parsers)) {
-        if (!schema[key] || schema[key].success) {
-          // get the validator for the column type
-          const validator = VALIDATORS[parsers[key].type]
-
-          try {
-            // allow null/undefined values
-            schema[key].success = !row[key] || validator(row[key])
-          } catch (err) {
-            schema[key].success = false
-          }
-        }
-      }
-    })
-    result.on("done", error => {
-      if (error) {
-        console.error(error)
-        reject(error)
-      }
-
-      resolve(schema)
-    })
-  })
-}
-
-export function updateSchema({
-  schema,
-  existingTable,
-}: {
-  schema?: Record<string, any>
-  existingTable?: Table
-}) {
-  if (!schema) {
-    return schema
-  }
-  const finalSchema: Record<string, FieldSchema> = {}
-  const schemaKeyMap: Record<string, any> = {}
-  Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
-  for (let [key, field] of Object.entries(existingTable?.schema || {})) {
-    const lcKey = key.toLowerCase()
-    const foundKey: string = schemaKeyMap[lcKey]
-    if (foundKey) {
-      finalSchema[key] = schema[foundKey]
-      finalSchema[key].type = field.type
-    }
-  }
-  return finalSchema
-}
-
-export async function transform({
-  schema,
-  csvString,
-  existingTable,
-}: CsvParseOpts) {
-  if (!schema || !csvString) {
-    throw new Error("Unable to transform CSV without schema")
-  }
-  const colParser: any = {}
-
-  // make sure the table has all the columns required for import
-  if (existingTable) {
-    schema = updateSchema({ schema, existingTable })
-  }
-
-  for (let [key, field] of Object.entries(schema || {})) {
-    // don't import data to auto columns
-    if (!field.autocolumn) {
-      colParser[key] = PARSERS[field.type] || field.type
-    }
-  }
-
-  try {
-    const data = await csv({ colParser }).fromString(csvString)
-    const schemaKeyMap: any = {}
-    Object.keys(schema || {}).forEach(
-      key => (schemaKeyMap[key.toLowerCase()] = key)
-    )
-    for (let element of data) {
-      if (!data) {
-        continue
-      }
-      for (let key of Object.keys(element)) {
-        const mappedKey = schemaKeyMap[key.toLowerCase()]
-        // isn't a column in the table, remove it
-        if (mappedKey == null) {
-          delete element[key]
-        }
-        // casing is different, fix it in row
-        else if (key !== mappedKey) {
-          element[mappedKey] = element[key]
-          delete element[key]
-        }
-      }
-    }
-    return data
-  } catch (err) {
-    console.error(`Error transforming CSV to JSON for data import`, err)
-    throw err
-  }
-}

packages/server/src/utilities/schema.ts

@@ -0,0 +1,141 @@
+import { FieldTypes } from "../constants"
+
+interface SchemaColumn {
+  readonly name: string
+  readonly type: FieldTypes
+  readonly autocolumn?: boolean
+}
+
+interface Schema {
+  readonly [index: string]: SchemaColumn
+}
+
+interface Row {
+  [index: string]: any
+}
+
+type Rows = Array<Row>
+
+interface SchemaValidation {
+  [index: string]: boolean
+}
+
+interface ValidationResults {
+  schemaValidation: SchemaValidation
+  allValid: boolean
+  invalidColumns: Array<string>
+}
+
+const PARSERS: any = {
+  [FieldTypes.NUMBER]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return Number(attribute)
+  },
+  [FieldTypes.DATETIME]: (attribute?: string) => {
+    if (!attribute) {
+      return attribute
+    }
+    return new Date(attribute).toISOString()
+  },
+}
+
+export function isSchema(schema: any): schema is Schema {
+  return (
+    typeof schema === "object" &&
+    Object.values(schema).every(rawColumn => {
+      const column = rawColumn as SchemaColumn
+
+      return (
+        column !== null &&
+        typeof column === "object" &&
+        typeof column.type === "string" &&
+        Object.values(FieldTypes).includes(column.type as FieldTypes)
+      )
+    })
+  )
+}
+
+export function isRows(rows: any): rows is Rows {
+  return Array.isArray(rows) && rows.every(row => typeof row === "object")
+}
+
+export function validate(rows: Rows, schema: Schema): ValidationResults {
+  const results: ValidationResults = {
+    schemaValidation: {},
+    allValid: false,
+    invalidColumns: [],
+  }
+
+  rows.forEach(row => {
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      const columnType = schema[columnName]?.type
+      const isAutoColumn = schema[columnName]?.autocolumn
+
+      // If the columnType is not a string, then it's not present in the schema, and should be added to the invalid columns array
+      if (typeof columnType !== "string") {
+        results.invalidColumns.push(columnName)
+      } else if (
+        // If there's no data for this field don't bother with further checks
+        // If the field is already marked as invalid there's no need for further checks
+        results.schemaValidation[columnName] === false ||
+        columnData == null ||
+        isAutoColumn
+      ) {
+        return
+      } else if (
+        columnType === FieldTypes.NUMBER &&
+        isNaN(Number(columnData))
+      ) {
+        // If provided must be a valid number
+        results.schemaValidation[columnName] = false
+      } else if (
+        // If provided must be a valid date
+        columnType === FieldTypes.DATETIME &&
+        isNaN(new Date(columnData).getTime())
+      ) {
+        results.schemaValidation[columnName] = false
+      } else {
+        results.schemaValidation[columnName] = true
+      }
+    })
+  })
+
+  results.allValid =
+    Object.values(results.schemaValidation).length > 0 &&
+    Object.values(results.schemaValidation).every(column => column)
+
+  // Select unique values
+  results.invalidColumns = [...new Set(results.invalidColumns)]
+  return results
+}
+
+export function parse(rows: Rows, schema: Schema): Rows {
+  return rows.map(row => {
+    const parsedRow: Row = {}
+
+    Object.entries(row).forEach(([columnName, columnData]) => {
+      if (!(columnName in schema) || schema[columnName]?.autocolumn) {
+        // Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
+        return
+      }
+
+      const columnType = schema[columnName].type
+
+      if (columnType === FieldTypes.NUMBER) {
+        // If provided must be a valid number
+        parsedRow[columnName] = columnData ? Number(columnData) : columnData
+      } else if (columnType === FieldTypes.DATETIME) {
+        // If provided must be a valid date
+        parsedRow[columnName] = columnData
+          ? new Date(columnData).toISOString()
+          : columnData
+      } else {
+        parsedRow[columnName] = columnData
+      }
+    })
+
+    return parsedRow
+  })
+}

packages/server/src/utilities/tests/__snapshots__/csvParser.spec.js.snap

@@ -1,15 +0,0 @@
-// Jest Snapshot v1, https://goo.gl/fbAQLP
-
-exports[`CSV Parser transformation transforms a CSV file into JSON 1`] = `
-Array [
-  Object {
-    "Age": 4324,
-  },
-  Object {
-    "Age": 34,
-  },
-  Object {
-    "Age": 23423,
-  },
-]
-`;

packages/server/src/utilities/tests/csvParser.spec.js

@@ -1,112 +0,0 @@
-const { readFileSync } = require("../fileSystem")
-const csvParser = require("../csvParser")
-
-const CSV_PATH = __dirname + "/test.csv"
-
-const SCHEMAS = {
-  VALID: {
-    Age: {
-      type: "number",
-    },
-  },
-  INVALID: {
-    Address: {
-      type: "number",
-    },
-    Age: {
-      type: "number",
-    },
-  },
-  IGNORE: {
-    Address: {
-      type: "omit",
-    },
-    Age: {
-      type: "omit",
-    },
-    Name: {
-      type: "string",
-    },
-  },
-  BROKEN: {
-    Address: {
-      type: "datetime",
-    },
-  },
-}
-
-describe("CSV Parser", () => {
-  const csvString = readFileSync(CSV_PATH, "utf8")
-
-  describe("parsing", () => {
-    it("returns status and types for a valid CSV transformation", async () => {
-      expect(await csvParser.parse(csvString, SCHEMAS.VALID)).toEqual({
-        Address: {
-          success: true,
-          type: "string",
-        },
-        Age: {
-          success: true,
-          type: "number",
-        },
-        Name: {
-          success: true,
-          type: "string",
-        },
-      })
-    })
-
-    it("returns status and types for an invalid CSV transformation", async () => {
-      expect(await csvParser.parse(csvString, SCHEMAS.INVALID)).toEqual({
-        Address: {
-          success: false,
-          type: "number",
-        },
-        Age: {
-          success: true,
-          type: "number",
-        },
-        Name: {
-          success: true,
-          type: "string",
-        },
-      })
-    })
-  })
-
-  describe("transformation", () => {
-    it("transforms a CSV file into JSON", async () => {
-      expect(
-        await csvParser.transform({
-          schema: SCHEMAS.VALID,
-          csvString,
-        })
-      ).toMatchSnapshot()
-    })
-
-    it("transforms a CSV file into JSON ignoring certain fields", async () => {
-      expect(
-        await csvParser.transform({
-          schema: SCHEMAS.IGNORE,
-          csvString,
-        })
-      ).toEqual([
-        {
-          Name: "Bertå",
-        },
-        {
-          Name: "Ernie",
-        },
-        {
-          Name: "Big Bird",
-        },
-      ])
-    })
-
-    it("throws an error on invalid schema", async () => {
-      await expect(
-        csvParser.transform({ schema: SCHEMAS.BROKEN, csvString })
-      ).rejects.toThrow()
-    })
-  })
-})

packages/server/yarn.lock

@@ -1273,13 +1273,13 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.2.12-alpha.16":
+"@budibase/backend-core@2.2.12-alpha.32":
-  version "2.2.12-alpha.16"
+  version "2.2.12-alpha.32"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.16.tgz#9ebfa7308fc97b34d6a076e4300fbcf996160d66"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.32.tgz#5d53fdde38e80fdade73c59cb81623041984a5fc"
-  integrity sha512-rHMryIOb71U7W5jZtn39vuBI7xSZ6XA4l6P7lc2bBT1lI10G/zQRoQWjsWaUWo+RVBQ5zki3Ok05tFS9Yx/7fA==
+  integrity sha512-XoaqGrttx01wlblI0+O23R9uS5FTMgK07juY6mUkdReCK46IvmG7FFnuYu0euKVRbhChSf5X8S+fvKIAEi1ZFw==
   dependencies:
     "@budibase/nano" "10.1.1"
-    "@budibase/types" "2.2.12-alpha.16"
+    "@budibase/types" "2.2.12-alpha.32"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -1374,13 +1374,13 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
-"@budibase/pro@2.2.12-alpha.16":
+"@budibase/pro@2.2.12-alpha.32":
-  version "2.2.12-alpha.16"
+  version "2.2.12-alpha.32"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.16.tgz#db5a345b072e725765cd01edcad4a930ae875eba"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.32.tgz#4f35b0ada97458cc69ee41ebb40d56b361518065"
-  integrity sha512-GBXdOQMIbxU0TGgGQ4+npNGtuFvanNVFrZBqwB7+3x6rIku313WkbgJJji5uemtU6B8XFh/QqS6AA0R0PS2Kmg==
+  integrity sha512-Kn/IahgIb9Ydzmasv4Bhlh3rvyAC2tBVSoI33OZ/6PKF0vPYJNJkFds3iGELNV7SyWoLbVWb2z5SQ4SSaOYsJw==
   dependencies:
-    "@budibase/backend-core" "2.2.12-alpha.16"
+    "@budibase/backend-core" "2.2.12-alpha.32"
-    "@budibase/types" "2.2.12-alpha.16"
+    "@budibase/types" "2.2.12-alpha.32"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
@@ -1405,10 +1405,10 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
-"@budibase/types@2.2.12-alpha.16":
+"@budibase/types@2.2.12-alpha.32":
-  version "2.2.12-alpha.16"
+  version "2.2.12-alpha.32"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.16.tgz#6fb42d4be88fbd8054a0a3264cf9c4b4a7248893"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.32.tgz#71af8e8cff66acbfd65aa87e66183dd95d76ce6f"
-  integrity sha512-pXn/r3tA0A30f2dJVJfzldMGXAEhpObBfqbONn8AStiD6Qm8Hu9H6aFaCPqS8DDaWBuwY/tMqSry2E0saRaSwg==
+  integrity sha512-ZZwmO+0ORGEFbU/EQvtnjo1VonUbBdsciFkTOiopVupU5iNY2oCKgbYTQiTZZisQrRuiKdPV6P17uV+YTkjQSQ==
 
 "@bull-board/api@3.7.0":
   version "3.7.0"
@@ -5301,15 +5301,10 @@ color@^3.1.3:
     color-convert "^1.9.3"
     color-string "^1.6.0"
 
-colorette@2.0.16:
+colorette@2.0.19, colorette@^2.0.14:
-  version "2.0.16"
+  version "2.0.19"
-  resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.16.tgz#713b9af84fdb000139f04546bd4a93f62a5085da"
+  resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.19.tgz#cdf044f47ad41a0f4b56b3a0d5b4e6e1a2d5a798"
-  integrity sha512-hUewv7oMjCp+wkBv5Rm0v87eJhq4woh5rSR+42YSQJKecCqgIqNkZ6lAlQms/BwHPJA5NKMRlpxPRv0n8HQW6g==
+  integrity sha512-3tlv/dIP7FWvj3BsbHrGLJ6l/oKh1O3TcgBqMn+yyCagOxc23fyzDS6HypQbgxWbkpDnf52p1LuR4eWDQ/K9WQ==
-
-colorette@^2.0.14:
-  version "2.0.17"
-  resolved "https://registry.yarnpkg.com/colorette/-/colorette-2.0.17.tgz#5dd4c0d15e2984b7433cb4a9f2ead45063b80c47"
-  integrity sha512-hJo+3Bkn0NCHybn9Tu35fIeoOKGOk5OCC32y4Hz2It+qlCO2Q3DeQ1hRn/tDDMQKRYUEzqsl7jbF6dYKjlE60g==
 
 colorspace@1.1.x:
   version "1.1.4"
@@ -5346,7 +5341,7 @@ commander@^5.1.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-5.1.0.tgz#46abbd1652f8e059bddaef99bbdcb2ad9cf179ae"
   integrity sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==
 
-commander@^7.0.0, commander@^7.1.0:
+commander@^7.0.0:
   version "7.2.0"
   resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
   integrity sha512-QrWXB+ZQSVPmIWIhtEO9H+gwHaMGYiF5ChvoJ+K9ZGHG/sVsa6yiesAD1GC/x46sET00Xlwo1u49RVVVzvcSkw==
@@ -5356,6 +5351,11 @@ commander@^8.3.0:
   resolved "https://registry.yarnpkg.com/commander/-/commander-8.3.0.tgz#4837ea1b2da67b9c616a67afbb0fafee567bca66"
   integrity sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==
 
+commander@^9.1.0:
+  version "9.5.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-9.5.0.tgz#bc08d1eb5cedf7ccb797a96199d41c7bc3e60d30"
+  integrity sha512-KRs7WVDKg86PWiuAqhDrAQnTXZKraVcCc6vFdL14qrZ/DcWwuRo7VoiYXalXO7S5GKpqYiVEwCbgFDfxNHKJBQ==
+
 commoner@^0.10.1:
   version "0.10.8"
   resolved "https://registry.yarnpkg.com/commoner/-/commoner-0.10.8.tgz#34fc3672cd24393e8bb47e70caa0293811f4f2c5"
@@ -5659,20 +5659,13 @@ dayjs@^1.10.4, dayjs@^1.10.5:
   resolved "https://registry.yarnpkg.com/dayjs/-/dayjs-1.11.3.tgz#4754eb694a624057b9ad2224b67b15d552589258"
   integrity sha512-xxwlswWOlGhzgQ4TKzASQkUhqERI3egRNqgV4ScR8wlANA/A9tZ7miXa44vTTKEq5l7vWoL5G57bG3zA+Kow0A==
 
-debug@4, debug@^4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
+debug@4, debug@4.3.4, debug@^4, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.2, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2:
   version "4.3.4"
   resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865"
   integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ==
   dependencies:
     ms "2.1.2"
 
-debug@4.3.2:
-  version "4.3.2"
-  resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.2.tgz#f0a49c18ac8779e31d4a0c6029dfb76873c7428b"
-  integrity sha512-mOp8wKcvj7XxC78zLgw/ZA+6TSgkoE2C/ienthhRD298T7UNwAg9diBpLRxC0mOezLl4B0xV7M0cCO6P/O0Xhw==
-  dependencies:
-    ms "2.1.2"
-
 debug@^2.2.0, debug@^2.3.3:
   version "2.6.9"
   resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f"
@@ -7441,10 +7434,10 @@ get-value@^2.0.3, get-value@^2.0.6:
   resolved "https://registry.yarnpkg.com/get-value/-/get-value-2.0.6.tgz#dc15ca1c672387ca76bd37ac0a395ba2042a2c28"
   integrity sha512-Ln0UQDlxH1BapMu3GPtf7CuYNwRZf2gwCuPqbyG6pB8WfmFpzqcy4xtAaAMUhnNqjMKTiCPZG2oMT3YSx8U2NA==
 
-getopts@2.2.5:
+getopts@2.3.0:
-  version "2.2.5"
+  version "2.3.0"
-  resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.2.5.tgz#67a0fe471cacb9c687d817cab6450b96dde8313b"
+  resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.3.0.tgz#71e5593284807e03e2427449d4f6712a268666f4"
-  integrity sha512-9jb7AW5p3in+IiJWhQiZmmwkpLaR/ccTWdWQCtZM66HJcHHLegowh4q4tSD7gouUyeNvFWRavfK9GXosQHDpFA==
+  integrity sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA==
 
 getpass@^0.1.1:
   version "0.1.7"
@@ -9825,23 +9818,24 @@ kleur@^3.0.3:
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-3.0.3.tgz#a79c9ecc86ee1ce3fa6206d1216c501f147fc07e"
   integrity sha512-eTIzlVOSUR+JxdDFepEYcBMtZ9Qqdef+rnzWdRZuMbOywu5tO2w2N7rqjoANZ5k9vywhL6Br1VRjUIgTQx4E8w==
 
-knex@0.95.15:
+knex@2.4.0:
-  version "0.95.15"
+  version "2.4.0"
-  resolved "https://registry.yarnpkg.com/knex/-/knex-0.95.15.tgz#39d7e7110a6e2ad7de5d673d2dea94143015e0e7"
+  resolved "https://registry.yarnpkg.com/knex/-/knex-2.4.0.tgz#7d33cc36f320cdac98741010544b4c6a98b8b19e"
-  integrity sha512-Loq6WgHaWlmL2bfZGWPsy4l8xw4pOE+tmLGkPG0auBppxpI0UcK+GYCycJcqz9W54f2LiGewkCVLBm3Wq4ur/w==
+  integrity sha512-i0GWwqYp1Hs2yvc2rlDO6nzzkLhwdyOZKRdsMTB8ZxOs2IXQyL5rBjSbS1krowCh6V65T4X9CJaKtuIfkaPGSA==
   dependencies:
-    colorette "2.0.16"
+    colorette "2.0.19"
-    commander "^7.1.0"
+    commander "^9.1.0"
-    debug "4.3.2"
+    debug "4.3.4"
     escalade "^3.1.1"
     esm "^3.2.25"
-    getopts "2.2.5"
+    get-package-type "^0.1.0"
+    getopts "2.3.0"
     interpret "^2.2.0"
     lodash "^4.17.21"
     pg-connection-string "2.5.0"
-    rechoir "0.7.0"
+    rechoir "^0.8.0"
     resolve-from "^5.0.0"
-    tarn "^3.0.1"
+    tarn "^3.0.2"
     tildify "2.0.0"
 
 koa-body@4.2.0:
@@ -12561,13 +12555,6 @@ recast@^0.11.17:
     private "~0.1.5"
     source-map "~0.5.0"
 
-rechoir@0.7.0:
-  version "0.7.0"
-  resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.0.tgz#32650fd52c21ab252aa5d65b19310441c7e03aca"
-  integrity sha512-ADsDEH2bvbjltXEP+hTIAmeFekTFK0V2BTxMkok6qILyAJEXV0AFfoWcAq4yfll5VdIMd/RVXq0lR+wQi5ZU3Q==
-  dependencies:
-    resolve "^1.9.0"
-
 rechoir@^0.7.0:
   version "0.7.1"
   resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.7.1.tgz#9478a96a1ca135b5e88fc027f03ee92d6c645686"
@@ -12575,6 +12562,13 @@ rechoir@^0.7.0:
   dependencies:
     resolve "^1.9.0"
 
+rechoir@^0.8.0:
+  version "0.8.0"
+  resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.8.0.tgz#49f866e0d32146142da3ad8f0eff352b3215ff22"
+  integrity sha512-/vxpCXddiX8NGfGO/mTafwjq4aFa/71pvamip0++IQk3zG8cbCj0fifNPrjjF1XMXUne91jL9OoxmdykoEtifQ==
+  dependencies:
+    resolve "^1.20.0"
+
 redis-commands@1.7.0, redis-commands@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
@@ -14051,7 +14045,7 @@ tarn@^1.1.5:
   resolved "https://registry.yarnpkg.com/tarn/-/tarn-1.1.5.tgz#7be88622e951738b9fa3fb77477309242cdddc2d"
   integrity sha512-PMtJ3HCLAZeedWjJPgGnCvcphbCOMbtZpjKgLq3qM5Qq9aQud+XHrL0WlrlgnTyS8U+jrjGbEXprFcQrxPy52g==
 
-tarn@^3.0.1:
+tarn@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/tarn/-/tarn-3.0.2.tgz#73b6140fbb881b71559c4f8bfde3d9a4b3d27693"
   integrity sha512-51LAVKUSZSVfI05vjPESNc5vwqqZpbXCsU+/+wxlOrUjk2SnFTt97v9ZgQrD4YmxYW1Px6w2KjaDitCfkvgxMQ==

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/types/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/types",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase types",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

packages/types/src/api/web/user.ts

@@ -57,3 +57,7 @@ export interface CreateAdminUserRequest {
   password: string
   tenantId: string
 }
+
+export interface SyncUserRequest {
+  previousUser?: User
+}

packages/types/src/documents/app/table.ts

@@ -69,7 +69,7 @@ export interface Table extends Document {
   constrained?: string[]
   sql?: boolean
   indexes?: { [key: string]: any }
-  dataImport?: { [key: string]: any }
+  rows?: { [key: string]: any }
 }
 
 export interface TableRequest extends Table {

packages/types/src/documents/global/user.ts

@@ -70,3 +70,7 @@ export interface AdminUser extends User {
     global: boolean
   }
 }
+
+export function isUser(user: object): user is User {
+  return !!(user as User).roles
+}

packages/types/src/sdk/events/event.ts

@@ -188,6 +188,4 @@ export interface BaseEvent {
   hosting?: Hosting
 }
 
-export type RowImportFormat = "csv"
 export type TableExportFormat = "json" | "csv"
-export type TableImportFormat = "csv"

packages/types/src/sdk/events/rows.ts

@@ -1,8 +1,7 @@
-import { BaseEvent, RowImportFormat } from "./event"
+import { BaseEvent } from "./event"
 
 export interface RowsImportedEvent extends BaseEvent {
   tableId: string
-  format: RowImportFormat
   count: number
 }
 

packages/types/src/sdk/events/table.ts

@@ -1,4 +1,4 @@
-import { BaseEvent, TableExportFormat, TableImportFormat } from "./event"
+import { BaseEvent, TableExportFormat } from "./event"
 
 export interface TableCreatedEvent extends BaseEvent {
   tableId: string
@@ -19,5 +19,4 @@ export interface TableExportedEvent extends BaseEvent {
 
 export interface TableImportedEvent extends BaseEvent {
   tableId: string
-  format: TableImportFormat
 }

packages/types/src/sdk/koa.ts

@@ -41,7 +41,7 @@ export interface UserCtx<RequestBody = any, ResponseBody = any>
 }
 
 /**
- * Deprecated: Use UserCtx / Ctx appropriately
+ * @deprecated: Use UserCtx / Ctx appropriately
  * Authenticated context.
  */
 export interface BBContext extends Ctx {

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.16",
+  "version": "2.2.12-alpha.32",
   "description": "Budibase background service",
   "main": "src/index.ts",
   "repository": {
@@ -36,10 +36,10 @@
   "author": "Budibase",
   "license": "GPL-3.0",
   "dependencies": {
-    "@budibase/backend-core": "2.2.12-alpha.16",
+    "@budibase/backend-core": "2.2.12-alpha.32",
-    "@budibase/pro": "2.2.12-alpha.16",
+    "@budibase/pro": "2.2.12-alpha.32",
-    "@budibase/string-templates": "2.2.12-alpha.16",
+    "@budibase/string-templates": "2.2.12-alpha.32",
-    "@budibase/types": "2.2.12-alpha.16",
+    "@budibase/types": "2.2.12-alpha.32",
     "@koa/router": "8.0.8",
     "@sentry/node": "6.17.7",
     "@techpass/passport-openidconnect": "0.3.2",

packages/worker/src/api/routes/global/configs.ts

@@ -34,8 +34,8 @@ function settingValidation() {
 function googleValidation() {
   // prettier-ignore
   return Joi.object({
-    clientID: Joi.string().required(),
+    clientID: Joi.when('activated', { is: true, then: Joi.string().required() }),
-    clientSecret: Joi.string().required(),
+    clientSecret: Joi.when('activated', { is: true, then: Joi.string().required() }),
     activated: Joi.boolean().required(),
   }).unknown(true)
 }
@@ -45,12 +45,12 @@ function oidcValidation() {
   return Joi.object({
     configs: Joi.array().items(
       Joi.object({
-        clientID: Joi.string().required(),
+        clientID: Joi.when('activated', { is: true, then: Joi.string().required() }),
-        clientSecret: Joi.string().required(),
+        clientSecret: Joi.when('activated', { is: true, then: Joi.string().required() }),
-        configUrl: Joi.string().required(),
+        configUrl: Joi.when('activated', { is: true, then: Joi.string().required() }),
         logo: Joi.string().allow("", null),
         name: Joi.string().allow("", null),
-        uuid: Joi.string().required(),
+        uuid: Joi.when('activated', { is: true, then: Joi.string().required() }),
         activated: Joi.boolean().required(),
         scopes: Joi.array().optional()
       })

packages/worker/src/sdk/users/users.ts

@@ -31,6 +31,7 @@ import {
   SearchUsersRequest,
   User,
   ThirdPartyUser,
+  isUser,
 } from "@budibase/types"
 import { sendEmail } from "../../utilities/email"
 import { EmailTemplatePurpose } from "../../constants"
@@ -265,8 +266,9 @@ export const save = async (
     await eventHelpers.handleSaveEvents(builtUser, dbUser)
     await addTenant(tenantId, _id, email)
     await cache.user.invalidateUser(response.id)
+
     // let server know to sync user
-    await apps.syncUserInApps(_id)
+    await apps.syncUserInApps(_id, dbUser)
 
     await Promise.all(groupPromises)
 
@@ -572,7 +574,7 @@ export const destroy = async (id: string, currentUser: any) => {
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "deletion" })
   // let server know to sync user
-  await apps.syncUserInApps(userId)
+  await apps.syncUserInApps(userId, dbUser)
 }
 
 const bulkDeleteProcessing = async (dbUser: User) => {
@@ -582,7 +584,7 @@ const bulkDeleteProcessing = async (dbUser: User) => {
   await cache.user.invalidateUser(userId)
   await sessions.invalidateSessions(userId, { reason: "bulk-deletion" })
   // let server know to sync user
-  await apps.syncUserInApps(userId)
+  await apps.syncUserInApps(userId, dbUser)
 }
 
 export const invite = async (

packages/worker/src/utilities/appService.ts

@@ -2,6 +2,7 @@ import fetch from "node-fetch"
 import { constants, tenancy, logging } from "@budibase/backend-core"
 import { checkSlashesInUrl } from "../utilities"
 import env from "../environment"
+import { SyncUserRequest, User } from "@budibase/types"
 
 async function makeAppRequest(url: string, method: string, body: any) {
   if (env.isTest()) {
@@ -24,11 +25,15 @@ async function makeAppRequest(url: string, method: string, body: any) {
   return fetch(checkSlashesInUrl(env.APPS_URL + url), request)
 }
 
-export async function syncUserInApps(userId: string) {
+export async function syncUserInApps(userId: string, previousUser?: User) {
+  const body: SyncUserRequest = {
+    previousUser,
+  }
+
   const response = await makeAppRequest(
     `/api/users/metadata/sync/${userId}`,
     "POST",
-    {}
+    body
   )
   if (response && response.status !== 200) {
     throw "Unable to sync user."

packages/worker/yarn.lock

@@ -470,13 +470,13 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@2.2.12-alpha.16":
+"@budibase/backend-core@2.2.12-alpha.32":
-  version "2.2.12-alpha.16"
+  version "2.2.12-alpha.32"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.16.tgz#9ebfa7308fc97b34d6a076e4300fbcf996160d66"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.2.12-alpha.32.tgz#5d53fdde38e80fdade73c59cb81623041984a5fc"
-  integrity sha512-rHMryIOb71U7W5jZtn39vuBI7xSZ6XA4l6P7lc2bBT1lI10G/zQRoQWjsWaUWo+RVBQ5zki3Ok05tFS9Yx/7fA==
+  integrity sha512-XoaqGrttx01wlblI0+O23R9uS5FTMgK07juY6mUkdReCK46IvmG7FFnuYu0euKVRbhChSf5X8S+fvKIAEi1ZFw==
   dependencies:
     "@budibase/nano" "10.1.1"
-    "@budibase/types" "2.2.12-alpha.16"
+    "@budibase/types" "2.2.12-alpha.32"
     "@shopify/jest-koa-mocks" "5.0.1"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-cloudfront-sign "2.2.0"
@@ -521,23 +521,23 @@
     qs "^6.11.0"
     tough-cookie "^4.1.2"
 
-"@budibase/pro@2.2.12-alpha.16":
+"@budibase/pro@2.2.12-alpha.32":
-  version "2.2.12-alpha.16"
+  version "2.2.12-alpha.32"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.16.tgz#db5a345b072e725765cd01edcad4a930ae875eba"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.2.12-alpha.32.tgz#4f35b0ada97458cc69ee41ebb40d56b361518065"
-  integrity sha512-GBXdOQMIbxU0TGgGQ4+npNGtuFvanNVFrZBqwB7+3x6rIku313WkbgJJji5uemtU6B8XFh/QqS6AA0R0PS2Kmg==
+  integrity sha512-Kn/IahgIb9Ydzmasv4Bhlh3rvyAC2tBVSoI33OZ/6PKF0vPYJNJkFds3iGELNV7SyWoLbVWb2z5SQ4SSaOYsJw==
   dependencies:
-    "@budibase/backend-core" "2.2.12-alpha.16"
+    "@budibase/backend-core" "2.2.12-alpha.32"
-    "@budibase/types" "2.2.12-alpha.16"
+    "@budibase/types" "2.2.12-alpha.32"
     "@koa/router" "8.0.8"
     bull "4.10.1"
     joi "17.6.0"
     jsonwebtoken "8.5.1"
     node-fetch "^2.6.1"
 
-"@budibase/types@2.2.12-alpha.16":
+"@budibase/types@2.2.12-alpha.32":
-  version "2.2.12-alpha.16"
+  version "2.2.12-alpha.32"
-  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.16.tgz#6fb42d4be88fbd8054a0a3264cf9c4b4a7248893"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.32.tgz#71af8e8cff66acbfd65aa87e66183dd95d76ce6f"
-  integrity sha512-pXn/r3tA0A30f2dJVJfzldMGXAEhpObBfqbONn8AStiD6Qm8Hu9H6aFaCPqS8DDaWBuwY/tMqSry2E0saRaSwg==
+  integrity sha512-ZZwmO+0ORGEFbU/EQvtnjo1VonUbBdsciFkTOiopVupU5iNY2oCKgbYTQiTZZisQrRuiKdPV6P17uV+YTkjQSQ==
 
 "@cspotcode/source-map-support@^0.8.0":
   version "0.8.1"

qa-core/src/config/internal-api/TestConfiguration/applications.ts

@@ -117,7 +117,7 @@ export default class AppApi {
     return [response, json]
   }
 
-  async update(
+  async rename(
     appId: string,
     oldName: string,
     body: any
@@ -153,4 +153,27 @@ export default class AppApi {
     expect(response).toHaveStatusCode(204)
     return [response]
   }
+
+  async unlock(appId: string): Promise<[Response, responseMessage]> {
+    const response = await this.api.del(`/dev/${appId}/lock`)
+    const json = await response.json()
+    expect(response).toHaveStatusCode(200)
+    expect(json.message).toEqual("Lock released successfully.")
+    return [response, json]
+  }
+
+  async updateIcon(appId: string): Promise<[Response, Application]> {
+    const body = {
+      icon: {
+        name: "ConversionFunnel",
+        color: "var(--spectrum-global-color-red-400)",
+      },
+    }
+    const response = await this.api.put(`/applications/${appId}`, { body })
+    const json = await response.json()
+    expect(response).toHaveStatusCode(200)
+    expect(json.icon.name).toEqual(body.icon.name)
+    expect(json.icon.color).toEqual(body.icon.color)
+    return [response, json]
+  }
 }

qa-core/src/config/internal-api/TestConfiguration/rows.ts

@@ -15,7 +15,7 @@ export default class RowsApi {
     const json = await response.json()
     if (this.rowAdded) {
       expect(response).toHaveStatusCode(200)
-      expect(json.length).toEqual(1)
+      expect(json.length).toBeGreaterThanOrEqual(1)
     }
     return [response, json]
   }
@@ -36,4 +36,27 @@ export default class RowsApi {
     expect(response).toHaveStatusCode(200)
     return [response, json]
   }
+
+  async searchNoPagination(
+    tableId: string,
+    body: any
+  ): Promise<[Response, Row[]]> {
+    const response = await this.api.post(`/${tableId}/search`, { body })
+    const json = await response.json()
+    expect(response).toHaveStatusCode(200)
+    expect(json.hasNextPage).toEqual(false)
+    return [response, json.rows]
+  }
+
+  async searchWithPagination(
+    tableId: string,
+    body: any
+  ): Promise<[Response, Row[]]> {
+    const response = await this.api.post(`/${tableId}/search`, { body })
+    const json = await response.json()
+    expect(response).toHaveStatusCode(200)
+    expect(json.hasNextPage).toEqual(true)
+    expect(json.rows.length).toEqual(10)
+    return [response, json.rows]
+  }
 }

qa-core/src/config/internal-api/fixtures/rows.ts

@@ -6,3 +6,27 @@ export const generateNewRowForTable = (tableId: string): Row => {
     tableId: tableId,
   }
 }
+
+export const searchBody = (primaryDisplay: string): any => {
+  return {
+    bookmark: null,
+    limit: 10,
+    paginate: true,
+    query: {
+      contains: {},
+      containsAny: {},
+      empty: {},
+      equal: {},
+      fuzzy: {},
+      notContains: {},
+      notEmpty: {},
+      notEqual: {},
+      oneOf: {},
+      range: {},
+      string: {},
+    },
+    sort: primaryDisplay,
+    sortOrder: "ascending",
+    sortType: "string",
+  }
+}

qa-core/src/config/internal-api/fixtures/table.ts

@@ -6,10 +6,6 @@ export const generateTable = (): Table => {
     schema: {},
     sourceId: "bb_internal",
     type: "internal",
-    dataImport: {
-      valid: true,
-      schema: {},
-    },
   }
 }
 

qa-core/src/tests/internal-api/applications/applications.spec.ts

@@ -69,7 +69,7 @@ describe("Internal API - Application creation, update, publish and delete", () =
     await config.applications.unpublish(<string>app.appId)
   })
 
-  it("POST - Sync application before deployment", async () => {
+  it("Sync application before deployment", async () => {
     const app = await config.applications.create(generateApp())
     config.applications.api.appId = app.appId
 
@@ -81,7 +81,7 @@ describe("Internal API - Application creation, update, publish and delete", () =
     })
   })
 
-  it("POST - Sync application after deployment", async () => {
+  it("Sync application after deployment", async () => {
     const app = await config.applications.create(generateApp())
     config.applications.api.appId = app.appId
 
@@ -96,24 +96,32 @@ describe("Internal API - Application creation, update, publish and delete", () =
     })
   })
 
-  it("PUT - Update an application", async () => {
+  it("Rename an application", async () => {
     const app = await config.applications.create(generateApp())
 
     config.applications.api.appId = app.appId
 
-    await config.applications.update(<string>app.appId, <string>app.name, {
+    await config.applications.rename(<string>app.appId, <string>app.name, {
       name: generator.word(),
     })
   })
 
-  it("POST - Revert Changes without changes", async () => {
+  it("Update the icon and color of an application", async () => {
+    const app = await config.applications.create(generateApp())
+
+    config.applications.api.appId = app.appId
+
+    await config.applications.updateIcon(<string>app.appId)
+  })
+
+  it("Revert Changes without changes", async () => {
     const app = await config.applications.create(generateApp())
     config.applications.api.appId = app.appId
 
     await config.applications.revertUnpublished(<string>app.appId)
   })
 
-  it("POST - Revert Changes", async () => {
+  it("Revert Changes", async () => {
     const app = await config.applications.create(generateApp())
     config.applications.api.appId = app.appId
 
@@ -126,11 +134,12 @@ describe("Internal API - Application creation, update, publish and delete", () =
     // // Revert the app to published state
     await config.applications.revertPublished(<string>app.appId)
 
+    await config.applications.unlock(<string>app.appId)
     // Check screen is removed
     await config.applications.getRoutes()
   })
 
-  it("DELETE - Delete an application", async () => {
+  it("Delete an application", async () => {
     const app = await config.applications.create(generateApp())
 
     await config.applications.delete(<string>app.appId)

qa-core/src/tests/internal-api/screens/screens.spec.ts

@@ -21,7 +21,7 @@ describe("Internal API - /screens endpoints", () => {
     await config.afterAll()
   })
 
-  it("POST - Create a screen with each role type", async () => {
+  it("Create a screen with each role type", async () => {
     // Create app
     const app = await appConfig.applications.create(generateApp())
 
@@ -35,7 +35,7 @@ describe("Internal API - /screens endpoints", () => {
     }
   })
 
-  it("GET - Fetch screens", async () => {
+  it("Get screens", async () => {
     // Create app
     const app = await appConfig.applications.create(generateApp())
 
@@ -47,7 +47,7 @@ describe("Internal API - /screens endpoints", () => {
     await appConfig.applications.getRoutes(true)
   })
 
-  it("DELETE - Delete a screen", async () => {
+  it("Delete a screen", async () => {
     // Create app
     const app = await appConfig.applications.create(generateApp())
 

qa-core/src/tests/internal-api/tables/tables.spec.ts

@@ -6,9 +6,12 @@ import {
   generateTable,
   generateNewColumnForTable,
 } from "../../../config/internal-api/fixtures/table"
-import { generateNewRowForTable } from "../../../config/internal-api/fixtures/rows"
+import {
+  generateNewRowForTable,
+  searchBody,
+} from "../../../config/internal-api/fixtures/rows"
 
-describe("Internal API - Application creation, update, publish and delete", () => {
+describe("Internal API - Table Operations", () => {
   const api = new InternalAPIClient()
   const config = new TestConfiguration<Application>(api)
 
@@ -31,7 +34,7 @@ describe("Internal API - Application creation, update, publish and delete", () =
     })
   }
 
-  it("Operations on Tables", async () => {
+  it("Create and delete table, columns and rows", async () => {
     // create the app
     const appName = generator.word()
     const app = await createAppFromTemplate()
@@ -86,4 +89,70 @@ describe("Internal API - Application creation, update, publish and delete", () =
     //Table was deleted
     await config.tables.getAll(2)
   })
+
+  it("Search and pagination", async () => {
+    // create the app
+    const appName = generator.word()
+    const app = await createAppFromTemplate()
+    config.applications.api.appId = app.appId
+
+    // Get current tables: expect 2 in this template
+    await config.tables.getAll(2)
+
+    // Add new table
+    const [createdTableResponse, createdTableData] = await config.tables.save(
+      generateTable()
+    )
+
+    //Table was added
+    await config.tables.getAll(3)
+
+    //Get information about the table
+    await config.tables.getTableById(<string>createdTableData._id)
+
+    //Add Column to table
+    const newColumn = generateNewColumnForTable(createdTableData)
+    const [addColumnResponse, addColumnData] = await config.tables.save(
+      newColumn,
+      true
+    )
+
+    //Add Row to table
+    let newRow = generateNewRowForTable(<string>addColumnData._id)
+    await config.rows.add(<string>addColumnData._id, newRow)
+
+    //Search single row
+    await config.rows.searchNoPagination(
+      <string>createdTableData._id,
+      searchBody(<string>createdTableData.primaryDisplay)
+    )
+
+    //Add 10 more rows
+    for (let i = 0; i < 10; i++) {
+      let newRow = generateNewRowForTable(<string>addColumnData._id)
+      await config.rows.add(<string>addColumnData._id, newRow)
+    }
+
+    //Search rows with pagination
+    const [allRowsResponse, allRowsJson] =
+      await config.rows.searchWithPagination(
+        <string>createdTableData._id,
+        searchBody(<string>createdTableData.primaryDisplay)
+      )
+
+    //Delete Rows from table
+    const rowToDelete = {
+      rows: [allRowsJson],
+    }
+    const [deleteRowResponse, deleteRowData] = await config.rows.delete(
+      <string>createdTableData._id,
+      rowToDelete
+    )
+
+    //Search single row
+    await config.rows.searchWithPagination(
+      <string>createdTableData._id,
+      searchBody(<string>createdTableData.primaryDisplay)
+    )
+  })
 })

qa-core/src/tests/internal-api/userManagement/appSpecificRoles.spec.ts

@@ -28,9 +28,12 @@ describe("Internal API - App Specific Roles & Permissions", () => {
   })
 
   it("Add BASIC user to app", async () => {
+    // Create a user with BASIC role and check if it was created successfully
     const appUser = generateUser()
     expect(appUser[0].builder?.global).toEqual(false)
     expect(appUser[0].admin?.global).toEqual(false)
+
+    // Add the user to the tenant.
     const [createUserResponse, createUserJson] = await config.users.addMultiple(
       appUser
     )
@@ -38,9 +41,12 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     const app = await config.applications.create(appFromTemplate())
     config.applications.api.appId = app.appId
 
+    // Get all the information from the create user
     const [userInfoResponse, userInfoJson] = await config.users.getInfo(
       createUserJson.created.successful[0]._id
     )
+
+    // Create the body with the information from the user and add the role to the app
     const body: User = {
       ...userInfoJson,
       roles: {
@@ -49,6 +55,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     }
     await config.users.updateInfo(body)
 
+    // Get the user information again and check if the role was added
     const [changedUserInfoResponse, changedUserInfoJson] =
       await config.users.getInfo(createUserJson.created.successful[0]._id)
     expect(changedUserInfoJson.roles[<string>app.appId]).toBeDefined()
@@ -56,6 +63,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
   })
 
   it("Add ADMIN user to app", async () => {
+    // Create a user with ADMIN role and check if it was created successfully
     const adminUser = generateUser(1, "admin")
     expect(adminUser[0].builder?.global).toEqual(true)
     expect(adminUser[0].admin?.global).toEqual(true)
@@ -63,15 +71,15 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       adminUser
     )
 
-    //const app = await config.applications.create(generateApp())
-    //config.applications.api.appId = app.appId
-
     const app = await config.applications.create(appFromTemplate())
     config.applications.api.appId = app.appId
 
+    // Get all the information from the create user
     const [userInfoResponse, userInfoJson] = await config.users.getInfo(
       createUserJson.created.successful[0]._id
     )
+
+    // Create the body with the information from the user and add the role to the app
     const body: User = {
       ...userInfoJson,
       roles: {
@@ -80,6 +88,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     }
     await config.users.updateInfo(body)
 
+    // Get the user information again and check if the role was added
     const [changedUserInfoResponse, changedUserInfoJson] =
       await config.users.getInfo(createUserJson.created.successful[0]._id)
     expect(changedUserInfoJson.roles[<string>app.appId]).toBeDefined()
@@ -93,9 +102,9 @@ describe("Internal API - App Specific Roles & Permissions", () => {
   })
 
   it("Add POWER user to app", async () => {
+    // Create a user with POWER role and check if it was created successfully
     const powerUser = generateUser(1, "developer")
     expect(powerUser[0].builder?.global).toEqual(true)
-
     const [createUserResponse, createUserJson] = await config.users.addMultiple(
       powerUser
     )
@@ -103,9 +112,12 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     const app = await config.applications.create(generateApp())
     config.applications.api.appId = app.appId
 
+    // Get all the information from the create user
     const [userInfoResponse, userInfoJson] = await config.users.getInfo(
       createUserJson.created.successful[0]._id
     )
+
+    // Create the body with the information from the user and add the role to the app
     const body: User = {
       ...userInfoJson,
       roles: {
@@ -114,6 +126,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     }
     await config.users.updateInfo(body)
 
+    // Get the user information again and check if the role was added
     const [changedUserInfoResponse, changedUserInfoJson] =
       await config.users.getInfo(createUserJson.created.successful[0]._id)
     expect(changedUserInfoJson.roles[<string>app.appId]).toBeDefined()
@@ -122,6 +135,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
 
   describe("Check Access for default roles", () => {
     it("Check Table access for app user", async () => {
+      // Create a user with BASIC role and check if it was created successfully
       const appUser = generateUser()
       expect(appUser[0].builder?.global).toEqual(false)
       expect(appUser[0].admin?.global).toEqual(false)
@@ -131,9 +145,12 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       const app = await config.applications.create(generateApp())
       config.applications.api.appId = app.appId
 
+      // Get all the information from the create user
       const [userInfoResponse, userInfoJson] = await config.users.getInfo(
         createUserJson.created.successful[0]._id
       )
+
+      // Create the body with the information from the user and add the role to the app
       const body: User = {
         ...userInfoJson,
         roles: {
@@ -142,14 +159,18 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       }
       await config.users.updateInfo(body)
 
+      // Get the user information again and check if the role was added
       const [changedUserInfoResponse, changedUserInfoJson] =
         await config.users.getInfo(createUserJson.created.successful[0]._id)
       expect(changedUserInfoJson.roles[<string>app.appId]).toBeDefined()
       expect(changedUserInfoJson.roles[<string>app.appId]).toEqual("BASIC")
 
+      // Create a table
       const [createdTableResponse, createdTableData] = await config.tables.save(
         generateTable()
       )
+
+      // Login with the user created and try to create a column
       await config.login(<string>appUser[0].email, <string>appUser[0].password)
       const newColumn = generateNewColumnForTable(createdTableData)
       await config.tables.forbiddenSave(newColumn)
@@ -157,6 +178,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     })
 
     it("Check Table access for developer", async () => {
+      // Create a user with POWER role and check if it was created successfully
       const developer = generateUser(1, "developer")
       expect(developer[0].builder?.global).toEqual(true)
 
@@ -166,9 +188,12 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       const app = await config.applications.create(generateApp())
       config.applications.api.appId = app.appId
 
+      // Get all the information from the create user
       const [userInfoResponse, userInfoJson] = await config.users.getInfo(
         createUserJson.created.successful[0]._id
       )
+
+      // Create the body with the information from the user and add the role to the app
       const body: User = {
         ...userInfoJson,
         roles: {
@@ -177,14 +202,18 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       }
       await config.users.updateInfo(body)
 
+      // Get the user information again and check if the role was added
       const [changedUserInfoResponse, changedUserInfoJson] =
         await config.users.getInfo(createUserJson.created.successful[0]._id)
       expect(changedUserInfoJson.roles[<string>app.appId]).toBeDefined()
       expect(changedUserInfoJson.roles[<string>app.appId]).toEqual("POWER")
 
+      // Create a table
       const [createdTableResponse, createdTableData] = await config.tables.save(
         generateTable()
       )
+
+      // Login with the user created and try to create a column
       await config.login(
         <string>developer[0].email,
         <string>developer[0].password
@@ -197,6 +226,7 @@ describe("Internal API - App Specific Roles & Permissions", () => {
     })
 
     it("Check Table access for admin", async () => {
+      // Create a user with ADMIN role and check if it was created successfully
       const adminUser = generateUser(1, "admin")
       expect(adminUser[0].builder?.global).toEqual(true)
       expect(adminUser[0].admin?.global).toEqual(true)
@@ -206,9 +236,12 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       const app = await config.applications.create(generateApp())
       config.applications.api.appId = app.appId
 
+      // Get all the information from the create user
       const [userInfoResponse, userInfoJson] = await config.users.getInfo(
         createUserJson.created.successful[0]._id
       )
+
+      // Create the body with the information from the user and add the role to the app
       const body: User = {
         ...userInfoJson,
         roles: {
@@ -217,11 +250,13 @@ describe("Internal API - App Specific Roles & Permissions", () => {
       }
       await config.users.updateInfo(body)
 
+      // Get the user information again and check if the role was added
       const [changedUserInfoResponse, changedUserInfoJson] =
         await config.users.getInfo(createUserJson.created.successful[0]._id)
       expect(changedUserInfoJson.roles[<string>app.appId]).toBeDefined()
       expect(changedUserInfoJson.roles[<string>app.appId]).toEqual("ADMIN")
 
+      // Login with the created user and create a table
       await config.login(
         <string>adminUser[0].email,
         <string>adminUser[0].password

qa-core/src/tests/internal-api/userManagement/userManagement.spec.ts

@@ -18,9 +18,13 @@ describe("Internal API - User Management & Permissions", () => {
   })
 
   it("Add Users with different roles", async () => {
+    // Get all users
     await config.users.search()
+
+    // Get all roles
     await config.users.getRoles()
 
+    // Add users with each role
     const admin = generateUser(1, "admin")
     expect(admin[0].builder?.global).toEqual(true)
     expect(admin[0].admin?.global).toEqual(true)
@@ -34,6 +38,7 @@ describe("Internal API - User Management & Permissions", () => {
 
     await config.users.addMultiple(userList)
 
+    // Check users are added
     const [allUsersResponse, allUsersJson] = await config.users.getAll()
     expect(allUsersJson.length).toBeGreaterThan(0)
   })