Merge branch 'develop' of github.com:Budibase/budibase into ak-fixes

hosting/kubernetes/budibase/templates/worker-service-deployment.yaml

@@ -89,6 +89,8 @@ spec:
           value: {{ .Values.globals.selfHosted | quote }}
         - name: ACCOUNT_PORTAL_URL
           value: {{ .Values.globals.accountPortalUrl | quote }}
+        - name: ACCOUNT_PORTAL_API_KEY
+          value: {{ .Values.globals.accountPortalApiKey | quote }}
         - name: COOKIE_DOMAIN
           value: {{ .Values.globals.cookieDomain | quote }}
         image: budibase/worker

hosting/kubernetes/budibase/values.yaml

@@ -90,6 +90,7 @@ globals:
   logLevel: info
   selfHosted: 1
   accountPortalUrL: ""
+  accountPortalApiKey: ""
   cookieDomain: ""
   createSecrets: true # creates an internal API key, JWT secrets and redis password for you
 

hosting/kubernetes/envoy/envoy.yaml

@@ -50,6 +50,11 @@ static_resources:
                 route:
                   cluster: app-service
 
+              - match: { path: "/api/deploy" }
+                route:
+                  timeout: 60s
+                  cluster: app-service
+
               # special case for when API requests are made, can just forward, not to minio
               - match: { prefix: "/api/" }
                 route:

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

package.json

@@ -50,6 +50,8 @@
     "multi:disable": "lerna run multi:disable",
     "selfhost:enable": "lerna run selfhost:enable",
     "selfhost:disable": "lerna run selfhost:disable",
+    "localdomain:enable": "lerna run localdomain:enable",
+    "localdomain:disable": "lerna run localdomain:disable",
     "postinstall": "husky install"
   }
 }

packages/auth/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/auth",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "description": "Authentication middlewares for budibase builder and apps",
   "main": "src/index.js",
   "author": "Budibase",

packages/auth/src/cloud/accounts.js

@@ -1,16 +1,18 @@
 const API = require("./api")
 const env = require("../environment")
+const { Headers } = require("../constants")
 
 const api = new API(env.ACCOUNT_PORTAL_URL)
 
-// TODO: Authorization
-
 exports.getAccount = async email => {
   const payload = {
     email,
   }
   const response = await api.post(`/api/accounts/search`, {
     body: payload,
+    headers: {
+      [Headers.API_KEY]: env.ACCOUNT_PORTAL_API_KEY,
+    },
   })
   const json = await response.json()
 

packages/auth/src/environment.js

@@ -21,6 +21,7 @@ module.exports = {
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
   MULTI_TENANCY: process.env.MULTI_TENANCY,
   ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  ACCOUNT_PORTAL_API_KEY: process.env.ACCOUNT_PORTAL_API_KEY,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
   SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
   COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,

packages/auth/src/middleware/matchers.js

@@ -7,6 +7,7 @@ exports.buildMatcherRegex = patterns => {
   return patterns.map(pattern => {
     const isObj = typeof pattern === "object" && pattern.route
     const method = isObj ? pattern.method : "GET"
+    const strict = pattern.strict ? pattern.strict : false
     let route = isObj ? pattern.route : pattern
 
     const matches = route.match(PARAM_REGEX)
@@ -16,13 +17,19 @@ exports.buildMatcherRegex = patterns => {
         route = route.replace(match, pattern)
       }
     }
-    return { regex: new RegExp(route), method }
+    return { regex: new RegExp(route), method, strict, route }
   })
 }
 
 exports.matches = (ctx, options) => {
-  return options.find(({ regex, method }) => {
-    const urlMatch = regex.test(ctx.request.url)
+  return options.find(({ regex, method, strict, route }) => {
+    let urlMatch
+    if (strict) {
+      urlMatch = ctx.request.url === route
+    } else {
+      urlMatch = regex.test(ctx.request.url)
+    }
+
     const methodMatch =
       method === "ALL"
         ? true

packages/bbui/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/bbui",
   "description": "A UI solution used in the different Budibase projects.",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "license": "AGPL-3.0",
   "svelte": "src/index.js",
   "module": "dist/bbui.es.js",

packages/builder/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/builder",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "license": "AGPL-3.0",
   "private": true,
   "scripts": {
@@ -65,10 +65,10 @@
     }
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.148-alpha.7",
-    "@budibase/client": "^0.9.148-alpha.7",
+    "@budibase/bbui": "^0.9.150-alpha.0",
+    "@budibase/client": "^0.9.150-alpha.0",
     "@budibase/colorpicker": "1.1.2",
-    "@budibase/string-templates": "^0.9.148-alpha.7",
+    "@budibase/string-templates": "^0.9.150-alpha.0",
     "@sentry/browser": "5.19.1",
     "@spectrum-css/page": "^3.0.1",
     "@spectrum-css/vars": "^3.0.1",

packages/builder/src/components/automation/Shared/WebhookDisplay.svelte

@@ -1,22 +1,14 @@
 <script>
   import { Input, Icon, notifications } from "@budibase/bbui"
-  import { store, hostingStore } from "builderStore"
 
   export let value
-  export let production = false
-
-  $: appId = $store.appId
-  $: appUrl = $hostingStore.appUrl
 
   function fullWebhookURL(uri) {
     if (!uri) {
       return ""
     }
-    if (production) {
-      return `${appUrl}/${uri}`
-    } else {
-      return `${window.location.origin}/${uri}`
-    }
+
+    return `${window.location.origin}/${uri}`
   }
 
   function copyToClipboard() {

packages/builder/src/components/backend/DatasourceNavigator/modals/CreateDatasourceModal.svelte

@@ -75,7 +75,7 @@
     }}
   >
     <Layout noPadding>
-      <Body size="XS"
+      <Body size="S"
         >All apps need data. You can connect to a data source below, or add data
         to your app using Budibase's built-in database.
       </Body>

packages/builder/src/components/design/NavigationPanel/NewScreenModal.svelte

@@ -5,6 +5,7 @@
   import { Input, Select, ModalContent, Toggle } from "@budibase/bbui"
   import getTemplates from "builderStore/store/screenTemplates"
   import analytics, { Events } from "analytics"
+  import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
 
   const CONTAINER = "@budibase/standard-components/container"
 
@@ -84,7 +85,7 @@
     if (!event.detail.startsWith("/")) {
       route = "/" + event.detail
     }
-    route = route.replace(/ +/g, "-")
+    route = sanitizeUrl(route)
   }
 </script>
 

packages/builder/src/components/design/PropertiesPanel/ScreenSettingsSection.svelte

@@ -7,6 +7,7 @@
   import RoleSelect from "./PropertyControls/RoleSelect.svelte"
   import { currentAsset, store } from "builderStore"
   import { FrontendTypes } from "constants"
+  import sanitizeUrl from "builderStore/store/screenTemplates/utils/sanitizeUrl"
 
   export let componentInstance
   export let bindings
@@ -37,7 +38,12 @@
       key: "routing.route",
       label: "Route",
       control: Input,
-      parser: val => val.replace(/ +/g, "-"),
+      parser: val => {
+        if (!val.startsWith("/")) {
+          val = "/" + val
+        }
+        return sanitizeUrl(val)
+      },
     },
     { key: "routing.roleId", label: "Access", control: RoleSelect },
     { key: "layoutId", label: "Layout", control: LayoutSelect },

packages/builder/src/components/upgrade/UpgradeModal.svelte

@@ -1,10 +1,11 @@
 <script>
   import { Modal, ModalContent, Button } from "@budibase/bbui"
+  import { admin } from "stores/portal"
 
   let upgradeModal
 
   const onConfirm = () => {
-    window.open("https://account.budibase.app/portal/install", "_blank")
+    window.open(`${$admin.accountPortalUrl}/portal/install`, "_blank")
   }
 </script>
 
@@ -25,8 +26,8 @@
     confirmText="Self-host Budibase"
   >
     <span>
-      Self-host Budibase for free, and get SSO, unlimited apps, and more - and
-      it only takes a few minutes!
+      Self-host budibase for free to get unlimited apps and more - and it only
+      takes a few minutes!
     </span>
   </ModalContent>
 </Modal>

packages/builder/src/pages/builder/_layout.svelte

@@ -14,16 +14,30 @@
   $: useAccountPortal = cloud && !$admin.disableAccountPortal
 
   const validateTenantId = async () => {
-    // set the tenant from the url in the cloud
-    const tenantId = window.location.host.split(".")[0]
+    const host = window.location.host
+    if (host.includes("localhost:")) {
+      // ignore local dev
+      return
+    }
 
-    if (!tenantId.includes("localhost:")) {
-      // user doesn't have permission to access this tenant - kick them out
-      if (user && user.tenantId !== tenantId) {
-        await auth.logout()
-        await auth.setOrganisation(null)
+    if (user && user.tenantId) {
+      let urlTenantId
+      const hostParts = host.split(".")
+
+      // only run validation when we know we are in a tenant url
+      // not when we visit the root budibase.app domain
+      // e.g. ['tenant', 'budibase', 'app'] vs ['budibase', 'app']
+      if (hostParts.length > 2) {
+        urlTenantId = hostParts[0]
       } else {
-        await auth.setOrganisation(tenantId)
+        // no tenant in the url - send to account portal to fix this
+        window.location.href = $admin.accountPortalUrl
+        return
+      }
+
+      if (user.tenantId !== urlTenantId) {
+        // user should not be here - play it safe and log them out
+        await auth.logout()
       }
     }
   }
@@ -32,7 +46,7 @@
     await auth.checkAuth()
     await admin.init()
 
-    if (cloud && multiTenancyEnabled) {
+    if (useAccountPortal && multiTenancyEnabled) {
       await validateTenantId()
     }
 

packages/builder/src/pages/builder/app/[application]/_layout.svelte

@@ -92,7 +92,7 @@
         <ActionGroup />
       </div>
       <div class="toprightnav">
-        {#if $admin.cloud}
+        {#if $admin.cloud && $auth.user.account}
           <UpgradeModal />
         {/if}
         <VersionModal />

packages/builder/src/pages/builder/app/[application]/data/datasource/[selectedDatasource]/CreateEditRelationship/CreateEditRelationship.svelte

@@ -156,6 +156,8 @@
         ...relateTo,
         through: through._id,
         fieldName: fromTable.primary[0],
+        throughFrom: relateFrom.throughTo,
+        throughTo: relateFrom.throughFrom,
       }
     } else {
       // the relateFrom.fieldName should remain the same, as it is the foreignKey in the other
@@ -251,6 +253,22 @@
       bind:error={errors.through}
       bind:value={fromRelationship.through}
     />
+    {#if fromTable && toTable && through}
+      <Select
+        label={`Foreign Key (${fromTable?.name})`}
+        options={Object.keys(through?.schema)}
+        on:change={() => ($touched.fromForeign = true)}
+        bind:error={errors.fromForeign}
+        bind:value={fromRelationship.throughTo}
+      />
+      <Select
+        label={`Foreign Key (${toTable?.name})`}
+        options={Object.keys(through?.schema)}
+        on:change={() => ($touched.toForeign = true)}
+        bind:error={errors.toForeign}
+        bind:value={fromRelationship.throughFrom}
+      />
+    {/if}
   {:else if fromRelationship?.relationshipType && toTable}
     <Select
       label={`Foreign Key (${toTable?.name})`}

packages/builder/src/pages/builder/portal/apps/index.svelte

@@ -327,6 +327,13 @@
     gap: 10px;
   }
 
+  @media only screen and (max-width: 560px) {
+    .title {
+      flex-direction: column;
+      align-items: flex-start;
+    }
+  }
+
   .select {
     display: grid;
     grid-template-columns: 1fr 1fr;

packages/builder/src/pages/builder/portal/manage/users/[userId].svelte

@@ -52,11 +52,11 @@
 
   async function deleteUser() {
     const res = await users.delete(userId)
-    if (res.message) {
+    if (res.status === 200) {
       notifications.success(`User ${$userFetch?.data?.email} deleted.`)
       $goto("./")
     } else {
-      notifications.error("Failed to delete user.")
+      notifications.error(res?.message ? res.message : "Failed to delete user.")
     }
   }
 

packages/builder/src/stores/portal/users.js

@@ -55,7 +55,11 @@ export function createUsersStore() {
   async function del(id) {
     const response = await api.delete(`/api/global/users/${id}`)
     update(users => users.filter(user => user._id !== id))
-    return await response.json()
+    const json = await response.json()
+    return {
+      ...json,
+      status: response.status,
+    }
   }
 
   async function save(data) {

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/cli",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "description": "Budibase CLI, for developers, self hosting and migrations.",
   "main": "src/index.js",
   "bin": {

packages/client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/client",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "license": "MPL-2.0",
   "module": "dist/budibase-client.js",
   "main": "dist/budibase-client.js",
@@ -19,9 +19,9 @@
     "dev:builder": "rollup -cw"
   },
   "dependencies": {
-    "@budibase/bbui": "^0.9.148-alpha.7",
+    "@budibase/bbui": "^0.9.150-alpha.0",
     "@budibase/standard-components": "^0.9.139",
-    "@budibase/string-templates": "^0.9.148-alpha.7",
+    "@budibase/string-templates": "^0.9.150-alpha.0",
     "regexparam": "^1.3.0",
     "shortid": "^2.2.15",
     "svelte-spa-router": "^3.0.5"

packages/server/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -27,7 +27,9 @@
     "multi:enable": "node scripts/multiTenancy.js enable",
     "multi:disable": "node scripts/multiTenancy.js disable",
     "selfhost:enable": "node scripts/selfhost.js enable",
-    "selfhost:disable": "node scripts/selfhost.js disable"
+    "selfhost:disable": "node scripts/selfhost.js disable",
+    "localdomain:enable": "node scripts/localdomain.js enable",
+    "localdomain:disable": "node scripts/localdomain.js disable"
   },
   "jest": {
     "preset": "ts-jest",
@@ -64,9 +66,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.148-alpha.7",
-    "@budibase/client": "^0.9.148-alpha.7",
-    "@budibase/string-templates": "^0.9.148-alpha.7",
+    "@budibase/auth": "^0.9.150-alpha.0",
+    "@budibase/client": "^0.9.150-alpha.0",
+    "@budibase/string-templates": "^0.9.150-alpha.0",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",

packages/server/scripts/integrations/customer-categories/docker-compose.yml

@@ -0,0 +1,28 @@
+version: "3.8"
+services:
+  db:
+    container_name: postgres
+    image: postgres
+    restart: always
+    environment:
+      POSTGRES_USER: root
+      POSTGRES_PASSWORD: root
+      POSTGRES_DB: main
+    ports:
+      - "5432:5432"
+    volumes:
+      #- pg_data:/var/lib/postgresql/data/
+      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
+
+  pgadmin:
+    container_name: pgadmin-pg
+    image: dpage/pgadmin4
+    restart: always
+    environment:
+      PGADMIN_DEFAULT_EMAIL: root@root.com
+      PGADMIN_DEFAULT_PASSWORD: root
+    ports:
+      - "5050:80"
+
+#volumes:
+#  pg_data:

packages/server/scripts/integrations/customer-categories/init.sql

@@ -0,0 +1,41 @@
+SELECT 'CREATE DATABASE main'
+WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = 'main')\gexec
+
+CREATE TABLE categories
+(
+    name text COLLATE pg_catalog."default",
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    CONSTRAINT categories_pkey PRIMARY KEY (id)
+);
+
+CREATE TABLE customers
+(
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    name text COLLATE pg_catalog."default",
+    email text COLLATE pg_catalog."default",
+    age integer,
+    "dateOfBirth" date,
+    CONSTRAINT customers_pkey PRIMARY KEY (id)
+);
+
+CREATE TABLE customer_category
+(
+    customer_id integer,
+    category_id integer,
+    notes text COLLATE pg_catalog."default",
+    id integer NOT NULL GENERATED ALWAYS AS IDENTITY ( INCREMENT 1 START 1 MINVALUE 1 MAXVALUE 2147483647 CACHE 1 ),
+    CONSTRAINT "Category" FOREIGN KEY (category_id)
+        REFERENCES public.categories (id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+        NOT VALID,
+    CONSTRAINT "Customer" FOREIGN KEY (customer_id)
+        REFERENCES public.customers (id) MATCH SIMPLE
+        ON UPDATE NO ACTION
+        ON DELETE NO ACTION
+        NOT VALID
+);
+
+
+INSERT INTO customers (name, email, age) VALUES ('Mike', 'mike@mike.com', 30);
+INSERT INTO categories (name) VALUES ('Books');

packages/server/scripts/integrations/customer-categories/reset.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+docker-compose down
+docker volume prune -f

packages/server/scripts/localdomain.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+const updateDotEnv = require("update-dotenv")
+
+const arg = process.argv.slice(2)[0]
+
+/**
+ * For testing multi tenancy sub domains locally.
+ *
+ * Relies on an entry in /etc/hosts e.g:
+ *
+ * 127.0.0.1   local.com
+ *
+ * and an entry for each tenant you wish to test locally e.g:
+ *
+ * 127.0.0.1   t1.local.com
+ * 127.0.0.1   t2.local.com
+ */
+updateDotEnv({
+  ACCOUNT_PORTAL_URL:
+    arg === "enable" ? "http://local.com:10001" : "http://localhost:10001",
+  COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
+}).then(() => console.log("Updated worker!"))

packages/server/src/api/controllers/analytics.js

@@ -7,14 +7,19 @@ if (env.POSTHOG_TOKEN && env.ENABLE_ANALYTICS && !env.SELF_HOSTED) {
   posthogClient = new PostHog(env.POSTHOG_TOKEN)
 }
 
-exports.isEnabled = async function (ctx) {
+exports.isEnabled = async ctx => {
   ctx.body = {
     enabled: !env.SELF_HOSTED && env.ENABLE_ANALYTICS === "true",
   }
 }
 
-exports.endUserPing = async (ctx, next) => {
-  if (!posthogClient) return next()
+exports.endUserPing = async ctx => {
+  if (!posthogClient) {
+    ctx.body = {
+      ping: false,
+    }
+    return
+  }
 
   posthogClient.capture("budibase:end_user_ping", {
     userId: ctx.user && ctx.user._id,

packages/server/src/api/controllers/application.js

@@ -82,7 +82,7 @@ async function getAppUrlIfNotInUse(ctx) {
   if (!env.SELF_HOSTED) {
     return url
   }
-  const deployedApps = await getDeployedApps(ctx)
+  const deployedApps = await getDeployedApps()
   if (
     url &&
     deployedApps[url] != null &&

packages/server/src/api/controllers/deploy/index.js

@@ -64,6 +64,7 @@ async function storeDeploymentHistory(deployment) {
 
 async function initDeployedApp(prodAppId) {
   const db = new CouchDB(prodAppId)
+  console.log("Reading automation docs")
   const automations = (
     await db.allDocs(
       getAutomationParams(null, {
@@ -71,12 +72,17 @@ async function initDeployedApp(prodAppId) {
       })
     )
   ).rows.map(row => row.doc)
+  console.log("You have " + automations.length + " automations")
   const promises = []
+  console.log("Disabling prod crons..")
   await disableAllCrons(prodAppId)
+  console.log("Prod Cron triggers disabled..")
+  console.log("Enabling cron triggers for deployed app..")
   for (let automation of automations) {
     promises.push(enableCronTrigger(prodAppId, automation))
   }
   await Promise.all(promises)
+  console.log("Enabled cron triggers for deployed app..")
 }
 
 async function deployApp(deployment) {
@@ -88,13 +94,18 @@ async function deployApp(deployment) {
       target: productionAppId,
     })
 
+    console.log("Replication object created")
+
     await replication.replicate()
+    console.log("replication complete.. replacing app meta doc")
     const db = new CouchDB(productionAppId)
     const appDoc = await db.get(DocumentTypes.APP_METADATA)
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    console.log("New app doc written successfully.")
 
+    console.log("Setting up live repl between dev and prod")
     // Set up live sync between the live and dev instances
     const liveReplication = new Replication({
       source: productionAppId,
@@ -105,8 +116,11 @@ async function deployApp(deployment) {
         return doc._id !== DocumentTypes.APP_METADATA
       },
     })
+    console.log("Set up live repl between dev and prod")
 
+    console.log("Initialising deployed app")
     await initDeployedApp(productionAppId)
+    console.log("Init complete, setting deployment to successful")
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeDeploymentHistory(deployment)
   } catch (err) {
@@ -153,9 +167,13 @@ exports.deploymentProgress = async function (ctx) {
 
 exports.deployApp = async function (ctx) {
   let deployment = new Deployment(ctx.appId)
+  console.log("Deployment object created")
   deployment.setStatus(DeploymentStatus.PENDING)
+  console.log("Deployment object set to pending")
   deployment = await storeDeploymentHistory(deployment)
+  console.log("Stored deployment history")
 
+  console.log("Deploying app...")
   await deployApp(deployment)
 
   ctx.body = deployment

packages/server/src/api/controllers/hosting.js

@@ -18,5 +18,5 @@ exports.fetchUrls = async ctx => {
 }
 
 exports.getDeployedApps = async ctx => {
-  ctx.body = await getDeployedApps(ctx)
+  ctx.body = await getDeployedApps()
 }

packages/server/src/api/controllers/row/ExternalRequest.ts

@@ -205,9 +205,13 @@ module External {
         } else {
           // we're not inserting a doc, will be a bunch of update calls
           const isUpdate = !field.through
-          const thisKey: string = isUpdate ? "id" : linkTablePrimary
+          const thisKey: string = isUpdate
+            ? "id"
+            : field.throughTo || linkTablePrimary
           // @ts-ignore
-          const otherKey: string = isUpdate ? field.fieldName : tablePrimary
+          const otherKey: string = isUpdate
+            ? field.fieldName
+            : field.throughFrom || tablePrimary
           row[key].map((relationship: any) => {
             // we don't really support composite keys for relationships, this is why [0] is used
             manyRelationships.push({
@@ -328,12 +332,11 @@ module External {
         if (!table.primary || !linkTable.primary) {
           continue
         }
-        const definition = {
+        const definition: any = {
           // if no foreign key specified then use the name of the field in other table
           from: field.foreignKey || table.primary[0],
           to: field.fieldName,
           tableName: linkTableName,
-          through: undefined,
           // need to specify where to put this back into
           column: fieldName,
         }
@@ -343,8 +346,10 @@ module External {
           )
           definition.through = throughTableName
           // don't support composite keys for relationships
-          definition.from = table.primary[0]
-          definition.to = linkTable.primary[0]
+          definition.from = field.throughFrom || table.primary[0]
+          definition.to = field.throughTo || linkTable.primary[0]
+          definition.fromPrimary = table.primary[0]
+          definition.toPrimary = linkTable.primary[0]
         }
         relationships.push(definition)
       }
@@ -369,7 +374,8 @@ module External {
         }
         const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
         const tableId = isMany ? field.through : field.tableId
-        const fieldName = isMany ? primaryKey : field.fieldName
+        const manyKey = field.throughFrom || primaryKey
+        const fieldName = isMany ? manyKey : field.fieldName
         const response = await makeExternalQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {

packages/server/src/api/controllers/static/index.js

@@ -40,7 +40,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
 async function checkForSelfHostedURL(ctx) {
   // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-  const apps = await getDeployedApps(ctx)
+  const apps = await getDeployedApps()
   if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
     return apps[possibleAppUrl].appId
   } else {

packages/server/src/api/routes/analytics.js

@@ -3,7 +3,8 @@ const controller = require("../controllers/analytics")
 
 const router = Router()
 
-router.get("/api/analytics", controller.isEnabled)
-router.post("/api/analytics/ping", controller.endUserPing)
+router
+  .get("/api/analytics", controller.isEnabled)
+  .post("/api/analytics/ping", controller.endUserPing)
 
 module.exports = router

packages/server/src/definitions/common.ts

@@ -15,6 +15,8 @@ export interface FieldSchema {
   through?: string
   foreignKey?: string
   autocolumn?: boolean
+  throughFrom?: string
+  throughTo?: string
   constraints?: {
     type?: string
     email?: boolean

packages/server/src/definitions/datasource.ts

@@ -121,6 +121,8 @@ export interface RelationshipsJson {
   through?: string
   from?: string
   to?: string
+  fromPrimary?: string
+  toPrimary?: string
   tableName: string
   column: string
 }

packages/server/src/integrations/base/sql.ts

@@ -112,14 +112,16 @@ function addRelationships(
       )
     } else {
       const throughTable = relationship.through
+      const fromPrimary = relationship.fromPrimary
+      const toPrimary = relationship.toPrimary
       query = query
         // @ts-ignore
         .leftJoin(
           throughTable,
-          `${fromTable}.${from}`,
+          `${fromTable}.${fromPrimary}`,
           `${throughTable}.${from}`
         )
-        .leftJoin(toTable, `${toTable}.${to}`, `${throughTable}.${to}`)
+        .leftJoin(toTable, `${toTable}.${toPrimary}`, `${throughTable}.${to}`)
     }
   }
   return query

packages/server/src/utilities/rowProcessor/index.js

@@ -99,6 +99,7 @@ function processAutoColumn(
   row,
   opts = { reprocessing: false, noAutoRelationships: false }
 ) {
+  let noUser = !user || !user.userId
   let now = new Date().toISOString()
   // if a row doesn't have a revision then it doesn't exist yet
   const creating = !row._rev
@@ -108,7 +109,12 @@ function processAutoColumn(
     }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
-        if (creating && !opts.reprocessing && !opts.noAutoRelationships) {
+        if (
+          creating &&
+          !opts.reprocessing &&
+          !opts.noAutoRelationships &&
+          !noUser
+        ) {
           row[key] = [user.userId]
         }
         break
@@ -118,7 +124,7 @@ function processAutoColumn(
         }
         break
       case AutoFieldSubTypes.UPDATED_BY:
-        if (!opts.reprocessing && !opts.noAutoRelationships) {
+        if (!opts.reprocessing && !opts.noAutoRelationships && !noUser) {
           row[key] = [user.userId]
         }
         break

packages/server/src/utilities/workerRequests.js

@@ -52,16 +52,17 @@ exports.sendSmtpEmail = async (to, from, subject, contents, automation) => {
   )
 
   if (response.status !== 200) {
-    throw "Unable to send email."
+    const error = await response.text()
+    throw `Unable to send email - ${error}`
   }
   return response.json()
 }
 
-exports.getDeployedApps = async ctx => {
+exports.getDeployedApps = async () => {
   try {
     const response = await fetch(
       checkSlashesInUrl(env.WORKER_URL + `/api/apps`),
-      request(ctx, {
+      request(null, {
         method: "GET",
       })
     )

packages/string-templates/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@budibase/string-templates",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "description": "Handlebars wrapper for Budibase templating.",
   "main": "src/index.cjs",
   "module": "dist/bundle.mjs",

packages/worker/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/worker",
   "email": "hi@budibase.com",
-  "version": "0.9.148-alpha.7",
+  "version": "0.9.150-alpha.0",
   "description": "Budibase background service",
   "main": "src/index.js",
   "repository": {
@@ -20,13 +20,15 @@
     "multi:enable": "node scripts/multiTenancy.js enable",
     "multi:disable": "node scripts/multiTenancy.js disable",
     "selfhost:enable": "node scripts/selfhost.js enable",
-    "selfhost:disable": "node scripts/selfhost.js disable"
+    "selfhost:disable": "node scripts/selfhost.js disable",
+    "localdomain:enable": "node scripts/localdomain.js enable",
+    "localdomain:disable": "node scripts/localdomain.js disable"
   },
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.148-alpha.7",
-    "@budibase/string-templates": "^0.9.148-alpha.7",
+    "@budibase/auth": "^0.9.150-alpha.0",
+    "@budibase/string-templates": "^0.9.150-alpha.0",
     "@koa/router": "^8.0.0",
     "@techpass/passport-openidconnect": "^0.3.0",
     "aws-sdk": "^2.811.0",

packages/worker/scripts/dev/manage.js

@@ -23,6 +23,8 @@ async function init() {
       MULTI_TENANCY: "",
       DISABLE_ACCOUNT_PORTAL: "",
       ACCOUNT_PORTAL_URL: "http://localhost:10001",
+      ACCOUNT_PORTAL_API_KEY: "budibase",
+      PLATFORM_URL: "http://localhost:10000",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

packages/worker/scripts/localdomain.js

@@ -0,0 +1,24 @@
+#!/usr/bin/env node
+const updateDotEnv = require("update-dotenv")
+
+const arg = process.argv.slice(2)[0]
+
+/**
+ * For testing multi tenancy sub domains locally.
+ *
+ * Relies on an entry in /etc/hosts e.g:
+ *
+ * 127.0.0.1   local.com
+ *
+ * and an entry for each tenant you wish to test locally e.g:
+ *
+ * 127.0.0.1   t1.local.com
+ * 127.0.0.1   t2.local.com
+ */
+updateDotEnv({
+  ACCOUNT_PORTAL_URL:
+    arg === "enable" ? "http://local.com:10001" : "http://localhost:10001",
+  COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
+  PLATFORM_URL:
+    arg === "enable" ? "http://local.com:10000" : "http://localhost:10000",
+}).then(() => console.log("Updated worker!"))

packages/worker/src/api/controllers/global/users.js

@@ -205,6 +205,18 @@ exports.adminUser = async ctx => {
 exports.destroy = async ctx => {
   const db = getGlobalDB()
   const dbUser = await db.get(ctx.params.id)
+
+  // root account holder can't be deleted from inside budibase
+  const email = dbUser.email
+  const account = await accounts.getAccount(email)
+  if (account) {
+    if (email === ctx.user.email) {
+      ctx.throw(400, 'Please visit "Account" to delete this user')
+    } else {
+      ctx.throw(400, "Account holder cannot be deleted")
+    }
+  }
+
   await removeUserFromInfoDB(dbUser)
   await db.remove(dbUser._id, dbUser._rev)
   await userCache.invalidateUser(dbUser._id)

packages/worker/src/api/index.js

@@ -87,7 +87,7 @@ router
     if (ctx.publicEndpoint) {
       return next()
     }
-    if (!ctx.isAuthenticated || !ctx.user.budibaseAccess) {
+    if ((!ctx.isAuthenticated || !ctx.user.budibaseAccess) && !ctx.internal) {
       ctx.throw(403, "Unauthorized - no public worker access")
     }
     return next()

packages/worker/src/api/routes/global/users.js

@@ -3,6 +3,7 @@ const controller = require("../../controllers/global/users")
 const joiValidator = require("../../../middleware/joi-validator")
 const adminOnly = require("../../../middleware/adminOnly")
 const Joi = require("joi")
+const cloudRestricted = require("../../../middleware/cloudRestricted")
 
 const router = Router()
 
@@ -90,6 +91,7 @@ router
   )
   .post(
     "/api/global/users/init",
+    cloudRestricted,
     buildAdminInitValidation(),
     controller.adminUser
   )

packages/worker/src/environment.js

@@ -40,6 +40,7 @@ module.exports = {
   SMTP_HOST: process.env.SMTP_HOST,
   SMTP_PORT: process.env.SMTP_PORT,
   SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS,
+  PLATFORM_URL: process.env.PLATFORM_URL,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value

packages/worker/src/middleware/cloudRestricted.js

@@ -0,0 +1,17 @@
+const env = require("../environment")
+const { Headers } = require("@budibase/auth").constants
+
+/**
+ * This is a restricted endpoint in the cloud.
+ * Ensure that the correct API key has been supplied.
+ */
+module.exports = async (ctx, next) => {
+  if (!env.SELF_HOSTED) {
+    const apiKey = ctx.request.headers[Headers.API_KEY]
+    if (apiKey !== env.INTERNAL_API_KEY) {
+      ctx.throw(403, "Unauthorized")
+    }
+  }
+
+  return next()
+}

packages/worker/src/utilities/templates.js

@@ -8,8 +8,6 @@ const {
 const { checkSlashesInUrl } = require("./index")
 const env = require("../environment")
 const { getGlobalDB, addTenantToUrl } = require("@budibase/auth/tenancy")
-
-const LOCAL_URL = `http://localhost:${env.CLUSTER_PORT || 10000}`
 const BASE_COMPANY = "Budibase"
 
 exports.getSettingsTemplateContext = async (purpose, code = null) => {
@@ -17,7 +15,7 @@ exports.getSettingsTemplateContext = async (purpose, code = null) => {
   // TODO: use more granular settings in the future if required
   let settings = (await getScopedConfig(db, { type: Configs.SETTINGS })) || {}
   if (!settings || !settings.platformUrl) {
-    settings.platformUrl = LOCAL_URL
+    settings.platformUrl = env.PLATFORM_URL
   }
   const URL = settings.platformUrl
   const context = {