Merge branch 'develop' of github.com:Budibase/budibase into data-section-multidev

2024-07-11 09:15:48 +12:00 · 2023-05-24 15:14:32 +01:00 · 2023-05-24 15:14:32 +01:00 · 5cf4753853
parent 9ab5c8792a d05a48db05
commit 5cf4753853
65 changed files with 2540 additions and 1167 deletions
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@ -44,9 +44,9 @@ jobs:
          node-version: 14.x
          cache: "yarn"
      - run: yarn
-      - run: yarn build
+      - run: yarn nx run-many -t=build --configuration=production

-  test:
+  test-libraries:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
@ -59,8 +59,27 @@ jobs:
          node-version: 14.x
          cache: "yarn"
      - run: yarn
-      - run: yarn build --scope=@budibase/types --scope=@budibase/shared-core --scope=@budibase/string-templates
-      - run: yarn test --ignore=@budibase/pro
+      - run: yarn test --ignore=@budibase/worker --ignore=@budibase/server --ignore=@budibase/pro
+      - uses: codecov/codecov-action@v3
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
+          name: codecov-umbrella
+          verbose: true
+
+  test-services:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: true
+          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: 14.x
+          cache: "yarn"
+      - run: yarn
+      - run: yarn test --scope=@budibase/worker --scope=@budibase/server
      - uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -51,7 +51,7 @@ jobs:
          node scripts/syncLocalDependencies.js $version
          echo "Syncing yarn workspace"
          yarn
-      - run: yarn build
+      - run: yarn build --configuration=production
      - run: yarn build:sdk

      - name: Publish budibase packages to NPM
--- a/.github/workflows/release-master.yml
+++ b/.github/workflows/release-master.yml
@ -57,7 +57,7 @@ jobs:
          echo "Syncing yarn workspace"
          yarn
      - run: yarn lint
-      - run: yarn build
+      - run: yarn build --configuration=production
      - run: yarn build:sdk

      - name: Publish budibase packages to NPM
--- a/.nvmrc
+++ b/.nvmrc
@ -1 +1 @@
-v14.20.1
+v14.20.1
--- a/.python-version
+++ b/.python-version
@ -1 +1 @@
-3.10.0
+3.10.0
--- a/hosting/docker-compose.build.yaml
+++ b/hosting/docker-compose.build.yaml
@ -0,0 +1,77 @@
+version: "3"
+
+# optional ports are specified throughout for more advanced use cases.
+
+services:
+  app-service:
+    build: ../packages/server
+    container_name: build-bbapps
+    environment:
+      SELF_HOSTED: 1
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+      BB_ADMIN_USER_EMAIL: ${BB_ADMIN_USER_EMAIL}
+      BB_ADMIN_USER_PASSWORD: ${BB_ADMIN_USER_PASSWORD}
+      PLUGINS_DIR: ${PLUGINS_DIR}
+    depends_on:
+      - worker-service
+      - redis-service
+  #    volumes:
+  #      - /some/path/to/plugins:/plugins
+
+  worker-service:
+    build: ../packages/worker
+    container_name: build-bbworker
+    environment:
+      SELF_HOSTED: 1
+      PORT: 4003
+      CLUSTER_PORT: ${MAIN_PORT}
+      API_ENCRYPTION_KEY: ${API_ENCRYPTION_KEY}
+      JWT_SECRET: ${JWT_SECRET}
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      MINIO_URL: http://minio-service:9000
+      APPS_URL: http://app-service:4002
+      COUCH_DB_USERNAME: ${COUCH_DB_USER}
+      COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD}
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    depends_on:
+      - redis-service
+      - minio-service
+
+  proxy-service-docker:
+    ports:
+      - "${MAIN_PORT}:10000"
+    container_name: build-bbproxy
+    image: budibase/proxy
+    environment:
+      - PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
+      - PROXY_RATE_LIMIT_API_PER_SECOND=20
+      - APPS_UPSTREAM_URL=http://app-service:4002
+      - WORKER_UPSTREAM_URL=http://worker-service:4003
+      - MINIO_UPSTREAM_URL=http://minio-service:9000
+      - COUCHDB_UPSTREAM_URL=http://couchdb-service:5984
+      - WATCHTOWER_UPSTREAM_URL=http://watchtower-service:8080
+      - RESOLVER=127.0.0.11
+    depends_on:
+      - minio-service
+      - worker-service
+      - app-service
+      - couchdb-service
--- a/lerna.json
+++ b/lerna.json
@ -1,5 +1,5 @@
 {
-  "version": "2.6.19-alpha.2",
+  "version": "2.6.19-alpha.6",
  "npmClient": "yarn",
  "packages": [
    "packages/backend-core",
--- a/nx.json
+++ b/nx.json
@ -6,5 +6,15 @@
        "cacheableOperations": ["build", "test"]
      }
    }
+  },
+  "targetDefaults": {
+    "dev:builder": {
+      "dependsOn": [
+        {
+          "projects": ["@budibase/string-templates"],
+          "target": "build"
+        }
+      ]
+    }
  }
 }
--- a/package.json
+++ b/package.json
@ -2,17 +2,23 @@
  "name": "root",
  "private": true,
  "devDependencies": {
+    "@esbuild-plugins/node-resolve": "^0.2.2",
+    "@nx/esbuild": "16.2.1",
+    "@nx/js": "16.2.1",
    "@rollup/plugin-json": "^4.0.2",
    "@typescript-eslint/parser": "5.45.0",
    "babel-eslint": "^10.0.3",
+    "esbuild": "^0.17.18",
    "eslint": "^7.28.0",
    "eslint-plugin-cypress": "^2.11.3",
    "eslint-plugin-svelte3": "^3.2.0",
    "husky": "^8.0.3",
    "js-yaml": "^4.1.0",
    "kill-port": "^1.6.1",
-    "lerna": "^6.6.1",
+    "lerna": "7.0.0-alpha.0",
    "madge": "^6.0.0",
+    "minimist": "^1.2.8",
+    "nx": "^16.2.1",
    "prettier": "^2.3.1",
    "prettier-plugin-svelte": "^2.3.0",
    "rimraf": "^3.0.2",
@ -24,8 +30,8 @@
  "scripts": {
    "preinstall": "node scripts/syncProPackage.js",
    "setup": "git config submodule.recurse true && git submodule update && node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
-    "bootstrap": "./scripts/bootstrap.sh && lerna link && ./scripts/link-dependencies.sh",
-    "build": "lerna run --stream build",
+    "bootstrap": "./scripts/bootstrap.sh  && ./scripts/link-dependencies.sh",
+    "build": "yarn nx run-many -t=build",
    "build:dev": "lerna run --stream prebuild && yarn nx run-many --target=build --output-style=dynamic --watch --preserveWatchOutput",
    "backend:bootstrap": "./scripts/scopeBackend.sh && yarn run bootstrap",
    "backend:build": "./scripts/scopeBackend.sh 'lerna run --stream build'",
@ -41,10 +47,11 @@
    "kill-builder": "kill-port 3000",
    "kill-server": "kill-port 4001 4002",
    "kill-all": "yarn run kill-builder && yarn run kill-server",
-    "dev": "yarn run kill-all && lerna link && lerna run --stream --parallel dev:builder --concurrency 1 --stream",
-    "dev:noserver": "yarn run kill-builder && lerna link && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --concurrency 1 --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --concurrency 1 --scope @budibase/backend-core --scope @budibase/worker --scope @budibase/server",
+    "dev": "yarn run kill-all  && lerna run --stream --parallel dev:builder --stream",
+    "dev:noserver": "yarn run kill-builder  && lerna run --stream dev:stack:up && lerna run --stream --parallel dev:builder --ignore @budibase/backend-core --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --stream --parallel dev:builder --scope @budibase/worker --scope @budibase/server",
    "dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream --parallel dev:built",
+    "dev:docker": "yarn build && docker-compose -f hosting/docker-compose.dev.yaml -f hosting/docker-compose.build.yaml up --build --scale proxy-service=0 ",
    "test": "lerna run --stream test --stream",
    "lint:eslint": "eslint packages && eslint qa-core",
    "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\" && prettier --check \"qa-core/**/*.{js,ts,svelte}\"",
@ -101,5 +108,6 @@
      "packages/worker",
      "packages/pro/packages/pro"
    ]
-  }
+  },
+  "dependencies": {}
 }
--- a/packages/backend-core/package.json
+++ b/packages/backend-core/package.json
@ -88,5 +88,19 @@
    "tsconfig-paths": "4.0.0",
    "typescript": "4.7.3"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/bbui/package.json
+++ b/packages/bbui/package.json
@ -90,5 +90,19 @@
  "resolutions": {
    "loader-utils": "1.4.1"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/bbui/src/ActionButton/ActionButton.svelte
+++ b/packages/bbui/src/ActionButton/ActionButton.svelte
@ -102,7 +102,9 @@
    margin-left: 0;
    transition: color ease-out 130ms;
  }
-  .is-selected:not(.spectrum-ActionButton--emphasized):not(.spectrum-ActionButton--quiet) {
+  .is-selected:not(.spectrum-ActionButton--emphasized):not(
+      .spectrum-ActionButton--quiet
+    ) {
    background: var(--spectrum-global-color-gray-300);
    border-color: var(--spectrum-global-color-gray-500);
  }
--- a/packages/builder/package.json
+++ b/packages/builder/package.json
@ -9,7 +9,7 @@
    "dev:builder": "routify -c dev:vite",
    "dev:vite": "vite --host 0.0.0.0",
    "rollup": "rollup -c -w",
-    "test": "vitest"
+    "test": "vitest run"
  },
  "jest": {
    "globals": {
@ -117,5 +117,31 @@
    "vite": "^3.0.8",
    "vitest": "^0.29.2"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      },
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/shared-core",
+              "@budibase/string-templates"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "115189f72a850bfb52b65ec61d932531bf327072"
 }
--- a/packages/builder/src/components/common/CodeMirrorEditor.svelte
+++ b/packages/builder/src/components/common/CodeMirrorEditor.svelte
@ -146,15 +146,18 @@

  /* Override default active line highlight colour in dark theme */
  div
-    :global(.CodeMirror-focused.cm-s-tomorrow-night-eighties
-      .CodeMirror-activeline-background) {
+    :global(
+      .CodeMirror-focused.cm-s-tomorrow-night-eighties
+        .CodeMirror-activeline-background
+    ) {
    background: rgba(255, 255, 255, 0.075);
  }

  /* Remove active line styling when not focused */
  div
-    :global(.CodeMirror:not(.CodeMirror-focused)
-      .CodeMirror-activeline-background) {
+    :global(
+      .CodeMirror:not(.CodeMirror-focused) .CodeMirror-activeline-background
+    ) {
    background: unset;
  }

--- a/packages/builder/src/pages/builder/portal/settings/auth/scim.svelte
+++ b/packages/builder/src/pages/builder/portal/settings/auth/scim.svelte
@ -115,27 +115,6 @@
    align-items: center;
  }

-  input[type="file"] {
-    display: none;
-  }
-  .sso-link-icon {
-    padding-top: 4px;
-    margin-left: 3px;
-  }
-  .sso-link {
-    margin-top: 12px;
-    display: flex;
-    flex-direction: row;
-    align-items: center;
-  }
-  .enforce-sso-title {
-    margin-right: 10px;
-  }
-  .enforce-sso-heading-container {
-    display: flex;
-    flex-direction: row;
-    align-items: start;
-  }
  .provider-title {
    display: flex;
    flex-direction: row;
@ -143,9 +122,6 @@
    align-items: center;
    gap: var(--spacing-m);
  }
-  .provider-title span {
-    flex: 1 1 auto;
-  }
  .inputContainer {
    display: flex;
    flex-direction: row;
--- a/packages/builder/src/stores/backend/datasources.js
+++ b/packages/builder/src/stores/backend/datasources.js
@ -25,6 +25,8 @@ export function createDatasourcesStore() {
    store.update(state => ({
      ...state,
      selectedDatasourceId: id,
+      // Remove any possible schema error
+      schemaError: null,
    }))
  }

--- a/packages/cli/package.json
+++ b/packages/cli/package.json
@ -63,5 +63,19 @@
    "renamer": "^4.0.0",
    "ts-node": "^10.9.1",
    "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/backend-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
  }
 }
--- a/packages/client/package.json
+++ b/packages/client/package.json
@ -65,5 +65,20 @@
  "resolutions": {
    "loader-utils": "1.4.1"
  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/client/src/components/app/charts/ApexChart.svelte
+++ b/packages/client/src/components/app/charts/ApexChart.svelte
@ -54,8 +54,9 @@
    color: white;
  }
  div
-    :global(.apexcharts-theme-dark
-      .apexcharts-tooltip-series-group.apexcharts-active) {
+    :global(
+      .apexcharts-theme-dark .apexcharts-tooltip-series-group.apexcharts-active
+    ) {
    padding-bottom: 0;
  }
 </style>
--- a/packages/client/src/components/app/forms/JSONField.svelte
+++ b/packages/client/src/components/app/forms/JSONField.svelte
@ -72,9 +72,11 @@
  :global(.spectrum-Form-itemField .spectrum-Textfield--multiline) {
    min-height: calc(var(--height) - 24px);
  }
-  :global(.spectrum-Form--labelsAbove
-      .spectrum-Form-itemField
-      .spectrum-Textfield--multiline) {
+  :global(
+      .spectrum-Form--labelsAbove
+        .spectrum-Form-itemField
+        .spectrum-Textfield--multiline
+    ) {
    min-height: calc(var(--height) - 24px);
  }
 </style>
--- a/packages/pro
+++ b/packages/pro
@ -1 +1 @@
-Subproject commit a590dc237a16983b8f39dc8e65005b7736d23467
+Subproject commit aea8a4acb0bae6a1036520bf4c6d8cae428cc7d9
--- a/packages/server/.dockerignore
+++ b/packages/server/.dockerignore
@ -1,6 +1,7 @@
-node_modules
-npm-debug.log
-Dockerfile
-.dockerignore
-.git
-.gitignore
+*
+!/dist/
+!/scripts/integrations/oracle/
+!/package.json
+!/docker_run.sh
+!/builder/
+!/client/
--- a/packages/server/Dockerfile
+++ b/packages/server/Dockerfile
@ -15,22 +15,28 @@ ENV POSTHOG_TOKEN=phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
 ENV TENANT_FEATURE_FLAGS=*:LICENSING,*:USER_GROUPS,*:ONBOARDING_TOUR
 ENV ACCOUNT_PORTAL_URL=https://account.budibase.app

-# copy files and install dependencies
-COPY . ./
+
 # handle node-gyp
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends g++ make python \
-    && yarn \
-    && yarn cache clean \
-    && apt-get remove -y --purge --auto-remove g++ make python \
-    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+    && apt-get install -y --no-install-recommends g++ make python
 RUN yarn global add pm2
-RUN yarn build

 # Install client for oracle datasource
 RUN apt-get install unzip libaio1
+COPY scripts/integrations/oracle/ scripts/integrations/oracle/
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 

+COPY package.json .
+RUN yarn 
+# Remove unneeded data from file system to reduce image size
+RUN yarn cache clean && apt-get remove -y --purge --auto-remove g++ make python \
+    && rm -rf /tmp/* /root/.node-gyp /usr/local/lib/node_modules/npm/node_modules/node-gyp
+
+COPY dist/ dist/
+COPY docker_run.sh .
+COPY builder/ builder/
+COPY client/ client/
+
 EXPOSE 4001

 # have to add node environment production after install
@ -38,4 +44,6 @@ EXPOSE 4001
 # which are actually needed to get this environment up and running
 ENV NODE_ENV=production
 ENV CLUSTER_MODE=${CLUSTER_MODE}
+ENV TOP_LEVEL_PATH=/app
+
 CMD ["./docker_run.sh"]
--- a/packages/server/jest.config.ts
+++ b/packages/server/jest.config.ts
@ -1,6 +1,7 @@
 import { Config } from "@jest/types"

 import * as fs from "fs"
+import { join } from "path"
 const preset = require("ts-jest/jest-preset")

 const baseConfig: Config.InitialProjectOptions = {
@ -49,4 +50,6 @@ const config: Config.InitialOptions = {
  coverageReporters: ["lcov", "json", "clover"],
 }

+process.env.TOP_LEVEL_PATH = join(__dirname, "..", "..")
+
 export default config
--- a/packages/server/nodemon.json
+++ b/packages/server/nodemon.json
@ -6,5 +6,5 @@
    "src/**/*.spec.js",
    "../backend-core/dist/**/*"
  ],
-  "exec": "ts-node src/index.ts"
+  "exec": "node ../../scripts/build && node ./dist/index.js"
 }
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -10,22 +10,22 @@
  },
  "scripts": {
    "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json && mv dist/src/* dist/ && rimraf dist/src/",
+    "build": "node ./scripts/build.js",
+    "postbuild": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
    "debug": "yarn build && node --expose-gc --inspect=9222 dist/index.js",
-    "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
    "test": "bash scripts/test.sh",
    "test:memory": "jest --maxWorkers=2 --logHeapUsage --forceExit",
    "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
-    "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
    "build:docs": "node ./scripts/docs/generate.js open",
    "run:docker": "node dist/index.js",
    "run:docker:cluster": "pm2-runtime start pm2.config.js",
    "dev:stack:up": "node scripts/dev/manage.js up",
    "dev:stack:down": "node scripts/dev/manage.js down",
    "dev:stack:nuke": "node scripts/dev/manage.js nuke",
-    "dev:builder": "yarn run dev:stack:up && nodemon",
+    "dev:builder": "yarn run dev:stack:up && rimraf dist/ && nodemon",
    "dev:built": "yarn run dev:stack:up && yarn run run:docker",
    "specs": "ts-node specs/generate.ts && openapi-typescript specs/openapi.yaml --output src/definitions/openapi.ts",
    "initialise": "node scripts/initialise.js",
@ -118,8 +118,8 @@
    "validate.js": "0.13.1",
    "vm2": "3.9.17",
    "worker-farm": "1.7.0",
-    "xml2js": "0.5.0",
-    "yargs": "13.2.4"
+    "yargs": "13.2.4",
+    "xml2js": "0.5.0"
  },
  "devDependencies": {
    "@babel/core": "7.17.4",
@ -177,5 +177,20 @@
  "optionalDependencies": {
    "oracledb": "5.3.0"
  },
+  "nx": {
+    "targets": {
+      "test": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/string-templates",
+              "@budibase/shared-core"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
+  },
  "gitHead": "d1836a898cab3f8ab80ee6d8f42be1a9eed7dcdc"
 }
--- a/packages/server/pm2.config.js
+++ b/packages/server/pm2.config.js
@ -1,7 +1,7 @@
 module.exports = {
  apps: [
    {
-      script: "dist/index.js",
+      script: "./dist/index.js",
      instances: "max",
      exec_mode: "cluster",
    },
--- a/packages/server/scripts/build.js
+++ b/packages/server/scripts/build.js
@ -0,0 +1,48 @@
+#!/usr/bin/node
+const { join } = require("path")
+const fs = require("fs")
+const coreBuild = require("../../../scripts/build")
+
+const dir = join(__dirname, "..")
+const entryPath = join(dir, "src")
+const outfilePath = join(dir, "dist")
+
+/**
+ * The reasoning for this is that now our built version is simple
+ * dist/index.js - any kind of threaded approach in Node.js requires
+ * a runner file to work from - I played around with a lot of
+ * different methods, but we really want to be able to use forks.
+ *
+ * Rather than trying to rewrite so that forks run the whole system,
+ * I instead went down a path of building the individual threads so
+ * that we have runner files for each of them e.g. dist/automations.js
+ * and dist/query.js - these can be ran totally independently and then
+ * the parent process can pass down data for processing to them.
+ *
+ * The ignoring is simply to remove the files which really don't need
+ * to be built - they could be built and it wouldn't cause any issues,
+ * but this just means if any further threads are added in future
+ * they will naturally work (rather than including, which would mean
+ * adjustments to the build files).
+ */
+const ignoredFiles = ["definitions", "index", "utils"]
+const threadNames = fs
+  .readdirSync(join(dir, "src", "threads"))
+  .filter(path => !ignoredFiles.find(file => path.includes(file)))
+  .map(path => path.replace(".ts", ""))
+const files = [
+  {
+    entry: join(entryPath, "index.ts"),
+    out: join(outfilePath, "index.js"),
+  },
+]
+for (let name of threadNames) {
+  files.push({
+    entry: join(entryPath, "threads", `${name}.ts`),
+    out: join(outfilePath, `${name}.js`),
+  })
+}
+
+for (let file of files) {
+  coreBuild(file.entry, file.out)
+}
--- a/packages/server/src/api/controllers/datasource.ts
+++ b/packages/server/src/api/controllers/datasource.ts
@ -21,6 +21,7 @@ import {
  CreateDatasourceRequest,
  VerifyDatasourceRequest,
  VerifyDatasourceResponse,
+  FetchDatasourceInfoResponse,
  IntegrationBase,
  DatasourcePlus,
 } from "@budibase/types"
@ -154,6 +155,21 @@ export async function verify(
  }
 }

+export async function information(
+  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+) {
+  const datasourceId = ctx.params.datasourceId
+  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
+  const connector = (await getConnector(datasource)) as DatasourcePlus
+  if (!connector.getTableNames) {
+    ctx.throw(400, "Table name fetching not supported by datasource")
+  }
+  const tableNames = await connector.getTableNames()
+  ctx.body = {
+    tableNames,
+  }
+}
+
 export async function buildSchemaFromDb(ctx: UserCtx) {
  const db = context.getAppDB()
  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
--- a/packages/server/src/api/controllers/static/index.ts
+++ b/packages/server/src/api/controllers/static/index.ts
@ -134,7 +134,7 @@ export const serveApp = async function (ctx: any) {
            ? objectStore.getGlobalFileUrl("settings", "logoUrl")
            : "",
      })
-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
      ctx.body = await processString(appHbs, {
        head,
        body: html,
@ -161,7 +161,7 @@ export const serveApp = async function (ctx: any) {
            : "",
      })

-      const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
+      const appHbs = loadHandlebarsFile(`${__dirname}/app.hbs`)
      ctx.body = await processString(appHbs, {
        head,
        body: html,
@ -177,7 +177,7 @@ export const serveBuilderPreview = async function (ctx: any) {

  if (!env.isJest()) {
    let appId = context.getAppId()
-    const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
+    const previewHbs = loadHandlebarsFile(`${__dirname}/preview.hbs`)
    ctx.body = await processString(previewHbs, {
      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
    })
--- a/packages/server/src/api/routes/datasource.ts
+++ b/packages/server/src/api/routes/datasource.ts
@ -20,6 +20,11 @@ router
    authorized(permissions.BUILDER),
    datasourceController.verify
  )
+  .get(
+    "/api/datasources/:datasourceId/info",
+    authorized(permissions.BUILDER),
+    datasourceController.information
+  )
  .get(
    "/api/datasources/:datasourceId",
    authorized(
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@ -5,6 +5,7 @@ import authorized from "../../middleware/authorized"
 import { permissions } from "@budibase/backend-core"
 import env from "../../environment"
 import { paramResource } from "../../middleware/resourceId"
+import { devClientLibPath } from "../../utilities/fileSystem"
 const { BUILDER, PermissionType, PermissionLevel } = permissions

 const router: Router = new Router()
@ -17,7 +18,8 @@ router.param("file", async (file: any, ctx: any, next: any) => {
  }
  // test serves from require
  if (env.isTest()) {
-    ctx.devPath = require.resolve("@budibase/client").split(ctx.file)[0]
+    const path = devClientLibPath()
+    ctx.devPath = path.split(ctx.file)[0]
  } else if (env.isDev()) {
    // Serving the client library from your local dir in dev
    ctx.devPath = budibaseTempDir()
--- a/packages/server/src/api/routes/tests/datasource.spec.ts
+++ b/packages/server/src/api/routes/tests/datasource.spec.ts
@ -87,7 +87,7 @@ describe("/datasources", () => {
        expect(contents.rows.length).toEqual(1)

        // update the datasource to remove the variables
-        datasource.config.dynamicVariables = []
+        datasource.config!.dynamicVariables = []
        const res = await request
          .put(`/api/datasources/${datasource._id}`)
          .send(datasource)
--- a/packages/server/src/constants/index.ts
+++ b/packages/server/src/constants/index.ts
@ -1,4 +1,5 @@
 import { objectStore, roles, constants } from "@budibase/backend-core"
+import { FieldType as FieldTypes } from "@budibase/types"
 export { FieldType as FieldTypes, RelationshipTypes } from "@budibase/types"

 export enum FilterTypes {
@ -24,14 +25,14 @@ export const NoEmptyFilterStrings = [
 ]

 export const CanSwitchTypes = [
-  [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
+  [FieldTypes.JSON, FieldTypes.ARRAY],
  [
-    exports.FieldTypes.STRING,
-    exports.FieldTypes.OPTIONS,
-    exports.FieldTypes.LONGFORM,
-    exports.FieldTypes.BARCODEQR,
+    FieldTypes.STRING,
+    FieldTypes.OPTIONS,
+    FieldTypes.LONGFORM,
+    FieldTypes.BARCODEQR,
  ],
-  [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
+  [FieldTypes.BOOLEAN, FieldTypes.NUMBER],
 ]

 export const SwitchableTypes = CanSwitchTypes.reduce((prev, current) =>
@ -77,9 +78,9 @@ export const USERS_TABLE_SCHEMA = {
  // TODO: ADMIN PANEL - when implemented this doesn't need to be carried out
  schema: {
    email: {
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        email: true,
        length: {
          maximum: "",
@ -92,27 +93,27 @@ export const USERS_TABLE_SCHEMA = {
    firstName: {
      name: "firstName",
      fieldName: "firstName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
      },
    },
    lastName: {
      name: "lastName",
      fieldName: "lastName",
-      type: exports.FieldTypes.STRING,
+      type: FieldTypes.STRING,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
      },
    },
    roleId: {
      fieldName: "roleId",
      name: "roleId",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
        inclusion: Object.values(roles.BUILTIN_ROLE_IDS),
      },
@ -120,9 +121,9 @@ export const USERS_TABLE_SCHEMA = {
    status: {
      fieldName: "status",
      name: "status",
-      type: exports.FieldTypes.OPTIONS,
+      type: FieldTypes.OPTIONS,
      constraints: {
-        type: exports.FieldTypes.STRING,
+        type: FieldTypes.STRING,
        presence: false,
        inclusion: Object.values(constants.UserStatus),
      },
--- a/packages/server/src/db/inMemoryView.ts
+++ b/packages/server/src/db/inMemoryView.ts
@ -25,7 +25,9 @@ export async function runView(
      }))
    )
    let fn = (doc: Document, emit: any) => emit(doc._id)
-    eval("fn = " + view?.map?.replace("function (doc)", "function (doc, emit)"))
+    ;(0, eval)(
+      "fn = " + view?.map?.replace("function (doc)", "function (doc, emit)")
+    )
    const queryFns: any = {
      meta: view.meta,
      map: fn,
--- a/packages/server/src/environment.ts
+++ b/packages/server/src/environment.ts
@ -96,6 +96,7 @@ const environment = {
  isInThread: () => {
    return process.env.FORKED_PROCESS
  },
+  TOP_LEVEL_PATH: process.env.TOP_LEVEL_PATH,
 }

 // threading can cause memory issues with node-ts in development
--- a/packages/server/src/integration-test/postgres.spec.ts
+++ b/packages/server/src/integration-test/postgres.spec.ts
@ -26,7 +26,7 @@ jest.setTimeout(30000)

 jest.unmock("pg")

-describe("row api - postgres", () => {
+describe("postgres integrations", () => {
  let makeRequest: MakeRequestResponse,
    postgresDatasource: Datasource,
    primaryPostgresTable: Table,
@ -52,8 +52,8 @@ describe("row api - postgres", () => {
    makeRequest = generateMakeRequest(apiKey, true)
  })

-  beforeEach(async () => {
-    postgresDatasource = await config.createDatasource({
+  function pgDatasourceConfig() {
+    return {
      datasource: {
        type: "datasource",
        source: SourceName.POSTGRES,
@ -70,7 +70,11 @@ describe("row api - postgres", () => {
          ca: false,
        },
      },
-    })
+    }
+  }
+
+  beforeEach(async () => {
+    postgresDatasource = await config.createDatasource(pgDatasourceConfig())

    async function createAuxTable(prefix: string) {
      return await config.createTable({
@ -1024,4 +1028,43 @@ describe("row api - postgres", () => {
      })
    })
  })
+
+  describe("POST /api/datasources/verify", () => {
+    it("should be able to verify the connection", async () => {
+      const config = pgDatasourceConfig()
+      const response = await makeRequest(
+        "post",
+        "/api/datasources/verify",
+        config
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(true)
+    })
+
+    it("should state an invalid datasource cannot connect", async () => {
+      const config = pgDatasourceConfig()
+      config.datasource.config.password = "wrongpassword"
+      const response = await makeRequest(
+        "post",
+        "/api/datasources/verify",
+        config
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.connected).toBe(false)
+      expect(response.body.error).toBeDefined()
+    })
+  })
+
+  describe("GET /api/datasources/:datasourceId/info", () => {
+    it("should fetch information about postgres datasource", async () => {
+      const primaryName = primaryPostgresTable.name
+      const response = await makeRequest(
+        "get",
+        `/api/datasources/${postgresDatasource._id}/info`
+      )
+      expect(response.status).toBe(200)
+      expect(response.body.tableNames).toBeDefined()
+      expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)
+    })
+  })
 })
--- a/packages/server/src/integrations/base/sqlTable.ts
+++ b/packages/server/src/integrations/base/sqlTable.ts
@ -206,4 +206,3 @@ class SqlTableQueryBuilder {
 }

 export default SqlTableQueryBuilder
-module.exports = SqlTableQueryBuilder
--- a/packages/server/src/integrations/googlesheets.ts
+++ b/packages/server/src/integrations/googlesheets.ts
@ -63,10 +63,13 @@ const SCHEMA: Integration = {
  relationships: false,
  docs: "https://developers.google.com/sheets/api/quickstart/nodejs",
  description:
-    "Create and collaborate on online spreadsheets in real-time and from any device. ",
+    "Create and collaborate on online spreadsheets in real-time and from any device.",
  friendlyName: "Google Sheets",
  type: "Spreadsheet",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    spreadsheetId: {
      display: "Google Sheet URL",
@ -145,7 +148,6 @@ class GoogleSheetsIntegration implements DatasourcePlus {
  async testConnection(): Promise<ConnectionInfo> {
    try {
      await this.connect()
-      await this.client.loadInfo()
      return { connected: true }
    } catch (e: any) {
      return {
@ -240,6 +242,12 @@ class GoogleSheetsIntegration implements DatasourcePlus {
    }
  }

+  async getTableNames(): Promise<string[]> {
+    await this.connect()
+    const sheets = this.client.sheetsByIndex
+    return sheets.map(s => s.title)
+  }
+
  getTableSchema(title: string, headerValues: string[], id?: string) {
    // base table
    const table: Table = {
--- a/packages/server/src/integrations/microsoftSqlServer.ts
+++ b/packages/server/src/integrations/microsoftSqlServer.ts
@ -20,7 +20,6 @@ import {
 } from "./utils"
 import Sql from "./base/sql"
 import { MSSQLTablesResponse, MSSQLColumn } from "./base/types"
-
 const sqlServer = require("mssql")
 const DEFAULT_SCHEMA = "dbo"

@ -41,7 +40,10 @@ const SCHEMA: Integration = {
    "Microsoft SQL Server is a relational database management system developed by Microsoft. ",
  friendlyName: "MS SQL Server",
  type: "Relational",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    user: {
      type: DatasourceFieldType.STRING,
@ -284,6 +286,20 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    this.schemaErrors = final.errors
  }

+  async queryTableNames() {
+    let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
+    const schema = this.config.schema || DEFAULT_SCHEMA
+    return tableInfo
+      .filter((record: any) => record.TABLE_SCHEMA === schema)
+      .map((record: any) => record.TABLE_NAME)
+      .filter((name: string) => this.MASTER_TABLES.indexOf(name) === -1)
+  }
+
+  async getTableNames() {
+    await this.connect()
+    return this.queryTableNames()
+  }
+
  async read(query: SqlQuery | string) {
    await this.connect()
    const response = await this.internalQuery(getSqlQuery(query))
--- a/packages/server/src/integrations/mysql.ts
+++ b/packages/server/src/integrations/mysql.ts
@ -36,7 +36,10 @@ const SCHEMA: Integration = {
  type: "Relational",
  description:
    "MySQL Database Service is a fully managed database service to deploy cloud-native applications. ",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    host: {
      type: DatasourceFieldType.STRING,
@ -214,20 +217,11 @@ class MySQLIntegration extends Sql implements DatasourcePlus {

  async buildSchema(datasourceId: string, entities: Record<string, Table>) {
    const tables: { [key: string]: Table } = {}
-    const database = this.config.database
    await this.connect()

    try {
      // get the tables first
-      const tablesResp: Record<string, string>[] = await this.internalQuery(
-        { sql: "SHOW TABLES;" },
-        { connect: false }
-      )
-      const tableNames: string[] = tablesResp.map(
-        (obj: any) =>
-          obj[`Tables_in_${database}`] ||
-          obj[`Tables_in_${database.toLowerCase()}`]
-      )
+      const tableNames = await this.queryTableNames()
      for (let tableName of tableNames) {
        const primaryKeys = []
        const schema: TableSchema = {}
@ -274,6 +268,28 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
    this.schemaErrors = final.errors
  }

+  async queryTableNames() {
+    const database = this.config.database
+    const tablesResp: Record<string, string>[] = await this.internalQuery(
+      { sql: "SHOW TABLES;" },
+      { connect: false }
+    )
+    return tablesResp.map(
+      (obj: any) =>
+        obj[`Tables_in_${database}`] ||
+        obj[`Tables_in_${database.toLowerCase()}`]
+    )
+  }
+
+  async getTableNames() {
+    await this.connect()
+    try {
+      return this.queryTableNames()
+    } finally {
+      await this.disconnect()
+    }
+  }
+
  async create(query: SqlQuery | string) {
    const results = await this.internalQuery(getSqlQuery(query))
    return results.length ? results : [{ created: true }]
--- a/packages/server/src/integrations/oracle.ts
+++ b/packages/server/src/integrations/oracle.ts
@ -50,7 +50,10 @@ const SCHEMA: Integration = {
  type: "Relational",
  description:
    "Oracle Database is an object-relational database management system developed by Oracle Corporation",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    host: {
      type: DatasourceFieldType.STRING,
@ -323,6 +326,13 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    this.schemaErrors = final.errors
  }

+  async getTableNames() {
+    const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
+      sql: this.COLUMNS_SQL,
+    })
+    return (columnsResponse.rows || []).map(row => row.TABLE_NAME)
+  }
+
  async testConnection() {
    const response: ConnectionInfo = {
      connected: false,
--- a/packages/server/src/integrations/postgres.ts
+++ b/packages/server/src/integrations/postgres.ts
@ -52,7 +52,10 @@ const SCHEMA: Integration = {
  type: "Relational",
  description:
    "PostgreSQL, also known as Postgres, is a free and open-source relational database management system emphasizing extensibility and SQL compliance.",
-  features: [DatasourceFeature.CONNECTION_CHECKING],
+  features: [
+    DatasourceFeature.CONNECTION_CHECKING,
+    DatasourceFeature.FETCH_TABLE_NAMES,
+  ],
  datasource: {
    host: {
      type: DatasourceFieldType.STRING,
@ -126,14 +129,15 @@ class PostgresIntegration extends Sql implements DatasourcePlus {

  COLUMNS_SQL!: string

-  PRIMARY_KEYS_SQL = `
-  select tc.table_schema, tc.table_name, kc.column_name as primary_key 
-  from information_schema.table_constraints tc
-  join 
-    information_schema.key_column_usage kc on kc.table_name = tc.table_name 
-    and kc.table_schema = tc.table_schema 
-    and kc.constraint_name = tc.constraint_name
-  where tc.constraint_type = 'PRIMARY KEY';
+  PRIMARY_KEYS_SQL = () => `
+  SELECT pg_namespace.nspname table_schema
+     , pg_class.relname table_name
+     , pg_attribute.attname primary_key
+  FROM pg_class
+  JOIN pg_index ON pg_class.oid = pg_index.indrelid AND pg_index.indisprimary
+  JOIN pg_attribute ON pg_attribute.attrelid = pg_class.oid AND pg_attribute.attnum = ANY(pg_index.indkey)
+  JOIN pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+  WHERE pg_namespace.nspname = '${this.config.schema}';
  `

  constructor(config: PostgresConfig) {
@ -239,7 +243,9 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    let tableKeys: { [key: string]: string[] } = {}
    await this.openConnection()
    try {
-      const primaryKeysResponse = await this.client.query(this.PRIMARY_KEYS_SQL)
+      const primaryKeysResponse = await this.client.query(
+        this.PRIMARY_KEYS_SQL()
+      )
      for (let table of primaryKeysResponse.rows) {
        const tableName = table.table_name
        if (!tableKeys[tableName]) {
@ -311,6 +317,17 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
    }
  }

+  async getTableNames() {
+    try {
+      await this.openConnection()
+      const columnsResponse: { rows: PostgresColumn[] } =
+        await this.client.query(this.COLUMNS_SQL)
+      return columnsResponse.rows.map(row => row.table_name)
+    } finally {
+      await this.closeConnection()
+    }
+  }
+
  async create(query: SqlQuery | string) {
    const response = await this.internalQuery(getSqlQuery(query))
    return response.rows.length ? response.rows : [{ created: true }]
--- a/packages/server/src/integrations/tests/googlesheets.spec.ts
+++ b/packages/server/src/integrations/tests/googlesheets.spec.ts
@ -17,14 +17,15 @@ jest.mock("google-spreadsheet")
 const { GoogleSpreadsheet } = require("google-spreadsheet")

 const sheetsByTitle: { [title: string]: GoogleSpreadsheetWorksheet } = {}
+const sheetsByIndex: GoogleSpreadsheetWorksheet[] = []
+const mockGoogleIntegration = {
+  useOAuth2Client: jest.fn(),
+  loadInfo: jest.fn(),
+  sheetsByTitle,
+  sheetsByIndex,
+}

-GoogleSpreadsheet.mockImplementation(() => {
-  return {
-    useOAuth2Client: jest.fn(),
-    loadInfo: jest.fn(),
-    sheetsByTitle,
-  }
-})
+GoogleSpreadsheet.mockImplementation(() => mockGoogleIntegration)

 import { structures } from "@budibase/backend-core/tests"
 import TestConfiguration from "../../tests/utilities/TestConfiguration"
@ -53,6 +54,8 @@ describe("Google Sheets Integration", () => {
      },
    })
    await config.init()
+
+    jest.clearAllMocks()
  })

  function createBasicTable(name: string, columns: string[]): Table {
@ -88,7 +91,7 @@ describe("Google Sheets Integration", () => {
  }

  describe("update table", () => {
-    test("adding a new field will be adding a new header row", async () => {
+    it("adding a new field will be adding a new header row", async () => {
      await config.doInContext(structures.uuid(), async () => {
        const tableColumns = ["name", "description", "new field"]
        const table = createBasicTable(structures.uuid(), tableColumns)
@ -103,7 +106,7 @@ describe("Google Sheets Integration", () => {
      })
    })

-    test("removing an existing field will remove the header from the google sheet", async () => {
+    it("removing an existing field will remove the header from the google sheet", async () => {
      const sheet = await config.doInContext(structures.uuid(), async () => {
        const tableColumns = ["name"]
        const table = createBasicTable(structures.uuid(), tableColumns)
@ -123,4 +126,33 @@ describe("Google Sheets Integration", () => {
      expect((sheet.setHeaderRow as any).mock.calls[0][0]).toHaveLength(1)
    })
  })
+
+  describe("getTableNames", () => {
+    it("can fetch table names", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const sheetNames: string[] = []
+        for (let i = 0; i < 5; i++) {
+          const sheet = createSheet({ headerValues: [] })
+          sheetsByIndex.push(sheet)
+          sheetNames.push(sheet.title)
+        }
+
+        const res = await integration.getTableNames()
+
+        expect(mockGoogleIntegration.loadInfo).toBeCalledTimes(1)
+        expect(res).toEqual(sheetNames)
+      })
+    })
+  })
+
+  describe("testConnection", () => {
+    it("can test successful connections", async () => {
+      await config.doInContext(structures.uuid(), async () => {
+        const res = await integration.testConnection()
+
+        expect(mockGoogleIntegration.loadInfo).toBeCalledTimes(1)
+        expect(res).toEqual({ connected: true })
+      })
+    })
+  })
 })
--- a/packages/server/src/utilities/fileSystem/app.ts
+++ b/packages/server/src/utilities/fileSystem/app.ts
@ -35,7 +35,7 @@ export const getComponentLibraryManifest = async (library: string) => {
  const filename = "manifest.json"

  if (env.isDev() || env.isTest()) {
-    const path = join(TOP_LEVEL_PATH, "../client", filename)
+    const path = join(TOP_LEVEL_PATH, "packages/client", filename)
    // always load from new so that updates are refreshed
    delete require.cache[require.resolve(path)]
    return require(path)
--- a/packages/server/src/utilities/fileSystem/clientLibrary.ts
+++ b/packages/server/src/utilities/fileSystem/clientLibrary.ts
@ -1,4 +1,4 @@
-import { join } from "path"
+import path, { join } from "path"
 import { ObjectStoreBuckets } from "../../constants"
 import fs from "fs"
 import { objectStore } from "@budibase/backend-core"
@ -6,6 +6,10 @@ import { resolve } from "../centralPath"
 import env from "../../environment"
 import { TOP_LEVEL_PATH } from "./filesystem"

+export function devClientLibPath() {
+  return require.resolve("@budibase/client")
+}
+
 /**
 * Client library paths in the object store:
 * Previously, the entire client library package was downloaded from NPM
@ -89,9 +93,10 @@ export async function updateClientLibrary(appId: string) {
  let manifest, client

  if (env.isDev()) {
+    const clientPath = devClientLibPath()
    // Load the symlinked version in dev which is always the newest
-    manifest = require.resolve("@budibase/client/manifest.json")
-    client = require.resolve("@budibase/client")
+    manifest = join(path.dirname(path.dirname(clientPath)), "manifest.json")
+    client = clientPath
  } else {
    // Load the bundled version in prod
    manifest = resolve(TOP_LEVEL_PATH, "client", "manifest.json")
--- a/packages/server/src/utilities/fileSystem/filesystem.ts
+++ b/packages/server/src/utilities/fileSystem/filesystem.ts
@ -4,9 +4,11 @@ import { budibaseTempDir } from "../budibaseDir"
 import { join } from "path"
 import env from "../../environment"
 import tar from "tar"
+import environment from "../../environment"
 const uuid = require("uuid/v4")

-export const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
+export const TOP_LEVEL_PATH =
+  environment.TOP_LEVEL_PATH || join(__dirname, "..", "..", "..")

 /**
 * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
--- a/packages/server/tsconfig.build.json
+++ b/packages/server/tsconfig.build.json
@ -10,7 +10,15 @@
    "incremental": true,
    "types": ["node", "jest"],
    "outDir": "dist/src",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
  },
  "include": ["src/**/*"],
  "exclude": [
--- a/packages/server/tsconfig.json
+++ b/packages/server/tsconfig.json
@ -18,12 +18,6 @@
    "require": ["tsconfig-paths/register"],
    "swc": true
  },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../shared-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
  "include": ["src/**/*", "specs"],
  "exclude": ["node_modules", "dist"]
 }
--- a/packages/shared-core/package.json
+++ b/packages/shared-core/package.json
@ -26,5 +26,19 @@
    "concurrently": "^7.6.0",
    "rimraf": "3.0.2",
    "typescript": "4.7.3"
+  },
+  "nx": {
+    "targets": {
+      "build": {
+        "dependsOn": [
+          {
+            "projects": [
+              "@budibase/types"
+            ],
+            "target": "build"
+          }
+        ]
+      }
+    }
  }
 }
--- a/packages/shared-core/src/filters.ts
+++ b/packages/shared-core/src/filters.ts
@ -328,8 +328,8 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
      return (
        docValue == null ||
        docValue === "" ||
-        docValue < testValue.low ||
-        docValue > testValue.high
+        +docValue < testValue.low ||
+        +docValue > testValue.high
      )
    }
  )
--- a/packages/string-templates/tsconfig.json
+++ b/packages/string-templates/tsconfig.json
@ -1,12 +1,11 @@
 {
  "include": ["src/**/*"],
-
  "compilerOptions": {
    "allowJs": true,
    "declaration": true,
    "emitDeclarationOnly": true,
    "outDir": "dist",
    "esModuleInterop": true,
-    "types": ["node"]
+    "types": ["node", "jest"]
  }
 }
--- a/packages/types/src/api/web/app/datasource.ts
+++ b/packages/types/src/api/web/app/datasource.ts
@ -23,6 +23,10 @@ export interface VerifyDatasourceResponse {
  error?: string
 }

+export interface FetchDatasourceInfoResponse {
+  tableNames: string[]
+}
+
 export interface UpdateDatasourceRequest extends Datasource {
  datasource: Datasource
 }
--- a/packages/types/src/sdk/datasources.ts
+++ b/packages/types/src/sdk/datasources.ts
@ -75,6 +75,7 @@ export enum FilterType {

 export enum DatasourceFeature {
  CONNECTION_CHECKING = "connection",
+  FETCH_TABLE_NAMES = "fetch_table_names",
 }

 export interface StepDefinition {
@ -150,4 +151,5 @@ export interface DatasourcePlus extends IntegrationBase {
  getBindingIdentifier(): string
  getStringConcat(parts: string[]): string
  buildSchema(datasourceId: string, entities: Record<string, Table>): any
+  getTableNames(): Promise<string[]>
 }
--- a/packages/worker/.dockerignore
+++ b/packages/worker/.dockerignore
@ -1,7 +1,3 @@
-node_modules
-npm-debug.log
-Dockerfile
-.dockerignore
-.git
-.gitignore
-
+*
+!/dist/
+!/docker_run.sh
--- a/packages/worker/Dockerfile
+++ b/packages/worker/Dockerfile
@ -7,13 +7,20 @@ LABEL com.centurylinklabs.watchtower.lifecycle.post-check="scripts/watchtower-ho

 WORKDIR /app

-# copy files and install dependencies
-COPY . ./
 # handle node-gyp
-RUN apk add --no-cache --virtual .gyp python3 make g++ \
-    && yarn && apk del .gyp
+RUN apk add --no-cache --virtual .gyp python3 make g++
 RUN yarn global add pm2

+
+COPY dist/package.json .
+RUN yarn
+# Remove unneeded data from file system to reduce image size
+RUN apk del .gyp \
+    && yarn cache clean
+
+COPY dist/ dist/
+COPY docker_run.sh .
+
 EXPOSE 4001

 # have to add node environment production after install
--- a/packages/worker/nodemon.json
+++ b/packages/worker/nodemon.json
@ -6,5 +6,5 @@
    "src/**/*.spec.js",
    "../backend-core/dist/**/*"
  ],
-  "exec": "ts-node src/index.ts"
+  "exec": "yarn build && node dist/index.js"
 }
--- a/packages/worker/package.json
+++ b/packages/worker/package.json
@ -13,15 +13,15 @@
  ],
  "scripts": {
    "prebuild": "rimraf dist/",
-    "build": "tsc -p tsconfig.build.json",
-    "postbuild": "copyfiles -u 1 src/**/*.hbs dist/",
+    "build": "cd ../.. && nx build @budibase/worker",
    "build:dev": "yarn prebuild && tsc --build --watch --preserveWatchOutput",
    "run:docker": "node dist/index.js",
    "debug": "yarn build && node --expose-gc --inspect=9223 dist/index.js",
    "run:docker:cluster": "pm2-runtime start pm2.config.js",
-    "build:docker": "docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "predocker": "yarn build --configuration=production",
+    "build:docker": "yarn predocker && docker build . -t worker-service --label version=$BUDIBASE_RELEASE_VERSION",
    "dev:stack:init": "node ./scripts/dev/manage.js init",
-    "dev:builder": "npm run dev:stack:init && nodemon",
+    "dev:builder": "npm run dev:stack:init && rimraf dist/ && nodemon",
    "dev:built": "yarn run dev:stack:init && yarn run run:docker",
    "test": "bash scripts/test.sh",
    "test:watch": "jest --watch",
--- a/packages/worker/project.json
+++ b/packages/worker/project.json
@ -0,0 +1,47 @@
+{
+  "name": "@budibase/worker",
+  "$schema": "../../node_modules/nx/schemas/project-schema.json",
+  "targets": {
+    "build": {
+      "executor": "@nx/esbuild:esbuild",
+      "outputs": ["{options.outputPath}"],
+      "options": {
+        "main": "packages/worker/src/index.ts",
+        "outputPath": "packages/worker/dist",
+        "outputFileName": "index.js",
+        "tsConfig": "packages/worker/tsconfig.build.json",
+        "assets": [
+          {
+            "glob": "**/*.hbs",
+            "input": "packages/worker/src/constants/templates",
+            "output": "."
+          }
+        ],
+        "format": ["cjs"],
+        "esbuildOptions": {
+          "outExtension": {
+            ".js": ".js"
+          },
+          "sourcemap": true
+        },
+        "minify": true,
+        "generatePackageJson": true,
+        "skipTypeCheck": true
+      },
+      "configurations": {
+        "production": {
+          "skipTypeCheck": false,
+          "esbuildOptions": {
+            "sourcemap": false
+          }
+        }
+      },
+      "dependsOn": [
+        {
+          "projects": ["@budibase/types", "@budibase/string-templates"],
+          "target": "build"
+        }
+      ]
+    }
+  }
+}
--- a/packages/worker/tsconfig.build.json
+++ b/packages/worker/tsconfig.build.json
@ -8,13 +8,18 @@
    "esModuleInterop": true,
    "resolveJsonModule": true,
    "incremental": true,
-    "types": [ "node", "jest" ],
+    "types": ["node", "jest"],
    "outDir": "dist",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
  },
-  "include": [
-    "src/**/*"
-  ],
+  "include": ["src/**/*"],
  "exclude": [
    "node_modules",
    "dist",
@ -22,4 +27,4 @@
    "**/*.spec.js",
    "**/*.spec.ts"
  ]
-}
+}
--- a/packages/worker/tsconfig.json
+++ b/packages/worker/tsconfig.json
@ -16,7 +16,6 @@
    "require": ["tsconfig-paths/register"],
    "swc": true
  },
-  "references": [{ "path": "../types" }, { "path": "../backend-core" }],
  "include": ["src/**/*"],
  "exclude": ["dist"]
 }
--- a/qa-core/tsconfig.json
+++ b/qa-core/tsconfig.json
@ -22,10 +22,6 @@
  "ts-node": {
    "require": ["tsconfig-paths/register"]
  },
-  "references": [
-    { "path": "../packages/types" },
-    { "path": "../packages/backend-core" }
-  ],
  "include": ["src/**/*"],
  "exclude": ["node_modules", "dist"]
 }
--- a/scripts/bootstrap.sh
+++ b/scripts/bootstrap.sh
@ -3,5 +3,4 @@ if [ -d "packages/pro/packages" ]; then

  yarn
  lerna bootstrap
-  yarn setup
 fi
--- a/scripts/build.js
+++ b/scripts/build.js
@ -0,0 +1,69 @@
+#!/usr/bin/node
+
+const start = Date.now()
+
+const glob = require("glob")
+const fs = require("fs")
+const path = require("path")
+
+const { build } = require("esbuild")
+
+const { default: NodeResolve } = require("@esbuild-plugins/node-resolve")
+
+var argv = require("minimist")(process.argv.slice(2))
+
+function runBuild(entry, outfile) {
+  const isDev = process.env.NODE_ENV !== "production"
+  const tsconfig = argv["p"] || `tsconfig.build.json`
+
+  const sharedConfig = {
+    entryPoints: [entry],
+    bundle: true,
+    minify: !isDev,
+    sourcemap: isDev,
+    tsconfig,
+    plugins: [
+      NodeResolve({
+        extensions: [".ts", ".js"],
+        onResolved: resolved => {
+          if (resolved.includes("node_modules")) {
+            return {
+              external: true,
+            }
+          }
+          return resolved
+        },
+      }),
+    ],
+    target: "node14",
+    preserveSymlinks: true,
+    loader: {
+      ".svelte": "copy",
+    },
+  }
+
+  build({
+    ...sharedConfig,
+    platform: "node",
+    outfile,
+  }).then(() => {
+    glob(`${process.cwd()}/src/**/*.hbs`, {}, (err, files) => {
+      for (const file of files) {
+        fs.copyFileSync(file, `${process.cwd()}/dist/${path.basename(file)}`)
+      }
+
+      console.log(
+        "\x1b[32m%s\x1b[0m",
+        `Build successfully in ${(Date.now() - start) / 1000} seconds`
+      )
+    })
+  })
+}
+
+if (require.main === module) {
+  const entry = argv["e"] || "./src/index.ts"
+  const outfile = `dist/${entry.split("/").pop().replace(".ts", ".js")}`
+  runBuild(entry, outfile)
+} else {
+  module.exports = runBuild
+}
--- a/yarn.lock
+++ b/yarn.lock
 @ -1 +1 @@
 .10.0
 .10.0