Merge remote-tracking branch 'origin/master' into feature/toggle-all-formblock-fields

.github/workflows/deploy-cloud.yaml

@@ -1,48 +0,0 @@
-name: Budibase Deploy Production
-
-on:
-  workflow_dispatch:
-    inputs:
-      version:
-        description: Budibase release version. For example - 1.0.0
-        required: false
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then
-            echo "Workflow Dispatch can only be run on tags"
-            exit 1
-          fi
-      - uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          if [ -z "${{ github.event.inputs.version }}" ]; then
-            release_version=$(cat lerna.json | jq -r '.version')
-          else
-            release_version=${{ github.event.inputs.version }}
-          fi
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - uses: passeidireto/trigger-external-workflow-action@main
-        env:
-          PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }}
-        with:
-          repository: budibase/budibase-deploys
-          event: budicloud-prod-deploy
-          github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

.github/workflows/release-master.yml

@@ -1,178 +0,0 @@
-name: Budibase Release
-concurrency:
-  group: release
-  cancel-in-progress: false
-
-on:
-  push:
-    tags:
-      - "[0-9]+.[0-9]+.[0-9]+"
-      # Exclude all pre-releases
-      - "!*[0-9]+.[0-9]+.[0-9]+-*"
-
-env:
-  # Posthog token used by ui at build time
-  POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU
-  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-jobs:
-  release-images:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - uses: actions/setup-node@v1
-        with:
-          node-version: 18.x
-          cache: yarn
-
-      - run: yarn install --frozen-lockfile
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - run: yarn lint
-      - run: yarn build
-      - run: yarn build:sdk
-
-      - name: Publish budibase packages to NPM
-        env:
-          NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: |
-          # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
-          git config --global user.name "Budibase Release Bot"
-          git config --global user.email "<>"
-          git submodule foreach git commit -a -m 'Release process'
-          git commit -a -m 'Release process'
-          echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
-          yarn release
-
-      - name: "Get Current tag"
-        id: currenttag
-        run: |
-          version=$(./scripts/getCurrentVersion.sh)
-          echo "Using tag $version"
-          echo "version=$version" >> "$GITHUB_OUTPUT"
-
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-
-      - name: Docker login
-        run: |
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-
-      - name: Build worker docker
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
-          file: ./packages/worker/Dockerfile.v2
-          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-        env:
-          IMAGE_NAME: budibase/worker
-          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
-          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
-
-      - name: Build server docker
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
-          build-args: |
-            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
-          file: ./packages/server/Dockerfile.v2
-          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-        env:
-          IMAGE_NAME: budibase/apps
-          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
-          BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }}
-
-      - name: Build proxy docker
-        uses: docker/build-push-action@v5
-        with:
-          context: ./hosting/proxy
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }}
-          file: ./hosting/proxy/Dockerfile
-          cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest
-          cache-to: type=inline
-        env:
-          IMAGE_NAME: budibase/proxy
-          IMAGE_TAG: ${{ steps.currenttag.outputs.version }}
-
-  release-helm-chart:
-    needs: [release-images]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        id: helm-install
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
-      # we need to create new package in a different dir, merge the index and move the package back
-      - name: Build and release helm chart
-        run: |
-          git config user.name "Budibase Helm Bot"
-          git config user.email "<>"
-          git reset --hard
-          git fetch
-          mkdir sync
-          echo "Packaging chart to sync dir"
-          helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync
-          echo "Packaging successful"
-          git checkout gh-pages
-          echo "Indexing helm repo"         
-          helm repo index --merge docs/index.yaml sync
-          mv -f sync/* docs
-          rm -rf sync
-          echo "Pushing new helm release"
-          git add -A
-          git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
-          git push
-
-  trigger-deploy-to-qa-env:
-    needs: [release-helm-chart]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-
-      - uses: peter-evans/repository-dispatch@v2
-        with:
-          repository: budibase/budibase-deploys
-          event-type: budicloud-qa-deploy
-          token: ${{ secrets.GH_ACCESS_TOKEN }}
-          client-payload: |-
-            {
-              "VERSION": "${{ github.ref_name }}",
-              "REF_NAME": "${{ github.ref_name}}"
-            }

.github/workflows/release-selfhost.yml

@@ -1,125 +0,0 @@
-name: Budibase Release Selfhost
-
-on:
-  workflow_dispatch:
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-
-      - uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-          fetch-depth: 0
-
-      - name: Fail if tag is not in master
-        run: |
-          if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
-            echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
-            exit 1
-          fi
-
-      - name: Use Node.js 18.x
-        uses: actions/setup-node@v1
-        with:
-          node-version: 18.x
-
-      - name: Get the latest budibase release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-
-      - name: Tag and release Docker images (Self Host)
-        run: |
-          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-
-          release_tag=${{ env.RELEASE_VERSION }}
-
-          # Pull apps and worker images
-          docker pull budibase/apps:$release_tag
-          docker pull budibase/worker:$release_tag
-          docker pull budibase/proxy:$release_tag
-
-          # Tag apps and worker images
-          docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG
-          docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG
-          docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG
-
-          # Push images
-          docker push budibase/apps:$SELFHOST_TAG
-          docker push budibase/worker:$SELFHOST_TAG
-          docker push budibase/proxy:$SELFHOST_TAG
-        env:
-          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
-          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
-          SELFHOST_TAG: latest
-
-      - name: Bootstrap and build (CLI)
-        run: |
-          yarn
-          yarn build
-
-      - name: Build OpenAPI spec
-        run: |
-          pushd packages/server
-          yarn
-          yarn specs
-          popd
-
-      - name: Setup Helm
-        uses: azure/setup-helm@v1
-        id: helm-install
-
-      # due to helm repo index issue: https://github.com/helm/helm/issues/7363
-      # we need to create new package in a different dir, merge the index and move the package back
-      - name: Build and release helm chart
-        run: |
-          git config user.name "Budibase Helm Bot"
-          git config user.email "<>"
-          git reset --hard
-          git fetch
-          mkdir sync
-          echo "Packaging chart to sync dir"
-          helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync
-          echo "Packaging successful"
-          git checkout gh-pages
-          echo "Indexing helm repo"
-          helm repo index --merge docs/index.yaml sync
-          mv -f sync/* docs
-          rm -rf sync
-          echo "Pushing new helm release"
-          git add -A
-          git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
-          git push
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Perform Github Release
-        uses: softprops/action-gh-release@v1
-        with:
-          name: ${{ env.RELEASE_VERSION }}
-          tag_name: ${{ env.RELEASE_VERSION }}
-          generate_release_notes: true
-          files: |
-            packages/cli/build/cli-win.exe
-            packages/cli/build/cli-linux
-            packages/cli/build/cli-macos 
-            packages/server/specs/openapi.yaml
-            packages/server/specs/openapi.json
-
-      - name: Discord Webhook Action
-        uses: tsickert/discord-webhook@v4.0.0
-        with:
-          webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
-          content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host."
-          embed-title: ${{ env.RELEASE_VERSION }}

.github/workflows/release-singleimage.yml

@@ -1,86 +0,0 @@
-name: Deploy Budibase Single Container Image to DockerHub
-
-on:
-  workflow_dispatch:
-
-env:
-  CI: true
-  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-  REGISTRY_URL: registry.hub.docker.com
-jobs:
-  build:
-    name: "build"
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        node-version: [18.x]
-    steps:
-      - name: Maximize build space
-        uses: easimon/maximize-build-space@master
-        with:
-          root-reserve-mb: 30000
-          swap-size-mb: 1024
-          remove-android: "true"
-          remove-dotnet: "true"
-      - name: Fail if not a tag
-        run: |
-          if [[ $GITHUB_REF != refs/tags/* ]]; then 
-            echo "Workflow Dispatch can only be run on tags" 
-            exit 1 
-          fi
-      - name: "Checkout"
-        uses: actions/checkout@v2
-        with:
-          submodules: true
-          token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
-
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
-        with:
-          node-version: ${{ matrix.node-version }}
-      - name: Setup QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Setup Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Run Yarn
-        run: yarn
-      - name: Update versions
-        run: ./scripts/updateVersions.sh
-      - name: Run Yarn Build
-        run: yarn build
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_API_KEY }}
-      - name: Get the latest release version
-        id: version
-        run: |
-          release_version=$(cat lerna.json | jq -r '.version')
-          echo $release_version
-          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
-      - name: Tag and release Budibase service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64,linux/arm64
-          build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile.v2
-        env:
-          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}
-      - name: Tag and release Budibase Azure App Service docker image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          push: true
-          platforms: linux/amd64
-          build-args: |
-            TARGETBUILD=aas
-            BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }}
-          tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }}
-          file: ./hosting/single/Dockerfile.v2
-        env:
-          BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }}

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.13.5",
+  "version": "2.13.7",
   "npmClient": "yarn",
   "packages": [
     "packages/*"

packages/backend-core/src/cache/appMetadata.ts

@@ -19,7 +19,7 @@ async function populateFromDB(appId: string) {
   return doWithDB(
     appId,
     (db: Database) => {
-      return db.get(DocumentType.APP_METADATA)
+      return db.get<App>(DocumentType.APP_METADATA)
     },
     { skip_setup: true }
   )

packages/backend-core/src/context/Context.ts

@@ -4,7 +4,7 @@ import { ContextMap } from "./types"
 export default class Context {
   static storage = new AsyncLocalStorage<ContextMap>()
 
-  static run(context: ContextMap, func: any) {
+  static run<T>(context: ContextMap, func: () => T) {
     return Context.storage.run(context, () => func())
   }
 

packages/backend-core/src/context/mainContext.ts

@@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap {
   return context
 }
 
-async function newContext(updates: ContextMap, task: any) {
+async function newContext<T>(updates: ContextMap, task: () => T) {
   // see if there already is a context setup
   let context: ContextMap = updateContext(updates)
   return Context.run(context, task)
 }
 
-export async function doInAutomationContext(params: {
+export async function doInAutomationContext<T>(params: {
   appId: string
   automationId: string
-  task: any
-}): Promise<any> {
+  task: () => T
+}): Promise<T> {
   const tenantId = getTenantIDFromAppID(params.appId)
   return newContext(
     {
@@ -144,10 +144,10 @@ export async function doInTenant<T>(
   return newContext(updates, task)
 }
 
-export async function doInAppContext(
+export async function doInAppContext<T>(
   appId: string | null,
-  task: any
-): Promise<any> {
+  task: () => T
+): Promise<T> {
   if (!appId && !env.isTest()) {
     throw new Error("appId is required")
   }
@@ -165,10 +165,10 @@ export async function doInAppContext(
   return newContext(updates, task)
 }
 
-export async function doInIdentityContext(
+export async function doInIdentityContext<T>(
   identity: IdentityContext,
-  task: any
-): Promise<any> {
+  task: () => T
+): Promise<T> {
   if (!identity) {
     throw new Error("identity is required")
   }
@@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database {
  */
 export function getAppDB(opts?: any): Database {
   const appId = getAppId()
+  if (!appId) {
+    throw new Error("Unable to retrieve app DB - no app ID.")
+  }
   return getDB(appId, opts)
 }
 

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -48,10 +48,7 @@ export class DatabaseImpl implements Database {
 
   private readonly couchInfo = getCouchInfo()
 
-  constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) {
-    if (dbName == null) {
-      throw new Error("Database name cannot be undefined.")
-    }
+  constructor(dbName: string, opts?: DatabaseOpts, connection?: string) {
     this.name = dbName
     this.pouchOpts = opts || {}
     if (connection) {
@@ -112,7 +109,7 @@ export class DatabaseImpl implements Database {
     }
   }
 
-  async get<T>(id?: string): Promise<T | any> {
+  async get<T extends Document>(id?: string): Promise<T> {
     const db = await this.checkSetup()
     if (!id) {
       throw new Error("Unable to get doc without a valid _id.")
@@ -120,6 +117,28 @@ export class DatabaseImpl implements Database {
     return this.updateOutput(() => db.get(id))
   }
 
+  async getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean }
+  ): Promise<T[]> {
+    // get unique
+    ids = [...new Set(ids)]
+    const response = await this.allDocs<T>({
+      keys: ids,
+      include_docs: true,
+    })
+    const NOT_FOUND = "not_found"
+    const rows = response.rows.filter(row => row.error !== NOT_FOUND)
+    const someMissing = rows.length !== response.rows.length
+    // some were filtered out - means some missing
+    if (!opts?.allowMissing && someMissing) {
+      const missing = response.rows.filter(row => row.error === NOT_FOUND)
+      const missingIds = missing.map(row => row.key).join(", ")
+      throw new Error(`Unable to get documents: ${missingIds}`)
+    }
+    return rows.map(row => row.doc!)
+  }
+
   async remove(idOrDoc: string | Document, rev?: string) {
     const db = await this.checkSetup()
     let _id: string

packages/backend-core/src/db/db.ts

@@ -1,10 +1,7 @@
-import env from "../environment"
 import { directCouchQuery, DatabaseImpl } from "./couch"
-import { CouchFindOptions, Database } from "@budibase/types"
+import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types"
 
-const dbList = new Set()
-
-export function getDB(dbName?: string, opts?: any): Database {
+export function getDB(dbName: string, opts?: DatabaseOpts): Database {
   return new DatabaseImpl(dbName, opts)
 }
 
@@ -14,7 +11,7 @@ export function getDB(dbName?: string, opts?: any): Database {
 export async function doWithDB<T>(
   dbName: string,
   cb: (db: Database) => Promise<T>,
-  opts = {}
+  opts?: DatabaseOpts
 ) {
   const db = getDB(dbName, opts)
   // need this to be async so that we can correctly close DB after all
@@ -22,13 +19,6 @@ export async function doWithDB<T>(
   return await cb(db)
 }
 
-export function allDbs() {
-  if (!env.isTest()) {
-    throw new Error("Cannot be used outside test environment.")
-  }
-  return [...dbList]
-}
-
 export async function directCouchAllDbs(queryString?: string) {
   let couchPath = "/_all_dbs"
   if (queryString) {

packages/builder/src/components/integration/RestQueryViewer.svelte

@@ -404,7 +404,7 @@
     datasource = $datasources.list.find(ds => ds._id === query?.datasourceId)
     const datasourceUrl = datasource?.config.url
     const qs = query?.fields.queryString
-    breakQs = restUtils.breakQueryString(qs)
+    breakQs = restUtils.breakQueryString(encodeURI(qs))
     breakQs = runtimeToReadableMap(mergedBindings, breakQs)
 
     const path = query.fields.path
@@ -652,7 +652,7 @@
     <div class="bottom">
       <Layout paddingY="S" gap="S">
         <Divider />
-        {#if !response && Object.keys(schema).length === 0}
+        {#if !response && Object.keys(schema || {}).length === 0}
           <Heading size="M">Response</Heading>
           <div class="placeholder">
             <div class="placeholder-internal">

packages/client/src/components/app/Text.svelte

@@ -94,7 +94,7 @@
   .align--right {
     text-align: right;
   }
-  .align-justify {
+  .align--justify {
     text-align: justify;
   }
 </style>

packages/pro

@@ -1 +1 @@
-Subproject commit ad9a0085bee0c4f3184acd86cadd872ea9917e88
+Subproject commit e202f415d9fa540d08cc2ba6e27394fbc22f357b

packages/server/src/api/controllers/datasource.ts

@@ -337,7 +337,7 @@ export async function destroy(ctx: UserCtx) {
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
     await destroyInternalTablesBySourceId(datasourceId)
   } else {
-    const queries = await db.allDocs(getQueryParams(datasourceId, null))
+    const queries = await db.allDocs(getQueryParams(datasourceId))
     await db.bulkDocs(
       queries.rows.map((row: any) => ({
         _id: row.id,

packages/server/src/api/controllers/row/internal.ts

@@ -1,9 +1,5 @@
 import * as linkRows from "../../../db/linkedRows"
-import {
-  generateRowID,
-  getMultiIDParams,
-  InternalTables,
-} from "../../../db/utils"
+import { generateRowID, InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import {
   cleanupAttachments,
@@ -240,8 +236,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
   const linkVals = links as LinkDocumentValue[]
 
   // look up the actual rows based on the ids
-  const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id))
-  let linkedRows = (await db.allDocs<Row>(params)).rows.map(row => row.doc!)
+  let linkedRows = await db.getMultiple<Row>(
+    linkVals.map(linkVal => linkVal.id),
+    { allowMissing: true }
+  )
 
   // get the linked tables
   const linkTableIds = getLinkedTableIDs(table as Table)

packages/server/src/api/controllers/row/utils.ts

@@ -1,21 +1,9 @@
 import { InternalTables } from "../../../db/utils"
 import * as userController from "../user"
 import { context } from "@budibase/backend-core"
-import {
-  Ctx,
-  FieldType,
-  ManyToOneRelationshipFieldMetadata,
-  OneToManyRelationshipFieldMetadata,
-  Row,
-  SearchFilters,
-  Table,
-  UserCtx,
-} from "@budibase/types"
-import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
-import sdk from "../../../sdk"
+import { Ctx, Row, UserCtx } from "@budibase/types"
 
 import validateJs from "validate.js"
-import { cloneDeep } from "lodash/fp"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {

packages/server/src/automations/triggers.ts

@@ -94,7 +94,7 @@ export async function externalTrigger(
   automation: Automation,
   params: { fields: Record<string, any>; timeout?: number },
   { getResponses }: { getResponses?: boolean } = {}
-) {
+): Promise<any> {
   if (
     automation.definition != null &&
     automation.definition.trigger != null &&

packages/server/src/db/linkedRows/index.ts

@@ -8,7 +8,7 @@ import {
   getLinkedTable,
 } from "./linkUtils"
 import flatten from "lodash/flatten"
-import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils"
+import { USER_METDATA_PREFIX } from "../utils"
 import partition from "lodash/partition"
 import { getGlobalUsersFromMetadata } from "../../utilities/global"
 import { processFormulas } from "../../utilities/rowProcessor"
@@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) {
   const db = context.getAppDB()
   const linkedRowIds = links.map(link => link.id)
   const uniqueRowIds = [...new Set(linkedRowIds)]
-  let dbRows = (await db.allDocs<Row>(getMultiIDParams(uniqueRowIds))).rows.map(
-    row => row.doc!
-  )
+  let dbRows = await db.getMultiple<Row>(uniqueRowIds, { allowMissing: true })
   // convert the unique db rows back to a full list of linked rows
   const linked = linkedRowIds
     .map(id => dbRows.find(row => row && row._id === id))

packages/server/src/db/utils.ts

@@ -6,6 +6,7 @@ import {
   RelationshipFieldMetadata,
   VirtualDocumentType,
   INTERNAL_TABLE_SOURCE_ID,
+  DatabaseQueryOpts,
 } from "@budibase/types"
 import { FieldTypes } from "../constants"
 export { DocumentType, VirtualDocumentType } from "@budibase/types"
@@ -229,7 +230,10 @@ export function getAutomationMetadataParams(otherProps: any = {}) {
 /**
  * Gets parameters for retrieving a query, this is a utility function for the getDocParams function.
  */
-export function getQueryParams(datasourceId?: Optional, otherProps: any = {}) {
+export function getQueryParams(
+  datasourceId?: Optional,
+  otherProps: Partial<DatabaseQueryOpts> = {}
+) {
   if (datasourceId == null) {
     return getDocParams(DocumentType.QUERY, null, otherProps)
   }
@@ -256,7 +260,7 @@ export function generateMetadataID(type: string, entityId: string) {
 export function getMetadataParams(
   type: string,
   entityId?: Optional,
-  otherProps: any = {}
+  otherProps: Partial<DatabaseQueryOpts> = {}
 ) {
   let docId = `${type}${SEPARATOR}`
   if (entityId != null) {
@@ -269,7 +273,9 @@ export function generateMemoryViewID(viewName: string) {
   return `${DocumentType.MEM_VIEW}${SEPARATOR}${viewName}`
 }
 
-export function getMemoryViewParams(otherProps: any = {}) {
+export function getMemoryViewParams(
+  otherProps: Partial<DatabaseQueryOpts> = {}
+) {
   return getDocParams(DocumentType.MEM_VIEW, null, otherProps)
 }
 
@@ -277,16 +283,6 @@ export function generatePluginID(name: string) {
   return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
 }
 
-/**
- * This can be used with the db.allDocs to get a list of IDs
- */
-export function getMultiIDParams(ids: string[]) {
-  return {
-    keys: ids,
-    include_docs: true,
-  }
-}
-
 /**
  * Generates a new view ID.
  * @returns The new view ID which the view doc can be stored under.

packages/server/src/integrations/redis.ts

@@ -165,10 +165,22 @@ class RedisIntegration {
       // commands split line by line
       const commands = query.json.trim().split("\n")
       let pipelineCommands = []
+      let tokenised
 
       // process each command separately
       for (let command of commands) {
-        const tokenised = command.trim().split(" ")
+        const valueToken = command.trim().match(/".*"/)
+        if (valueToken?.[0]) {
+          tokenised = [
+            ...command
+              .substring(0, command.indexOf(valueToken[0]) - 1)
+              .trim()
+              .split(" "),
+            valueToken?.[0],
+          ]
+        } else {
+          tokenised = command.trim().split(" ")
+        }
         // Pipeline only accepts lower case commands
         tokenised[0] = tokenised[0].toLowerCase()
         pipelineCommands.push(tokenised)

packages/server/src/integrations/tests/redis.spec.ts

@@ -85,4 +85,21 @@ describe("Redis Integration", () => {
       ["get", "foo"],
     ])
   })
+
+  it("calls the pipeline method with double quoted phrase values", async () => {
+    const body = {
+      json: 'SET foo "What a wonderful world!"\nGET foo',
+    }
+
+    // ioredis-mock doesn't support pipelines
+    config.integration.client.pipeline = jest.fn(() => ({
+      exec: jest.fn(() => [[]]),
+    }))
+
+    await config.integration.command(body)
+    expect(config.integration.client.pipeline).toHaveBeenCalledWith([
+      ["set", "foo", '"What a wonderful world!"'],
+      ["get", "foo"],
+    ])
+  })
 })

packages/server/src/sdk/app/tables/getters.ts

@@ -1,5 +1,5 @@
 import { context } from "@budibase/backend-core"
-import { getMultiIDParams, getTableParams } from "../../../db/utils"
+import { getTableParams } from "../../../db/utils"
 import {
   breakExternalTableId,
   isExternalTableID,
@@ -17,6 +17,9 @@ import datasources from "../datasources"
 import sdk from "../../../sdk"
 
 export function processTable(table: Table): Table {
+  if (!table) {
+    return table
+  }
   if (table._id && isExternalTableID(table._id)) {
     return {
       ...table,
@@ -73,6 +76,9 @@ export async function getExternalTable(
   tableName: string
 ): Promise<Table> {
   const entities = await getExternalTablesInDatasource(datasourceId)
+  if (!entities[tableName]) {
+    throw new Error(`Unable to find table named "${tableName}"`)
+  }
   return processTable(entities[tableName])
 }
 
@@ -124,10 +130,10 @@ export async function getTables(tableIds: string[]): Promise<Table[]> {
   }
   if (internalTableIds.length) {
     const db = context.getAppDB()
-    const internalTableDocs = await db.allDocs<Table>(
-      getMultiIDParams(internalTableIds)
-    )
-    tables = tables.concat(internalTableDocs.rows.map(row => row.doc!))
+    const internalTables = await db.getMultiple<Table>(internalTableIds, {
+      allowMissing: true,
+    })
+    tables = tables.concat(internalTables)
   }
   return processTables(tables)
 }

packages/server/src/sdk/tests/tables.spec.ts

@@ -0,0 +1,39 @@
+import TestConfig from "../../tests/utilities/TestConfiguration"
+import { basicTable } from "../../tests/utilities/structures"
+import { Table } from "@budibase/types"
+import sdk from "../"
+
+describe("tables", () => {
+  const config = new TestConfig()
+  let table: Table
+
+  beforeAll(async () => {
+    await config.init()
+    table = await config.api.table.create(basicTable())
+  })
+
+  describe("getTables", () => {
+    it("should be able to retrieve tables", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!])
+        expect(tables.length).toBe(1)
+        expect(tables[0]._id).toBe(table._id)
+        expect(tables[0].name).toBe(table.name)
+      })
+    })
+
+    it("shouldn't fail when retrieving tables that don't exist", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables(["unknown"])
+        expect(tables.length).toBe(0)
+      })
+    })
+
+    it("should de-duplicate the IDs", async () => {
+      await config.doInContext(config.appId, async () => {
+        const tables = await sdk.tables.getTables([table._id!, table._id!])
+        expect(tables.length).toBe(1)
+      })
+    })
+  })
+})

packages/server/src/tests/utilities/TestConfiguration.ts

@@ -510,13 +510,14 @@ class TestConfiguration {
     // create dev app
     // clear any old app
     this.appId = null
-    await context.doInAppContext(null, async () => {
-      this.app = await this._req(
+    this.app = await context.doInAppContext(null, async () => {
+      const app = await this._req(
         { name: appName },
         null,
         controllers.app.create
       )
-      this.appId = this.app?.appId!
+      this.appId = app.appId!
+      return app
     })
     return await context.doInAppContext(this.appId, async () => {
       // create production app
@@ -525,7 +526,7 @@ class TestConfiguration {
       this.allApps.push(this.prodApp)
       this.allApps.push(this.app)
 
-      return this.app
+      return this.app!
     })
   }
 
@@ -537,7 +538,7 @@ class TestConfiguration {
 
     return context.doInAppContext(prodAppId, async () => {
       const db = context.getProdAppDB()
-      return await db.get(dbCore.DocumentType.APP_METADATA)
+      return await db.get<App>(dbCore.DocumentType.APP_METADATA)
     })
   }
 

packages/server/src/threads/automation.ts

@@ -241,7 +241,7 @@ class Orchestrator {
     })
   }
 
-  async execute() {
+  async execute(): Promise<any> {
     // this will retrieve from context created at start of thread
     this._context.env = await sdkUtils.getEnvironmentVariables()
     let automation = this._automation

packages/server/src/utilities/global.ts

@@ -1,4 +1,4 @@
-import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import {
   roles,
   db as dbCore,
@@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise<User[]> {
   const db = tenancy.getGlobalDB()
   let globalUsers: User[]
   if (userIds) {
-    globalUsers = (await db.allDocs<User>(getMultiIDParams(userIds))).rows.map(
-      row => row.doc!
-    )
+    globalUsers = await db.getMultiple<User>(userIds, { allowMissing: true })
   } else {
     globalUsers = (
       await db.allDocs<User>(

packages/types/src/sdk/db.ts

@@ -122,7 +122,11 @@ export interface Database {
 
   exists(): Promise<boolean>
   checkSetup(): Promise<Nano.DocumentScope<any>>
-  get<T>(id?: string): Promise<T>
+  get<T extends Document>(id?: string): Promise<T>
+  getMultiple<T extends Document>(
+    ids: string[],
+    opts?: { allowMissing?: boolean }
+  ): Promise<T[]>
   remove(
     id: string | Document,
     rev?: string

packages/types/src/sdk/licensing/plan.ts

@@ -7,7 +7,9 @@ export enum PlanType {
   /** @deprecated */
   PREMIUM = "premium",
   PREMIUM_PLUS = "premium_plus",
+  /** @deprecated */
   BUSINESS = "business",
+  ENTERPRISE_BASIC = "enterprise_basic",
   ENTERPRISE = "enterprise",
 }