diff --git a/.github/workflows/deploy-cloud.yaml b/.github/workflows/deploy-cloud.yaml deleted file mode 100644 index 389b10f7d3..0000000000 --- a/.github/workflows/deploy-cloud.yaml +++ /dev/null @@ -1,48 +0,0 @@ -name: Budibase Deploy Production - -on: - workflow_dispatch: - inputs: - version: - description: Budibase release version. For example - 1.0.0 - required: false - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Get the latest budibase release version - id: version - run: | - if [ -z "${{ github.event.inputs.version }}" ]; then - release_version=$(cat lerna.json | jq -r '.version') - else - release_version=${{ github.event.inputs.version }} - fi - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - uses: passeidireto/trigger-external-workflow-action@main - env: - PAYLOAD_VERSION: ${{ env.RELEASE_VERSION }} - with: - repository: budibase/budibase-deploys - event: budicloud-prod-deploy - github_pat: ${{ secrets.GH_ACCESS_TOKEN }} diff --git a/.github/workflows/release-master.yml b/.github/workflows/release-master.yml deleted file mode 100644 index 2edb470405..0000000000 --- a/.github/workflows/release-master.yml +++ /dev/null @@ -1,178 +0,0 @@ -name: Budibase Release -concurrency: - group: release - cancel-in-progress: false - -on: - push: - tags: - - "[0-9]+.[0-9]+.[0-9]+" - # Exclude all pre-releases - - "!*[0-9]+.[0-9]+.[0-9]+-*" - -env: - # Posthog token used by ui at build time - POSTHOG_TOKEN: phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU - INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }} - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - -jobs: - release-images: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - uses: actions/setup-node@v1 - with: - node-version: 18.x - cache: yarn - - - run: yarn install --frozen-lockfile - - name: Update versions - run: ./scripts/updateVersions.sh - - run: yarn lint - - run: yarn build - - run: yarn build:sdk - - - name: Publish budibase packages to NPM - env: - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} - run: | - # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default - git config --global user.name "Budibase Release Bot" - git config --global user.email "<>" - git submodule foreach git commit -a -m 'Release process' - git commit -a -m 'Release process' - echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc - yarn release - - - name: "Get Current tag" - id: currenttag - run: | - version=$(./scripts/getCurrentVersion.sh) - echo "Using tag $version" - echo "version=$version" >> "$GITHUB_OUTPUT" - - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - - name: Docker login - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - - - name: Build worker docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/worker/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/worker - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build server docker - uses: docker/build-push-action@v5 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: | - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./packages/server/Dockerfile.v2 - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/apps - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - BUDIBASE_VERSION: ${{ steps.currenttag.outputs.version }} - - - name: Build proxy docker - uses: docker/build-push-action@v5 - with: - context: ./hosting/proxy - push: true - platforms: linux/amd64,linux/arm64 - tags: ${{ env.IMAGE_NAME }}:${{ env.IMAGE_TAG }} - file: ./hosting/proxy/Dockerfile - cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:latest - cache-to: type=inline - env: - IMAGE_NAME: budibase/proxy - IMAGE_TAG: ${{ steps.currenttag.outputs.version }} - - release-helm-chart: - needs: [release-images] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - - trigger-deploy-to-qa-env: - needs: [release-helm-chart] - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - uses: peter-evans/repository-dispatch@v2 - with: - repository: budibase/budibase-deploys - event-type: budicloud-qa-deploy - token: ${{ secrets.GH_ACCESS_TOKEN }} - client-payload: |- - { - "VERSION": "${{ github.ref_name }}", - "REF_NAME": "${{ github.ref_name}}" - } diff --git a/.github/workflows/release-selfhost.yml b/.github/workflows/release-selfhost.yml deleted file mode 100644 index d2689a0ea0..0000000000 --- a/.github/workflows/release-selfhost.yml +++ /dev/null @@ -1,125 +0,0 @@ -name: Budibase Release Selfhost - -on: - workflow_dispatch: - -jobs: - release: - runs-on: ubuntu-latest - - steps: - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - fetch-depth: 0 - - - name: Fail if tag is not in master - run: | - if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then - echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch" - exit 1 - fi - - - name: Use Node.js 18.x - uses: actions/setup-node@v1 - with: - node-version: 18.x - - - name: Get the latest budibase release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - - name: Tag and release Docker images (Self Host) - run: | - docker login -u $DOCKER_USER -p $DOCKER_PASSWORD - - release_tag=${{ env.RELEASE_VERSION }} - - # Pull apps and worker images - docker pull budibase/apps:$release_tag - docker pull budibase/worker:$release_tag - docker pull budibase/proxy:$release_tag - - # Tag apps and worker images - docker tag budibase/apps:$release_tag budibase/apps:$SELFHOST_TAG - docker tag budibase/worker:$release_tag budibase/worker:$SELFHOST_TAG - docker tag budibase/proxy:$release_tag budibase/proxy:$SELFHOST_TAG - - # Push images - docker push budibase/apps:$SELFHOST_TAG - docker push budibase/worker:$SELFHOST_TAG - docker push budibase/proxy:$SELFHOST_TAG - env: - DOCKER_USER: ${{ secrets.DOCKER_USERNAME }} - DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }} - SELFHOST_TAG: latest - - - name: Bootstrap and build (CLI) - run: | - yarn - yarn build - - - name: Build OpenAPI spec - run: | - pushd packages/server - yarn - yarn specs - popd - - - name: Setup Helm - uses: azure/setup-helm@v1 - id: helm-install - - # due to helm repo index issue: https://github.com/helm/helm/issues/7363 - # we need to create new package in a different dir, merge the index and move the package back - - name: Build and release helm chart - run: | - git config user.name "Budibase Helm Bot" - git config user.email "<>" - git reset --hard - git fetch - mkdir sync - echo "Packaging chart to sync dir" - helm package charts/budibase --version "$RELEASE_VERSION" --app-version "$RELEASE_VERSION" --destination sync - echo "Packaging successful" - git checkout gh-pages - echo "Indexing helm repo" - helm repo index --merge docs/index.yaml sync - mv -f sync/* docs - rm -rf sync - echo "Pushing new helm release" - git add -A - git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}" - git push - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Perform Github Release - uses: softprops/action-gh-release@v1 - with: - name: ${{ env.RELEASE_VERSION }} - tag_name: ${{ env.RELEASE_VERSION }} - generate_release_notes: true - files: | - packages/cli/build/cli-win.exe - packages/cli/build/cli-linux - packages/cli/build/cli-macos - packages/server/specs/openapi.yaml - packages/server/specs/openapi.json - - - name: Discord Webhook Action - uses: tsickert/discord-webhook@v4.0.0 - with: - webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }} - content: "Self Host Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Self Host." - embed-title: ${{ env.RELEASE_VERSION }} diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml deleted file mode 100644 index 16b1da186a..0000000000 --- a/.github/workflows/release-singleimage.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: Deploy Budibase Single Container Image to DockerHub - -on: - workflow_dispatch: - -env: - CI: true - PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - REGISTRY_URL: registry.hub.docker.com -jobs: - build: - name: "build" - runs-on: ubuntu-latest - strategy: - matrix: - node-version: [18.x] - steps: - - name: Maximize build space - uses: easimon/maximize-build-space@master - with: - root-reserve-mb: 30000 - swap-size-mb: 1024 - remove-android: "true" - remove-dotnet: "true" - - name: Fail if not a tag - run: | - if [[ $GITHUB_REF != refs/tags/* ]]; then - echo "Workflow Dispatch can only be run on tags" - exit 1 - fi - - name: "Checkout" - uses: actions/checkout@v2 - with: - submodules: true - token: ${{ secrets.PERSONAL_ACCESS_TOKEN }} - - - name: Use Node.js ${{ matrix.node-version }} - uses: actions/setup-node@v1 - with: - node-version: ${{ matrix.node-version }} - - name: Setup QEMU - uses: docker/setup-qemu-action@v1 - - name: Setup Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v1 - - name: Run Yarn - run: yarn - - name: Update versions - run: ./scripts/updateVersions.sh - - name: Run Yarn Build - run: yarn build - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_API_KEY }} - - name: Get the latest release version - id: version - run: | - release_version=$(cat lerna.json | jq -r '.version') - echo $release_version - echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV - - name: Tag and release Budibase service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64,linux/arm64 - build-args: BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - - name: Tag and release Budibase Azure App Service docker image - uses: docker/build-push-action@v2 - with: - context: . - push: true - platforms: linux/amd64 - build-args: | - TARGETBUILD=aas - BUDIBASE_VERSION=${{ env.BUDIBASE_VERSION }} - tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} - file: ./hosting/single/Dockerfile.v2 - env: - BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} diff --git a/lerna.json b/lerna.json index 5605642877..c0559d8346 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.13.5", + "version": "2.13.7", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/src/cache/appMetadata.ts b/packages/backend-core/src/cache/appMetadata.ts index bd3efc20db..d442511fb8 100644 --- a/packages/backend-core/src/cache/appMetadata.ts +++ b/packages/backend-core/src/cache/appMetadata.ts @@ -19,7 +19,7 @@ async function populateFromDB(appId: string) { return doWithDB( appId, (db: Database) => { - return db.get(DocumentType.APP_METADATA) + return db.get(DocumentType.APP_METADATA) }, { skip_setup: true } ) diff --git a/packages/backend-core/src/context/Context.ts b/packages/backend-core/src/context/Context.ts index d29b6935a8..a59f5c6503 100644 --- a/packages/backend-core/src/context/Context.ts +++ b/packages/backend-core/src/context/Context.ts @@ -4,7 +4,7 @@ import { ContextMap } from "./types" export default class Context { static storage = new AsyncLocalStorage() - static run(context: ContextMap, func: any) { + static run(context: ContextMap, func: () => T) { return Context.storage.run(context, () => func()) } diff --git a/packages/backend-core/src/context/mainContext.ts b/packages/backend-core/src/context/mainContext.ts index 609c18abb5..d2259cfcab 100644 --- a/packages/backend-core/src/context/mainContext.ts +++ b/packages/backend-core/src/context/mainContext.ts @@ -98,17 +98,17 @@ function updateContext(updates: ContextMap): ContextMap { return context } -async function newContext(updates: ContextMap, task: any) { +async function newContext(updates: ContextMap, task: () => T) { // see if there already is a context setup let context: ContextMap = updateContext(updates) return Context.run(context, task) } -export async function doInAutomationContext(params: { +export async function doInAutomationContext(params: { appId: string automationId: string - task: any -}): Promise { + task: () => T +}): Promise { const tenantId = getTenantIDFromAppID(params.appId) return newContext( { @@ -144,10 +144,10 @@ export async function doInTenant( return newContext(updates, task) } -export async function doInAppContext( +export async function doInAppContext( appId: string | null, - task: any -): Promise { + task: () => T +): Promise { if (!appId && !env.isTest()) { throw new Error("appId is required") } @@ -165,10 +165,10 @@ export async function doInAppContext( return newContext(updates, task) } -export async function doInIdentityContext( +export async function doInIdentityContext( identity: IdentityContext, - task: any -): Promise { + task: () => T +): Promise { if (!identity) { throw new Error("identity is required") } @@ -276,6 +276,9 @@ export function getAuditLogsDB(): Database { */ export function getAppDB(opts?: any): Database { const appId = getAppId() + if (!appId) { + throw new Error("Unable to retrieve app DB - no app ID.") + } return getDB(appId, opts) } diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 330b15e680..6a1e575ac9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -48,10 +48,7 @@ export class DatabaseImpl implements Database { private readonly couchInfo = getCouchInfo() - constructor(dbName?: string, opts?: DatabaseOpts, connection?: string) { - if (dbName == null) { - throw new Error("Database name cannot be undefined.") - } + constructor(dbName: string, opts?: DatabaseOpts, connection?: string) { this.name = dbName this.pouchOpts = opts || {} if (connection) { @@ -112,7 +109,7 @@ export class DatabaseImpl implements Database { } } - async get(id?: string): Promise { + async get(id?: string): Promise { const db = await this.checkSetup() if (!id) { throw new Error("Unable to get doc without a valid _id.") @@ -120,6 +117,28 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } + async getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise { + // get unique + ids = [...new Set(ids)] + const response = await this.allDocs({ + keys: ids, + include_docs: true, + }) + const NOT_FOUND = "not_found" + const rows = response.rows.filter(row => row.error !== NOT_FOUND) + const someMissing = rows.length !== response.rows.length + // some were filtered out - means some missing + if (!opts?.allowMissing && someMissing) { + const missing = response.rows.filter(row => row.error === NOT_FOUND) + const missingIds = missing.map(row => row.key).join(", ") + throw new Error(`Unable to get documents: ${missingIds}`) + } + return rows.map(row => row.doc!) + } + async remove(idOrDoc: string | Document, rev?: string) { const db = await this.checkSetup() let _id: string diff --git a/packages/backend-core/src/db/db.ts b/packages/backend-core/src/db/db.ts index 9aae64b892..3e69d49f0e 100644 --- a/packages/backend-core/src/db/db.ts +++ b/packages/backend-core/src/db/db.ts @@ -1,10 +1,7 @@ -import env from "../environment" import { directCouchQuery, DatabaseImpl } from "./couch" -import { CouchFindOptions, Database } from "@budibase/types" +import { CouchFindOptions, Database, DatabaseOpts } from "@budibase/types" -const dbList = new Set() - -export function getDB(dbName?: string, opts?: any): Database { +export function getDB(dbName: string, opts?: DatabaseOpts): Database { return new DatabaseImpl(dbName, opts) } @@ -14,7 +11,7 @@ export function getDB(dbName?: string, opts?: any): Database { export async function doWithDB( dbName: string, cb: (db: Database) => Promise, - opts = {} + opts?: DatabaseOpts ) { const db = getDB(dbName, opts) // need this to be async so that we can correctly close DB after all @@ -22,13 +19,6 @@ export async function doWithDB( return await cb(db) } -export function allDbs() { - if (!env.isTest()) { - throw new Error("Cannot be used outside test environment.") - } - return [...dbList] -} - export async function directCouchAllDbs(queryString?: string) { let couchPath = "/_all_dbs" if (queryString) { diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte index e6913b0953..9634cd9746 100644 --- a/packages/builder/src/components/integration/RestQueryViewer.svelte +++ b/packages/builder/src/components/integration/RestQueryViewer.svelte @@ -404,7 +404,7 @@ datasource = $datasources.list.find(ds => ds._id === query?.datasourceId) const datasourceUrl = datasource?.config.url const qs = query?.fields.queryString - breakQs = restUtils.breakQueryString(qs) + breakQs = restUtils.breakQueryString(encodeURI(qs)) breakQs = runtimeToReadableMap(mergedBindings, breakQs) const path = query.fields.path @@ -652,7 +652,7 @@
- {#if !response && Object.keys(schema).length === 0} + {#if !response && Object.keys(schema || {}).length === 0} Response
diff --git a/packages/client/src/components/app/Text.svelte b/packages/client/src/components/app/Text.svelte index 6c16db25fd..1037725ff8 100644 --- a/packages/client/src/components/app/Text.svelte +++ b/packages/client/src/components/app/Text.svelte @@ -94,7 +94,7 @@ .align--right { text-align: right; } - .align-justify { + .align--justify { text-align: justify; } diff --git a/packages/pro b/packages/pro index ad9a0085be..e202f415d9 160000 --- a/packages/pro +++ b/packages/pro @@ -1 +1 @@ -Subproject commit ad9a0085bee0c4f3184acd86cadd872ea9917e88 +Subproject commit e202f415d9fa540d08cc2ba6e27394fbc22f357b diff --git a/packages/server/src/api/controllers/datasource.ts b/packages/server/src/api/controllers/datasource.ts index 5d024d51b6..39bc612b32 100644 --- a/packages/server/src/api/controllers/datasource.ts +++ b/packages/server/src/api/controllers/datasource.ts @@ -337,7 +337,7 @@ export async function destroy(ctx: UserCtx) { if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) { await destroyInternalTablesBySourceId(datasourceId) } else { - const queries = await db.allDocs(getQueryParams(datasourceId, null)) + const queries = await db.allDocs(getQueryParams(datasourceId)) await db.bulkDocs( queries.rows.map((row: any) => ({ _id: row.id, diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index fe7d94547a..0907c22f0e 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,9 +1,5 @@ import * as linkRows from "../../../db/linkedRows" -import { - generateRowID, - getMultiIDParams, - InternalTables, -} from "../../../db/utils" +import { generateRowID, InternalTables } from "../../../db/utils" import * as userController from "../user" import { cleanupAttachments, @@ -240,8 +236,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const linkVals = links as LinkDocumentValue[] // look up the actual rows based on the ids - const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) - let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc!) + let linkedRows = await db.getMultiple( + linkVals.map(linkVal => linkVal.id), + { allowMissing: true } + ) // get the linked tables const linkTableIds = getLinkedTableIDs(table as Table) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index cd311fdf0f..ed6ccd4c53 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,21 +1,9 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { context } from "@budibase/backend-core" -import { - Ctx, - FieldType, - ManyToOneRelationshipFieldMetadata, - OneToManyRelationshipFieldMetadata, - Row, - SearchFilters, - Table, - UserCtx, -} from "@budibase/types" -import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" -import sdk from "../../../sdk" +import { Ctx, Row, UserCtx } from "@budibase/types" import validateJs from "validate.js" -import { cloneDeep } from "lodash/fp" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index f0eca759f5..ac977bbefb 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -94,7 +94,7 @@ export async function externalTrigger( automation: Automation, params: { fields: Record; timeout?: number }, { getResponses }: { getResponses?: boolean } = {} -) { +): Promise { if ( automation.definition != null && automation.definition.trigger != null && diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7324fa1d94..7af3f9392f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -8,7 +8,7 @@ import { getLinkedTable, } from "./linkUtils" import flatten from "lodash/flatten" -import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils" +import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" @@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( - row => row.doc! - ) + let dbRows = await db.getMultiple(uniqueRowIds, { allowMissing: true }) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index d532d8a8b2..a5569f8166 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -6,6 +6,7 @@ import { RelationshipFieldMetadata, VirtualDocumentType, INTERNAL_TABLE_SOURCE_ID, + DatabaseQueryOpts, } from "@budibase/types" import { FieldTypes } from "../constants" export { DocumentType, VirtualDocumentType } from "@budibase/types" @@ -229,7 +230,10 @@ export function getAutomationMetadataParams(otherProps: any = {}) { /** * Gets parameters for retrieving a query, this is a utility function for the getDocParams function. */ -export function getQueryParams(datasourceId?: Optional, otherProps: any = {}) { +export function getQueryParams( + datasourceId?: Optional, + otherProps: Partial = {} +) { if (datasourceId == null) { return getDocParams(DocumentType.QUERY, null, otherProps) } @@ -256,7 +260,7 @@ export function generateMetadataID(type: string, entityId: string) { export function getMetadataParams( type: string, entityId?: Optional, - otherProps: any = {} + otherProps: Partial = {} ) { let docId = `${type}${SEPARATOR}` if (entityId != null) { @@ -269,7 +273,9 @@ export function generateMemoryViewID(viewName: string) { return `${DocumentType.MEM_VIEW}${SEPARATOR}${viewName}` } -export function getMemoryViewParams(otherProps: any = {}) { +export function getMemoryViewParams( + otherProps: Partial = {} +) { return getDocParams(DocumentType.MEM_VIEW, null, otherProps) } @@ -277,16 +283,6 @@ export function generatePluginID(name: string) { return `${DocumentType.PLUGIN}${SEPARATOR}${name}` } -/** - * This can be used with the db.allDocs to get a list of IDs - */ -export function getMultiIDParams(ids: string[]) { - return { - keys: ids, - include_docs: true, - } -} - /** * Generates a new view ID. * @returns The new view ID which the view doc can be stored under. diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 879a790550..6a6331ccd4 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -165,10 +165,22 @@ class RedisIntegration { // commands split line by line const commands = query.json.trim().split("\n") let pipelineCommands = [] + let tokenised // process each command separately for (let command of commands) { - const tokenised = command.trim().split(" ") + const valueToken = command.trim().match(/".*"/) + if (valueToken?.[0]) { + tokenised = [ + ...command + .substring(0, command.indexOf(valueToken[0]) - 1) + .trim() + .split(" "), + valueToken?.[0], + ] + } else { + tokenised = command.trim().split(" ") + } // Pipeline only accepts lower case commands tokenised[0] = tokenised[0].toLowerCase() pipelineCommands.push(tokenised) diff --git a/packages/server/src/integrations/tests/redis.spec.ts b/packages/server/src/integrations/tests/redis.spec.ts index 9521d58a51..942da99530 100644 --- a/packages/server/src/integrations/tests/redis.spec.ts +++ b/packages/server/src/integrations/tests/redis.spec.ts @@ -85,4 +85,21 @@ describe("Redis Integration", () => { ["get", "foo"], ]) }) + + it("calls the pipeline method with double quoted phrase values", async () => { + const body = { + json: 'SET foo "What a wonderful world!"\nGET foo', + } + + // ioredis-mock doesn't support pipelines + config.integration.client.pipeline = jest.fn(() => ({ + exec: jest.fn(() => [[]]), + })) + + await config.integration.command(body) + expect(config.integration.client.pipeline).toHaveBeenCalledWith([ + ["set", "foo", '"What a wonderful world!"'], + ["get", "foo"], + ]) + }) }) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a7074f95b2..72a6ab61f1 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,5 +1,5 @@ import { context } from "@budibase/backend-core" -import { getMultiIDParams, getTableParams } from "../../../db/utils" +import { getTableParams } from "../../../db/utils" import { breakExternalTableId, isExternalTableID, @@ -17,6 +17,9 @@ import datasources from "../datasources" import sdk from "../../../sdk" export function processTable(table: Table): Table { + if (!table) { + return table + } if (table._id && isExternalTableID(table._id)) { return { ...table, @@ -73,6 +76,9 @@ export async function getExternalTable( tableName: string ): Promise { const entities = await getExternalTablesInDatasource(datasourceId) + if (!entities[tableName]) { + throw new Error(`Unable to find table named "${tableName}"`) + } return processTable(entities[tableName]) } @@ -124,10 +130,10 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTableDocs = await db.allDocs
( - getMultiIDParams(internalTableIds) - ) - tables = tables.concat(internalTableDocs.rows.map(row => row.doc!)) + const internalTables = await db.getMultiple
(internalTableIds, { + allowMissing: true, + }) + tables = tables.concat(internalTables) } return processTables(tables) } diff --git a/packages/server/src/sdk/tests/tables.spec.ts b/packages/server/src/sdk/tests/tables.spec.ts new file mode 100644 index 0000000000..0e3cd73cfd --- /dev/null +++ b/packages/server/src/sdk/tests/tables.spec.ts @@ -0,0 +1,39 @@ +import TestConfig from "../../tests/utilities/TestConfiguration" +import { basicTable } from "../../tests/utilities/structures" +import { Table } from "@budibase/types" +import sdk from "../" + +describe("tables", () => { + const config = new TestConfig() + let table: Table + + beforeAll(async () => { + await config.init() + table = await config.api.table.create(basicTable()) + }) + + describe("getTables", () => { + it("should be able to retrieve tables", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!]) + expect(tables.length).toBe(1) + expect(tables[0]._id).toBe(table._id) + expect(tables[0].name).toBe(table.name) + }) + }) + + it("shouldn't fail when retrieving tables that don't exist", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables(["unknown"]) + expect(tables.length).toBe(0) + }) + }) + + it("should de-duplicate the IDs", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!, table._id!]) + expect(tables.length).toBe(1) + }) + }) + }) +}) diff --git a/packages/server/src/tests/utilities/TestConfiguration.ts b/packages/server/src/tests/utilities/TestConfiguration.ts index 6877561fcb..3a14a87d2a 100644 --- a/packages/server/src/tests/utilities/TestConfiguration.ts +++ b/packages/server/src/tests/utilities/TestConfiguration.ts @@ -510,13 +510,14 @@ class TestConfiguration { // create dev app // clear any old app this.appId = null - await context.doInAppContext(null, async () => { - this.app = await this._req( + this.app = await context.doInAppContext(null, async () => { + const app = await this._req( { name: appName }, null, controllers.app.create ) - this.appId = this.app?.appId! + this.appId = app.appId! + return app }) return await context.doInAppContext(this.appId, async () => { // create production app @@ -525,7 +526,7 @@ class TestConfiguration { this.allApps.push(this.prodApp) this.allApps.push(this.app) - return this.app + return this.app! }) } @@ -537,7 +538,7 @@ class TestConfiguration { return context.doInAppContext(prodAppId, async () => { const db = context.getProdAppDB() - return await db.get(dbCore.DocumentType.APP_METADATA) + return await db.get(dbCore.DocumentType.APP_METADATA) }) } diff --git a/packages/server/src/threads/automation.ts b/packages/server/src/threads/automation.ts index 9241289e86..d1fcc2be72 100644 --- a/packages/server/src/threads/automation.ts +++ b/packages/server/src/threads/automation.ts @@ -241,7 +241,7 @@ class Orchestrator { }) } - async execute() { + async execute(): Promise { // this will retrieve from context created at start of thread this._context.env = await sdkUtils.getEnvironmentVariables() let automation = this._automation diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index cdc2d84513..bbb84c1882 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -1,4 +1,4 @@ -import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils" +import { getGlobalIDFromUserMetadataID } from "../db/utils" import { roles, db as dbCore, @@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() let globalUsers: User[] if (userIds) { - globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( - row => row.doc! - ) + globalUsers = await db.getMultiple(userIds, { allowMissing: true }) } else { globalUsers = ( await db.allDocs( diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 26807d99ce..7613ac6aeb 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -122,7 +122,11 @@ export interface Database { exists(): Promise checkSetup(): Promise> - get(id?: string): Promise + get(id?: string): Promise + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise remove( id: string | Document, rev?: string diff --git a/packages/types/src/sdk/licensing/plan.ts b/packages/types/src/sdk/licensing/plan.ts index 1604dfb8af..5ac8b1c9f6 100644 --- a/packages/types/src/sdk/licensing/plan.ts +++ b/packages/types/src/sdk/licensing/plan.ts @@ -7,7 +7,9 @@ export enum PlanType { /** @deprecated */ PREMIUM = "premium", PREMIUM_PLUS = "premium_plus", + /** @deprecated */ BUSINESS = "business", + ENTERPRISE_BASIC = "enterprise_basic", ENTERPRISE = "enterprise", }