diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 1ace60ed5b..6a1e575ac9 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -109,7 +109,7 @@ export class DatabaseImpl implements Database { } } - async get(id?: string): Promise { + async get(id?: string): Promise { const db = await this.checkSetup() if (!id) { throw new Error("Unable to get doc without a valid _id.") @@ -117,6 +117,28 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.get(id)) } + async getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise { + // get unique + ids = [...new Set(ids)] + const response = await this.allDocs({ + keys: ids, + include_docs: true, + }) + const NOT_FOUND = "not_found" + const rows = response.rows.filter(row => row.error !== NOT_FOUND) + const someMissing = rows.length !== response.rows.length + // some were filtered out - means some missing + if (!opts?.allowMissing && someMissing) { + const missing = response.rows.filter(row => row.error === NOT_FOUND) + const missingIds = missing.map(row => row.key).join(", ") + throw new Error(`Unable to get documents: ${missingIds}`) + } + return rows.map(row => row.doc!) + } + async remove(idOrDoc: string | Document, rev?: string) { const db = await this.checkSetup() let _id: string diff --git a/packages/builder/src/components/integration/RestQueryViewer.svelte b/packages/builder/src/components/integration/RestQueryViewer.svelte index e6913b0953..9634cd9746 100644 --- a/packages/builder/src/components/integration/RestQueryViewer.svelte +++ b/packages/builder/src/components/integration/RestQueryViewer.svelte @@ -404,7 +404,7 @@ datasource = $datasources.list.find(ds => ds._id === query?.datasourceId) const datasourceUrl = datasource?.config.url const qs = query?.fields.queryString - breakQs = restUtils.breakQueryString(qs) + breakQs = restUtils.breakQueryString(encodeURI(qs)) breakQs = runtimeToReadableMap(mergedBindings, breakQs) const path = query.fields.path @@ -652,7 +652,7 @@
- {#if !response && Object.keys(schema).length === 0} + {#if !response && Object.keys(schema || {}).length === 0} Response
diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index fe7d94547a..0907c22f0e 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -1,9 +1,5 @@ import * as linkRows from "../../../db/linkedRows" -import { - generateRowID, - getMultiIDParams, - InternalTables, -} from "../../../db/utils" +import { generateRowID, InternalTables } from "../../../db/utils" import * as userController from "../user" import { cleanupAttachments, @@ -240,8 +236,10 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const linkVals = links as LinkDocumentValue[] // look up the actual rows based on the ids - const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) - let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc!) + let linkedRows = await db.getMultiple( + linkVals.map(linkVal => linkVal.id), + { allowMissing: true } + ) // get the linked tables const linkTableIds = getLinkedTableIDs(table as Table) diff --git a/packages/server/src/api/controllers/row/utils.ts b/packages/server/src/api/controllers/row/utils.ts index cd311fdf0f..ed6ccd4c53 100644 --- a/packages/server/src/api/controllers/row/utils.ts +++ b/packages/server/src/api/controllers/row/utils.ts @@ -1,21 +1,9 @@ import { InternalTables } from "../../../db/utils" import * as userController from "../user" import { context } from "@budibase/backend-core" -import { - Ctx, - FieldType, - ManyToOneRelationshipFieldMetadata, - OneToManyRelationshipFieldMetadata, - Row, - SearchFilters, - Table, - UserCtx, -} from "@budibase/types" -import { FieldTypes, NoEmptyFilterStrings } from "../../../constants" -import sdk from "../../../sdk" +import { Ctx, Row, UserCtx } from "@budibase/types" import validateJs from "validate.js" -import { cloneDeep } from "lodash/fp" validateJs.extend(validateJs.validators.datetime, { parse: function (value: string) { diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7324fa1d94..7af3f9392f 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -8,7 +8,7 @@ import { getLinkedTable, } from "./linkUtils" import flatten from "lodash/flatten" -import { getMultiIDParams, USER_METDATA_PREFIX } from "../utils" +import { USER_METDATA_PREFIX } from "../utils" import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" @@ -79,9 +79,7 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( - row => row.doc! - ) + let dbRows = await db.getMultiple(uniqueRowIds, { allowMissing: true }) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) diff --git a/packages/server/src/db/utils.ts b/packages/server/src/db/utils.ts index 715db552c9..a5569f8166 100644 --- a/packages/server/src/db/utils.ts +++ b/packages/server/src/db/utils.ts @@ -283,16 +283,6 @@ export function generatePluginID(name: string) { return `${DocumentType.PLUGIN}${SEPARATOR}${name}` } -/** - * This can be used with the db.allDocs to get a list of IDs - */ -export function getMultiIDParams(ids: string[]) { - return { - keys: ids, - include_docs: true, - } -} - /** * Generates a new view ID. * @returns The new view ID which the view doc can be stored under. diff --git a/packages/server/src/integrations/redis.ts b/packages/server/src/integrations/redis.ts index 879a790550..6a6331ccd4 100644 --- a/packages/server/src/integrations/redis.ts +++ b/packages/server/src/integrations/redis.ts @@ -165,10 +165,22 @@ class RedisIntegration { // commands split line by line const commands = query.json.trim().split("\n") let pipelineCommands = [] + let tokenised // process each command separately for (let command of commands) { - const tokenised = command.trim().split(" ") + const valueToken = command.trim().match(/".*"/) + if (valueToken?.[0]) { + tokenised = [ + ...command + .substring(0, command.indexOf(valueToken[0]) - 1) + .trim() + .split(" "), + valueToken?.[0], + ] + } else { + tokenised = command.trim().split(" ") + } // Pipeline only accepts lower case commands tokenised[0] = tokenised[0].toLowerCase() pipelineCommands.push(tokenised) diff --git a/packages/server/src/integrations/tests/redis.spec.ts b/packages/server/src/integrations/tests/redis.spec.ts index 9521d58a51..942da99530 100644 --- a/packages/server/src/integrations/tests/redis.spec.ts +++ b/packages/server/src/integrations/tests/redis.spec.ts @@ -85,4 +85,21 @@ describe("Redis Integration", () => { ["get", "foo"], ]) }) + + it("calls the pipeline method with double quoted phrase values", async () => { + const body = { + json: 'SET foo "What a wonderful world!"\nGET foo', + } + + // ioredis-mock doesn't support pipelines + config.integration.client.pipeline = jest.fn(() => ({ + exec: jest.fn(() => [[]]), + })) + + await config.integration.command(body) + expect(config.integration.client.pipeline).toHaveBeenCalledWith([ + ["set", "foo", '"What a wonderful world!"'], + ["get", "foo"], + ]) + }) }) diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index a7074f95b2..72a6ab61f1 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -1,5 +1,5 @@ import { context } from "@budibase/backend-core" -import { getMultiIDParams, getTableParams } from "../../../db/utils" +import { getTableParams } from "../../../db/utils" import { breakExternalTableId, isExternalTableID, @@ -17,6 +17,9 @@ import datasources from "../datasources" import sdk from "../../../sdk" export function processTable(table: Table): Table { + if (!table) { + return table + } if (table._id && isExternalTableID(table._id)) { return { ...table, @@ -73,6 +76,9 @@ export async function getExternalTable( tableName: string ): Promise { const entities = await getExternalTablesInDatasource(datasourceId) + if (!entities[tableName]) { + throw new Error(`Unable to find table named "${tableName}"`) + } return processTable(entities[tableName]) } @@ -124,10 +130,10 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTableDocs = await db.allDocs
( - getMultiIDParams(internalTableIds) - ) - tables = tables.concat(internalTableDocs.rows.map(row => row.doc!)) + const internalTables = await db.getMultiple
(internalTableIds, { + allowMissing: true, + }) + tables = tables.concat(internalTables) } return processTables(tables) } diff --git a/packages/server/src/sdk/tests/tables.spec.ts b/packages/server/src/sdk/tests/tables.spec.ts new file mode 100644 index 0000000000..0e3cd73cfd --- /dev/null +++ b/packages/server/src/sdk/tests/tables.spec.ts @@ -0,0 +1,39 @@ +import TestConfig from "../../tests/utilities/TestConfiguration" +import { basicTable } from "../../tests/utilities/structures" +import { Table } from "@budibase/types" +import sdk from "../" + +describe("tables", () => { + const config = new TestConfig() + let table: Table + + beforeAll(async () => { + await config.init() + table = await config.api.table.create(basicTable()) + }) + + describe("getTables", () => { + it("should be able to retrieve tables", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!]) + expect(tables.length).toBe(1) + expect(tables[0]._id).toBe(table._id) + expect(tables[0].name).toBe(table.name) + }) + }) + + it("shouldn't fail when retrieving tables that don't exist", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables(["unknown"]) + expect(tables.length).toBe(0) + }) + }) + + it("should de-duplicate the IDs", async () => { + await config.doInContext(config.appId, async () => { + const tables = await sdk.tables.getTables([table._id!, table._id!]) + expect(tables.length).toBe(1) + }) + }) + }) +}) diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index cdc2d84513..bbb84c1882 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -1,4 +1,4 @@ -import { getMultiIDParams, getGlobalIDFromUserMetadataID } from "../db/utils" +import { getGlobalIDFromUserMetadataID } from "../db/utils" import { roles, db as dbCore, @@ -96,9 +96,7 @@ export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() let globalUsers: User[] if (userIds) { - globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( - row => row.doc! - ) + globalUsers = await db.getMultiple(userIds, { allowMissing: true }) } else { globalUsers = ( await db.allDocs( diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 26807d99ce..7613ac6aeb 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -122,7 +122,11 @@ export interface Database { exists(): Promise checkSetup(): Promise> - get(id?: string): Promise + get(id?: string): Promise + getMultiple( + ids: string[], + opts?: { allowMissing?: boolean } + ): Promise remove( id: string | Document, rev?: string