diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 0bbed77458..6120290d0d 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -109,7 +109,7 @@ jobs: - name: Pull testcontainers images run: | docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb & + docker pull budibase/couchdb:v3.2.1-sql & docker pull redis & wait $(jobs -p) @@ -173,7 +173,7 @@ jobs: docker pull mongo:7.0-jammy & docker pull mariadb:lts & docker pull testcontainers/ryuk:0.5.1 & - docker pull budibase/couchdb & + docker pull budibase/couchdb:v3.2.1-sql & docker pull redis & wait $(jobs -p) diff --git a/globalSetup.ts b/globalSetup.ts index 7bf5e2152c..115796c395 100644 --- a/globalSetup.ts +++ b/globalSetup.ts @@ -13,8 +13,8 @@ export default async function setup() { } try { - let couchdb = new GenericContainer("budibase/couchdb") - .withExposedPorts(5984) + let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs") + .withExposedPorts(5984, 4984) .withEnvironment({ COUCHDB_PASSWORD: "budibase", COUCHDB_USER: "budibase", diff --git a/packages/backend-core/tests/core/utilities/testContainerUtils.ts b/packages/backend-core/tests/core/utilities/testContainerUtils.ts index 951a6f0517..d0dd2c9b4d 100644 --- a/packages/backend-core/tests/core/utilities/testContainerUtils.ts +++ b/packages/backend-core/tests/core/utilities/testContainerUtils.ts @@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) { throw new Error("CouchDB port not found") } + const couchSqlPort = getExposedV4Port(couch, 4984) + if (!couchSqlPort) { + throw new Error("CouchDB SQL port not found") + } + const configs = [ { key: "COUCH_DB_PORT", value: `${couchPort}` }, { key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` }, + { key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` }, ] for (const config of configs.filter(x => !!x.value)) { diff --git a/packages/server/src/api/controllers/row/utils/basic.ts b/packages/server/src/api/controllers/row/utils/basic.ts index 2fb67b07d5..1fc84de9c7 100644 --- a/packages/server/src/api/controllers/row/utils/basic.ts +++ b/packages/server/src/api/controllers/row/utils/basic.ts @@ -1,6 +1,7 @@ // need to handle table name + field or just field, depending on if relationships used import { FieldType, Row, Table } from "@budibase/types" import { generateRowIdField } from "../../../../integrations/utils" +import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils" function extractFieldValue({ row, @@ -20,6 +21,15 @@ function extractFieldValue({ return value } +export function getInternalRowId(row: Row, table: Table): string { + return extractFieldValue({ + row, + tableName: table._id!, + fieldName: "_id", + isLinked: false, + }) +} + export function generateIdForRow( row: Row | undefined, table: Table, @@ -78,6 +88,15 @@ export function basicProcessing({ thisRow._id = generateIdForRow(row, table, isLinked) thisRow.tableId = table._id thisRow._rev = "rev" + } else { + for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) { + thisRow[internalColumn] = extractFieldValue({ + row, + tableName: table._id!, + fieldName: internalColumn, + isLinked: false, + }) + } } return thisRow } diff --git a/packages/server/src/api/controllers/row/utils/sqlUtils.ts b/packages/server/src/api/controllers/row/utils/sqlUtils.ts index 2df829154f..6f9837e0ab 100644 --- a/packages/server/src/api/controllers/row/utils/sqlUtils.ts +++ b/packages/server/src/api/controllers/row/utils/sqlUtils.ts @@ -32,7 +32,7 @@ export async function updateRelationshipColumns( row: Row, rows: { [key: string]: Row }, relationships: RelationshipsJson[], - opts?: { internal?: boolean } + opts?: { sqs?: boolean } ) { const columns: { [key: string]: any } = {} for (let relationship of relationships) { @@ -55,7 +55,7 @@ export async function updateRelationshipColumns( row, table: linkedTable, isLinked: true, - internal: opts?.internal, + internal: opts?.sqs, }) if (!linked._id) { continue diff --git a/packages/server/src/api/controllers/row/utils/utils.ts b/packages/server/src/api/controllers/row/utils/utils.ts index c77ccf63a9..f387a468cf 100644 --- a/packages/server/src/api/controllers/row/utils/utils.ts +++ b/packages/server/src/api/controllers/row/utils/utils.ts @@ -15,7 +15,12 @@ import { processFormulas, } from "../../../../utilities/rowProcessor" import { updateRelationshipColumns } from "./sqlUtils" -import { basicProcessing, generateIdForRow, fixArrayTypes } from "./basic" +import { + basicProcessing, + generateIdForRow, + fixArrayTypes, + getInternalRowId, +} from "./basic" import sdk from "../../../../sdk" import validateJs from "validate.js" @@ -117,7 +122,7 @@ export async function sqlOutputProcessing( table: Table, tables: Record, relationships: RelationshipsJson[], - opts?: { internal?: boolean } + opts?: { sqs?: boolean } ): Promise { if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) { return [] @@ -125,7 +130,9 @@ export async function sqlOutputProcessing( let finalRows: { [key: string]: Row } = {} for (let row of rows as Row[]) { let rowId = row._id - if (!rowId) { + if (opts?.sqs) { + rowId = getInternalRowId(row, table) + } else if (!rowId) { rowId = generateIdForRow(row, table) row._id = rowId } @@ -146,7 +153,7 @@ export async function sqlOutputProcessing( row, table, isLinked: false, - internal: opts?.internal, + internal: opts?.sqs, }), table ) diff --git a/packages/server/src/api/routes/tests/search.spec.ts b/packages/server/src/api/routes/tests/search.spec.ts new file mode 100644 index 0000000000..a281d624dc --- /dev/null +++ b/packages/server/src/api/routes/tests/search.spec.ts @@ -0,0 +1,74 @@ +import { tableForDatasource } from "../../../tests/utilities/structures" +import { DatabaseName, getDatasource } from "../../../integrations/tests/utils" + +import * as setup from "./utilities" +import { Datasource, FieldType, Table } from "@budibase/types" + +jest.unmock("mssql") + +describe.each([ + ["internal", undefined], + ["internal-sqs", undefined], + [DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)], + [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)], + [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)], + [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)], +])("/api/:sourceId/search (%s)", (name, dsProvider) => { + const isSqs = name === "internal-sqs" + const config = setup.getConfig() + + let envCleanup: (() => void) | undefined + let table: Table + let datasource: Datasource | undefined + + beforeAll(async () => { + if (isSqs) { + envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" }) + } + await config.init() + if (dsProvider) { + datasource = await config.createDatasource({ + datasource: await dsProvider, + }) + } + }) + + afterAll(async () => { + setup.afterAll() + if (envCleanup) { + envCleanup() + } + }) + + beforeEach(async () => { + table = await config.api.table.save( + tableForDatasource(datasource, { + schema: { + name: { + name: "name", + type: FieldType.STRING, + }, + }, + }) + ) + }) + + it("should return rows", async () => { + const rows = await Promise.all([ + config.api.row.save(table._id!, { name: "foo" }), + config.api.row.save(table._id!, { name: "bar" }), + ]) + + const result = await config.api.row.search(table._id!, { + tableId: table._id!, + query: {}, + }) + + expect(result.rows).toEqual( + expect.arrayContaining([ + expect.objectContaining({ _id: rows[0]._id }), + expect.objectContaining({ _id: rows[1]._id }), + ]) + ) + }) +}) diff --git a/packages/server/src/sdk/app/rows/search/sqs.ts b/packages/server/src/sdk/app/rows/search/sqs.ts index 8aba94c886..89dae7628f 100644 --- a/packages/server/src/sdk/app/rows/search/sqs.ts +++ b/packages/server/src/sdk/app/rows/search/sqs.ts @@ -179,7 +179,7 @@ export async function search( allTablesMap, relationships, { - internal: true, + sqs: true, } ), }