Remove aws sdk global mock and update tests (#13637)
* Remove aws sdk global mock and update tests * add awaits * Minio healthcheck in tests. * Bind to 127.0.0.1 instead of 0.0.0.0 * Fix port fetching for minio container. * Actually fix port mapping this time. * Pull minio container before running tests. * Enable testcontainers debug logging. * Promote minio container to always running in tests, like CouchDB. * Remove testcontainers debug logging. --------- Co-authored-by: Sam Rose <hello@samwho.dev>
This commit is contained in:
parent
68cb2636df
commit
1d300c2577
17 changed files with 57 additions and 74 deletions
3
.github/workflows/budibase_ci.yml
vendored
3
.github/workflows/budibase_ci.yml
vendored
|
@ -170,7 +170,8 @@ jobs:
|
||||||
docker pull mongo:7.0-jammy &
|
docker pull mongo:7.0-jammy &
|
||||||
docker pull mariadb:lts &
|
docker pull mariadb:lts &
|
||||||
docker pull testcontainers/ryuk:0.5.1 &
|
docker pull testcontainers/ryuk:0.5.1 &
|
||||||
docker pull budibase/couchdb:v3.2.1-sql &
|
docker pull budibase/couchdb:v3.2.1-sqs &
|
||||||
|
docker pull minio/minio &
|
||||||
docker pull redis &
|
docker pull redis &
|
||||||
|
|
||||||
wait $(jobs -p)
|
wait $(jobs -p)
|
||||||
|
|
|
@ -46,7 +46,7 @@ export default async function setup() {
|
||||||
await killContainers(containers)
|
await killContainers(containers)
|
||||||
|
|
||||||
try {
|
try {
|
||||||
let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
|
const couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
|
||||||
.withExposedPorts(5984, 4984)
|
.withExposedPorts(5984, 4984)
|
||||||
.withEnvironment({
|
.withEnvironment({
|
||||||
COUCHDB_PASSWORD: "budibase",
|
COUCHDB_PASSWORD: "budibase",
|
||||||
|
@ -69,7 +69,20 @@ export default async function setup() {
|
||||||
).withStartupTimeout(20000)
|
).withStartupTimeout(20000)
|
||||||
)
|
)
|
||||||
|
|
||||||
await couchdb.start()
|
const minio = new GenericContainer("minio/minio")
|
||||||
|
.withExposedPorts(9000)
|
||||||
|
.withCommand(["server", "/data"])
|
||||||
|
.withEnvironment({
|
||||||
|
MINIO_ACCESS_KEY: "budibase",
|
||||||
|
MINIO_SECRET_KEY: "budibase",
|
||||||
|
})
|
||||||
|
.withLabels({ "com.budibase": "true" })
|
||||||
|
.withReuse()
|
||||||
|
.withWaitStrategy(
|
||||||
|
Wait.forHttp("/minio/health/ready", 9000).withStartupTimeout(10000)
|
||||||
|
)
|
||||||
|
|
||||||
|
await Promise.all([couchdb.start(), minio.start()])
|
||||||
} finally {
|
} finally {
|
||||||
lockfile.unlockSync(lockPath)
|
lockfile.unlockSync(lockPath)
|
||||||
}
|
}
|
||||||
|
|
|
@ -83,7 +83,7 @@ export function ObjectStore(
|
||||||
bucket: string,
|
bucket: string,
|
||||||
opts: { presigning: boolean } = { presigning: false }
|
opts: { presigning: boolean } = { presigning: false }
|
||||||
) {
|
) {
|
||||||
const config: any = {
|
const config: AWS.S3.ClientConfiguration = {
|
||||||
s3ForcePathStyle: true,
|
s3ForcePathStyle: true,
|
||||||
signatureVersion: "v4",
|
signatureVersion: "v4",
|
||||||
apiVersion: "2006-03-01",
|
apiVersion: "2006-03-01",
|
||||||
|
|
|
@ -4,6 +4,3 @@ export { generator } from "./structures"
|
||||||
export * as testContainerUtils from "./testContainerUtils"
|
export * as testContainerUtils from "./testContainerUtils"
|
||||||
export * as utils from "./utils"
|
export * as utils from "./utils"
|
||||||
export * from "./jestUtils"
|
export * from "./jestUtils"
|
||||||
import * as minio from "./minio"
|
|
||||||
|
|
||||||
export const objectStoreTestProviders = { minio }
|
|
||||||
|
|
|
@ -1,34 +0,0 @@
|
||||||
import { GenericContainer, Wait, StartedTestContainer } from "testcontainers"
|
|
||||||
import { AbstractWaitStrategy } from "testcontainers/build/wait-strategies/wait-strategy"
|
|
||||||
import env from "../../../src/environment"
|
|
||||||
|
|
||||||
let container: StartedTestContainer | undefined
|
|
||||||
|
|
||||||
class ObjectStoreWaitStrategy extends AbstractWaitStrategy {
|
|
||||||
async waitUntilReady(container: any, boundPorts: any, startTime?: Date) {
|
|
||||||
const logs = Wait.forListeningPorts()
|
|
||||||
await logs.waitUntilReady(container, boundPorts, startTime)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function start(): Promise<void> {
|
|
||||||
container = await new GenericContainer("minio/minio")
|
|
||||||
.withExposedPorts(9000)
|
|
||||||
.withCommand(["server", "/data"])
|
|
||||||
.withEnvironment({
|
|
||||||
MINIO_ACCESS_KEY: "budibase",
|
|
||||||
MINIO_SECRET_KEY: "budibase",
|
|
||||||
})
|
|
||||||
.withWaitStrategy(new ObjectStoreWaitStrategy().withStartupTimeout(30000))
|
|
||||||
.start()
|
|
||||||
|
|
||||||
const port = container.getMappedPort(9000)
|
|
||||||
env._set("MINIO_URL", `http://0.0.0.0:${port}`)
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function stop() {
|
|
||||||
if (container) {
|
|
||||||
await container.stop()
|
|
||||||
container = undefined
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -86,10 +86,18 @@ export function setupEnv(...envs: any[]) {
|
||||||
throw new Error("CouchDB SQL port not found")
|
throw new Error("CouchDB SQL port not found")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const minio = getContainerByImage("minio/minio")
|
||||||
|
|
||||||
|
const minioPort = getExposedV4Port(minio, 9000)
|
||||||
|
if (!minioPort) {
|
||||||
|
throw new Error("Minio port not found")
|
||||||
|
}
|
||||||
|
|
||||||
const configs = [
|
const configs = [
|
||||||
{ key: "COUCH_DB_PORT", value: `${couchPort}` },
|
{ key: "COUCH_DB_PORT", value: `${couchPort}` },
|
||||||
{ key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
|
{ key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
|
||||||
{ key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
|
{ key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
|
||||||
|
{ key: "MINIO_URL", value: `http://127.0.0.1:${minioPort}` },
|
||||||
]
|
]
|
||||||
|
|
||||||
for (const config of configs.filter(x => !!x.value)) {
|
for (const config of configs.filter(x => !!x.value)) {
|
||||||
|
|
|
@ -4,10 +4,12 @@ import { APIError } from "@budibase/types"
|
||||||
describe("/api/applications/:appId/sync", () => {
|
describe("/api/applications/:appId/sync", () => {
|
||||||
let config = setup.getConfig()
|
let config = setup.getConfig()
|
||||||
|
|
||||||
afterAll(setup.afterAll)
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await config.init()
|
await config.init()
|
||||||
})
|
})
|
||||||
|
afterAll(async () => {
|
||||||
|
setup.afterAll()
|
||||||
|
})
|
||||||
|
|
||||||
describe("/api/attachments/process", () => {
|
describe("/api/attachments/process", () => {
|
||||||
it("should accept an image file upload", async () => {
|
it("should accept an image file upload", async () => {
|
||||||
|
@ -18,7 +20,8 @@ describe("/api/applications/:appId/sync", () => {
|
||||||
expect(resp.length).toBe(1)
|
expect(resp.length).toBe(1)
|
||||||
|
|
||||||
let upload = resp[0]
|
let upload = resp[0]
|
||||||
expect(upload.url.endsWith(".jpg")).toBe(true)
|
|
||||||
|
expect(upload.url.split("?")[0].endsWith(".jpg")).toBe(true)
|
||||||
expect(upload.extension).toBe("jpg")
|
expect(upload.extension).toBe("jpg")
|
||||||
expect(upload.size).toBe(1)
|
expect(upload.size).toBe(1)
|
||||||
expect(upload.name).toBe("1px.jpg")
|
expect(upload.name).toBe("1px.jpg")
|
||||||
|
|
|
@ -1,16 +1,18 @@
|
||||||
|
import { mocks } from "@budibase/backend-core/tests"
|
||||||
import tk from "timekeeper"
|
import tk from "timekeeper"
|
||||||
import * as setup from "./utilities"
|
import * as setup from "./utilities"
|
||||||
import { events } from "@budibase/backend-core"
|
import { events } from "@budibase/backend-core"
|
||||||
import sdk from "../../../sdk"
|
import sdk from "../../../sdk"
|
||||||
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
|
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
|
||||||
import { mocks } from "@budibase/backend-core/tests"
|
|
||||||
|
|
||||||
mocks.licenses.useBackups()
|
mocks.licenses.useBackups()
|
||||||
|
|
||||||
describe("/backups", () => {
|
describe("/backups", () => {
|
||||||
let config = setup.getConfig()
|
let config = setup.getConfig()
|
||||||
|
|
||||||
afterAll(setup.afterAll)
|
afterAll(async () => {
|
||||||
|
setup.afterAll()
|
||||||
|
})
|
||||||
|
|
||||||
beforeEach(async () => {
|
beforeEach(async () => {
|
||||||
tk.reset()
|
tk.reset()
|
||||||
|
|
|
@ -856,7 +856,7 @@ describe.each([
|
||||||
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
|
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
|
||||||
return context.doInAppContext(config.getAppId(), async () => {
|
return context.doInAppContext(config.getAppId(), async () => {
|
||||||
const enriched = await outputProcessing(table, [row])
|
const enriched = await outputProcessing(table, [row])
|
||||||
expect((enriched as Row[])[0].attachment.url).toBe(
|
expect((enriched as Row[])[0].attachment.url.split("?")[0]).toBe(
|
||||||
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
|
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
@ -889,7 +889,7 @@ describe.each([
|
||||||
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
|
await config.withEnv({ SELF_HOSTED: "true" }, async () => {
|
||||||
return context.doInAppContext(config.getAppId(), async () => {
|
return context.doInAppContext(config.getAppId(), async () => {
|
||||||
const enriched = await outputProcessing(table, [row])
|
const enriched = await outputProcessing(table, [row])
|
||||||
expect((enriched as Row[])[0].attachment[0].url).toBe(
|
expect((enriched as Row[])[0].attachment[0].url.split("?")[0]).toBe(
|
||||||
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
|
`/files/signed/prod-budi-app-assets/${config.getProdAppId()}/attachments/${attachmentId}`
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
|
|
|
@ -1,3 +1,13 @@
|
||||||
|
// Directly mock the AWS SDK
|
||||||
|
jest.mock("aws-sdk", () => ({
|
||||||
|
S3: jest.fn(() => ({
|
||||||
|
getSignedUrl: jest.fn(
|
||||||
|
(operation, params) => `http://example.com/${params.Bucket}/${params.Key}`
|
||||||
|
),
|
||||||
|
upload: jest.fn(() => ({ Contents: {} })),
|
||||||
|
})),
|
||||||
|
}))
|
||||||
|
|
||||||
const setup = require("./utilities")
|
const setup = require("./utilities")
|
||||||
const { constants } = require("@budibase/backend-core")
|
const { constants } = require("@budibase/backend-core")
|
||||||
|
|
||||||
|
|
|
@ -1,6 +1,3 @@
|
||||||
import fs from "fs"
|
|
||||||
import { join } from "path"
|
|
||||||
|
|
||||||
const response = (body: any, extra?: any) => () => ({
|
const response = (body: any, extra?: any) => () => ({
|
||||||
promise: () => body,
|
promise: () => body,
|
||||||
...extra,
|
...extra,
|
||||||
|
@ -62,9 +59,7 @@ class S3 {
|
||||||
Body: "",
|
Body: "",
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
createReadStream: jest
|
createReadStream: jest.fn().mockReturnValue("stream"),
|
||||||
.fn()
|
|
||||||
.mockReturnValue(fs.createReadStream(join(__dirname, "aws-sdk.ts"))),
|
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
)
|
)
|
|
@ -1,7 +1,6 @@
|
||||||
|
jest.mock("aws-sdk", () => require("./aws-sdk.mock"))
|
||||||
import { default as DynamoDBIntegration } from "../dynamodb"
|
import { default as DynamoDBIntegration } from "../dynamodb"
|
||||||
|
|
||||||
jest.mock("aws-sdk")
|
|
||||||
|
|
||||||
class TestConfiguration {
|
class TestConfiguration {
|
||||||
integration: any
|
integration: any
|
||||||
|
|
||||||
|
|
|
@ -28,7 +28,6 @@ jest.mock("uuid", () => ({ v4: () => "00000000-0000-0000-0000-000000000000" }))
|
||||||
import { default as RestIntegration } from "../rest"
|
import { default as RestIntegration } from "../rest"
|
||||||
import { RestAuthType } from "@budibase/types"
|
import { RestAuthType } from "@budibase/types"
|
||||||
import fetch from "node-fetch"
|
import fetch from "node-fetch"
|
||||||
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
|
|
||||||
import { Readable } from "stream"
|
import { Readable } from "stream"
|
||||||
|
|
||||||
const FormData = require("form-data")
|
const FormData = require("form-data")
|
||||||
|
@ -627,15 +626,6 @@ describe("REST Integration", () => {
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("File Handling", () => {
|
describe("File Handling", () => {
|
||||||
beforeAll(async () => {
|
|
||||||
jest.unmock("aws-sdk")
|
|
||||||
await objectStoreTestProviders.minio.start()
|
|
||||||
})
|
|
||||||
|
|
||||||
afterAll(async () => {
|
|
||||||
await objectStoreTestProviders.minio.stop()
|
|
||||||
})
|
|
||||||
|
|
||||||
it("uploads file to object store and returns signed URL", async () => {
|
it("uploads file to object store and returns signed URL", async () => {
|
||||||
const responseData = Buffer.from("teest file contnt")
|
const responseData = Buffer.from("teest file contnt")
|
||||||
const filename = "test.tar.gz"
|
const filename = "test.tar.gz"
|
||||||
|
|
|
@ -1,7 +1,6 @@
|
||||||
|
jest.mock("aws-sdk", () => require("./aws-sdk.mock"))
|
||||||
import { default as S3Integration } from "../s3"
|
import { default as S3Integration } from "../s3"
|
||||||
|
|
||||||
jest.mock("aws-sdk")
|
|
||||||
|
|
||||||
class TestConfiguration {
|
class TestConfiguration {
|
||||||
integration: any
|
integration: any
|
||||||
|
|
||||||
|
|
|
@ -14,3 +14,6 @@ process.env.WORKER_URL = "http://localhost:10000"
|
||||||
process.env.COUCH_DB_PASSWORD = "budibase"
|
process.env.COUCH_DB_PASSWORD = "budibase"
|
||||||
process.env.COUCH_DB_USER = "budibase"
|
process.env.COUCH_DB_USER = "budibase"
|
||||||
process.env.JWT_SECRET = "jwtsecret"
|
process.env.JWT_SECRET = "jwtsecret"
|
||||||
|
process.env.MINIO_URL = "http://localhost"
|
||||||
|
process.env.MINIO_ACCESS_KEY = "budibase"
|
||||||
|
process.env.MINIO_SECRET_KEY = "budibase"
|
||||||
|
|
|
@ -100,13 +100,13 @@ describe("rowProcessor - outputProcessing", () => {
|
||||||
}
|
}
|
||||||
|
|
||||||
const output = await outputProcessing(table, row, { squash: false })
|
const output = await outputProcessing(table, row, { squash: false })
|
||||||
expect(output.attach[0].url).toBe(
|
expect(output.attach[0].url?.split("?")[0]).toBe(
|
||||||
"/files/signed/prod-budi-app-assets/test.jpg"
|
"/files/signed/prod-budi-app-assets/test.jpg"
|
||||||
)
|
)
|
||||||
|
|
||||||
row.attach[0].url = ""
|
row.attach[0].url = ""
|
||||||
const output2 = await outputProcessing(table, row, { squash: false })
|
const output2 = await outputProcessing(table, row, { squash: false })
|
||||||
expect(output2.attach[0].url).toBe(
|
expect(output2.attach[0].url?.split("?")[0]).toBe(
|
||||||
"/files/signed/prod-budi-app-assets/test.jpg"
|
"/files/signed/prod-budi-app-assets/test.jpg"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -141,13 +141,13 @@ describe("rowProcessor - outputProcessing", () => {
|
||||||
}
|
}
|
||||||
|
|
||||||
const output = await outputProcessing(table, row, { squash: false })
|
const output = await outputProcessing(table, row, { squash: false })
|
||||||
expect(output.attach.url).toBe(
|
expect(output.attach.url?.split("?")[0]).toBe(
|
||||||
"/files/signed/prod-budi-app-assets/test.jpg"
|
"/files/signed/prod-budi-app-assets/test.jpg"
|
||||||
)
|
)
|
||||||
|
|
||||||
row.attach.url = ""
|
row.attach.url = ""
|
||||||
const output2 = await outputProcessing(table, row, { squash: false })
|
const output2 = await outputProcessing(table, row, { squash: false })
|
||||||
expect(output2.attach.url).toBe(
|
expect(output2.attach?.url?.split("?")[0]).toBe(
|
||||||
"/files/signed/prod-budi-app-assets/test.jpg"
|
"/files/signed/prod-budi-app-assets/test.jpg"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
@ -2,7 +2,6 @@ jest.unmock("node-fetch")
|
||||||
jest.unmock("aws-sdk")
|
jest.unmock("aws-sdk")
|
||||||
import { TestConfiguration } from "../../../../tests"
|
import { TestConfiguration } from "../../../../tests"
|
||||||
import { EmailTemplatePurpose } from "../../../../constants"
|
import { EmailTemplatePurpose } from "../../../../constants"
|
||||||
import { objectStoreTestProviders } from "@budibase/backend-core/tests"
|
|
||||||
import { objectStore } from "@budibase/backend-core"
|
import { objectStore } from "@budibase/backend-core"
|
||||||
import tk from "timekeeper"
|
import tk from "timekeeper"
|
||||||
import { EmailAttachment } from "@budibase/types"
|
import { EmailAttachment } from "@budibase/types"
|
||||||
|
@ -19,12 +18,10 @@ describe("/api/global/email", () => {
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
tk.reset()
|
tk.reset()
|
||||||
await objectStoreTestProviders.minio.start()
|
|
||||||
await config.beforeAll()
|
await config.beforeAll()
|
||||||
})
|
})
|
||||||
|
|
||||||
afterAll(async () => {
|
afterAll(async () => {
|
||||||
await objectStoreTestProviders.minio.stop()
|
|
||||||
await config.afterAll()
|
await config.afterAll()
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue