From 678033cc8b7276e5823536925cfebc577527f764 Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Thu, 5 Oct 2023 17:39:40 +0100 Subject: [PATCH 01/14] License Key - Activate & Manage Tests There are two test files, license.activate.spec.ts and license.manage.spec.ts These test files each contain a test: - Creates, activates, and deletes an online license for a self hosted account - license.activate.spec.ts - Retrieves plans, creates checkout session, and updates license - license.manage.spec.ts Updated and created API files - StripeAPI - LicenseAPI - internal-api LicenseAPI - index & AccountInternalAPI also updated to reflect API file changes --- .../src/account-api/api/AccountInternalAPI.ts | 4 +- .../src/account-api/api/apis/LicenseAPI.ts | 137 +++++++++++++----- qa-core/src/account-api/api/apis/StripeAPI.ts | 64 ++++++++ qa-core/src/account-api/api/apis/index.ts | 1 + .../tests/licensing/license.activate.spec.ts | 74 ++++++++++ .../tests/licensing/license.manage.spec.ts | 57 ++++++++ .../src/internal-api/api/apis/LicenseAPI.ts | 40 +++-- 7 files changed, 329 insertions(+), 48 deletions(-) create mode 100644 qa-core/src/account-api/api/apis/StripeAPI.ts create mode 100644 qa-core/src/account-api/tests/licensing/license.activate.spec.ts create mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts diff --git a/qa-core/src/account-api/api/AccountInternalAPI.ts b/qa-core/src/account-api/api/AccountInternalAPI.ts index 3813ad2c9e..ef2c39d5a4 100644 --- a/qa-core/src/account-api/api/AccountInternalAPI.ts +++ b/qa-core/src/account-api/api/AccountInternalAPI.ts @@ -1,5 +1,5 @@ import AccountInternalAPIClient from "./AccountInternalAPIClient" -import { AccountAPI, LicenseAPI, AuthAPI } from "./apis" +import { AccountAPI, LicenseAPI, AuthAPI, StripeAPI } from "./apis" import { State } from "../../types" export default class AccountInternalAPI { @@ -8,11 +8,13 @@ export default class AccountInternalAPI { auth: AuthAPI accounts: AccountAPI licenses: LicenseAPI + stripe: StripeAPI constructor(state: State) { this.client = new AccountInternalAPIClient(state) this.auth = new AuthAPI(this.client) this.accounts = new AccountAPI(this.client) this.licenses = new LicenseAPI(this.client) + this.stripe = new StripeAPI((this.client)) } } diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index 44579f867b..9f06ec7198 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -2,25 +2,23 @@ import AccountInternalAPIClient from "../AccountInternalAPIClient" import { Account, CreateOfflineLicenseRequest, + GetLicenseKeyResponse, GetOfflineLicenseResponse, UpdateLicenseRequest, } from "@budibase/types" import { Response } from "node-fetch" import BaseAPI from "./BaseAPI" import { APIRequestOpts } from "../../../types" - export default class LicenseAPI extends BaseAPI { client: AccountInternalAPIClient - constructor(client: AccountInternalAPIClient) { super() this.client = client } - async updateLicense( - accountId: string, - body: UpdateLicenseRequest, - opts: APIRequestOpts = { status: 200 } + accountId: string, + body: UpdateLicenseRequest, + opts: APIRequestOpts = { status: 200 } ): Promise<[Response, Account]> { return this.doRequest(() => { return this.client.put(`/api/accounts/${accountId}/license`, { @@ -29,44 +27,111 @@ export default class LicenseAPI extends BaseAPI { }) }, opts) } - // TODO: Better approach for setting tenant id header - async createOfflineLicense( - accountId: string, - tenantId: string, - body: CreateOfflineLicenseRequest, - opts: { status?: number } = {} + accountId: string, + tenantId: string, + body: CreateOfflineLicenseRequest, + opts: { status?: number } = {} ): Promise { const [response, json] = await this.client.post( - `/api/internal/accounts/${accountId}/license/offline`, - { - body, - internal: true, - headers: { - "x-budibase-tenant-id": tenantId, + `/api/internal/accounts/${accountId}/license/offline`, + { + body, + internal: true, + headers: { + "x-budibase-tenant-id": tenantId, + }, + } + ) + expect(response.status).toBe(opts.status ? opts.status : 201) + return response + } + async getOfflineLicense( + accountId: string, + tenantId: string, + opts: { status?: number } = {} + ): Promise<[Response, GetOfflineLicenseResponse]> { + const [response, json] = await this.client.get( + `/api/internal/accounts/${accountId}/license/offline`, + { + internal: true, + headers: { + "x-budibase-tenant-id": tenantId, + }, + } + ) + expect(response.status).toBe(opts.status ? opts.status : 200) + return [response, json] + } + async getLicenseKey( + opts: { status?: number } = {} + ): Promise<[Response, GetLicenseKeyResponse]> { + const [response, json] = await this.client.get(`/api/license/key`) + expect(response.status).toBe(opts.status ? opts.status : 200) + return [response, json] + } + async activateLicense( + apiKey: string, + tenantId: string, + licenseKey: string, + opts: APIRequestOpts = { status: 200 } + ) { + return this.doRequest(() => { + return this.client.post(`/api/license/activate`, { + body: { + apiKey: apiKey, + tenantId: tenantId, + licenseKey: licenseKey, }, - } + }) + }, opts) + } + async regenerateLicenseKey(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.post(`/api/license/key/regenerate`, {}) + }, opts) + } + + async getPlans(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/plans`) + }, opts) + } + + async updatePlan(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.put(`/api/license/plan`) + }, opts) + } + + async refreshAccountLicense( + accountId: string, + opts: { status?: number } = {} + ): Promise { + const [response, json] = await this.client.post( + `/api/accounts/${accountId}/license/refresh`, + { + internal: true, + } ) expect(response.status).toBe(opts.status ? opts.status : 201) return response } - async getOfflineLicense( - accountId: string, - tenantId: string, - opts: { status?: number } = {} - ): Promise<[Response, GetOfflineLicenseResponse]> { - const [response, json] = await this.client.get( - `/api/internal/accounts/${accountId}/license/offline`, - { - internal: true, - headers: { - "x-budibase-tenant-id": tenantId, - }, - } - ) - expect(response.status).toBe(opts.status ? opts.status : 200) - return [response, json] + async getLicenseUsage(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/license/usage`) + }, opts) } -} + + async licenseUsageTriggered( + opts: { status?: number } = {} + ): Promise { + const [response, json] = await this.client.post( + `/api/license/usage/triggered` + ) + expect(response.status).toBe(opts.status ? opts.status : 201) + return response + } +} \ No newline at end of file diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts new file mode 100644 index 0000000000..ffa96b3c2b --- /dev/null +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -0,0 +1,64 @@ +import AccountInternalAPIClient from "../AccountInternalAPIClient" +import BaseAPI from "./BaseAPI" +import { APIRequestOpts } from "../../../types" + +export default class StripeAPI extends BaseAPI { + client: AccountInternalAPIClient + + constructor(client: AccountInternalAPIClient) { + super() + this.client = client + } + + async createCheckoutSession( + priceId: string, + opts: APIRequestOpts = { status: 200 } + ) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/checkout-session`, { + body: { priceId }, + }) + }, opts) + } + + async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/checkout-success`) + }, opts) + } + + async createPortalSession( + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 } + ) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/portal-session`, { + body: { stripeCustomerId }, + }) + }, opts) + } + + async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/link`) + }, opts) + } + + async getInvoices(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/stripe/invoices`) + }, opts) + } + + async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/stripe/upcoming-invoice`) + }, opts) + } + + async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/stripe/customers`) + }, opts) + } +} diff --git a/qa-core/src/account-api/api/apis/index.ts b/qa-core/src/account-api/api/apis/index.ts index 1137ac3e36..5b0cf55110 100644 --- a/qa-core/src/account-api/api/apis/index.ts +++ b/qa-core/src/account-api/api/apis/index.ts @@ -1,3 +1,4 @@ export { default as AuthAPI } from "./AuthAPI" export { default as AccountAPI } from "./AccountAPI" export { default as LicenseAPI } from "./LicenseAPI" +export { default as StripeAPI } from "./StripeAPI" diff --git a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts new file mode 100644 index 0000000000..709e2c33f0 --- /dev/null +++ b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts @@ -0,0 +1,74 @@ +import TestConfiguration from "../../config/TestConfiguration" +import * as fixures from "../../fixtures" +import { Feature, Hosting } from "@budibase/types" + +describe("license activation", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("creates, activates and deletes online license - self host", async () => { + // Remove existing license key + await config.internalApi.license.deleteLicenseKey() + + // Verify license key not found + await config.internalApi.license.getLicenseKey({ status: 404 }) + + // Create self host account + const createAccountRequest = fixures.accounts.generateAccount({ + hosting: Hosting.SELF, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { autoVerify: true }) + + let licenseKey: string = " " + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + // Retrieve license key + const [res, body] = + await config.accountsApi.licenses.getLicenseKey() + licenseKey = body.licenseKey + }) + + const accountId = account.accountId! + + // Update license to have paid feature + const [res, acc] = await config.accountsApi.licenses.updateLicense( + accountId, + { + overrides: { + features: [Feature.APP_BACKUPS], + }, + } + ) + + // Activate license key + await config.internalApi.license.activateLicenseKey({licenseKey}) + + // Verify license updated with new feature + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, body] = await config.api.accounts.self() + expect(body.license.features[0]).toBe("appBackups") + }) + + // Remove license key + await config.internalApi.license.deleteLicenseKey() + + // Verify license key not found + await config.internalApi.license.getLicenseKey({ status: 404 }) + + // Verify user downgraded to free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, body] = await config.api.accounts.self() + expect(body.license.plan.type).toBe("free") + }) + }) +}) diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts new file mode 100644 index 0000000000..967252a0f9 --- /dev/null +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -0,0 +1,57 @@ +import TestConfiguration from "../../config/TestConfiguration" +import * as fixtures from "../../fixtures" +import { Hosting, PlanType } from "@budibase/types" + +describe("license management", () => { + const config = new TestConfiguration() + + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, + }) + + // Self response has free license + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) + + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() + + // Select priceId from premium plan + let premiumPriceId = null + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + break + } + } + + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") + + // TODO: Mimic checkout success + // Create stripe customer + // Create subscription for premium plan + // Assert license updated from free to premium + + // Create portal session + //await config.api.stripe.createPortalSession() + + // Update from free to business license + + // License updated + }) +}) diff --git a/qa-core/src/internal-api/api/apis/LicenseAPI.ts b/qa-core/src/internal-api/api/apis/LicenseAPI.ts index 4c9d14c55e..268f8781c3 100644 --- a/qa-core/src/internal-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/internal-api/api/apis/LicenseAPI.ts @@ -1,45 +1,63 @@ import { Response } from "node-fetch" import { + ActivateLicenseKeyRequest, ActivateOfflineLicenseTokenRequest, + GetLicenseKeyResponse, GetOfflineIdentifierResponse, GetOfflineLicenseTokenResponse, } from "@budibase/types" import BudibaseInternalAPIClient from "../BudibaseInternalAPIClient" import BaseAPI from "./BaseAPI" +import { APIRequestOpts } from "../../../types" export default class LicenseAPI extends BaseAPI { constructor(client: BudibaseInternalAPIClient) { super(client) } - async getOfflineLicenseToken( - opts: { status?: number } = {} + opts: { status?: number } = {} ): Promise<[Response, GetOfflineLicenseTokenResponse]> { const [response, body] = await this.get( - `/global/license/offline`, - opts.status + `/global/license/offline`, + opts.status ) return [response, body] } - async deleteOfflineLicenseToken(): Promise<[Response]> { const [response] = await this.del(`/global/license/offline`, 204) return [response] } - async activateOfflineLicenseToken( - body: ActivateOfflineLicenseTokenRequest + body: ActivateOfflineLicenseTokenRequest ): Promise<[Response]> { const [response] = await this.post(`/global/license/offline`, body) return [response] } - async getOfflineIdentifier(): Promise< - [Response, GetOfflineIdentifierResponse] + [Response, GetOfflineIdentifierResponse] > { const [response, body] = await this.get( - `/global/license/offline/identifier` + `/global/license/offline/identifier` ) return [response, body] } -} + + async getLicenseKey( + opts: { status?: number } = {} + ): Promise<[Response, GetLicenseKeyResponse]> { + const [response, body] = await this.get(`/global/license/key`, opts.status) + return [response, body] + } + + async activateLicenseKey( + body: ActivateLicenseKeyRequest + ): Promise<[Response]> { + const [response] = await this.post(`/global/license/key`, body) + return [response] + } + + async deleteLicenseKey(): Promise<[Response]> { + const [response] = await this.del(`/global/license/key`, 204) + return [response] + } +} \ No newline at end of file From 5e16d0451936ed2b1c2c0c6d1928c146b45c65c1 Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Thu, 5 Oct 2023 17:43:25 +0100 Subject: [PATCH 02/14] lint --- .../src/account-api/api/AccountInternalAPI.ts | 2 +- .../src/account-api/api/apis/LicenseAPI.ts | 78 +++++------ qa-core/src/account-api/api/apis/StripeAPI.ts | 100 +++++++-------- .../tests/licensing/license.activate.spec.ts | 121 +++++++++--------- .../tests/licensing/license.manage.spec.ts | 82 ++++++------ .../src/internal-api/api/apis/LicenseAPI.ts | 18 +-- 6 files changed, 201 insertions(+), 200 deletions(-) diff --git a/qa-core/src/account-api/api/AccountInternalAPI.ts b/qa-core/src/account-api/api/AccountInternalAPI.ts index ef2c39d5a4..f89bf556f2 100644 --- a/qa-core/src/account-api/api/AccountInternalAPI.ts +++ b/qa-core/src/account-api/api/AccountInternalAPI.ts @@ -15,6 +15,6 @@ export default class AccountInternalAPI { this.auth = new AuthAPI(this.client) this.accounts = new AccountAPI(this.client) this.licenses = new LicenseAPI(this.client) - this.stripe = new StripeAPI((this.client)) + this.stripe = new StripeAPI(this.client) } } diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index 9f06ec7198..dba1a661d4 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -16,9 +16,9 @@ export default class LicenseAPI extends BaseAPI { this.client = client } async updateLicense( - accountId: string, - body: UpdateLicenseRequest, - opts: APIRequestOpts = { status: 200 } + accountId: string, + body: UpdateLicenseRequest, + opts: APIRequestOpts = { status: 200 } ): Promise<[Response, Account]> { return this.doRequest(() => { return this.client.put(`/api/accounts/${accountId}/license`, { @@ -29,53 +29,53 @@ export default class LicenseAPI extends BaseAPI { } // TODO: Better approach for setting tenant id header async createOfflineLicense( - accountId: string, - tenantId: string, - body: CreateOfflineLicenseRequest, - opts: { status?: number } = {} + accountId: string, + tenantId: string, + body: CreateOfflineLicenseRequest, + opts: { status?: number } = {} ): Promise { const [response, json] = await this.client.post( - `/api/internal/accounts/${accountId}/license/offline`, - { - body, - internal: true, - headers: { - "x-budibase-tenant-id": tenantId, - }, - } + `/api/internal/accounts/${accountId}/license/offline`, + { + body, + internal: true, + headers: { + "x-budibase-tenant-id": tenantId, + }, + } ) expect(response.status).toBe(opts.status ? opts.status : 201) return response } async getOfflineLicense( - accountId: string, - tenantId: string, - opts: { status?: number } = {} + accountId: string, + tenantId: string, + opts: { status?: number } = {} ): Promise<[Response, GetOfflineLicenseResponse]> { const [response, json] = await this.client.get( - `/api/internal/accounts/${accountId}/license/offline`, - { - internal: true, - headers: { - "x-budibase-tenant-id": tenantId, - }, - } + `/api/internal/accounts/${accountId}/license/offline`, + { + internal: true, + headers: { + "x-budibase-tenant-id": tenantId, + }, + } ) expect(response.status).toBe(opts.status ? opts.status : 200) return [response, json] } async getLicenseKey( - opts: { status?: number } = {} + opts: { status?: number } = {} ): Promise<[Response, GetLicenseKeyResponse]> { const [response, json] = await this.client.get(`/api/license/key`) expect(response.status).toBe(opts.status ? opts.status : 200) return [response, json] } async activateLicense( - apiKey: string, - tenantId: string, - licenseKey: string, - opts: APIRequestOpts = { status: 200 } + apiKey: string, + tenantId: string, + licenseKey: string, + opts: APIRequestOpts = { status: 200 } ) { return this.doRequest(() => { return this.client.post(`/api/license/activate`, { @@ -106,14 +106,14 @@ export default class LicenseAPI extends BaseAPI { } async refreshAccountLicense( - accountId: string, - opts: { status?: number } = {} + accountId: string, + opts: { status?: number } = {} ): Promise { const [response, json] = await this.client.post( - `/api/accounts/${accountId}/license/refresh`, - { - internal: true, - } + `/api/accounts/${accountId}/license/refresh`, + { + internal: true, + } ) expect(response.status).toBe(opts.status ? opts.status : 201) return response @@ -126,12 +126,12 @@ export default class LicenseAPI extends BaseAPI { } async licenseUsageTriggered( - opts: { status?: number } = {} + opts: { status?: number } = {} ): Promise { const [response, json] = await this.client.post( - `/api/license/usage/triggered` + `/api/license/usage/triggered` ) expect(response.status).toBe(opts.status ? opts.status : 201) return response } -} \ No newline at end of file +} diff --git a/qa-core/src/account-api/api/apis/StripeAPI.ts b/qa-core/src/account-api/api/apis/StripeAPI.ts index ffa96b3c2b..c9c776e89b 100644 --- a/qa-core/src/account-api/api/apis/StripeAPI.ts +++ b/qa-core/src/account-api/api/apis/StripeAPI.ts @@ -3,62 +3,62 @@ import BaseAPI from "./BaseAPI" import { APIRequestOpts } from "../../../types" export default class StripeAPI extends BaseAPI { - client: AccountInternalAPIClient + client: AccountInternalAPIClient - constructor(client: AccountInternalAPIClient) { - super() - this.client = client - } + constructor(client: AccountInternalAPIClient) { + super() + this.client = client + } - async createCheckoutSession( - priceId: string, - opts: APIRequestOpts = { status: 200 } - ) { - return this.doRequest(() => { - return this.client.post(`/api/stripe/checkout-session`, { - body: { priceId }, - }) - }, opts) - } + async createCheckoutSession( + priceId: string, + opts: APIRequestOpts = { status: 200 } + ) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/checkout-session`, { + body: { priceId }, + }) + }, opts) + } - async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) { - return this.doRequest(() => { - return this.client.post(`/api/stripe/checkout-success`) - }, opts) - } + async checkoutSuccess(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/checkout-success`) + }, opts) + } - async createPortalSession( - stripeCustomerId: string, - opts: APIRequestOpts = { status: 200 } - ) { - return this.doRequest(() => { - return this.client.post(`/api/stripe/portal-session`, { - body: { stripeCustomerId }, - }) - }, opts) - } + async createPortalSession( + stripeCustomerId: string, + opts: APIRequestOpts = { status: 200 } + ) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/portal-session`, { + body: { stripeCustomerId }, + }) + }, opts) + } - async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) { - return this.doRequest(() => { - return this.client.post(`/api/stripe/link`) - }, opts) - } + async linkStripeCustomer(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.post(`/api/stripe/link`) + }, opts) + } - async getInvoices(opts: APIRequestOpts = { status: 200 }) { - return this.doRequest(() => { - return this.client.get(`/api/stripe/invoices`) - }, opts) - } + async getInvoices(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/stripe/invoices`) + }, opts) + } - async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) { - return this.doRequest(() => { - return this.client.get(`/api/stripe/upcoming-invoice`) - }, opts) - } + async getUpcomingInvoice(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/stripe/upcoming-invoice`) + }, opts) + } - async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) { - return this.doRequest(() => { - return this.client.get(`/api/stripe/customers`) - }, opts) - } + async getStripeCustomers(opts: APIRequestOpts = { status: 200 }) { + return this.doRequest(() => { + return this.client.get(`/api/stripe/customers`) + }, opts) + } } diff --git a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts index 709e2c33f0..a494ceb354 100644 --- a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts @@ -3,72 +3,73 @@ import * as fixures from "../../fixtures" import { Feature, Hosting } from "@budibase/types" describe("license activation", () => { - const config = new TestConfiguration() + const config = new TestConfiguration() - beforeAll(async () => { - await config.beforeAll() + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("creates, activates and deletes online license - self host", async () => { + // Remove existing license key + await config.internalApi.license.deleteLicenseKey() + + // Verify license key not found + await config.internalApi.license.getLicenseKey({ status: 404 }) + + // Create self host account + const createAccountRequest = fixures.accounts.generateAccount({ + hosting: Hosting.SELF, + }) + const [createAccountRes, account] = + await config.accountsApi.accounts.create(createAccountRequest, { + autoVerify: true, + }) + + let licenseKey: string = " " + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + // Retrieve license key + const [res, body] = await config.accountsApi.licenses.getLicenseKey() + licenseKey = body.licenseKey }) - afterAll(async () => { - await config.afterAll() + const accountId = account.accountId! + + // Update license to have paid feature + const [res, acc] = await config.accountsApi.licenses.updateLicense( + accountId, + { + overrides: { + features: [Feature.APP_BACKUPS], + }, + } + ) + + // Activate license key + await config.internalApi.license.activateLicenseKey({ licenseKey }) + + // Verify license updated with new feature + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, body] = await config.api.accounts.self() + expect(body.license.features[0]).toBe("appBackups") }) - it("creates, activates and deletes online license - self host", async () => { - // Remove existing license key - await config.internalApi.license.deleteLicenseKey() + // Remove license key + await config.internalApi.license.deleteLicenseKey() - // Verify license key not found - await config.internalApi.license.getLicenseKey({ status: 404 }) + // Verify license key not found + await config.internalApi.license.getLicenseKey({ status: 404 }) - // Create self host account - const createAccountRequest = fixures.accounts.generateAccount({ - hosting: Hosting.SELF, - }) - const [createAccountRes, account] = - await config.accountsApi.accounts.create(createAccountRequest, { autoVerify: true }) - - let licenseKey: string = " " - await config.doInNewState(async () => { - await config.loginAsAccount(createAccountRequest) - // Retrieve license key - const [res, body] = - await config.accountsApi.licenses.getLicenseKey() - licenseKey = body.licenseKey - }) - - const accountId = account.accountId! - - // Update license to have paid feature - const [res, acc] = await config.accountsApi.licenses.updateLicense( - accountId, - { - overrides: { - features: [Feature.APP_BACKUPS], - }, - } - ) - - // Activate license key - await config.internalApi.license.activateLicenseKey({licenseKey}) - - // Verify license updated with new feature - await config.doInNewState(async () => { - await config.loginAsAccount(createAccountRequest) - const [selfRes, body] = await config.api.accounts.self() - expect(body.license.features[0]).toBe("appBackups") - }) - - // Remove license key - await config.internalApi.license.deleteLicenseKey() - - // Verify license key not found - await config.internalApi.license.getLicenseKey({ status: 404 }) - - // Verify user downgraded to free license - await config.doInNewState(async () => { - await config.loginAsAccount(createAccountRequest) - const [selfRes, body] = await config.api.accounts.self() - expect(body.license.plan.type).toBe("free") - }) + // Verify user downgraded to free license + await config.doInNewState(async () => { + await config.loginAsAccount(createAccountRequest) + const [selfRes, body] = await config.api.accounts.self() + expect(body.license.plan.type).toBe("free") }) + }) }) diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts index 967252a0f9..3f87838ee4 100644 --- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts @@ -3,55 +3,55 @@ import * as fixtures from "../../fixtures" import { Hosting, PlanType } from "@budibase/types" describe("license management", () => { - const config = new TestConfiguration() + const config = new TestConfiguration() - beforeAll(async () => { - await config.beforeAll() + beforeAll(async () => { + await config.beforeAll() + }) + + afterAll(async () => { + await config.afterAll() + }) + + it("retrieves plans, creates checkout session, and updates license", async () => { + // Create cloud account + const createAccountRequest = fixtures.accounts.generateAccount({ + hosting: Hosting.CLOUD, }) - afterAll(async () => { - await config.afterAll() - }) + // Self response has free license + const [selfRes, selfBody] = await config.api.accounts.self() + expect(selfBody.license.plan.type).toBe(PlanType.FREE) - it("retrieves plans, creates checkout session, and updates license", async () => { - // Create cloud account - const createAccountRequest = fixtures.accounts.generateAccount({ - hosting: Hosting.CLOUD, - }) + // Retrieve plans + const [plansRes, planBody] = await config.api.licenses.getPlans() - // Self response has free license - const [selfRes, selfBody] = await config.api.accounts.self() - expect(selfBody.license.plan.type).toBe(PlanType.FREE) + // Select priceId from premium plan + let premiumPriceId = null + for (const plan of planBody) { + if (plan.type === PlanType.PREMIUM) { + premiumPriceId = plan.prices[0].priceId + break + } + } - // Retrieve plans - const [plansRes, planBody] = await config.api.licenses.getPlans() + // Create checkout session for price + const checkoutSessionRes = await config.api.stripe.createCheckoutSession( + premiumPriceId + ) + const checkoutSessionUrl = checkoutSessionRes[1].url + expect(checkoutSessionUrl).toContain("checkout.stripe.com") - // Select priceId from premium plan - let premiumPriceId = null - for (const plan of planBody) { - if (plan.type === PlanType.PREMIUM) { - premiumPriceId = plan.prices[0].priceId - break - } - } + // TODO: Mimic checkout success + // Create stripe customer + // Create subscription for premium plan + // Assert license updated from free to premium - // Create checkout session for price - const checkoutSessionRes = await config.api.stripe.createCheckoutSession( - premiumPriceId - ) - const checkoutSessionUrl = checkoutSessionRes[1].url - expect(checkoutSessionUrl).toContain("checkout.stripe.com") + // Create portal session + //await config.api.stripe.createPortalSession() - // TODO: Mimic checkout success - // Create stripe customer - // Create subscription for premium plan - // Assert license updated from free to premium + // Update from free to business license - // Create portal session - //await config.api.stripe.createPortalSession() - - // Update from free to business license - - // License updated - }) + // License updated + }) }) diff --git a/qa-core/src/internal-api/api/apis/LicenseAPI.ts b/qa-core/src/internal-api/api/apis/LicenseAPI.ts index 268f8781c3..ef322e069a 100644 --- a/qa-core/src/internal-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/internal-api/api/apis/LicenseAPI.ts @@ -15,11 +15,11 @@ export default class LicenseAPI extends BaseAPI { super(client) } async getOfflineLicenseToken( - opts: { status?: number } = {} + opts: { status?: number } = {} ): Promise<[Response, GetOfflineLicenseTokenResponse]> { const [response, body] = await this.get( - `/global/license/offline`, - opts.status + `/global/license/offline`, + opts.status ) return [response, body] } @@ -28,29 +28,29 @@ export default class LicenseAPI extends BaseAPI { return [response] } async activateOfflineLicenseToken( - body: ActivateOfflineLicenseTokenRequest + body: ActivateOfflineLicenseTokenRequest ): Promise<[Response]> { const [response] = await this.post(`/global/license/offline`, body) return [response] } async getOfflineIdentifier(): Promise< - [Response, GetOfflineIdentifierResponse] + [Response, GetOfflineIdentifierResponse] > { const [response, body] = await this.get( - `/global/license/offline/identifier` + `/global/license/offline/identifier` ) return [response, body] } async getLicenseKey( - opts: { status?: number } = {} + opts: { status?: number } = {} ): Promise<[Response, GetLicenseKeyResponse]> { const [response, body] = await this.get(`/global/license/key`, opts.status) return [response, body] } async activateLicenseKey( - body: ActivateLicenseKeyRequest + body: ActivateLicenseKeyRequest ): Promise<[Response]> { const [response] = await this.post(`/global/license/key`, body) return [response] @@ -60,4 +60,4 @@ export default class LicenseAPI extends BaseAPI { const [response] = await this.del(`/global/license/key`, 204) return [response] } -} \ No newline at end of file +} From f3234f6bd63e0bf5307ffc752c9992bfc4813b0d Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Fri, 6 Oct 2023 16:22:56 +0100 Subject: [PATCH 03/14] Update license activate test I have removed the end of the test which was to 'Verify user downgraded to free license' - This is not needed I have also updated getLicenseKey - specifically how it handles the expected 200 response --- qa-core/src/account-api/api/apis/LicenseAPI.ts | 2 +- .../account-api/tests/licensing/license.activate.spec.ts | 7 ------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/qa-core/src/account-api/api/apis/LicenseAPI.ts b/qa-core/src/account-api/api/apis/LicenseAPI.ts index dba1a661d4..b371f00f05 100644 --- a/qa-core/src/account-api/api/apis/LicenseAPI.ts +++ b/qa-core/src/account-api/api/apis/LicenseAPI.ts @@ -68,7 +68,7 @@ export default class LicenseAPI extends BaseAPI { opts: { status?: number } = {} ): Promise<[Response, GetLicenseKeyResponse]> { const [response, json] = await this.client.get(`/api/license/key`) - expect(response.status).toBe(opts.status ? opts.status : 200) + expect(response.status).toBe(opts.status || 200) return [response, json] } async activateLicense( diff --git a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts index a494ceb354..96c6eaea2a 100644 --- a/qa-core/src/account-api/tests/licensing/license.activate.spec.ts +++ b/qa-core/src/account-api/tests/licensing/license.activate.spec.ts @@ -64,12 +64,5 @@ describe("license activation", () => { // Verify license key not found await config.internalApi.license.getLicenseKey({ status: 404 }) - - // Verify user downgraded to free license - await config.doInNewState(async () => { - await config.loginAsAccount(createAccountRequest) - const [selfRes, body] = await config.api.accounts.self() - expect(body.license.plan.type).toBe("free") - }) }) }) From a06451222400312d9928e67618e870cc28f2dbf7 Mon Sep 17 00:00:00 2001 From: Dean Date: Wed, 18 Oct 2023 11:14:16 +0100 Subject: [PATCH 04/14] Added description field to the formblock --- packages/client/manifest.json | 6 ++ .../app/blocks/form/FormBlock.svelte | 2 + .../app/blocks/form/InnerFormBlock.svelte | 101 ++++++++++-------- 3 files changed, 67 insertions(+), 42 deletions(-) diff --git a/packages/client/manifest.json b/packages/client/manifest.json index 8d0a4e456f..ed69b8868f 100644 --- a/packages/client/manifest.json +++ b/packages/client/manifest.json @@ -5305,6 +5305,12 @@ "key": "title", "nested": true }, + { + "type": "text", + "label": "Description", + "key": "description", + "nested": true + }, { "section": true, "dependsOn": { diff --git a/packages/client/src/components/app/blocks/form/FormBlock.svelte b/packages/client/src/components/app/blocks/form/FormBlock.svelte index 5d57d10ab6..f905227af9 100644 --- a/packages/client/src/components/app/blocks/form/FormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/FormBlock.svelte @@ -12,6 +12,7 @@ export let fields export let labelPosition export let title + export let description export let showDeleteButton export let showSaveButton export let saveButtonLabel @@ -98,6 +99,7 @@ fields: fieldsOrDefault, labelPosition, title, + description, saveButtonLabel: saveLabel, deleteButtonLabel: deleteLabel, schema, diff --git a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte index ec5daa21b1..e65d2cf90b 100644 --- a/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte +++ b/packages/client/src/components/app/blocks/form/InnerFormBlock.svelte @@ -11,6 +11,7 @@ export let fields export let labelPosition export let title + export let description export let saveButtonLabel export let deleteButtonLabel export let schema @@ -160,55 +161,71 @@ - {#if renderButtons} + > + {#if renderButtons} + + {#if renderDeleteButton} + + {/if} + {#if renderSaveButton} + + {/if} + + {/if} + + {#if description} + - {#if renderDeleteButton} - - {/if} - {#if renderSaveButton} - - {/if} - + /> {/if} {/if} From b53158ac083046f0522855a3cb7628e6bd76d526 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Oct 2023 16:25:14 +0100 Subject: [PATCH 05/14] Moving the client library to be retrieved using the same mechanism as we do in development, through the server instance (same as the builder itself). This should help with the CDN issues that we have been having, although attachments are still currently affected. --- .../src/objectStore/buckets/app.ts | 50 ++++++++------- .../src/objectStore/buckets/plugins.ts | 16 ++--- .../src/objectStore/objectStore.ts | 62 ++++++++++++------- .../src/api/controllers/static/index.ts | 49 +++++++++------ 4 files changed, 102 insertions(+), 75 deletions(-) diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index be9fddeaa6..931cbeb78b 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -1,37 +1,41 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" +import qs from "querystring" + +export function clientLibraryPath(appId: string) { + return `${objectStore.sanitizeKey(appId)}/budibase-client.js` +} /** - * In production the client library is stored in the object store, however in development - * we use the symlinked version produced by lerna, located in node modules. We link to this - * via a specific endpoint (under /api/assets/client). - * @param appId In production we need the appId to look up the correct bucket, as the - * version of the client lib may differ between apps. - * @param version The version to retrieve. - * @return The URL to be inserted into appPackage response or server rendered - * app index file. + * Previously we used to serve the client library directly from Cloudfront, however + * due to issues with the domain we were unable to continue doing this - keeping + * incase we are able to switch back to CDN path again in future. */ -export const clientLibraryUrl = (appId: string, version: string) => { - if (env.isProd()) { - let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js` - if (env.CLOUDFRONT_CDN) { - // append app version to bust the cache - if (version) { - file += `?v=${version}` - } - // don't need to use presigned for client with cloudfront - // file is public - return cloudfront.getUrl(file) - } else { - return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) +export function clientLibraryCDNUrl(appId: string, version: string) { + let file = clientLibraryPath(appId) + if (env.CLOUDFRONT_CDN) { + // append app version to bust the cache + if (version) { + file += `?v=${version}` } + // don't need to use presigned for client with cloudfront + // file is public + return cloudfront.getUrl(file) } else { - return `/api/assets/client` + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) } } -export const getAppFileUrl = (s3Key: string) => { +export function clientLibraryUrl(appId: string, version: string) { + const queryString = qs.encode({ + appId, + version, + }) + return `/api/assets/client?${queryString}` +} + +export function getAppFileUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts index f7721afb23..6f1b7116ae 100644 --- a/packages/backend-core/src/objectStore/buckets/plugins.ts +++ b/packages/backend-core/src/objectStore/buckets/plugins.ts @@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types" // URLS -export const enrichPluginURLs = (plugins: Plugin[]) => { +export function enrichPluginURLs(plugins: Plugin[]) { if (!plugins || !plugins.length) { return [] } @@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => { }) } -const getPluginJSUrl = (plugin: Plugin) => { +function getPluginJSUrl(plugin: Plugin) { const s3Key = getPluginJSKey(plugin) return getPluginUrl(s3Key) } -const getPluginIconUrl = (plugin: Plugin): string | undefined => { +function getPluginIconUrl(plugin: Plugin): string | undefined { const s3Key = getPluginIconKey(plugin) if (!s3Key) { return @@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => { return getPluginUrl(s3Key) } -const getPluginUrl = (s3Key: string) => { +function getPluginUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { @@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => { // S3 KEYS -export const getPluginJSKey = (plugin: Plugin) => { +export function getPluginJSKey(plugin: Plugin) { return getPluginS3Key(plugin, "plugin.min.js") } -export const getPluginIconKey = (plugin: Plugin) => { +export function getPluginIconKey(plugin: Plugin) { // stored iconUrl is deprecated - hardcode to icon.svg in this case const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName if (!iconFileName) { @@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => { return getPluginS3Key(plugin, iconFileName) } -const getPluginS3Key = (plugin: Plugin, fileName: string) => { +function getPluginS3Key(plugin: Plugin, fileName: string) { const s3Key = getPluginS3Dir(plugin.name) return `${s3Key}/${fileName}` } -export const getPluginS3Dir = (pluginName: string) => { +export function getPluginS3Dir(pluginName: string) { let s3Key = `${pluginName}` if (env.MULTI_TENANCY) { const tenantId = context.getTenantId() diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index c36a09915e..76d2dd6689 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -1,6 +1,6 @@ const sanitize = require("sanitize-s3-objectkey") import AWS from "aws-sdk" -import stream from "stream" +import stream, { Readable } from "stream" import fetch from "node-fetch" import tar from "tar-fs" import zlib from "zlib" @@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) { * @return an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = ( +export function ObjectStore( bucket: string, opts: { presigning: boolean } = { presigning: false } -) => { +) { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -104,7 +104,7 @@ export const ObjectStore = ( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: string) => { +export async function makeSureBucketExists(client: any, bucketName: string) { bucketName = sanitizeBucket(bucketName) try { await client @@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => { * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). */ -export const upload = async ({ +export async function upload({ bucket: bucketName, filename, path, type, metadata, -}: UploadParams) => { +}: UploadParams) { const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) @@ -180,12 +180,12 @@ export const upload = async ({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export const streamUpload = async ( +export async function streamUpload( bucketName: string, filename: string, stream: any, extra = {} -) => { +) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) @@ -215,7 +215,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: string, filepath: string) => { +export async function retrieve(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => { } } -export const listAllObjects = async (bucketName: string, path: string) => { +export async function listAllObjects(bucketName: string, path: string) { const objectStore = ObjectStore(bucketName) const list = (params: ListParams = {}) => { return objectStore @@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => { /** * Generate a presigned url with a default TTL of 1 hour */ -export const getPresignedUrl = ( +export function getPresignedUrl( bucketName: string, key: string, durationSeconds: number = 3600 -) => { +) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { Bucket: sanitizeBucket(bucketName), @@ -291,7 +291,7 @@ export const getPresignedUrl = ( /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: string, filepath: string) => { +export async function retrieveToTmp(bucketName: string, filepath: string) { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => { return outputPath } -export const retrieveDirectory = async (bucketName: string, path: string) => { +export async function retrieveDirectory(bucketName: string, path: string) { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) @@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => { /** * Delete a single file. */ -export const deleteFile = async (bucketName: string, filepath: string) => { +export async function deleteFile(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => { return objectStore.deleteObject(params).promise() } -export const deleteFiles = async (bucketName: string, filepaths: string[]) => { +export async function deleteFiles(bucketName: string, filepaths: string[]) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => { /** * Delete a path, including everything within. */ -export const deleteFolder = async ( +export async function deleteFolder( bucketName: string, folder: string -): Promise => { +): Promise { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) const client = ObjectStore(bucketName) @@ -383,11 +383,11 @@ export const deleteFolder = async ( } } -export const uploadDirectory = async ( +export async function uploadDirectory( bucketName: string, localPath: string, bucketPath: string -) => { +) { bucketName = sanitizeBucket(bucketName) let uploads = [] const files = fs.readdirSync(localPath, { withFileTypes: true }) @@ -404,11 +404,11 @@ export const uploadDirectory = async ( return files } -export const downloadTarballDirect = async ( +export async function downloadTarballDirect( url: string, path: string, headers = {} -) => { +) { path = sanitizeKey(path) const response = await fetch(url, { headers }) if (!response.ok) { @@ -418,11 +418,11 @@ export const downloadTarballDirect = async ( await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path)) } -export const downloadTarball = async ( +export async function downloadTarball( url: string, bucketName: string, path: string -) => { +) { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) @@ -438,3 +438,17 @@ export const downloadTarball = async ( // return the temporary path incase there is a use for it return tmpPath } + +export async function getReadStream( + bucketName: string, + path: string +): Promise { + bucketName = sanitizeBucket(bucketName) + path = sanitizeKey(path) + const client = ObjectStore(bucketName) + const params = { + Bucket: bucketName, + Key: path, + } + return client.getObject(params).createReadStream() +} diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index bbf9dd34f5..d2e887dcf8 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -16,7 +16,7 @@ import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" -import { App } from "@budibase/types" +import { App, Ctx } from "@budibase/types" const send = require("koa-send") @@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) { } } -export const toggleBetaUiFeature = async function (ctx: any) { +export const toggleBetaUiFeature = async function (ctx: Ctx) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) { } } -export const serveBuilder = async function (ctx: any) { +export const serveBuilder = async function (ctx: Ctx) { const builderPath = join(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) } -export const uploadFile = async function (ctx: any) { - let files = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] +export const uploadFile = async function (ctx: Ctx) { + const file = ctx.request?.files?.file + let files = file && Array.isArray(file) ? Array.from(file) : [file] const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() @@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) { ctx.body = await Promise.all(uploads) } -export const deleteObjects = async function (ctx: any) { +export const deleteObjects = async function (ctx: Ctx) { ctx.body = await objectStore.deleteFiles( ObjectStoreBuckets.APPS, ctx.request.body.keys ) } -export const serveApp = async function (ctx: any) { +export const serveApp = async function (ctx: Ctx) { const bbHeaderEmbed = ctx.request.get("x-budibase-embed")?.toLowerCase() === "true" @@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) { } } -export const serveBuilderPreview = async function (ctx: any) { +export const serveBuilderPreview = async function (ctx: Ctx) { const db = context.getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentType.APP_METADATA) @@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) { } } -export const serveClientLibrary = async function (ctx: any) { +export const serveClientLibrary = async function (ctx: Ctx) { + const appId = context.getAppId() || (ctx.request.query.appId as string) let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist") - // incase running from TS directly - if (env.isDev() && !fs.existsSync(rootPath)) { - rootPath = join(require.resolve("@budibase/client"), "..") + if (!appId) { + ctx.throw(400, "No app ID provided - cannot fetch client library.") + } + if (env.isProd()) { + ctx.body = await objectStore.getReadStream( + ObjectStoreBuckets.APPS, + objectStore.clientLibraryPath(appId!) + ) + } else if (env.isDev()) { + // incase running from TS directly + const tsPath = join(require.resolve("@budibase/client"), "..") + return send(ctx, "budibase-client.js", { + root: !fs.existsSync(rootPath) ? tsPath : rootPath, + }) + } else { + ctx.throw(500, "Unable to retrieve client library.") } - return send(ctx, "budibase-client.js", { - root: rootPath, - }) } -export const getSignedUploadURL = async function (ctx: any) { +export const getSignedUploadURL = async function (ctx: Ctx) { // Ensure datasource is valid let datasource try { @@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) { const params = { Bucket: bucket, Key: key } signedUrl = s3.getSignedUrl("putObject", params) publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}` - } catch (error) { + } catch (error: any) { ctx.throw(400, error) } } From 570eaa8e5347a41de09b78fb283b50c3d585def1 Mon Sep 17 00:00:00 2001 From: Mitch-Budibase Date: Tue, 24 Oct 2023 17:22:49 +0100 Subject: [PATCH 06/14] Update package.json + remove license.manage.spec.ts Update package.json - Updating "test:self:ci" - Now includes license. Needed to prevent license tests from running where an account portal is not setup Removing license.manage.spec.ts - This will be included in another PR - Work required to finish this test - Removing so the license activation test can be merged --- qa-core/package.json | 2 +- .../tests/licensing/license.manage.spec.ts | 57 ------------------- 2 files changed, 1 insertion(+), 58 deletions(-) delete mode 100644 qa-core/src/account-api/tests/licensing/license.manage.spec.ts diff --git a/qa-core/package.json b/qa-core/package.json index 3c789d89e6..d266ca9def 100644 --- a/qa-core/package.json +++ b/qa-core/package.json @@ -17,7 +17,7 @@ "test:notify": "node scripts/testResultsWebhook", "test:cloud:prod": "yarn run test --testPathIgnorePatterns=\\.integration\\.", "test:cloud:qa": "yarn run test", - "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\.", + "test:self:ci": "yarn run test --testPathIgnorePatterns=\\.integration\\. \\.cloud\\. \\.license\\.", "serve:test:self:ci": "start-server-and-test dev:built http://localhost:4001/health test:self:ci", "serve": "start-server-and-test dev:built http://localhost:4001/health", "dev:built": "cd ../ && yarn dev:built" diff --git a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts b/qa-core/src/account-api/tests/licensing/license.manage.spec.ts deleted file mode 100644 index 3f87838ee4..0000000000 --- a/qa-core/src/account-api/tests/licensing/license.manage.spec.ts +++ /dev/null @@ -1,57 +0,0 @@ -import TestConfiguration from "../../config/TestConfiguration" -import * as fixtures from "../../fixtures" -import { Hosting, PlanType } from "@budibase/types" - -describe("license management", () => { - const config = new TestConfiguration() - - beforeAll(async () => { - await config.beforeAll() - }) - - afterAll(async () => { - await config.afterAll() - }) - - it("retrieves plans, creates checkout session, and updates license", async () => { - // Create cloud account - const createAccountRequest = fixtures.accounts.generateAccount({ - hosting: Hosting.CLOUD, - }) - - // Self response has free license - const [selfRes, selfBody] = await config.api.accounts.self() - expect(selfBody.license.plan.type).toBe(PlanType.FREE) - - // Retrieve plans - const [plansRes, planBody] = await config.api.licenses.getPlans() - - // Select priceId from premium plan - let premiumPriceId = null - for (const plan of planBody) { - if (plan.type === PlanType.PREMIUM) { - premiumPriceId = plan.prices[0].priceId - break - } - } - - // Create checkout session for price - const checkoutSessionRes = await config.api.stripe.createCheckoutSession( - premiumPriceId - ) - const checkoutSessionUrl = checkoutSessionRes[1].url - expect(checkoutSessionUrl).toContain("checkout.stripe.com") - - // TODO: Mimic checkout success - // Create stripe customer - // Create subscription for premium plan - // Assert license updated from free to premium - - // Create portal session - //await config.api.stripe.createPortalSession() - - // Update from free to business license - - // License updated - }) -}) From ba5b1d7cfa0b9f9fc228a3d22c64b8b8bb53934a Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Oct 2023 17:27:58 +0100 Subject: [PATCH 07/14] Adding tenant ID to query string and fixing test cases. --- .../src/objectStore/buckets/app.ts | 19 ++++++--- .../src/objectStore/buckets/tests/app.spec.ts | 42 +++++++------------ 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index 931cbeb78b..95b73b9104 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -2,6 +2,7 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" import qs from "querystring" +import { getTenantId, DEFAULT_TENANT_ID } from "../../context" export function clientLibraryPath(appId: string) { return `${objectStore.sanitizeKey(appId)}/budibase-client.js` @@ -28,11 +29,19 @@ export function clientLibraryCDNUrl(appId: string, version: string) { } export function clientLibraryUrl(appId: string, version: string) { - const queryString = qs.encode({ - appId, - version, - }) - return `/api/assets/client?${queryString}` + let tenantId, qsParams: { appId: string; version: string; tenantId?: string } + try { + tenantId = getTenantId() + } finally { + qsParams = { + appId, + version, + } + } + if (tenantId) { + qsParams.tenantId = tenantId + } + return `/api/assets/client?${qs.encode(qsParams)}` } export function getAppFileUrl(s3Key: string) { diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts index aaa07ec9d3..cbbbee6255 100644 --- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -1,5 +1,4 @@ import * as app from "../app" -import { getAppFileUrl } from "../app" import { testEnv } from "../../../../tests/extra" describe("app", () => { @@ -7,6 +6,15 @@ describe("app", () => { testEnv.nodeJest() }) + function baseCheck(url: string, tenantId?: string) { + expect(url).toContain("/api/assets/client") + if (tenantId) { + expect(url).toContain(`tenantId=${tenantId}`) + } + expect(url).toContain("appId=app_123") + expect(url).toContain("version=2.0.0") + } + describe("clientLibraryUrl", () => { function getClientUrl() { return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0") @@ -20,31 +28,19 @@ describe("app", () => { it("gets url in dev", () => { testEnv.nodeDev() const url = getClientUrl() - expect(url).toBe("/api/assets/client") - }) - - it("gets url with embedded minio", () => { - testEnv.withMinio() - const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with custom S3", () => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with cloudfront + s3", () => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url) }) }) @@ -57,7 +53,7 @@ describe("app", () => { testEnv.nodeDev() await testEnv.withTenant(tenantId => { const url = getClientUrl() - expect(url).toBe("/api/assets/client") + baseCheck(url, tenantId) }) }) @@ -65,9 +61,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withMinio() const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -75,9 +69,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -85,9 +77,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url, tenantId) }) }) }) From 701d30abb4f3c292fbddd5ec50cf4f8c31fae789 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 24 Oct 2023 17:30:57 +0100 Subject: [PATCH 08/14] Don't pass through default tenant ID. --- packages/backend-core/src/objectStore/buckets/app.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index 95b73b9104..43bc965c65 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -2,7 +2,7 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" import qs from "querystring" -import { getTenantId, DEFAULT_TENANT_ID } from "../../context" +import { DEFAULT_TENANT_ID, getTenantId } from "../../context" export function clientLibraryPath(appId: string) { return `${objectStore.sanitizeKey(appId)}/budibase-client.js` @@ -38,7 +38,7 @@ export function clientLibraryUrl(appId: string, version: string) { version, } } - if (tenantId) { + if (tenantId && tenantId !== DEFAULT_TENANT_ID) { qsParams.tenantId = tenantId } return `/api/assets/client?${qs.encode(qsParams)}` From e2e5c9044c6fe708f520a1c66075f492c85f45b4 Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 24 Oct 2023 18:12:20 +0100 Subject: [PATCH 09/14] Revert "Client library fetching through server" --- .../src/objectStore/buckets/app.ts | 59 +++++++----------- .../src/objectStore/buckets/plugins.ts | 16 ++--- .../src/objectStore/buckets/tests/app.spec.ts | 42 ++++++++----- .../src/objectStore/objectStore.ts | 62 +++++++------------ .../src/api/controllers/static/index.ts | 49 ++++++--------- 5 files changed, 101 insertions(+), 127 deletions(-) diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index 43bc965c65..be9fddeaa6 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -1,50 +1,37 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" -import qs from "querystring" -import { DEFAULT_TENANT_ID, getTenantId } from "../../context" - -export function clientLibraryPath(appId: string) { - return `${objectStore.sanitizeKey(appId)}/budibase-client.js` -} /** - * Previously we used to serve the client library directly from Cloudfront, however - * due to issues with the domain we were unable to continue doing this - keeping - * incase we are able to switch back to CDN path again in future. + * In production the client library is stored in the object store, however in development + * we use the symlinked version produced by lerna, located in node modules. We link to this + * via a specific endpoint (under /api/assets/client). + * @param appId In production we need the appId to look up the correct bucket, as the + * version of the client lib may differ between apps. + * @param version The version to retrieve. + * @return The URL to be inserted into appPackage response or server rendered + * app index file. */ -export function clientLibraryCDNUrl(appId: string, version: string) { - let file = clientLibraryPath(appId) - if (env.CLOUDFRONT_CDN) { - // append app version to bust the cache - if (version) { - file += `?v=${version}` +export const clientLibraryUrl = (appId: string, version: string) => { + if (env.isProd()) { + let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js` + if (env.CLOUDFRONT_CDN) { + // append app version to bust the cache + if (version) { + file += `?v=${version}` + } + // don't need to use presigned for client with cloudfront + // file is public + return cloudfront.getUrl(file) + } else { + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) } - // don't need to use presigned for client with cloudfront - // file is public - return cloudfront.getUrl(file) } else { - return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) + return `/api/assets/client` } } -export function clientLibraryUrl(appId: string, version: string) { - let tenantId, qsParams: { appId: string; version: string; tenantId?: string } - try { - tenantId = getTenantId() - } finally { - qsParams = { - appId, - version, - } - } - if (tenantId && tenantId !== DEFAULT_TENANT_ID) { - qsParams.tenantId = tenantId - } - return `/api/assets/client?${qs.encode(qsParams)}` -} - -export function getAppFileUrl(s3Key: string) { +export const getAppFileUrl = (s3Key: string) => { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts index 6f1b7116ae..f7721afb23 100644 --- a/packages/backend-core/src/objectStore/buckets/plugins.ts +++ b/packages/backend-core/src/objectStore/buckets/plugins.ts @@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types" // URLS -export function enrichPluginURLs(plugins: Plugin[]) { +export const enrichPluginURLs = (plugins: Plugin[]) => { if (!plugins || !plugins.length) { return [] } @@ -17,12 +17,12 @@ export function enrichPluginURLs(plugins: Plugin[]) { }) } -function getPluginJSUrl(plugin: Plugin) { +const getPluginJSUrl = (plugin: Plugin) => { const s3Key = getPluginJSKey(plugin) return getPluginUrl(s3Key) } -function getPluginIconUrl(plugin: Plugin): string | undefined { +const getPluginIconUrl = (plugin: Plugin): string | undefined => { const s3Key = getPluginIconKey(plugin) if (!s3Key) { return @@ -30,7 +30,7 @@ function getPluginIconUrl(plugin: Plugin): string | undefined { return getPluginUrl(s3Key) } -function getPluginUrl(s3Key: string) { +const getPluginUrl = (s3Key: string) => { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { @@ -40,11 +40,11 @@ function getPluginUrl(s3Key: string) { // S3 KEYS -export function getPluginJSKey(plugin: Plugin) { +export const getPluginJSKey = (plugin: Plugin) => { return getPluginS3Key(plugin, "plugin.min.js") } -export function getPluginIconKey(plugin: Plugin) { +export const getPluginIconKey = (plugin: Plugin) => { // stored iconUrl is deprecated - hardcode to icon.svg in this case const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName if (!iconFileName) { @@ -53,12 +53,12 @@ export function getPluginIconKey(plugin: Plugin) { return getPluginS3Key(plugin, iconFileName) } -function getPluginS3Key(plugin: Plugin, fileName: string) { +const getPluginS3Key = (plugin: Plugin, fileName: string) => { const s3Key = getPluginS3Dir(plugin.name) return `${s3Key}/${fileName}` } -export function getPluginS3Dir(pluginName: string) { +export const getPluginS3Dir = (pluginName: string) => { let s3Key = `${pluginName}` if (env.MULTI_TENANCY) { const tenantId = context.getTenantId() diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts index cbbbee6255..aaa07ec9d3 100644 --- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -1,4 +1,5 @@ import * as app from "../app" +import { getAppFileUrl } from "../app" import { testEnv } from "../../../../tests/extra" describe("app", () => { @@ -6,15 +7,6 @@ describe("app", () => { testEnv.nodeJest() }) - function baseCheck(url: string, tenantId?: string) { - expect(url).toContain("/api/assets/client") - if (tenantId) { - expect(url).toContain(`tenantId=${tenantId}`) - } - expect(url).toContain("appId=app_123") - expect(url).toContain("version=2.0.0") - } - describe("clientLibraryUrl", () => { function getClientUrl() { return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0") @@ -28,19 +20,31 @@ describe("app", () => { it("gets url in dev", () => { testEnv.nodeDev() const url = getClientUrl() - baseCheck(url) + expect(url).toBe("/api/assets/client") + }) + + it("gets url with embedded minio", () => { + testEnv.withMinio() + const url = getClientUrl() + expect(url).toBe( + "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) }) it("gets url with custom S3", () => { testEnv.withS3() const url = getClientUrl() - baseCheck(url) + expect(url).toBe( + "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) }) it("gets url with cloudfront + s3", () => { testEnv.withCloudfront() const url = getClientUrl() - baseCheck(url) + expect(url).toBe( + "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" + ) }) }) @@ -53,7 +57,7 @@ describe("app", () => { testEnv.nodeDev() await testEnv.withTenant(tenantId => { const url = getClientUrl() - baseCheck(url, tenantId) + expect(url).toBe("/api/assets/client") }) }) @@ -61,7 +65,9 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withMinio() const url = getClientUrl() - baseCheck(url, tenantId) + expect(url).toBe( + "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) }) }) @@ -69,7 +75,9 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withS3() const url = getClientUrl() - baseCheck(url, tenantId) + expect(url).toBe( + "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" + ) }) }) @@ -77,7 +85,9 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withCloudfront() const url = getClientUrl() - baseCheck(url, tenantId) + expect(url).toBe( + "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" + ) }) }) }) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index 76d2dd6689..c36a09915e 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -1,6 +1,6 @@ const sanitize = require("sanitize-s3-objectkey") import AWS from "aws-sdk" -import stream, { Readable } from "stream" +import stream from "stream" import fetch from "node-fetch" import tar from "tar-fs" import zlib from "zlib" @@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) { * @return an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export function ObjectStore( +export const ObjectStore = ( bucket: string, opts: { presigning: boolean } = { presigning: false } -) { +) => { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -104,7 +104,7 @@ export function ObjectStore( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export async function makeSureBucketExists(client: any, bucketName: string) { +export const makeSureBucketExists = async (client: any, bucketName: string) => { bucketName = sanitizeBucket(bucketName) try { await client @@ -139,13 +139,13 @@ export async function makeSureBucketExists(client: any, bucketName: string) { * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). */ -export async function upload({ +export const upload = async ({ bucket: bucketName, filename, path, type, metadata, -}: UploadParams) { +}: UploadParams) => { const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) @@ -180,12 +180,12 @@ export async function upload({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export async function streamUpload( +export const streamUpload = async ( bucketName: string, filename: string, stream: any, extra = {} -) { +) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) @@ -215,7 +215,7 @@ export async function streamUpload( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export async function retrieve(bucketName: string, filepath: string) { +export const retrieve = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -230,7 +230,7 @@ export async function retrieve(bucketName: string, filepath: string) { } } -export async function listAllObjects(bucketName: string, path: string) { +export const listAllObjects = async (bucketName: string, path: string) => { const objectStore = ObjectStore(bucketName) const list = (params: ListParams = {}) => { return objectStore @@ -261,11 +261,11 @@ export async function listAllObjects(bucketName: string, path: string) { /** * Generate a presigned url with a default TTL of 1 hour */ -export function getPresignedUrl( +export const getPresignedUrl = ( bucketName: string, key: string, durationSeconds: number = 3600 -) { +) => { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { Bucket: sanitizeBucket(bucketName), @@ -291,7 +291,7 @@ export function getPresignedUrl( /** * Same as retrieval function but puts to a temporary file. */ -export async function retrieveToTmp(bucketName: string, filepath: string) { +export const retrieveToTmp = async (bucketName: string, filepath: string) => { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -300,7 +300,7 @@ export async function retrieveToTmp(bucketName: string, filepath: string) { return outputPath } -export async function retrieveDirectory(bucketName: string, path: string) { +export const retrieveDirectory = async (bucketName: string, path: string) => { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) @@ -324,7 +324,7 @@ export async function retrieveDirectory(bucketName: string, path: string) { /** * Delete a single file. */ -export async function deleteFile(bucketName: string, filepath: string) { +export const deleteFile = async (bucketName: string, filepath: string) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -334,7 +334,7 @@ export async function deleteFile(bucketName: string, filepath: string) { return objectStore.deleteObject(params).promise() } -export async function deleteFiles(bucketName: string, filepaths: string[]) { +export const deleteFiles = async (bucketName: string, filepaths: string[]) => { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -349,10 +349,10 @@ export async function deleteFiles(bucketName: string, filepaths: string[]) { /** * Delete a path, including everything within. */ -export async function deleteFolder( +export const deleteFolder = async ( bucketName: string, folder: string -): Promise { +): Promise => { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) const client = ObjectStore(bucketName) @@ -383,11 +383,11 @@ export async function deleteFolder( } } -export async function uploadDirectory( +export const uploadDirectory = async ( bucketName: string, localPath: string, bucketPath: string -) { +) => { bucketName = sanitizeBucket(bucketName) let uploads = [] const files = fs.readdirSync(localPath, { withFileTypes: true }) @@ -404,11 +404,11 @@ export async function uploadDirectory( return files } -export async function downloadTarballDirect( +export const downloadTarballDirect = async ( url: string, path: string, headers = {} -) { +) => { path = sanitizeKey(path) const response = await fetch(url, { headers }) if (!response.ok) { @@ -418,11 +418,11 @@ export async function downloadTarballDirect( await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path)) } -export async function downloadTarball( +export const downloadTarball = async ( url: string, bucketName: string, path: string -) { +) => { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) @@ -438,17 +438,3 @@ export async function downloadTarball( // return the temporary path incase there is a use for it return tmpPath } - -export async function getReadStream( - bucketName: string, - path: string -): Promise { - bucketName = sanitizeBucket(bucketName) - path = sanitizeKey(path) - const client = ObjectStore(bucketName) - const params = { - Bucket: bucketName, - Key: path, - } - return client.getObject(params).createReadStream() -} diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index d2e887dcf8..bbf9dd34f5 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -16,7 +16,7 @@ import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" -import { App, Ctx } from "@budibase/types" +import { App } from "@budibase/types" const send = require("koa-send") @@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) { } } -export const toggleBetaUiFeature = async function (ctx: Ctx) { +export const toggleBetaUiFeature = async function (ctx: any) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -67,14 +67,16 @@ export const toggleBetaUiFeature = async function (ctx: Ctx) { } } -export const serveBuilder = async function (ctx: Ctx) { +export const serveBuilder = async function (ctx: any) { const builderPath = join(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) } -export const uploadFile = async function (ctx: Ctx) { - const file = ctx.request?.files?.file - let files = file && Array.isArray(file) ? Array.from(file) : [file] +export const uploadFile = async function (ctx: any) { + let files = + ctx.request.files.file.length > 1 + ? Array.from(ctx.request.files.file) + : [ctx.request.files.file] const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() @@ -91,14 +93,14 @@ export const uploadFile = async function (ctx: Ctx) { ctx.body = await Promise.all(uploads) } -export const deleteObjects = async function (ctx: Ctx) { +export const deleteObjects = async function (ctx: any) { ctx.body = await objectStore.deleteFiles( ObjectStoreBuckets.APPS, ctx.request.body.keys ) } -export const serveApp = async function (ctx: Ctx) { +export const serveApp = async function (ctx: any) { const bbHeaderEmbed = ctx.request.get("x-budibase-embed")?.toLowerCase() === "true" @@ -179,7 +181,7 @@ export const serveApp = async function (ctx: Ctx) { } } -export const serveBuilderPreview = async function (ctx: Ctx) { +export const serveBuilderPreview = async function (ctx: any) { const db = context.getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentType.APP_METADATA) @@ -195,29 +197,18 @@ export const serveBuilderPreview = async function (ctx: Ctx) { } } -export const serveClientLibrary = async function (ctx: Ctx) { - const appId = context.getAppId() || (ctx.request.query.appId as string) +export const serveClientLibrary = async function (ctx: any) { let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist") - if (!appId) { - ctx.throw(400, "No app ID provided - cannot fetch client library.") - } - if (env.isProd()) { - ctx.body = await objectStore.getReadStream( - ObjectStoreBuckets.APPS, - objectStore.clientLibraryPath(appId!) - ) - } else if (env.isDev()) { - // incase running from TS directly - const tsPath = join(require.resolve("@budibase/client"), "..") - return send(ctx, "budibase-client.js", { - root: !fs.existsSync(rootPath) ? tsPath : rootPath, - }) - } else { - ctx.throw(500, "Unable to retrieve client library.") + // incase running from TS directly + if (env.isDev() && !fs.existsSync(rootPath)) { + rootPath = join(require.resolve("@budibase/client"), "..") } + return send(ctx, "budibase-client.js", { + root: rootPath, + }) } -export const getSignedUploadURL = async function (ctx: Ctx) { +export const getSignedUploadURL = async function (ctx: any) { // Ensure datasource is valid let datasource try { @@ -256,7 +247,7 @@ export const getSignedUploadURL = async function (ctx: Ctx) { const params = { Bucket: bucket, Key: key } signedUrl = s3.getSignedUrl("putObject", params) publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}` - } catch (error: any) { + } catch (error) { ctx.throw(400, error) } } From c6b377c5659067bb220006d94abfacc18a2b575b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 24 Oct 2023 18:13:36 +0100 Subject: [PATCH 10/14] Revert "Revert "Client library fetching through server"" --- .../src/objectStore/buckets/app.ts | 59 +++++++++++------- .../src/objectStore/buckets/plugins.ts | 16 ++--- .../src/objectStore/buckets/tests/app.spec.ts | 42 +++++-------- .../src/objectStore/objectStore.ts | 62 ++++++++++++------- .../src/api/controllers/static/index.ts | 49 +++++++++------ 5 files changed, 127 insertions(+), 101 deletions(-) diff --git a/packages/backend-core/src/objectStore/buckets/app.ts b/packages/backend-core/src/objectStore/buckets/app.ts index be9fddeaa6..43bc965c65 100644 --- a/packages/backend-core/src/objectStore/buckets/app.ts +++ b/packages/backend-core/src/objectStore/buckets/app.ts @@ -1,37 +1,50 @@ import env from "../../environment" import * as objectStore from "../objectStore" import * as cloudfront from "../cloudfront" +import qs from "querystring" +import { DEFAULT_TENANT_ID, getTenantId } from "../../context" + +export function clientLibraryPath(appId: string) { + return `${objectStore.sanitizeKey(appId)}/budibase-client.js` +} /** - * In production the client library is stored in the object store, however in development - * we use the symlinked version produced by lerna, located in node modules. We link to this - * via a specific endpoint (under /api/assets/client). - * @param appId In production we need the appId to look up the correct bucket, as the - * version of the client lib may differ between apps. - * @param version The version to retrieve. - * @return The URL to be inserted into appPackage response or server rendered - * app index file. + * Previously we used to serve the client library directly from Cloudfront, however + * due to issues with the domain we were unable to continue doing this - keeping + * incase we are able to switch back to CDN path again in future. */ -export const clientLibraryUrl = (appId: string, version: string) => { - if (env.isProd()) { - let file = `${objectStore.sanitizeKey(appId)}/budibase-client.js` - if (env.CLOUDFRONT_CDN) { - // append app version to bust the cache - if (version) { - file += `?v=${version}` - } - // don't need to use presigned for client with cloudfront - // file is public - return cloudfront.getUrl(file) - } else { - return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) +export function clientLibraryCDNUrl(appId: string, version: string) { + let file = clientLibraryPath(appId) + if (env.CLOUDFRONT_CDN) { + // append app version to bust the cache + if (version) { + file += `?v=${version}` } + // don't need to use presigned for client with cloudfront + // file is public + return cloudfront.getUrl(file) } else { - return `/api/assets/client` + return objectStore.getPresignedUrl(env.APPS_BUCKET_NAME, file) } } -export const getAppFileUrl = (s3Key: string) => { +export function clientLibraryUrl(appId: string, version: string) { + let tenantId, qsParams: { appId: string; version: string; tenantId?: string } + try { + tenantId = getTenantId() + } finally { + qsParams = { + appId, + version, + } + } + if (tenantId && tenantId !== DEFAULT_TENANT_ID) { + qsParams.tenantId = tenantId + } + return `/api/assets/client?${qs.encode(qsParams)}` +} + +export function getAppFileUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { diff --git a/packages/backend-core/src/objectStore/buckets/plugins.ts b/packages/backend-core/src/objectStore/buckets/plugins.ts index f7721afb23..6f1b7116ae 100644 --- a/packages/backend-core/src/objectStore/buckets/plugins.ts +++ b/packages/backend-core/src/objectStore/buckets/plugins.ts @@ -6,7 +6,7 @@ import { Plugin } from "@budibase/types" // URLS -export const enrichPluginURLs = (plugins: Plugin[]) => { +export function enrichPluginURLs(plugins: Plugin[]) { if (!plugins || !plugins.length) { return [] } @@ -17,12 +17,12 @@ export const enrichPluginURLs = (plugins: Plugin[]) => { }) } -const getPluginJSUrl = (plugin: Plugin) => { +function getPluginJSUrl(plugin: Plugin) { const s3Key = getPluginJSKey(plugin) return getPluginUrl(s3Key) } -const getPluginIconUrl = (plugin: Plugin): string | undefined => { +function getPluginIconUrl(plugin: Plugin): string | undefined { const s3Key = getPluginIconKey(plugin) if (!s3Key) { return @@ -30,7 +30,7 @@ const getPluginIconUrl = (plugin: Plugin): string | undefined => { return getPluginUrl(s3Key) } -const getPluginUrl = (s3Key: string) => { +function getPluginUrl(s3Key: string) { if (env.CLOUDFRONT_CDN) { return cloudfront.getPresignedUrl(s3Key) } else { @@ -40,11 +40,11 @@ const getPluginUrl = (s3Key: string) => { // S3 KEYS -export const getPluginJSKey = (plugin: Plugin) => { +export function getPluginJSKey(plugin: Plugin) { return getPluginS3Key(plugin, "plugin.min.js") } -export const getPluginIconKey = (plugin: Plugin) => { +export function getPluginIconKey(plugin: Plugin) { // stored iconUrl is deprecated - hardcode to icon.svg in this case const iconFileName = plugin.iconUrl ? "icon.svg" : plugin.iconFileName if (!iconFileName) { @@ -53,12 +53,12 @@ export const getPluginIconKey = (plugin: Plugin) => { return getPluginS3Key(plugin, iconFileName) } -const getPluginS3Key = (plugin: Plugin, fileName: string) => { +function getPluginS3Key(plugin: Plugin, fileName: string) { const s3Key = getPluginS3Dir(plugin.name) return `${s3Key}/${fileName}` } -export const getPluginS3Dir = (pluginName: string) => { +export function getPluginS3Dir(pluginName: string) { let s3Key = `${pluginName}` if (env.MULTI_TENANCY) { const tenantId = context.getTenantId() diff --git a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts index aaa07ec9d3..cbbbee6255 100644 --- a/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts +++ b/packages/backend-core/src/objectStore/buckets/tests/app.spec.ts @@ -1,5 +1,4 @@ import * as app from "../app" -import { getAppFileUrl } from "../app" import { testEnv } from "../../../../tests/extra" describe("app", () => { @@ -7,6 +6,15 @@ describe("app", () => { testEnv.nodeJest() }) + function baseCheck(url: string, tenantId?: string) { + expect(url).toContain("/api/assets/client") + if (tenantId) { + expect(url).toContain(`tenantId=${tenantId}`) + } + expect(url).toContain("appId=app_123") + expect(url).toContain("version=2.0.0") + } + describe("clientLibraryUrl", () => { function getClientUrl() { return app.clientLibraryUrl("app_123/budibase-client.js", "2.0.0") @@ -20,31 +28,19 @@ describe("app", () => { it("gets url in dev", () => { testEnv.nodeDev() const url = getClientUrl() - expect(url).toBe("/api/assets/client") - }) - - it("gets url with embedded minio", () => { - testEnv.withMinio() - const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with custom S3", () => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url) }) it("gets url with cloudfront + s3", () => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url) }) }) @@ -57,7 +53,7 @@ describe("app", () => { testEnv.nodeDev() await testEnv.withTenant(tenantId => { const url = getClientUrl() - expect(url).toBe("/api/assets/client") + baseCheck(url, tenantId) }) }) @@ -65,9 +61,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withMinio() const url = getClientUrl() - expect(url).toBe( - "/files/signed/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -75,9 +69,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withS3() const url = getClientUrl() - expect(url).toBe( - "http://s3.example.com/prod-budi-app-assets/app_123/budibase-client.js/budibase-client.js" - ) + baseCheck(url, tenantId) }) }) @@ -85,9 +77,7 @@ describe("app", () => { await testEnv.withTenant(tenantId => { testEnv.withCloudfront() const url = getClientUrl() - expect(url).toBe( - "http://cf.example.com/app_123/budibase-client.js/budibase-client.js?v=2.0.0" - ) + baseCheck(url, tenantId) }) }) }) diff --git a/packages/backend-core/src/objectStore/objectStore.ts b/packages/backend-core/src/objectStore/objectStore.ts index c36a09915e..76d2dd6689 100644 --- a/packages/backend-core/src/objectStore/objectStore.ts +++ b/packages/backend-core/src/objectStore/objectStore.ts @@ -1,6 +1,6 @@ const sanitize = require("sanitize-s3-objectkey") import AWS from "aws-sdk" -import stream from "stream" +import stream, { Readable } from "stream" import fetch from "node-fetch" import tar from "tar-fs" import zlib from "zlib" @@ -66,10 +66,10 @@ export function sanitizeBucket(input: string) { * @return an S3 object store object, check S3 Nodejs SDK for usage. * @constructor */ -export const ObjectStore = ( +export function ObjectStore( bucket: string, opts: { presigning: boolean } = { presigning: false } -) => { +) { const config: any = { s3ForcePathStyle: true, signatureVersion: "v4", @@ -104,7 +104,7 @@ export const ObjectStore = ( * Given an object store and a bucket name this will make sure the bucket exists, * if it does not exist then it will create it. */ -export const makeSureBucketExists = async (client: any, bucketName: string) => { +export async function makeSureBucketExists(client: any, bucketName: string) { bucketName = sanitizeBucket(bucketName) try { await client @@ -139,13 +139,13 @@ export const makeSureBucketExists = async (client: any, bucketName: string) => { * Uploads the contents of a file given the required parameters, useful when * temp files in use (for example file uploaded as an attachment). */ -export const upload = async ({ +export async function upload({ bucket: bucketName, filename, path, type, metadata, -}: UploadParams) => { +}: UploadParams) { const extension = filename.split(".").pop() const fileBytes = fs.readFileSync(path) @@ -180,12 +180,12 @@ export const upload = async ({ * Similar to the upload function but can be used to send a file stream * through to the object store. */ -export const streamUpload = async ( +export async function streamUpload( bucketName: string, filename: string, stream: any, extra = {} -) => { +) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) @@ -215,7 +215,7 @@ export const streamUpload = async ( * retrieves the contents of a file from the object store, if it is a known content type it * will be converted, otherwise it will be returned as a buffer stream. */ -export const retrieve = async (bucketName: string, filepath: string) => { +export async function retrieve(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) const params = { Bucket: sanitizeBucket(bucketName), @@ -230,7 +230,7 @@ export const retrieve = async (bucketName: string, filepath: string) => { } } -export const listAllObjects = async (bucketName: string, path: string) => { +export async function listAllObjects(bucketName: string, path: string) { const objectStore = ObjectStore(bucketName) const list = (params: ListParams = {}) => { return objectStore @@ -261,11 +261,11 @@ export const listAllObjects = async (bucketName: string, path: string) => { /** * Generate a presigned url with a default TTL of 1 hour */ -export const getPresignedUrl = ( +export function getPresignedUrl( bucketName: string, key: string, durationSeconds: number = 3600 -) => { +) { const objectStore = ObjectStore(bucketName, { presigning: true }) const params = { Bucket: sanitizeBucket(bucketName), @@ -291,7 +291,7 @@ export const getPresignedUrl = ( /** * Same as retrieval function but puts to a temporary file. */ -export const retrieveToTmp = async (bucketName: string, filepath: string) => { +export async function retrieveToTmp(bucketName: string, filepath: string) { bucketName = sanitizeBucket(bucketName) filepath = sanitizeKey(filepath) const data = await retrieve(bucketName, filepath) @@ -300,7 +300,7 @@ export const retrieveToTmp = async (bucketName: string, filepath: string) => { return outputPath } -export const retrieveDirectory = async (bucketName: string, path: string) => { +export async function retrieveDirectory(bucketName: string, path: string) { let writePath = join(budibaseTempDir(), v4()) fs.mkdirSync(writePath) const objects = await listAllObjects(bucketName, path) @@ -324,7 +324,7 @@ export const retrieveDirectory = async (bucketName: string, path: string) => { /** * Delete a single file. */ -export const deleteFile = async (bucketName: string, filepath: string) => { +export async function deleteFile(bucketName: string, filepath: string) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -334,7 +334,7 @@ export const deleteFile = async (bucketName: string, filepath: string) => { return objectStore.deleteObject(params).promise() } -export const deleteFiles = async (bucketName: string, filepaths: string[]) => { +export async function deleteFiles(bucketName: string, filepaths: string[]) { const objectStore = ObjectStore(bucketName) await makeSureBucketExists(objectStore, bucketName) const params = { @@ -349,10 +349,10 @@ export const deleteFiles = async (bucketName: string, filepaths: string[]) => { /** * Delete a path, including everything within. */ -export const deleteFolder = async ( +export async function deleteFolder( bucketName: string, folder: string -): Promise => { +): Promise { bucketName = sanitizeBucket(bucketName) folder = sanitizeKey(folder) const client = ObjectStore(bucketName) @@ -383,11 +383,11 @@ export const deleteFolder = async ( } } -export const uploadDirectory = async ( +export async function uploadDirectory( bucketName: string, localPath: string, bucketPath: string -) => { +) { bucketName = sanitizeBucket(bucketName) let uploads = [] const files = fs.readdirSync(localPath, { withFileTypes: true }) @@ -404,11 +404,11 @@ export const uploadDirectory = async ( return files } -export const downloadTarballDirect = async ( +export async function downloadTarballDirect( url: string, path: string, headers = {} -) => { +) { path = sanitizeKey(path) const response = await fetch(url, { headers }) if (!response.ok) { @@ -418,11 +418,11 @@ export const downloadTarballDirect = async ( await streamPipeline(response.body, zlib.createUnzip(), tar.extract(path)) } -export const downloadTarball = async ( +export async function downloadTarball( url: string, bucketName: string, path: string -) => { +) { bucketName = sanitizeBucket(bucketName) path = sanitizeKey(path) const response = await fetch(url) @@ -438,3 +438,17 @@ export const downloadTarball = async ( // return the temporary path incase there is a use for it return tmpPath } + +export async function getReadStream( + bucketName: string, + path: string +): Promise { + bucketName = sanitizeBucket(bucketName) + path = sanitizeKey(path) + const client = ObjectStore(bucketName) + const params = { + Bucket: bucketName, + Key: path, + } + return client.getObject(params).createReadStream() +} diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index bbf9dd34f5..d2e887dcf8 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -16,7 +16,7 @@ import AWS from "aws-sdk" import fs from "fs" import sdk from "../../../sdk" import * as pro from "@budibase/pro" -import { App } from "@budibase/types" +import { App, Ctx } from "@budibase/types" const send = require("koa-send") @@ -39,7 +39,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }: any) { } } -export const toggleBetaUiFeature = async function (ctx: any) { +export const toggleBetaUiFeature = async function (ctx: Ctx) { const cookieName = `beta:${ctx.params.feature}` if (ctx.cookies.get(cookieName)) { @@ -67,16 +67,14 @@ export const toggleBetaUiFeature = async function (ctx: any) { } } -export const serveBuilder = async function (ctx: any) { +export const serveBuilder = async function (ctx: Ctx) { const builderPath = join(TOP_LEVEL_PATH, "builder") await send(ctx, ctx.file, { root: builderPath }) } -export const uploadFile = async function (ctx: any) { - let files = - ctx.request.files.file.length > 1 - ? Array.from(ctx.request.files.file) - : [ctx.request.files.file] +export const uploadFile = async function (ctx: Ctx) { + const file = ctx.request?.files?.file + let files = file && Array.isArray(file) ? Array.from(file) : [file] const uploads = files.map(async (file: any) => { const fileExtension = [...file.name.split(".")].pop() @@ -93,14 +91,14 @@ export const uploadFile = async function (ctx: any) { ctx.body = await Promise.all(uploads) } -export const deleteObjects = async function (ctx: any) { +export const deleteObjects = async function (ctx: Ctx) { ctx.body = await objectStore.deleteFiles( ObjectStoreBuckets.APPS, ctx.request.body.keys ) } -export const serveApp = async function (ctx: any) { +export const serveApp = async function (ctx: Ctx) { const bbHeaderEmbed = ctx.request.get("x-budibase-embed")?.toLowerCase() === "true" @@ -181,7 +179,7 @@ export const serveApp = async function (ctx: any) { } } -export const serveBuilderPreview = async function (ctx: any) { +export const serveBuilderPreview = async function (ctx: Ctx) { const db = context.getAppDB({ skip_setup: true }) const appInfo = await db.get(DocumentType.APP_METADATA) @@ -197,18 +195,29 @@ export const serveBuilderPreview = async function (ctx: any) { } } -export const serveClientLibrary = async function (ctx: any) { +export const serveClientLibrary = async function (ctx: Ctx) { + const appId = context.getAppId() || (ctx.request.query.appId as string) let rootPath = join(NODE_MODULES_PATH, "@budibase", "client", "dist") - // incase running from TS directly - if (env.isDev() && !fs.existsSync(rootPath)) { - rootPath = join(require.resolve("@budibase/client"), "..") + if (!appId) { + ctx.throw(400, "No app ID provided - cannot fetch client library.") + } + if (env.isProd()) { + ctx.body = await objectStore.getReadStream( + ObjectStoreBuckets.APPS, + objectStore.clientLibraryPath(appId!) + ) + } else if (env.isDev()) { + // incase running from TS directly + const tsPath = join(require.resolve("@budibase/client"), "..") + return send(ctx, "budibase-client.js", { + root: !fs.existsSync(rootPath) ? tsPath : rootPath, + }) + } else { + ctx.throw(500, "Unable to retrieve client library.") } - return send(ctx, "budibase-client.js", { - root: rootPath, - }) } -export const getSignedUploadURL = async function (ctx: any) { +export const getSignedUploadURL = async function (ctx: Ctx) { // Ensure datasource is valid let datasource try { @@ -247,7 +256,7 @@ export const getSignedUploadURL = async function (ctx: any) { const params = { Bucket: bucket, Key: key } signedUrl = s3.getSignedUrl("putObject", params) publicUrl = `https://${bucket}.s3.${awsRegion}.amazonaws.com/${key}` - } catch (error) { + } catch (error: any) { ctx.throw(400, error) } } From 5fabe324345a4e20ae03130a8d7047ad33c5220b Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 24 Oct 2023 20:35:25 +0100 Subject: [PATCH 11/14] Making sure client endpoint available in prod. --- packages/server/src/api/routes/static.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/packages/server/src/api/routes/static.ts b/packages/server/src/api/routes/static.ts index 0012764b40..bd3f1aba2e 100644 --- a/packages/server/src/api/routes/static.ts +++ b/packages/server/src/api/routes/static.ts @@ -27,15 +27,9 @@ router.param("file", async (file: any, ctx: any, next: any) => { return next() }) -// only used in development for retrieving the client library, -// in production the client lib is always stored in the object store. -if (env.isDev()) { - router.get("/api/assets/client", controller.serveClientLibrary) -} - router - // TODO: for now this builder endpoint is not authorized/secured, will need to be .get("/builder/:file*", controller.serveBuilder) + .get("/api/assets/client", controller.serveClientLibrary) .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile) .post( "/api/attachments/delete", From 9983d67163e576d8e202b633017d394722efb84f Mon Sep 17 00:00:00 2001 From: Michael Drury Date: Tue, 24 Oct 2023 22:13:56 +0100 Subject: [PATCH 12/14] Adding mimetype for client library response (correcting from octet-stream). --- packages/server/src/api/controllers/static/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/src/api/controllers/static/index.ts b/packages/server/src/api/controllers/static/index.ts index d2e887dcf8..984cb16c06 100644 --- a/packages/server/src/api/controllers/static/index.ts +++ b/packages/server/src/api/controllers/static/index.ts @@ -206,6 +206,7 @@ export const serveClientLibrary = async function (ctx: Ctx) { ObjectStoreBuckets.APPS, objectStore.clientLibraryPath(appId!) ) + ctx.set("Content-Type", "application/javascript") } else if (env.isDev()) { // incase running from TS directly const tsPath = join(require.resolve("@budibase/client"), "..") From 65111c81efacc95b9389f24a952ca8fed4c0c219 Mon Sep 17 00:00:00 2001 From: jvcalderon Date: Wed, 25 Oct 2023 08:44:59 +0200 Subject: [PATCH 13/14] Reactivate sync quotas migration --- packages/types/src/sdk/migrations.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/types/src/sdk/migrations.ts b/packages/types/src/sdk/migrations.ts index 4667ed0c8f..0692b27f8e 100644 --- a/packages/types/src/sdk/migrations.ts +++ b/packages/types/src/sdk/migrations.ts @@ -46,7 +46,7 @@ export enum MigrationName { GLOBAL_INFO_SYNC_USERS = "global_info_sync_users", TABLE_SETTINGS_LINKS_TO_ACTIONS = "table_settings_links_to_actions", // increment this number to re-activate this migration - SYNC_QUOTAS = "sync_quotas_1", + SYNC_QUOTAS = "sync_quotas_2", } export interface MigrationDefinition { From 659cf3006024b938521f0fb18bbf07be3858e061 Mon Sep 17 00:00:00 2001 From: Budibase Staging Release Bot <> Date: Wed, 25 Oct 2023 08:59:28 +0000 Subject: [PATCH 14/14] Bump version to 2.11.44 --- lerna.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lerna.json b/lerna.json index 11e4e7627a..7d14875c97 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.11.43", + "version": "2.11.44", "npmClient": "yarn", "packages": [ "packages/*"