From 45349845f50a11c081ae58f2404add94198085a6 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Tue, 6 Sep 2022 18:07:18 +0100 Subject: [PATCH 01/30] Adding compaction before each replication to try to reduce size of database, as well as removing the possibility of app metadata being in conflict as per issue #7494. --- packages/backend-core/src/db/Replication.ts | 10 +++++++++ .../server/src/api/controllers/application.ts | 6 +----- .../src/api/controllers/deploy/index.ts | 21 +++++++++++++++---- 3 files changed, 28 insertions(+), 9 deletions(-) diff --git a/packages/backend-core/src/db/Replication.ts b/packages/backend-core/src/db/Replication.ts index b46f6072be..e0bd3c7a43 100644 --- a/packages/backend-core/src/db/Replication.ts +++ b/packages/backend-core/src/db/Replication.ts @@ -1,4 +1,5 @@ import { dangerousGetDB, closeDB } from "." +import { DocumentType } from "./constants" class Replication { source: any @@ -53,6 +54,14 @@ class Replication { return this.replication } + appReplicateOpts() { + return { + filter: (doc: any) => { + return doc._id !== DocumentType.APP_METADATA + }, + } + } + /** * Rollback the target DB back to the state of the source DB */ @@ -60,6 +69,7 @@ class Replication { await this.target.destroy() // Recreate the DB again this.target = dangerousGetDB(this.target.name) + // take the opportunity to remove deleted tombstones await this.replicate() } diff --git a/packages/server/src/api/controllers/application.ts b/packages/server/src/api/controllers/application.ts index 926fe0ec52..ce3649f082 100644 --- a/packages/server/src/api/controllers/application.ts +++ b/packages/server/src/api/controllers/application.ts @@ -553,11 +553,7 @@ export const sync = async (ctx: any, next: any) => { }) let error try { - await replication.replicate({ - filter: function (doc: any) { - return doc._id !== DocumentType.APP_METADATA - }, - }) + await replication.replicate(replication.appReplicateOpts()) } catch (err) { error = err } finally { diff --git a/packages/server/src/api/controllers/deploy/index.ts b/packages/server/src/api/controllers/deploy/index.ts index d63e167dfb..86718294de 100644 --- a/packages/server/src/api/controllers/deploy/index.ts +++ b/packages/server/src/api/controllers/deploy/index.ts @@ -15,6 +15,7 @@ import { getAppId, getAppDB, getProdAppDB, + getDevAppDB, } from "@budibase/backend-core/context" import { quotas } from "@budibase/pro" import { events } from "@budibase/backend-core" @@ -110,17 +111,29 @@ async function deployApp(deployment: any) { target: productionAppId, } replication = new Replication(config) - + const devDb = getDevAppDB() + console.log("Compacting development DB") + await devDb.compact() console.log("Replication object created") - await replication.replicate() + await replication.replicate(replication.appReplicateOpts()) console.log("replication complete.. replacing app meta doc") + // app metadata is excluded as it is likely to be in conflict + // replicate the app metadata document manually const db = getProdAppDB() - const appDoc = await db.get(DocumentType.APP_METADATA) + const appDoc = await devDb.get(DocumentType.APP_METADATA) + try { + const prodAppDoc = await db.get(DocumentType.APP_METADATA) + appDoc._rev = prodAppDoc._rev + } catch (err) { + // ignore the error - doesn't exist + } + // switch to production app ID deployment.appUrl = appDoc.url - appDoc.appId = productionAppId appDoc.instance._id = productionAppId + // remove automation errors if they exist + delete appDoc.automationErrors await db.put(appDoc) await appCache.invalidateAppMetadata(productionAppId) console.log("New app doc written successfully.") From 82795b50ef66e1c4ae4b4dccb04178560fa13993 Mon Sep 17 00:00:00 2001 From: ChukwuEmeka Date: Tue, 6 Sep 2022 21:51:45 +0100 Subject: [PATCH 02/30] switched from deprecated node-sass to sass library --- examples/nextjs-api-sales/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/nextjs-api-sales/package.json b/examples/nextjs-api-sales/package.json index 777d07f968..41ce52e952 100644 --- a/examples/nextjs-api-sales/package.json +++ b/examples/nextjs-api-sales/package.json @@ -12,7 +12,7 @@ "bulma": "^0.9.3", "next": "12.1.0", "node-fetch": "^3.2.10", - "node-sass": "^7.0.1", + "sass": "^1.52.3", "react": "17.0.2", "react-dom": "17.0.2", "react-notifications-component": "^3.4.1" @@ -24,4 +24,4 @@ "eslint-config-next": "12.1.0", "typescript": "4.6.2" } -} +} \ No newline at end of file From 8925cdf83ec617bb9460b6ee98b9519054ee09fb Mon Sep 17 00:00:00 2001 From: Andrew Kingston Date: Wed, 7 Sep 2022 10:40:00 +0100 Subject: [PATCH 03/30] Update preview endpoint to be prefixed with /app to not require an updated proxy config --- hosting/nginx.dev.conf.hbs | 4 ---- hosting/nginx.prod.conf.hbs | 4 ---- .../design/[screenId]/_components/AppPreview.svelte | 2 +- packages/server/src/api/routes/static.ts | 2 +- packages/server/src/api/routes/tests/static.spec.js | 4 ++-- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/hosting/nginx.dev.conf.hbs b/hosting/nginx.dev.conf.hbs index 20c4d3d182..148007d958 100644 --- a/hosting/nginx.dev.conf.hbs +++ b/hosting/nginx.dev.conf.hbs @@ -65,10 +65,6 @@ http { proxy_pass http://{{ address }}:4001; } - location /preview { - proxy_pass http://{{ address }}:4001; - } - location /builder { proxy_pass http://{{ address }}:3000; rewrite ^/builder(.*)$ /builder/$1 break; diff --git a/hosting/nginx.prod.conf.hbs b/hosting/nginx.prod.conf.hbs index 0ff986d0a7..c57fce291c 100644 --- a/hosting/nginx.prod.conf.hbs +++ b/hosting/nginx.prod.conf.hbs @@ -88,10 +88,6 @@ http { proxy_pass http://$apps:4002; } - location /preview { - proxy_pass http://$apps:4002; - } - location = / { proxy_pass http://$apps:4002; } diff --git a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte index eb8743bf39..7a16bb6ae2 100644 --- a/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte +++ b/packages/builder/src/pages/builder/app/[application]/design/[screenId]/_components/AppPreview.svelte @@ -290,7 +290,7 @@