From 71103bd6017f9130d1d7eb57f814b9f41066ab8f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Mar 2021 17:29:27 +0000 Subject: [PATCH 1/6] add docker-compose dep to package.json --- packages/server/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/server/package.json b/packages/server/package.json index a17da46c73..84b46e051f 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -137,6 +137,7 @@ "devDependencies": { "@budibase/standard-components": "^0.8.9", "@jest/test-sequencer": "^24.8.0", + "docker-compose": "^0.23.6", "electron": "10.1.3", "electron-builder": "^22.9.1", "electron-builder-notarize": "^1.1.2", From 555e6386a7ab6cdee20c8b02d4044c9259e58555 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 22 Mar 2021 19:35:29 +0000 Subject: [PATCH 2/6] better wording and tidy up of init script --- hosting/docker-compose.dev.yaml | 3 --- packages/server/scripts/dev/manage.js | 22 +++++++++++----------- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/hosting/docker-compose.dev.yaml b/hosting/docker-compose.dev.yaml index 8f6c5b504d..39fcb7ec83 100644 --- a/hosting/docker-compose.dev.yaml +++ b/hosting/docker-compose.dev.yaml @@ -61,9 +61,6 @@ services: container_name: budi-redis-dev restart: always image: redis - environment: - - COUCHDB_PASSWORD=${COUCH_DB_PASSWORD} - - COUCHDB_USER=${COUCH_DB_USER} ports: - "${REDIS_PORT}:6379" volumes: diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index f76b5ebd3d..88da3e5a0e 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -15,17 +15,6 @@ const Commands = { Nuke: "nuke", } -const managementCommand = process.argv.slice(2)[0] - -if ( - !managementCommand || - !Object.values(Commands).some(command => managementCommand === command) -) { - throw new Error( - "You must supply either an 'up' or 'down' commmand to manage the budibase dev env." - ) -} - async function up() { console.log("Spinning up your budibase dev environment... ๐Ÿ”งโœจ") try { @@ -55,6 +44,17 @@ async function nuke() { } } +const managementCommand = process.argv.slice(2)[0] + +if ( + !managementCommand || + !Object.values(Commands).some(command => managementCommand === command) +) { + throw new Error( + "You must supply either an 'up', 'down' or 'nuke' commmand to manage the budibase development environment." + ) +} + let command switch (managementCommand) { case Commands.Up: From afaa706e0707003814b23e7be798ce128dc5eb50 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 23 Mar 2021 11:01:33 +0000 Subject: [PATCH 3/6] simplify try catch in manage script --- packages/builder/cypress/setup.js | 28 +-- packages/server/package.json | 3 +- packages/server/scripts/dev/manage.js | 25 +-- packages/server/scripts/initialise.js | 17 -- packages/server/src/api/routes/static.js | 1 - packages/server/src/electron.js | 196 +++++++++--------- .../src/utilities/initialiseBudibase.js | 37 ---- 7 files changed, 121 insertions(+), 186 deletions(-) delete mode 100644 packages/server/scripts/initialise.js delete mode 100644 packages/server/src/utilities/initialiseBudibase.js diff --git a/packages/builder/cypress/setup.js b/packages/builder/cypress/setup.js index 6848d38cba..88c825bfc5 100644 --- a/packages/builder/cypress/setup.js +++ b/packages/builder/cypress/setup.js @@ -7,8 +7,6 @@ const { join, resolve } = require("path") const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase") const cypressConfig = require("../cypress.json") -const homedir = join(require("os").homedir(), ".budibase") - process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE" process.env.NODE_ENV = "cypress" process.env.ENABLE_ANALYTICS = "false" @@ -17,9 +15,8 @@ process.env.PORT = cypressConfig.env.PORT // Stop info logs polluting test outputs process.env.LOG_LEVEL = "error" -async function run(dir) { - process.env.BUDIBASE_DIR = resolve(dir) - require("dotenv").config({ path: resolve(dir, ".env") }) +async function run() { + // require("dotenv").config({ path: resolve(dir, ".env") }) // dont make this a variable or top level require // it will cause environment module to be loaded prematurely @@ -27,12 +24,15 @@ async function run(dir) { server.on("close", () => console.log("Server Closed")) } -initialiseBudibase({ dir: homedir, clientId: "cypress-test" }) - .then(() => { - delete require.cache[require.resolve("../../server/src/environment")] - const xPlatHomeDir = homedir.startsWith("~") - ? join(homedir(), homedir.substring(1)) - : homedir - run(xPlatHomeDir) - }) - .catch(e => console.error(e)) +run() + +// TODO: ensure that this still works +// initialiseBudibase({ dir: homedir, clientId: "cypress-test" }) +// .then(() => { +// delete require.cache[require.resolve("../../server/src/environment")] +// const xPlatHomeDir = homedir.startsWith("~") +// ? join(homedir(), homedir.substring(1)) +// : homedir +// run(xPlatHomeDir) +// }) +// .catch(e => console.error(e)) diff --git a/packages/server/package.json b/packages/server/package.json index 84b46e051f..c9150f11c9 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -65,8 +65,7 @@ "!src/db/tests/**/*", "!src/tests/**/*", "!src/automations/tests/**/*", - "!src/utilities/fileProcessor.js", - "!src/utilities/initialiseBudibase.js" + "!src/utilities/fileProcessor.js" ], "coverageReporters": [ "lcov", diff --git a/packages/server/scripts/dev/manage.js b/packages/server/scripts/dev/manage.js index 88da3e5a0e..ca67bd3420 100644 --- a/packages/server/scripts/dev/manage.js +++ b/packages/server/scripts/dev/manage.js @@ -17,31 +17,19 @@ const Commands = { async function up() { console.log("Spinning up your budibase dev environment... ๐Ÿ”งโœจ") - try { - await compose.upAll(CONFIG) - } catch (err) { - console.log("Something went wrong:", err.message) - } + await compose.upAll(CONFIG) } async function down() { console.log("Spinning down your budibase dev environment... ๐ŸŒ‡") - try { - await compose.stop(CONFIG) - } catch (err) { - console.log("Something went wrong:", err.message) - } + await compose.stop(CONFIG) } async function nuke() { console.log( "Clearing down your budibase dev environment, including all containers and volumes... ๐Ÿ’ฅ" ) - try { - await compose.down(CONFIG) - } catch (err) { - console.log("Something went wrong:", err.message) - } + await compose.down(CONFIG) } const managementCommand = process.argv.slice(2)[0] @@ -74,6 +62,9 @@ command() .then(() => { console.log("Done! ๐ŸŽ‰") }) - .catch(() => { - console.log("Error while managing budibase dev environment.") + .catch(err => { + console.error( + "Something went wrong while managing budibase dev environment:", + err.message + ) }) diff --git a/packages/server/scripts/initialise.js b/packages/server/scripts/initialise.js deleted file mode 100644 index bbba7a5c35..0000000000 --- a/packages/server/scripts/initialise.js +++ /dev/null @@ -1,17 +0,0 @@ -// const { join } = require("path") -// const { homedir } = require("os") - -// // const initialiseBudibase = require("../src/utilities/initialiseBudibase") -// // const DIRECTORY = "~/.budibase" - -// function run() { -// let opts = {} -// // let dir = DIRECTORY -// opts.quiet = true -// // opts.dir = dir.startsWith("~") ? join(homedir(), dir.substring(1)) : dir -// return initialiseBudibase(opts) -// } - -// run().then(() => { -// console.log("Init complete.") -// }) diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js index b376e1be9a..5088002df3 100644 --- a/packages/server/src/api/routes/static.js +++ b/packages/server/src/api/routes/static.js @@ -1,7 +1,6 @@ const Router = require("@koa/router") const controller = require("../controllers/static") const { budibaseTempDir } = require("../../utilities/budibaseDir") -const env = require("../../environment") const authorized = require("../../middleware/authorized") const { BUILDER } = require("../../utilities/security/permissions") const usage = require("../../middleware/usageQuota") diff --git a/packages/server/src/electron.js b/packages/server/src/electron.js index 290c13e6a0..49ef8758c3 100644 --- a/packages/server/src/electron.js +++ b/packages/server/src/electron.js @@ -1,112 +1,112 @@ -const { app, BrowserWindow, shell, dialog } = require("electron") -const { join } = require("./utilities/centralPath") -const isDev = require("electron-is-dev") -const { autoUpdater } = require("electron-updater") -const unhandled = require("electron-unhandled") -const { existsSync } = require("fs-extra") -const initialiseBudibase = require("./utilities/initialiseBudibase") -const { budibaseAppsDir } = require("./utilities/budibaseDir") -const { openNewGitHubIssue, debugInfo } = require("electron-util") -const eventEmitter = require("./events") +// const { app, BrowserWindow, shell, dialog } = require("electron") +// const { join } = require("./utilities/centralPath") +// const isDev = require("electron-is-dev") +// const { autoUpdater } = require("electron-updater") +// const unhandled = require("electron-unhandled") +// const { existsSync } = require("fs-extra") +// const initialiseBudibase = require("./utilities/initialiseBudibase") +// const { budibaseAppsDir } = require("./utilities/budibaseDir") +// const { openNewGitHubIssue, debugInfo } = require("electron-util") +// const eventEmitter = require("./events") -const budibaseDir = budibaseAppsDir() -const envFile = join(budibaseDir, ".env") +// const budibaseDir = budibaseAppsDir() +// const envFile = join(budibaseDir, ".env") -async function startApp() { - if (!existsSync(envFile)) { - await initialiseBudibase({ dir: budibaseDir }) - } - // evict environment from cache, so it reloads when next asked - delete require.cache[require.resolve("./environment")] - // store the port incase its going to get overridden - const port = process.env.PORT - require("dotenv").config({ path: envFile }) - // overwrite the port - don't want to use dotenv for the port - require("./environment")._set("PORT", port) +// async function startApp() { +// if (!existsSync(envFile)) { +// await initialiseBudibase({ dir: budibaseDir }) +// } +// // evict environment from cache, so it reloads when next asked +// delete require.cache[require.resolve("./environment")] +// // store the port incase its going to get overridden +// const port = process.env.PORT +// require("dotenv").config({ path: envFile }) +// // overwrite the port - don't want to use dotenv for the port +// require("./environment")._set("PORT", port) - unhandled({ - showDialog: true, - reportButton: error => { - openNewGitHubIssue({ - title: error.message, - user: "Budibase", - labels: ["error-report"], - repo: "budibase", - body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${ - error.stack - }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`, - }) - }, - }) +// unhandled({ +// showDialog: true, +// reportButton: error => { +// openNewGitHubIssue({ +// title: error.message, +// user: "Budibase", +// labels: ["error-report"], +// repo: "budibase", +// body: `### Error that occurred when using the budibase builder:\n\`\`\`\n${ +// error.stack +// }\n\`\`\`\n### Operating System Information:\n---\n\n${debugInfo()}`, +// }) +// }, +// }) - let win +// let win - function handleRedirect(e, url) { - e.preventDefault() - shell.openExternal(url) - } +// function handleRedirect(e, url) { +// e.preventDefault() +// shell.openExternal(url) +// } - async function createWindow() { - app.server = require("./app") - eventEmitter.on("internal:port", port => { - const APP_URL = `http://localhost:${port}/_builder` - const APP_TITLE = "Budibase Builder" - win = new BrowserWindow({ - width: 1920, - height: 1080, - icon: join(__dirname, "..", "build", "icons", "512x512.png"), - }) - win.setTitle(APP_TITLE) - win.loadURL(APP_URL) - if (isDev) { - win.webContents.openDevTools() - } else { - autoUpdater.checkForUpdatesAndNotify() - } +// async function createWindow() { +// app.server = require("./app") +// eventEmitter.on("internal:port", port => { +// const APP_URL = `http://localhost:${port}/_builder` +// const APP_TITLE = "Budibase Builder" +// win = new BrowserWindow({ +// width: 1920, +// height: 1080, +// icon: join(__dirname, "..", "build", "icons", "512x512.png"), +// }) +// win.setTitle(APP_TITLE) +// win.loadURL(APP_URL) +// if (isDev) { +// win.webContents.openDevTools() +// } else { +// autoUpdater.checkForUpdatesAndNotify() +// } - // open _blank in default browser - win.webContents.on("new-window", handleRedirect) - win.webContents.on("will-navigate", handleRedirect) - }) - } +// // open _blank in default browser +// win.webContents.on("new-window", handleRedirect) +// win.webContents.on("will-navigate", handleRedirect) +// }) +// } - app.whenReady().then(createWindow) +// app.whenReady().then(createWindow) - // Quit when all windows are closed. - app.on("window-all-closed", () => { - // On macOS it is common for applications and their menu bar - // to stay active until the user quits explicitly with Cmd + Q - if (process.platform !== "darwin") { - app.server.close() - app.quit() - } - }) +// // Quit when all windows are closed. +// app.on("window-all-closed", () => { +// // On macOS it is common for applications and their menu bar +// // to stay active until the user quits explicitly with Cmd + Q +// if (process.platform !== "darwin") { +// app.server.close() +// app.quit() +// } +// }) - app.on("activate", () => { - // On macOS it's common to re-create a window in the app when the - // dock icon is clicked and there are no other windows open. - if (win === null) createWindow() - }) -} +// app.on("activate", () => { +// // On macOS it's common to re-create a window in the app when the +// // dock icon is clicked and there are no other windows open. +// if (win === null) createWindow() +// }) +// } -autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => { - const dialogOpts = { - type: "info", - buttons: ["Restart", "Later"], - title: "Budibase Update Available", - message: process.platform === "win32" ? releaseNotes : releaseName, - detail: - "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.", - } +// autoUpdater.on("update-downloaded", (event, releaseNotes, releaseName) => { +// const dialogOpts = { +// type: "info", +// buttons: ["Restart", "Later"], +// title: "Budibase Update Available", +// message: process.platform === "win32" ? releaseNotes : releaseName, +// detail: +// "A new version of the budibase builder has been downloaded. Restart the application to apply the updates.", +// } - dialog.showMessageBox(dialogOpts).then(returnValue => { - if (returnValue.response === 0) autoUpdater.quitAndInstall() - }) -}) +// dialog.showMessageBox(dialogOpts).then(returnValue => { +// if (returnValue.response === 0) autoUpdater.quitAndInstall() +// }) +// }) -autoUpdater.on("error", message => { - console.error("There was a problem updating the application") - console.error(message) -}) +// autoUpdater.on("error", message => { +// console.error("There was a problem updating the application") +// console.error(message) +// }) -startApp() +// startApp() diff --git a/packages/server/src/utilities/initialiseBudibase.js b/packages/server/src/utilities/initialiseBudibase.js deleted file mode 100644 index 990b180372..0000000000 --- a/packages/server/src/utilities/initialiseBudibase.js +++ /dev/null @@ -1,37 +0,0 @@ -// const { existsSync, readFile, writeFile, ensureDir } = require("fs-extra") -// const { join, resolve } = require("./centralPath") -// const { processString } = require("@budibase/string-templates") -// const uuid = require("uuid") - -// module.exports = async opts => { -// // await ensureDir(opts.dir) -// await setCouchDbUrl(opts) - -// // need an env file -// await createDevEnvFile(opts) -// } - -// const setCouchDbUrl = async opts => { -// if (!opts.couchDbUrl) { -// const dataDir = join(opts.dir, ".data") -// await ensureDir(dataDir) -// opts.couchDbUrl = -// dataDir + (dataDir.endsWith("/") || dataDir.endsWith("\\") ? "" : "/") -// } -// } - -// const createDevEnvFile = async opts => { -// const destConfigFile = join(opts.dir, "./.env") -// let createConfig = !existsSync(destConfigFile) || opts.quiet -// if (createConfig) { -// const template = await readFile( -// resolve(__dirname, "..", "..", ".env.template"), -// { -// encoding: "utf8", -// } -// ) -// opts.cookieKey1 = opts.cookieKey1 || uuid.v4() -// const config = await processString(template, opts) -// await writeFile(destConfigFile, config, { flag: "w+" }) -// } -// } From c3b489dcef517a2aa9b9dda162ce5bf3d4b907be Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 23 Mar 2021 11:05:02 +0000 Subject: [PATCH 4/6] adding redis stack to dev --- hosting/docker-compose.yaml | 10 ++++++++++ hosting/envoy.yaml | 20 ++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index e8408d9a7d..78408c85e6 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -93,9 +93,19 @@ services: depends_on: - couchdb-service command: ["sh","-c","sleep 10 && $${PUT_CALL}/_users && $${PUT_CALL}/_replicator; fg;"] + + redis-service: + restart: always + image: redis + ports: + - "${REDIS_PORT}:6379" + volumes: + - redis_data:/data volumes: couchdb_data: driver: local minio_data: driver: local + redis_data: + driver: local diff --git a/hosting/envoy.yaml b/hosting/envoy.yaml index 11f5c81b99..8c6081d1a7 100644 --- a/hosting/envoy.yaml +++ b/hosting/envoy.yaml @@ -36,6 +36,11 @@ static_resources: cluster: worker-service prefix_rewrite: "/" + - match: { prefix: "/cache/" } + route: + cluster: redis-service + prefix_rewrite: "/" + - match: { prefix: "/db/" } route: cluster: couchdb-service @@ -107,3 +112,18 @@ static_resources: address: couchdb-service port_value: 5984 + - name: redis-service + connect_timeout: 0.25s + type: strict_dns + lb_policy: round_robin + load_assignment: + cluster_name: redis-service + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: redis-service + port_value: 6379 + + From 679da60b1c881f5383da40c1cd9835862337a381 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 24 Mar 2021 15:36:13 +0000 Subject: [PATCH 5/6] make cypress use leveldb --- packages/builder/cypress.json | 3 ++- packages/builder/cypress/setup.js | 12 +++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/builder/cypress.json b/packages/builder/cypress.json index 09b38ae985..84c50ae863 100644 --- a/packages/builder/cypress.json +++ b/packages/builder/cypress.json @@ -3,6 +3,7 @@ "video": true, "projectId": "bmbemn", "env": { - "PORT": "4001" + "PORT": "4001", + "JWT_SECRET": "test" } } diff --git a/packages/builder/cypress/setup.js b/packages/builder/cypress/setup.js index 88c825bfc5..4d2833f340 100644 --- a/packages/builder/cypress/setup.js +++ b/packages/builder/cypress/setup.js @@ -1,16 +1,14 @@ -// What this script does: -// 1. Removes the old test folder if it exists (.budibase) -// 2. Initialises using `.budibase` -// 3. Runs the server using said folder - -const { join, resolve } = require("path") -const initialiseBudibase = require("../../server/src/utilities/initialiseBudibase") const cypressConfig = require("../cypress.json") +const path = require("path") + +const tmpdir = path.join(require("os").tmpdir(), ".budibase") process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE" process.env.NODE_ENV = "cypress" process.env.ENABLE_ANALYTICS = "false" process.env.PORT = cypressConfig.env.PORT +process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET +process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` // Stop info logs polluting test outputs process.env.LOG_LEVEL = "error" From bf7031467d969442e0dcb92c8899a0b06e15ff3f Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Wed, 24 Mar 2021 21:15:14 +0000 Subject: [PATCH 6/6] cypress tests passing --- packages/builder/cypress/setup.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/builder/cypress/setup.js b/packages/builder/cypress/setup.js index 4d2833f340..7b849eb887 100644 --- a/packages/builder/cypress/setup.js +++ b/packages/builder/cypress/setup.js @@ -9,6 +9,12 @@ process.env.ENABLE_ANALYTICS = "false" process.env.PORT = cypressConfig.env.PORT process.env.JWT_SECRET = cypressConfig.env.JWT_SECRET process.env.COUCH_URL = `leveldb://${tmpdir}/.data/` +process.env.SELF_HOSTED = 1 +process.env.MINIO_URL = "http://localhost:10000/" +process.env.MINIO_ACCESS_KEY = "budibase" +process.env.MINIO_SECRET_KEY = "budibase" +process.env.COUCH_DB_USER = "budibase" +process.env.COUCH_DB_PASSWORD = "budibase" // Stop info logs polluting test outputs process.env.LOG_LEVEL = "error"