From b4e70301a0be39c006243dd66c122a4cb824a802 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Tue, 14 Jul 2020 21:07:53 +0100 Subject: [PATCH] download component libraries from NPM tarball --- packages/server/package.json | 3 +- .../server/src/api/controllers/application.js | 23 ++++--------- packages/server/src/api/controllers/static.js | 1 + packages/server/src/app.js | 2 +- packages/server/src/utilities/chmodr.js | 17 ---------- .../server/src/utilities/createAppPackage.js | 32 +++++++++++++++++-- packages/server/yarn.lock | 12 +++++-- 7 files changed, 50 insertions(+), 40 deletions(-) delete mode 100644 packages/server/src/utilities/chmodr.js diff --git a/packages/server/package.json b/packages/server/package.json index 4ece7c0048..a14479f954 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -49,6 +49,7 @@ "@sentry/node": "^5.19.2", "aws-sdk": "^2.706.0", "bcryptjs": "^2.4.3", + "chmodr": "^1.2.0", "dotenv": "^8.2.0", "electron-is-dev": "^1.2.0", "electron-unhandled": "^3.0.2", @@ -70,7 +71,7 @@ "pouchdb": "^7.2.1", "pouchdb-all-dbs": "^1.0.2", "squirrelly": "^7.5.0", - "tar-fs": "^2.0.0", + "tar-fs": "^2.1.0", "uuid": "^3.3.2", "validate.js": "^0.13.1", "yargs": "^13.2.4", diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 423960229d..91a9b29596 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -7,11 +7,13 @@ const instanceController = require("./instance") const { resolve, join } = require("path") const { copy, exists, readFile, writeFile } = require("fs-extra") const { budibaseAppsDir } = require("../../utilities/budibaseDir") -const { exec } = require("child_process") const sqrl = require("squirrelly") const setBuilderToken = require("../../utilities/builder/setBuilderToken") const fs = require("fs-extra") -const chmodr = require("../../utilities/chmodr") +const { promisify } = require("util") +const chmodr = require("chmodr") +const { downloadExtractComponentLibraries } = require("../../utilities/createAppPackage") + exports.fetch = async function(ctx) { const db = new CouchDB(ClientDb.name(getClientId(ctx))) @@ -84,7 +86,7 @@ exports.create = async function(ctx) { if (ctx.isDev) { const newAppFolder = await createEmptyAppPackage(ctx, newApplication) - await runNpmInstall(newAppFolder) + await downloadExtractComponentLibraries(newAppFolder) } ctx.status = 200 @@ -148,7 +150,8 @@ const createEmptyAppPackage = async (ctx, app) => { // this line allows full permission on copied files // we have an unknown problem without this, whereby the // files get weird permissions and cant be written to :( - await chmodr(newAppFolder, 0o777) + const chmodrPromise = promisify(chmodr) + await chmodrPromise(newAppFolder, 0o777) await updateJsonFile(join(appsFolder, app._id, "package.json"), { name: npmFriendlyAppName(app.name), @@ -216,18 +219,6 @@ const updateJsonFile = async (filePath, app) => { return JSON.parse(newJson) } -const runNpmInstall = async newAppFolder => { - return new Promise((resolve, reject) => { - const cmd = `cd ${newAppFolder} && npm install` - exec(cmd, (error, stdout, stderr) => { - if (error) { - reject(error) - } - resolve(stdout ? stdout : stderr) - }) - }) -} - const npmFriendlyAppName = name => name .replace(/_/g, "") diff --git a/packages/server/src/api/controllers/static.js b/packages/server/src/api/controllers/static.js index 1b9e49f9e0..ed19f3df53 100644 --- a/packages/server/src/api/controllers/static.js +++ b/packages/server/src/api/controllers/static.js @@ -83,6 +83,7 @@ exports.serveComponentLibrary = async function(ctx) { ctx.user.appId, "node_modules", decodeURI(ctx.query.library), + "package", "dist" ) diff --git a/packages/server/src/app.js b/packages/server/src/app.js index 3936225172..adf4c7e46d 100644 --- a/packages/server/src/app.js +++ b/packages/server/src/app.js @@ -27,7 +27,7 @@ app.context.eventEmitter = eventEmitter // api routes app.use(api.routes()) -if (electron.app && electron.app.isPackaged()) { +if (electron.app && electron.app.isPackaged) { Sentry.init() app.on("error", (err, ctx) => { diff --git a/packages/server/src/utilities/chmodr.js b/packages/server/src/utilities/chmodr.js deleted file mode 100644 index b5adc756f6..0000000000 --- a/packages/server/src/utilities/chmodr.js +++ /dev/null @@ -1,17 +0,0 @@ -const fs = require("fs-extra") -const { join } = require("path") - -const chmodr = async (dir, perms) => { - const children = await fs.readdir(dir) - for (let child of children) { - const fullChildPath = join(dir, child) - const stat = await fs.stat(join(dir, child)) - if (stat.isFile()) { - await fs.chmod(fullChildPath, perms) - } else { - await chmodr(fullChildPath, perms) - } - } -} - -module.exports = chmodr diff --git a/packages/server/src/utilities/createAppPackage.js b/packages/server/src/utilities/createAppPackage.js index 49dd7ea2af..f733d1caa9 100644 --- a/packages/server/src/utilities/createAppPackage.js +++ b/packages/server/src/utilities/createAppPackage.js @@ -1,7 +1,35 @@ const { resolve } = require("path") const { cwd } = require("process") +const stream = require("stream") +const fetch = require("node-fetch") +const tar = require("tar-fs") +const zlib = require("zlib") +const { promisify } = require("util") +const streamPipeline = promisify(stream.pipeline) -const appPackageFolder = (config, appname) => +exports.appPackageFolder = (config, appname) => resolve(cwd(), config.latestPackagesFolder, appname) -module.exports.appPackageFolder = appPackageFolder + +exports.downloadExtractComponentLibraries = async appFolder => { + + const LIBRARIES = [ + "materialdesign-components", + "standard-components" + ] + + // Need to download tarballs directly from NPM as our users may not have node on their machine + for (let lib of LIBRARIES) { + // download tarball + // TODO: make sure the latest version is always downloaded + const registryUrl = `https://registry.npmjs.org/@budibase/${lib}/-/${lib}-0.1.2.tgz`; + const response = await fetch(registryUrl) + if (!response.ok) throw new Error(`unexpected response ${response.statusText}`) + + await streamPipeline( + response.body, + zlib.Unzip(), + tar.extract(`${appFolder}/node_modules/@budibase/${lib}`) + ) + } +} \ No newline at end of file diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock index f5fcc09bcf..168b627818 100644 --- a/packages/server/yarn.lock +++ b/packages/server/yarn.lock @@ -1262,6 +1262,11 @@ chardet@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.7.0.tgz#90094849f0937f2eedc2425d0d28a9e5f0cbad9e" +chmodr@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/chmodr/-/chmodr-1.2.0.tgz#720e96caa09b7f1cdbb01529b7d0ab6bc5e118b9" + integrity sha512-Y5uI7Iq/Az6HgJEL6pdw7THVd7jbVOTPwsmcPOBjQL8e3N+pz872kzK5QxYGEy21iRys+iHWV0UZQXDFJo1hyA== + chokidar@^3.2.2: version "3.3.1" resolved "https://registry.yarnpkg.com/chokidar/-/chokidar-3.3.1.tgz#c84e5b3d18d9a4d77558fef466b1bf16bbeb3450" @@ -5742,9 +5747,10 @@ table@^5.2.3: slice-ansi "^2.1.0" string-width "^3.0.0" -tar-fs@^2.0.0: - version "2.0.1" - resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.0.1.tgz#e44086c1c60d31a4f0cf893b1c4e155dabfae9e2" +tar-fs@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-2.1.0.tgz#d1cdd121ab465ee0eb9ccde2d35049d3f3daf0d5" + integrity sha512-9uW5iDvrIMCVpvasdFHW0wJPez0K4JnMZtsuIeDI7HyMGJNxmDZDOCQROr7lXyS+iL/QMpj07qcjGYTSdRFXUg== dependencies: chownr "^1.1.1" mkdirp-classic "^0.5.2"