From c2321797f14e199fe2776b8d4aa53f9cc7fd05c4 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 28 Sep 2020 17:04:08 +0100 Subject: [PATCH] async templates working from S3 --- .../src/components/start/TemplateList.svelte | 14 ++--- packages/server/scripts/exportAppTemplate.js | 33 ++-------- .../server/src/api/controllers/application.js | 15 +++-- .../server/src/api/controllers/instance.js | 8 +-- packages/server/src/api/controllers/static.js | 4 +- .../server/src/api/controllers/templates.js | 44 ++++++++++++++ packages/server/src/api/index.js | 4 ++ packages/server/src/api/routes/index.js | 2 + packages/server/src/api/routes/templates.js | 17 ++++++ packages/server/src/utilities/templates.js | 60 +++++++++++++++++++ 10 files changed, 149 insertions(+), 52 deletions(-) create mode 100644 packages/server/src/api/controllers/templates.js create mode 100644 packages/server/src/api/routes/templates.js create mode 100644 packages/server/src/utilities/templates.js diff --git a/packages/builder/src/components/start/TemplateList.svelte b/packages/builder/src/components/start/TemplateList.svelte index 76b9f6dae2..8f056b1a0e 100644 --- a/packages/builder/src/components/start/TemplateList.svelte +++ b/packages/builder/src/components/start/TemplateList.svelte @@ -2,19 +2,13 @@ import { Button, Heading } from "@budibase/bbui" import AppCard from "./AppCard.svelte" import Spinner from "components/common/Spinner.svelte" + import api from "builderStore/api" export let onSelect - let templates = [] - - function fetchTemplates() { - return Promise.resolve([ - { - name: "Funky", - description: "Funky ass template", - minBuilderVersion: "", - }, - ]) + async function fetchTemplates() { + const response = await api.get("/api/templates?type=app") + return await response.json() } let templatesPromise = fetchTemplates() diff --git a/packages/server/scripts/exportAppTemplate.js b/packages/server/scripts/exportAppTemplate.js index c39b0df07a..78e6764ffa 100644 --- a/packages/server/scripts/exportAppTemplate.js +++ b/packages/server/scripts/exportAppTemplate.js @@ -1,36 +1,11 @@ #!/usr/bin/env node +const { exportTemplateFromApp } = require("../src/utilities/templates") -// Script to export a budibase app into a package -// Usage: foo +// Script to export a chosen budibase app into a package -const fs = require("fs-extra") -const path = require("path") -const os = require("os") -const replicationStream = require("pouchdb-replication-stream") +const [name, instanceId, appId] = process.argv.slice(1) -const PouchDB = require("../src/db") - -PouchDB.plugin(replicationStream.plugin) -PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) - -async function exportAppToTemplate({ instanceId, appId, templateName }) { - // Copy frontend files - console.log("Copying frontend definition...") - const appToExport = path.join(os.homedir(), ".budibase", appId, "pages") - const templateOutputPath = path.join(os.homedir(), ".budibase", templateName) - fs.copySync(appToExport, `${templateOutputPath}/pages`) - - const writeStream = fs.createWriteStream(`${templateOutputPath}/dump.txt`) - - // perform couch dump - const instanceDb = new PouchDB(instanceId) - - console.log("Performing database dump..") - await instanceDb.dump(writeStream) - console.log("Export complete!") -} - -exportAppToTemplate({ +exportTemplateFromApp({ templateName: "Funky", instanceId: "inst_b70abba_16feb394866140a1ac3f2e450e99f28a", appId: "b70abba3874546bf99a339911b579937", diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index 6794371c2b..d8668d7015 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -82,11 +82,10 @@ exports.create = async function(ctx) { }, }, } - // TODO: pass template into here from create InstCtx + await instanceController.create(createInstCtx) newApplication.instances.push(createInstCtx.body) - // TODO: if template is passed, create the app package from the template and seed the instance database if (process.env.NODE_ENV !== "jest") { const newAppFolder = await createEmptyAppPackage(ctx, newApplication) await downloadExtractComponentLibraries(newAppFolder) @@ -160,10 +159,16 @@ const createEmptyAppPackage = async (ctx, app) => { name: npmFriendlyAppName(app.name), }) - // Copy the frontend page definition files from the template directory - // if this app is being created from a template. + // if this app is being created from a template, + // copy the frontend page definition files from + // the template directory. if (app.template) { - const templatePageDefinitions = join(appsFolder, app.template.name, "pages") + const templatePageDefinitions = join( + appsFolder, + "templates", + app.template.key, + "pages" + ) await copy(templatePageDefinitions, join(appsFolder, app._id, "pages")) } diff --git a/packages/server/src/api/controllers/instance.js b/packages/server/src/api/controllers/instance.js index 3e40c14d7d..0099fe8e8f 100644 --- a/packages/server/src/api/controllers/instance.js +++ b/packages/server/src/api/controllers/instance.js @@ -3,6 +3,7 @@ const CouchDB = require("../../db") const client = require("../../db/clientDb") const newid = require("../../db/newid") const { budibaseAppsDir } = require("../../utilities/budibaseDir") +const { downloadTemplate } = require("../../utilities/templates") exports.create = async function(ctx) { const instanceName = ctx.request.body.name @@ -54,13 +55,10 @@ exports.create = async function(ctx) { budibaseApp.instances.push(instance) await clientDb.put(budibaseApp) - // TODO: download the chosen template tar file from s3 and unpack it // replicate the template data to the instance DB - // TODO: templates should be downloaded to .budibase/templates/something if (template) { - const dbDumpReadStream = fs.createReadStream( - `${budibaseAppsDir()}/${template.name}/dump.txt` - ) + const templatePath = await downloadTemplate(...template.key.split("/")) + const dbDumpReadStream = fs.createReadStream(`${templatePath}/db/dump.txt`) const { ok } = await db.load(dbDumpReadStream) if (!ok) { ctx.throw(500, "Error loading database dump from template.") diff --git a/packages/server/src/api/controllers/static.js b/packages/server/src/api/controllers/static.js index 24fb98208e..f0388fdb86 100644 --- a/packages/server/src/api/controllers/static.js +++ b/packages/server/src/api/controllers/static.js @@ -2,7 +2,7 @@ const send = require("koa-send") const { resolve, join } = require("path") const jwt = require("jsonwebtoken") const fetch = require("node-fetch") -const fs = require("fs") +const fs = require("fs-extra") const uuid = require("uuid") const AWS = require("aws-sdk") const { prepareUploadForS3 } = require("./deploy/aws") @@ -138,8 +138,6 @@ exports.performLocalFileProcessing = async function(ctx) { exports.serveApp = async function(ctx) { const mainOrAuth = ctx.isAuthenticated ? "main" : "unauthenticated" - console.log(ctx.user) - // default to homedir const appPath = resolve( budibaseAppsDir(), diff --git a/packages/server/src/api/controllers/templates.js b/packages/server/src/api/controllers/templates.js new file mode 100644 index 0000000000..c9750dee6f --- /dev/null +++ b/packages/server/src/api/controllers/templates.js @@ -0,0 +1,44 @@ +const fetch = require("node-fetch") +const { + downloadTemplate, + exportTemplateFromApp, +} = require("../../utilities/templates") + +const DEFAULT_TEMPLATES_BUCKET = + "prod-budi-templates.s3-eu-west-1.amazonaws.com" + +exports.fetch = async function(ctx) { + const { type = "app" } = ctx.query + + const response = await fetch( + `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json` + ) + const json = await response.json() + ctx.body = Object.values(json.templates[type]) +} + +exports.downloadTemplate = async function(ctx) { + const { type, name } = ctx.params + + await downloadTemplate(type, name) + + ctx.body = { + message: `template ${type}:${name} downloaded successfully.`, + } +} + +exports.exportTemplateFromApp = async function(ctx) { + const { appId, instanceId } = ctx.user.appId + const { templateName } = ctx.request.body + + await exportTemplateFromApp({ + appId, + instanceId, + templateName, + }) + + ctx.status = 200 + ctx.body = { + message: `Created template: ${templateName}`, + } +} diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index b7f156fb6a..286d08eea0 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -19,6 +19,7 @@ const { automationRoutes, accesslevelRoutes, apiKeysRoutes, + templatesRoutes, } = require("./routes") const router = new Router() @@ -89,6 +90,9 @@ router.use(automationRoutes.allowedMethods()) router.use(deployRoutes.routes()) router.use(deployRoutes.allowedMethods()) + +router.use(templatesRoutes.routes()) +router.use(templatesRoutes.allowedMethods()) // end auth routes router.use(pageRoutes.routes()) diff --git a/packages/server/src/api/routes/index.js b/packages/server/src/api/routes/index.js index a2b8d3bb6c..3ac7d811b5 100644 --- a/packages/server/src/api/routes/index.js +++ b/packages/server/src/api/routes/index.js @@ -13,6 +13,7 @@ const automationRoutes = require("./automation") const accesslevelRoutes = require("./accesslevel") const deployRoutes = require("./deploy") const apiKeysRoutes = require("./apikeys") +const templatesRoutes = require("./templates") module.exports = { deployRoutes, @@ -30,4 +31,5 @@ module.exports = { automationRoutes, accesslevelRoutes, apiKeysRoutes, + templatesRoutes, } diff --git a/packages/server/src/api/routes/templates.js b/packages/server/src/api/routes/templates.js new file mode 100644 index 0000000000..1dc5b46342 --- /dev/null +++ b/packages/server/src/api/routes/templates.js @@ -0,0 +1,17 @@ +const Router = require("@koa/router") +const controller = require("../controllers/templates") +const authorized = require("../../middleware/authorized") +const { BUILDER } = require("../../utilities/accessLevels") + +const router = Router() + +router + .get("/api/templates", authorized(BUILDER), controller.fetch) + .get( + "/api/templates/:type/:name", + // authorized(BUILDER), + controller.downloadTemplate + ) + .post("/api/templates", authorized(BUILDER), controller.exportTemplateFromApp) + +module.exports = router diff --git a/packages/server/src/utilities/templates.js b/packages/server/src/utilities/templates.js new file mode 100644 index 0000000000..e3d79a7943 --- /dev/null +++ b/packages/server/src/utilities/templates.js @@ -0,0 +1,60 @@ +const path = require("path") +const fs = require("fs-extra") +const os = require("os") +const fetch = require("node-fetch") +const stream = require("stream") +const tar = require("tar-fs") +const zlib = require("zlib") +const { promisify } = require("util") +const streamPipeline = promisify(stream.pipeline) +const { budibaseAppsDir } = require("./budibaseDir") +const CouchDB = require("../db") + +const DEFAULT_TEMPLATES_BUCKET = + "prod-budi-templates.s3-eu-west-1.amazonaws.com" + +exports.downloadTemplate = async function(type, name) { + const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz` + console.log(templateUrl, type, name) + const response = await fetch(templateUrl) + + if (!response.ok) { + throw new Error( + `Error downloading template ${type}:${name}: ${response.statusText}` + ) + } + + // stream the response, unzip and extract + await streamPipeline( + response.body, + zlib.Unzip(), + tar.extract(path.join(budibaseAppsDir(), "templates", type)) + ) + + return path.join(budibaseAppsDir(), "templates", type, name) +} + +exports.exportTemplateFromApp = async function({ + appId, + templateName, + instanceId, +}) { + // Copy frontend files + console.log("Copying frontend definition...") + const appToExport = path.join(os.homedir(), ".budibase", appId, "pages") + const templatesDir = path.join(os.homedir(), ".budibase", "templates") + fs.ensureDirSync(templatesDir) + + const templateOutputPath = path.join(templatesDir, templateName) + fs.copySync(appToExport, `${templateOutputPath}/pages`) + + fs.ensureDirSync(path.join(templateOutputPath, "db")) + const writeStream = fs.createWriteStream(`${templateOutputPath}/db/dump.txt`) + + // perform couch dump + const instanceDb = new CouchDB(instanceId) + + console.log("Performing database dump..") + await instanceDb.dump(writeStream) + console.log("Export complete!") +}