From 8ae6f0429000fee014307ebbb286d097264667f7 Mon Sep 17 00:00:00 2001 From: Martin McKeaveney Date: Mon, 6 Jul 2020 19:43:40 +0100 Subject: [PATCH] extra index template for production --- .github/workflows/budibase_ci.yml | 34 +++++++- .../builderStore/loadComponentLibraries.js | 3 +- .../pages/[application]/deploy/index.svelte | 35 ++++++-- packages/cli/src/commands/run/runHandler.js | 2 +- packages/client/src/index.js | 8 +- packages/server/.env.template | 6 +- packages/server/Dockerfile | 2 + packages/server/envfile | 17 ++++ .../server/src/api/controllers/deploy/aws.js | 87 ++++++++++++------- packages/server/src/api/controllers/static.js | 39 +++------ packages/server/src/api/index.js | 1 - .../server/src/utilities/builder/buildPage.js | 8 +- .../src/utilities/builder/index.template.html | 19 ++-- 13 files changed, 177 insertions(+), 84 deletions(-) create mode 100644 packages/server/envfile diff --git a/.github/workflows/budibase_ci.yml b/.github/workflows/budibase_ci.yml index 21d735fcbc..fde56b153a 100644 --- a/.github/workflows/budibase_ci.yml +++ b/.github/workflows/budibase_ci.yml @@ -32,4 +32,36 @@ jobs: - run: yarn test env: CI: true - name: Budibase CI \ No newline at end of file + name: Budibase CI + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Build, tag, and push image to Amazon ECR + id: build-image + env: + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: my-ecr-repo + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + echo "::set-output name=image::$ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG" + + - name: Fill in the new image ID in the Amazon ECS task definition + id: task-def + uses: aws-actions/amazon-ecs-render-task-definition@v1 + with: + task-definition: task-definition.json + container-name: my-container + image: ${{ steps.build-image.outputs.image }} + + - name: Deploy Amazon ECS task definition + uses: aws-actions/amazon-ecs-deploy-task-definition@v1 + with: + task-definition: ${{ steps.task-def.outputs.task-definition }} + service: my-service + cluster: my-cluster + wait-for-service-stability: true + \ No newline at end of file diff --git a/packages/builder/src/builderStore/loadComponentLibraries.js b/packages/builder/src/builderStore/loadComponentLibraries.js index 93fc5b3aac..9d534f86fe 100644 --- a/packages/builder/src/builderStore/loadComponentLibraries.js +++ b/packages/builder/src/builderStore/loadComponentLibraries.js @@ -23,8 +23,7 @@ export const fetchComponentLibModules = async application => { const allLibraries = {} for (let libraryName of application.componentLibraries) { - // const LIBRARY_URL = `/${application._id}/componentlibrary?library=${libraryName}` - const LIBRARY_URL = `/assets/componentlibrary/${libraryName}/dist/index.js` + const LIBRARY_URL = `/${application._id}/componentlibrary?library=${libraryName}` const libraryModule = await import(LIBRARY_URL) allLibraries[libraryName] = libraryModule } diff --git a/packages/builder/src/pages/[application]/deploy/index.svelte b/packages/builder/src/pages/[application]/deploy/index.svelte index 675980e7f2..56e32ac295 100644 --- a/packages/builder/src/pages/[application]/deploy/index.svelte +++ b/packages/builder/src/pages/[application]/deploy/index.svelte @@ -3,10 +3,15 @@ import { store } from "builderStore" import { notifier } from "builderStore/store/notifications" import api from "builderStore/api" + import Spinner from "components/common/Spinner.svelte" + + let deployed = false + let loading = false $: appId = $store.appId async function deployApp() { + loading = true const DEPLOY_URL = `/deploy` try { @@ -14,12 +19,15 @@ const response = await api.post(DEPLOY_URL) const json = await response.json() if (response.status !== 200) { - throw new Error + throw new Error() } - notifier.success(`Deployment Complete. View your app at blah URL https://${appId}.app.budi.live/${appId}`) + notifier.success(`Your Deployment is Complete.`) + deployed = true + loading = false } catch (err) { notifier.danger("Deployment unsuccessful. Please try again later.") + loading = false } } @@ -27,9 +35,18 @@

It's time to shine!

- + {#if deployed} + + View App + + {:else} + + {/if}
@@ -53,11 +70,11 @@ flex-direction: column; align-items: center; justify-content: center; - left: 0; - right: 0; + left: 0; + right: 0; top: 20%; - margin-left: auto; - margin-right: auto; + margin-left: auto; + margin-right: auto; width: 50%; } diff --git a/packages/cli/src/commands/run/runHandler.js b/packages/cli/src/commands/run/runHandler.js index e3f669cf53..b7c58e6e03 100644 --- a/packages/cli/src/commands/run/runHandler.js +++ b/packages/cli/src/commands/run/runHandler.js @@ -8,7 +8,7 @@ module.exports = ({ dir }) => { // dont make this a variable or top level require // ti will cause environment module to be loaded prematurely - require("@budibase/server/src/app")().then(server => { + return require("@budibase/server/src/app")().then(server => { server.on("close", () => console.log("Server Closed")) console.log(`Budibase running on ${JSON.stringify(server.address())}`) }) diff --git a/packages/client/src/index.js b/packages/client/src/index.js index a1f14f667e..f73370ab77 100644 --- a/packages/client/src/index.js +++ b/packages/client/src/index.js @@ -20,10 +20,10 @@ export const loadBudibase = async opts => { for (let library of libraries) { // fetch the JavaScript for the component libraries from the server - // componentLibraryModules[library] = await import( - // `/componentlibrary?library=${encodeURI(library)}` - // ) - componentLibraryModules[library] = await import(`/assets/componentlibrary/${library}/dist/index.js`) + componentLibraryModules[library] = await import( + `/componentlibrary?library=${encodeURI(library)}` + ) + // componentLibraryModules[library] = await import(`/assets/componentlibrary/${library}/dist/index.js`) } componentLibraryModules[builtinLibName] = builtins(_window) diff --git a/packages/server/.env.template b/packages/server/.env.template index f282d9c67f..30c28b1b2e 100644 --- a/packages/server/.env.template +++ b/packages/server/.env.template @@ -12,4 +12,8 @@ JWT_SECRET={{cookieKey1}} PORT=4001 # error level for koa-pino -LOG_LEVEL=error \ No newline at end of file +LOG_LEVEL=error + +DEPLOYMENT_CF_DISTRIBUTION_ID= +DEPLOYMENT_APP_ASSETS_BUCKET=g +DEPLOYMENT_CREDENTIALS_URL="https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/" \ No newline at end of file diff --git a/packages/server/Dockerfile b/packages/server/Dockerfile index 7ae24dfd62..bfdc52c6d3 100644 --- a/packages/server/Dockerfile +++ b/packages/server/Dockerfile @@ -2,6 +2,8 @@ FROM node:12-alpine WORKDIR /app +ENV CLOUD=1 + # copy files and install dependencies COPY . ./ RUN yarn diff --git a/packages/server/envfile b/packages/server/envfile new file mode 100644 index 0000000000..d38eaf1d56 --- /dev/null +++ b/packages/server/envfile @@ -0,0 +1,17 @@ +# url of couch db, including username and password +# http://admin:password@localhost:5984 +COUCH_DB_URL= +# identifies a client database - i.e. group of apps +CLIENT_ID=1 +# used to create cookie hashes +JWT_SECRET=4715888e-144f-4802-b8d8-862a1b8365dd +# port to run http server on +PORT=4001 + +# error level for koa-pino +LOG_LEVEL=error + +COUCH_DB_REMOTE=https://admin:afasdgafgF342G@couchdb.budi.live:5984 +BUDIBASE_APP_ASSETS_BUCKET=prod-budi-app-assets +BUDIBASE_API_KEY=d498278c-4ab4-144b-c212-b8f9e6da5c2b + diff --git a/packages/server/src/api/controllers/deploy/aws.js b/packages/server/src/api/controllers/deploy/aws.js index d9ea62769a..ae64c835b8 100644 --- a/packages/server/src/api/controllers/deploy/aws.js +++ b/packages/server/src/api/controllers/deploy/aws.js @@ -5,15 +5,28 @@ const { budibaseAppsDir, } = require("../../../utilities/budibaseDir") +async function invalidateCDN(appId) { + const cf = new AWS.CloudFront({}) + + return cf.createInvalidation({ + DistributionId: process.env.DEPLOYMENT_CF_DISTRIBUTION_ID, + InvalidationBatch: { + CallerReference: appId, + Paths: { + Quantity: 1, + Items: [ + `/assets/${appId}/*` + ] + } + } + }).promise() +} + async function fetchTemporaryCredentials() { - const CREDENTIALS_URL = "https://dt4mpwwap8.execute-api.eu-west-1.amazonaws.com/prod/" - - const BUDIBASE_API_KEY = process.env.BUDIBASE_API_KEY - - const response = await fetch(CREDENTIALS_URL, { + const response = await fetch(process.env.DEPLOYMENT_CREDENTIALS_URL, { method: "POST", body: JSON.stringify({ - apiKey: BUDIBASE_API_KEY + apiKey: process.env.BUDIBASE_API_KEY }) }) @@ -32,6 +45,27 @@ const CONTENT_TYPE_MAP = { js: "application/javascript" }; +/** + * Recursively walk a directory tree and execute a callback on all files. + * @param {Re} dirPath - Directory to traverse + * @param {*} callback - callback to execute on files + */ +function walkDir(dirPath, callback) { + for (let filename of fs.readdirSync(dirPath)) { + const filePath = `${dirPath}/${filename}` + const stat = fs.lstatSync(filePath) + + if (stat.isFile()) { + callback({ + bytes: fs.readFileSync(filePath), + filename + }) + } else { + walkDir(filePath, callback) + } + } +} + exports.uploadAppAssets = async function ({ appId }) { const { credentials, accountId } = await fetchTemporaryCredentials() @@ -43,7 +77,7 @@ exports.uploadAppAssets = async function ({ appId }) { const s3 = new AWS.S3({ params: { - Bucket: process.env.BUDIBASE_APP_ASSETS_BUCKET + Bucket: process.env.DEPLOYMENT_APP_ASSETS_BUCKET } }) @@ -54,34 +88,27 @@ exports.uploadAppAssets = async function ({ appId }) { const uploads = [] for (let page of appPages) { - for (let filename of fs.readdirSync(`${appAssetsPath}/${page}`)) { - const filePath = `${appAssetsPath}/${page}/${filename}` - const stat = await fs.lstatSync(filePath) - - // TODO: need to account for recursively traversing dirs - if (stat.isFile()) { - const fileBytes = fs.readFileSync(`${appAssetsPath}/${page}/${filename}`) + walkDir(`${appAssetsPath}/${page}`, function prepareUploadsForS3({ bytes, filename }) { + const fileExtension = [...filename.split(".")].pop() - console.log(`${appId}/${page}/${filename}`) + const upload = s3.upload({ + Key: `assets/${appId}/${page}/${filename}`, + Body: bytes, + ContentType: CONTENT_TYPE_MAP[fileExtension], + Metadata: { + accountId + } + }).promise() - const fileExtension = [...filename.split(".")].pop() - - const upload = s3.upload({ - Key: `assets/${appId}/${page}/${filename}`, - Body: fileBytes, - ContentType: CONTENT_TYPE_MAP[fileExtension], - Metadata: { - accountId - } - }).promise() - - uploads.push(upload) - } - } + uploads.push(upload) + }) } try { - return Promise.all(uploads) + const uploadAllFiles = Promise.all(uploads) + const invalidateCloudfront = invalidateCDN(appId) + await uploadAllFiles + await invalidateCloudfront } catch (err) { console.error("Error uploading budibase app assets to s3", err) throw err diff --git a/packages/server/src/api/controllers/static.js b/packages/server/src/api/controllers/static.js index 31d91672a3..6ce945dde4 100644 --- a/packages/server/src/api/controllers/static.js +++ b/packages/server/src/api/controllers/static.js @@ -44,19 +44,15 @@ exports.serveApp = async function(ctx) { }) } - const { file = "index.html" } = ctx - - - if (ctx.isCloud) { - const S3_URL = `https://${ctx.params.appId}.app.budi.live/assets/${ctx.params.appId}/${mainOrAuth}/${file}` - console.log("Serving: " + S3_URL) + if (process.env.CLOUD) { + const S3_URL = `https://${ctx.params.appId}.app.budi.live/assets/${ctx.params.appId}/${mainOrAuth}/${ctx.file || "index.production.html"}` const response = await fetch(S3_URL) const body = await response.text() ctx.body = body return } - await send(ctx, file, { root: ctx.devPath || appPath }) + await send(ctx, ctx.file || "index.html", { root: ctx.devPath || appPath }) } exports.serveAppAsset = async function(ctx) { @@ -70,15 +66,7 @@ exports.serveAppAsset = async function(ctx) { mainOrAuth ) - // if (ctx.isCloud) { - // const requestUrl = `${S3_URL_PREFIX}/${appId}/public/${mainOrAuth}/${ctx.file || "index.html"}` - // console.log('request url:' , requestUrl) - // const response = await fetch(requestUrl) - // const body = await response.text() - // ctx.body = body - // } else { - await send(ctx, ctx.file, { root: ctx.devPath || appPath }) - // } + await send(ctx, ctx.file, { root: ctx.devPath || appPath }) } exports.serveComponentLibrary = async function(ctx) { @@ -99,16 +87,15 @@ exports.serveComponentLibrary = async function(ctx) { ) } - // if (ctx.isCloud) { - // const appId = ctx.user.appId - // const requestUrl = encodeURI(`${S3_URL_PREFIX}/${appId}/node_modules/${ctx.query.library}/dist/index.js`) - // console.log('request url components: ', requestUrl) - // const response = await fetch(requestUrl) - // const body = await response.text() - // ctx.type = 'application/javascript' - // ctx.body = body; - // return - // } + if (process.env.CLOUD) { + const appId = ctx.user.appId + const S3_URL = encodeURI(`https://${appId}.app.budi.live/assets/componentlibrary/${ctx.query.library}/dist/index.js`) + const response = await fetch(S3_URL) + const body = await response.text() + ctx.type = 'application/javascript' + ctx.body = body; + return + } await send(ctx, "/index.js", { root: componentLibraryPath }) } diff --git a/packages/server/src/api/index.js b/packages/server/src/api/index.js index a3d7fbffc9..165a479a38 100644 --- a/packages/server/src/api/index.js +++ b/packages/server/src/api/index.js @@ -43,7 +43,6 @@ router useAppRootPath: true, } ctx.isDev = env.NODE_ENV !== "production" && env.NODE_ENV !== "jest" - // ctx.isCloud = true await next() }) .use(authenticated) diff --git a/packages/server/src/utilities/builder/buildPage.js b/packages/server/src/utilities/builder/buildPage.js index 96db612bf7..2c5ada76b3 100644 --- a/packages/server/src/utilities/builder/buildPage.js +++ b/packages/server/src/utilities/builder/buildPage.js @@ -57,8 +57,6 @@ const buildIndexHtml = async (config, appId, pageName, appPath, pkg) => { pageStyle: pkg.page._css, appId, pageName, - // TODO: don't hardcode - production: true } const indexHtmlTemplate = await readFile( @@ -67,10 +65,16 @@ const buildIndexHtml = async (config, appId, pageName, appPath, pkg) => { ) const indexHtmlPath = join(appPublicPath, "index.html") + const deployableHtmlPath = join(appPublicPath, "index.production.html") const indexHtml = sqrl.Render(indexHtmlTemplate, templateObj) + const deployableHtml = sqrl.Render(indexHtmlTemplate, { + ...templateObj, + production: true + }) await writeFile(indexHtmlPath, indexHtml, { flag: "w+" }) + await writeFile(deployableHtmlPath, deployableHtml, { flag: "w+" }) } const buildFrontendAppDefinition = async (config, appId, pageName, pkg) => { diff --git a/packages/server/src/utilities/builder/index.template.html b/packages/server/src/utilities/builder/index.template.html index 0f90b34002..e7cf633277 100644 --- a/packages/server/src/utilities/builder/index.template.html +++ b/packages/server/src/utilities/builder/index.template.html @@ -27,29 +27,34 @@ {{ /each }} {{ each(options.screenStyles) }} + {{ if(options.production) }} + + {{#else}} + {{ /if }} {{ /each }} {{ if(options.pageStyle) }} + {{ if(options.production) }} + + {{#else}} {{ /if }} + {{ /if }} {{ if(options.production) }} - - - {{ else }} - - - {{ /if }} + {{#else}} + + + {{ /if }} -