Merge pull request #6513 from Budibase/infra/beta-ui

Infra/beta UI
2024-06-28 11:00:55 +12:00 · 2022-06-30 13:11:21 +01:00 · 2022-06-30 13:11:21 +01:00 · d1ab765fff
parent b6aa4e7b21 e49a2147cd
commit d1ab765fff
19 changed files with 214 additions and 27 deletions
--- a/.github/workflows/budibase_ci.yml
+++ b/.github/workflows/budibase_ci.yml
@ -7,6 +7,7 @@ on:
    branches:
      - master
      - develop 
+      - new-design-ui
 pull_request:
    branches:
      - master
@ -59,3 +60,19 @@ jobs:
      with:
        install: false
        command: yarn test:e2e:ci
+    
+    - name: Configure AWS Credentials
+      uses: aws-actions/configure-aws-credentials@v1
+      with:
+        aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        aws-region: eu-west-1
+
+    - name: Upload to S3
+      if: github.ref == 'refs/heads/new-design-ui' 
+      run: | 
+        tar -czvf new_ui.tar.gz packages/server/assets packages/server/index.html 
+        aws s3 cp new_ui.tar.gz s3://prod-budi-app-assets/beta:design_ui/
+        aws s3 cp packages/client/dist/budibase-client.js s3://prod-budi-app-assets/beta:design_ui/budibase-client.js
+        aws cloudfront create-invalidation --distribution-id E3ELKP4RCEHVLW --paths "/beta:design_ui/*"
+        
--- a/.github/workflows/release-develop.yml
+++ b/.github/workflows/release-develop.yml
@ -21,7 +21,7 @@ env:
  # Posthog token used by ui at build time 
  POSTHOG_TOKEN: phc_uDYOfnFt6wAbBAXkC6STjcrTpAFiWIhqgFcsC1UVO5F
  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
-  PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
+  PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
      
 jobs:
  release:
--- a/packages/backend-core/src/objectStore/index.ts
+++ b/packages/backend-core/src/objectStore/index.ts
@ -294,6 +294,16 @@ export const uploadDirectory = async (
  await Promise.all(uploads)
 }

+exports.downloadTarballDirect = async (url: string, path: string) => {
+  path = sanitizeKey(path)
+  const response = await fetch(url)
+  if (!response.ok) {
+    throw new Error(`unexpected response ${response.statusText}`)
+  }
+
+  await streamPipeline(response.body, zlib.Unzip(), tar.extract(path))
+}
+
 export const downloadTarball = async (url: any, bucketName: any, path: any) => {
  bucketName = sanitizeBucket(bucketName)
  path = sanitizeKey(path)
--- a/packages/builder/src/pages/builder/app/[application]/_layout.svelte
+++ b/packages/builder/src/pages/builder/app/[application]/_layout.svelte
@ -1,7 +1,14 @@
 <script>
  import { store, automationStore } from "builderStore"
  import { roles, flags } from "stores/backend"
-  import { Icon, ActionGroup, Tabs, Tab, notifications } from "@budibase/bbui"
+  import {
+    Icon,
+    ActionGroup,
+    Tabs,
+    Tab,
+    notifications,
+    Banner,
+  } from "@budibase/bbui"
  import RevertModal from "components/deploy/RevertModal.svelte"
  import VersionModal from "components/deploy/VersionModal.svelte"
  import DeployNavigation from "components/deploy/DeployNavigation.svelte"
@ -17,6 +24,7 @@

  // Get Package and set store
  let promise = getPackage()
+  let betaAccess = false

  // Sync once when you load the app
  let hasSynced = false
@ -58,10 +66,18 @@
    })
  }

+  async function newDesignUi() {
+    await flags.toggleUiFeature("design_ui")
+    window.location.reload()
+  }
+
  onMount(async () => {
    if (!hasSynced && application) {
      try {
        await API.syncApp(application)
+        // check if user has beta access
+        const betaResponse = await API.checkBetaAccess($auth?.user?.email)
+        betaAccess = betaResponse.access
      } catch (error) {
        notifications.error("Failed to sync with production database")
      }
@ -79,6 +95,15 @@
  <div class="loading" />
 {:then _}
  <div class="root">
+    {#if betaAccess}
+      <Banner
+        extraButtonText="Try New UI (Beta)"
+        extraButtonAction={newDesignUi}
+      >
+        Try the <b>all new</b> budibase design interface. (Not recommended for existing
+        budibase apps)
+      </Banner>
+    {/if}
    <div class="top-nav">
      <div class="topleftnav">
        <button class="home-logo">
--- a/packages/builder/src/stores/backend/flags.js
+++ b/packages/builder/src/stores/backend/flags.js
@ -16,6 +16,9 @@ export function createFlagsStore() {
      })
      await actions.fetch()
    },
+    toggleUiFeature: async feature => {
+      await API.toggleUiFeature({ value: feature })
+    },
  }

  return {
--- a/packages/frontend-core/src/api/flags.js
+++ b/packages/frontend-core/src/api/flags.js
@ -22,4 +22,13 @@ export const buildFlagEndpoints = API => ({
      },
    })
  },
+  /**
+   * Allows us to experimentally toggle a beta UI feature through a cookie.
+   * @param value the feature to toggle
+   */
+  toggleUiFeature: async ({ value }) => {
+    return await API.post({
+      url: `/api/beta/${value}`,
+    })
+  },
 })
--- a/packages/frontend-core/src/api/other.js
+++ b/packages/frontend-core/src/api/other.js
@ -54,4 +54,13 @@ export const buildOtherEndpoints = API => ({
      url: "/api/permission/builtin",
    })
  },
+
+  /**
+   * Check if they are part of the budibase beta program.
+   */
+  checkBetaAccess: async email => {
+    return await API.get({
+      url: `/api/beta/access?email=${email}`,
+    })
+  },
 })
--- a/packages/server/.gitignore
+++ b/packages/server/.gitignore
@ -2,6 +2,7 @@ node_modules/
 myapps/
 .env
 builder/*
+new_design_ui/*
 client/*
 db/dev.db/
 dist
--- a/packages/server/src/api/controllers/application.ts
+++ b/packages/server/src/api/controllers/application.ts
@ -227,7 +227,11 @@ export const fetchAppPackage = async (ctx: any) => {
    application,
    screens,
    layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath: clientLibraryPath(
+      ctx.params.appId,
+      application.version,
+      ctx
+    ),
  }
 }

--- a/packages/server/src/api/controllers/dev.js
+++ b/packages/server/src/api/controllers/dev.js
@ -3,8 +3,12 @@ const env = require("../../environment")
 const { checkSlashesInUrl } = require("../../utilities")
 const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
-const { Replication, getProdAppID } = require("@budibase/backend-core/db")
-const { DocumentTypes } = require("../../db/utils")
+const {
+  Replication,
+  getProdAppID,
+  dangerousGetDB,
+} = require("@budibase/backend-core/db")
+const { DocumentTypes, getRowParams } = require("../../db/utils")
 const { app: appCache } = require("@budibase/backend-core/cache")
 const { getProdAppDB, getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
@ -133,3 +137,50 @@ exports.getBudibaseVersion = async ctx => {
  }
  await events.installation.versionChecked(version)
 }
+
+// TODO: remove as part of beta program
+exports.checkBetaAccess = async ctx => {
+  // go to the cloud platform if running self hosted
+  if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
+    let baseUrl = ""
+    if (env.ACCOUNT_PORTAL_URL) {
+      baseUrl = env.ACCOUNT_PORTAL_URL.replace("account.", "")
+    } else {
+      baseUrl = "https://budibase.app"
+    }
+
+    const response = await fetch(
+      `${baseUrl}/api/beta/access?email=${ctx.query.email}`
+    )
+    const json = await response.json()
+    ctx.body = json
+    return
+  }
+
+  const userToCheck = ctx.query.email
+  const BETA_USERS_DB = "app_bb_f9b77d06b9db4e3ca185476ab87a2364"
+  const BETA_USERS_TABLE = "ta_8c2c6df1c03f49cfb6340e85e066dd15"
+
+  try {
+    const db = dangerousGetDB(BETA_USERS_DB)
+    const betaUsers = (
+      await db.allDocs(
+        getRowParams(BETA_USERS_TABLE, null, {
+          include_docs: true,
+        })
+      )
+    ).rows.map(row => row.doc)
+
+    let access = false
+    for (let betaUser of betaUsers) {
+      if (betaUser["Email address"].trim() === userToCheck) {
+        access = true
+        break
+      }
+    }
+    ctx.body = { access }
+  } catch (err) {
+    console.error(err)
+    ctx.body = { access: false }
+  }
+}
--- a/packages/server/src/api/controllers/static/index.js
+++ b/packages/server/src/api/controllers/static/index.js
@ -16,9 +16,15 @@ const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes, isDevAppID } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
 const { events } = require("@budibase/backend-core")

+const fs = require("fs")
+const {
+  downloadTarballDirect,
+} = require("../../../utilities/fileSystem/utilities")
+
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
  const response = await upload({
    bucket,
@ -38,8 +44,41 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
  }
 }

+exports.toggleBetaUiFeature = async function (ctx) {
+  const cookieName = `beta:${ctx.params.feature}`
+
+  if (ctx.cookies.get(cookieName)) {
+    clearCookie(ctx, cookieName)
+    ctx.body = {
+      message: `${ctx.params.feature} disabled`,
+    }
+    return
+  }
+
+  let builderPath = resolve(TOP_LEVEL_PATH, "new_design_ui")
+
+  // // download it from S3
+  if (!fs.existsSync(builderPath)) {
+    fs.mkdirSync(builderPath)
+  }
+  await downloadTarballDirect(
+    "https://cdn.budi.live/beta:design_ui/new_ui.tar.gz",
+    builderPath
+  )
+  setCookie(ctx, {}, cookieName)
+
+  ctx.body = {
+    message: `${ctx.params.feature} enabled`,
+  }
+}
+
 exports.serveBuilder = async function (ctx) {
-  let builderPath = resolve(TOP_LEVEL_PATH, "builder")
+  // Temporary: New Design UI
+  const designUiCookie = ctx.cookies.get("beta:design_ui")
+  // TODO: get this from the tmp Dir that we downloaded from MinIO
+  const uiPath = designUiCookie ? "new_design_ui" : "builder"
+
+  let builderPath = resolve(TOP_LEVEL_PATH, uiPath)
  await send(ctx, ctx.file, { root: builderPath })
  if (!ctx.file.includes("assets/")) {
    await events.serve.servedBuilder()
@ -78,7 +117,7 @@ exports.serveApp = async function (ctx) {
      title: appInfo.name,
      production: env.isProd(),
      appId,
-      clientLibPath: clientLibraryPath(appId, appInfo.version),
+      clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
    })

    const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)
--- a/packages/server/src/api/routes/dev.js
+++ b/packages/server/src/api/routes/dev.js
@ -22,5 +22,6 @@ router
  .get("/api/dev/version", authorized(BUILDER), controller.getBudibaseVersion)
  .delete("/api/dev/:appId/lock", authorized(BUILDER), controller.clearLock)
  .post("/api/dev/:appId/revert", authorized(BUILDER), controller.revert)
+  .get("/api/beta/access", controller.checkBetaAccess)

 module.exports = router
--- a/packages/server/src/api/routes/static.ts
+++ b/packages/server/src/api/routes/static.ts
@ -38,6 +38,7 @@ router
  // TODO: for now this builder endpoint is not authorized/secured, will need to be
  .get("/builder/:file*", controller.serveBuilder)
  .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
+  .post("/api/beta/:feature", controller.toggleBetaUiFeature)
  .post(
    "/api/attachments/:tableId/upload",
    paramResource("tableId"),
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@ -67,7 +67,9 @@ module.exports = {
  SALT_ROUNDS: process.env.SALT_ROUNDS,
  LOGGER: process.env.LOGGER,
  LOG_LEVEL: process.env.LOG_LEVEL,
-  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
+  ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
+  AUTOMATION_MAX_ITERATIONS:
+    parseIntSafe(process.env.AUTOMATION_MAX_ITERATIONS) || 200,
  SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
  DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
  QUERY_THREAD_TIMEOUT: parseIntSafe(process.env.QUERY_THREAD_TIMEOUT),
--- a/packages/server/src/utilities/fileSystem/utilities.js
+++ b/packages/server/src/utilities/fileSystem/utilities.js
@ -9,6 +9,7 @@ const {
  deleteFolder,
  uploadDirectory,
  downloadTarball,
+  downloadTarballDirect,
 } = require("@budibase/backend-core/objectStore")

 /***********************************
@ -29,4 +30,5 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.downloadTarballDirect = downloadTarballDirect
 exports.deleteFiles = deleteFiles
--- a/packages/server/src/utilities/index.js
+++ b/packages/server/src/utilities/index.js
@ -54,11 +54,17 @@ exports.objectStoreUrl = () => {
 * @return {string} The URL to be inserted into appPackage response or server rendered
 * app index file.
 */
-exports.clientLibraryPath = (appId, version) => {
+exports.clientLibraryPath = (appId, version, ctx) => {
  if (env.isProd()) {
+    // TODO: remove - for beta testing UI
+    if (ctx && ctx.cookies.get("beta:design_ui")) {
+      return "https://cdn.budi.live/beta:design_ui/budibase-client.js"
+    }
+
    let url = `${exports.objectStoreUrl()}/${sanitizeKey(
      appId
    )}/budibase-client.js`
+
    // append app version to bust the cache
    if (version) {
      url += `?v=${version}`
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@ -1028,10 +1028,10 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212.tgz#7ce8bfa8c968f4aa7558bd2dd13ad3e1e3e40c8f"
-  integrity sha512-pCrAHr54d2onSbaUoCWP83LMJnm28PNIpBAmAhi2kNdSfaGTFY/Iw1sbGOE3G/9vNaB+RRXeibKEEPFjToOgAg==
+"@budibase/backend-core@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.213.tgz#da10d014b5e39457413a9b7f6ead54322d482855"
+  integrity sha512-ARqPhrev/da9WNXVIYSXN5M+cYLKSBYL7pvVVcwMXewp6KCR0gdUBHxuksnrmTbxqT43h7Uc/Zg1H/jYc1xQQQ==
  dependencies:
    "@techpass/passport-openidconnect" "0.3.2"
    aws-sdk "2.1030.0"
@ -1109,12 +1109,12 @@
    svelte-flatpickr "^3.2.3"
    svelte-portal "^1.0.0"

-"@budibase/pro@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212.tgz#d6d2528b9ec2ec47e79d3360d04feeb5d2900d1e"
-  integrity sha512-RcbDmz3pkReUHXAJDPzvgTYy0CBksw55XLbW0wNtDu2HVWP0ZXoiMMJjOxNDwMfL3q4eZitWgISa7QUDisDtMA==
+"@budibase/pro@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.213.tgz#80e6005bec51927d373d278dd8d8672c2f25a4d5"
+  integrity sha512-zhTMPZBv0IkQsdKz1ywnWaxmt/PMrw/EkW1dS8bIOAqHgFTUgawiMGrqrzH43Iw3JemMK7AvtI1EOhs+zrMWVg==
  dependencies:
-    "@budibase/backend-core" "1.0.212"
+    "@budibase/backend-core" "1.0.213"
    node-fetch "^2.6.1"

 "@budibase/standard-components@^0.9.139":
--- a/packages/worker/.dockerignore
+++ b/packages/worker/.dockerignore
@ -0,0 +1,7 @@
+node_modules
+npm-debug.log
+Dockerfile
+.dockerignore
+.git
+.gitignore
+
--- a/packages/worker/yarn.lock
+++ b/packages/worker/yarn.lock
@ -291,10 +291,10 @@
  resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
  integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==

-"@budibase/backend-core@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212.tgz#7ce8bfa8c968f4aa7558bd2dd13ad3e1e3e40c8f"
-  integrity sha512-pCrAHr54d2onSbaUoCWP83LMJnm28PNIpBAmAhi2kNdSfaGTFY/Iw1sbGOE3G/9vNaB+RRXeibKEEPFjToOgAg==
+"@budibase/backend-core@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.213.tgz#da10d014b5e39457413a9b7f6ead54322d482855"
+  integrity sha512-ARqPhrev/da9WNXVIYSXN5M+cYLKSBYL7pvVVcwMXewp6KCR0gdUBHxuksnrmTbxqT43h7Uc/Zg1H/jYc1xQQQ==
  dependencies:
    "@techpass/passport-openidconnect" "0.3.2"
    aws-sdk "2.1030.0"
@ -322,12 +322,12 @@
    uuid "8.3.2"
    zlib "1.0.5"

-"@budibase/pro@1.0.212":
-  version "1.0.212"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212.tgz#d6d2528b9ec2ec47e79d3360d04feeb5d2900d1e"
-  integrity sha512-RcbDmz3pkReUHXAJDPzvgTYy0CBksw55XLbW0wNtDu2HVWP0ZXoiMMJjOxNDwMfL3q4eZitWgISa7QUDisDtMA==
+"@budibase/pro@1.0.213":
+  version "1.0.213"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.213.tgz#80e6005bec51927d373d278dd8d8672c2f25a4d5"
+  integrity sha512-zhTMPZBv0IkQsdKz1ywnWaxmt/PMrw/EkW1dS8bIOAqHgFTUgawiMGrqrzH43Iw3JemMK7AvtI1EOhs+zrMWVg==
  dependencies:
-    "@budibase/backend-core" "1.0.212"
+    "@budibase/backend-core" "1.0.213"
    node-fetch "^2.6.1"

 "@cspotcode/source-map-consumer@0.8.0":