poc wip

.github/workflows/pro-release-develop.yml

@@ -0,0 +1,47 @@
+name: Budibase Release Staging (Pro)
+
+#  Temporary pipeline - eventualy this will be merged with the regular one
+
+on:
+  push:
+    branches:
+      - pro-develop
+    paths:
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"
+
+env:
+  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
+  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn lint
+      - run: yarn build
+      - run: yarn test
+
+      - name: Build/release Docker images (Pro)
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build:docker:develop:pro
+          yarn release:docker:pro develop
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

.github/workflows/pro-release-selfhost.yml

@@ -0,0 +1,41 @@
+name: Budibase Release Selfhost (Pro)
+
+#  Temporary pipeline - eventualy this will be merged with the regular one
+
+on:
+  workflow_dispatch:
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          node-version: 14.x
+          fetch_depth: 0
+
+      - name: Tag and release Docker images (Pro) (Self Host)
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+
+          # Get latest release version
+          release_version=$(cat lerna.json | jq -r '.version')
+          echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
+          release_tag=v$release_version
+
+          # Pull apps and worker images
+          docker pull budibase/apps-pro:$release_tag
+          docker pull budibase/worker-pro:$release_tag
+
+          # Tag apps and worker images
+          docker tag budibase/apps-pro:$release_tag budibase/apps-pro:$SELFHOST_TAG
+          docker tag budibase/worker-pro:$release_tag budibase/worker-pro:$SELFHOST_TAG
+
+          # Push images
+          docker push budibase/apps-pro:$SELFHOST_TAG
+          docker push budibase/worker-pro:$SELFHOST_TAG
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          SELFHOST_TAG: latest

.github/workflows/pro-release.yml

@@ -0,0 +1,53 @@
+name: Budibase Release (Pro)
+
+#  Temporary pipeline - eventualy this will be merged with the regular one
+
+on:
+  push:
+    branches:
+      - pro-master
+    paths:
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"
+
+env:
+  POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
+  INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
+  POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
+  SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-node@v1
+        with:
+          node-version: 14.x
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn lint
+      - run: yarn build
+      - run: yarn test
+
+      - name: "Get Previous tag"
+        id: previoustag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+
+      - name: Build/release Docker images
+        run: |
+          docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
+          yarn build:docker:pro
+          yarn release:docker:pro ${BUDIBASE_RELEASE_VERSION}
+        env:
+          DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
+          BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}

.github/workflows/release-develop.yml

@@ -1,38 +1,38 @@
 name: Budibase Release Staging
 
-on: 
- push:
+on:
+  push:
     branches:
       - develop
     paths:
-      - '.aws/**'
-      - '.github/**'
-      - 'charts/**'
-      - 'packages/**'
-      - 'scripts/**'
-      - 'package.json'
-      - 'yarn.lock'
-      - 'package.json'
-      - 'yarn.lock'
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"
 
 env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
   INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
   POSTHOG_URL: ${{ secrets.POSTHOG_URL }}
-      
+
 jobs:
   release:
-    runs-on: ubuntu-latest 
+    runs-on: ubuntu-latest
 
     steps:
       - uses: actions/checkout@v2
       - uses: actions/setup-node@v1
         with:
           node-version: 14.x
-      - run: yarn 
-      - run: yarn bootstrap 
-      - run: yarn lint 
-      - run: yarn build 
+      - run: yarn
+      - run: yarn bootstrap
+      - run: yarn lint
+      - run: yarn build
       - run: yarn test
 
       - name: Configure AWS Credentials
@@ -45,7 +45,7 @@ jobs:
       - name: Publish budibase packages to NPM
         env:
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
-        run: | 
+        run: |
           # setup the username and email. I tend to use 'GitHub Actions Bot' with no email by default
           git config user.name "Budibase Staging Release Bot"
           git config user.email "<>"
@@ -53,10 +53,10 @@ jobs:
           yarn release:develop
 
       - name: Build/release Docker images
-        run: | 
+        run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build
           yarn build:docker:develop
+          yarn release:docker develop
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

.github/workflows/release.yml

@@ -1,19 +1,19 @@
 name: Budibase Release
 
 on:
- push:
+  push:
     branches:
       - master
     paths:
-      - '.aws/**'
-      - '.github/**'
-      - 'charts/**'
-      - 'packages/**'
-      - 'scripts/**'
-      - 'package.json'
-      - 'yarn.lock'
-      - 'package.json'
-      - 'yarn.lock'
+      - ".aws/**"
+      - ".github/**"
+      - "charts/**"
+      - "packages/**"
+      - "scripts/**"
+      - "package.json"
+      - "yarn.lock"
+      - "package.json"
+      - "yarn.lock"
 
 env:
   POSTHOG_TOKEN: ${{ secrets.POSTHOG_TOKEN }}
@@ -53,15 +53,15 @@ jobs:
           echo //registry.npmjs.org/:_authToken=${NPM_TOKEN} >> .npmrc
           yarn release
 
-      - name: 'Get Previous tag'
+      - name: "Get Previous tag"
         id: previoustag
         uses: "WyriHaximus/github-action-get-previous-tag@v1"
 
       - name: Build/release Docker images
-        run: | 
+        run: |
           docker login -u $DOCKER_USER -p $DOCKER_PASSWORD
-          yarn build
           yarn build:docker
+          yarn release:docker ${BUDIBASE_RELEASE_VERSION}
         env:
           DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
           DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}

hosting/docker-compose.yaml

@@ -31,6 +31,34 @@ services:
       - worker-service
       - redis-service
 
+  app-service-pro:
+    restart: always
+    image: app-service-pro
+    container_name: bbapps-pro
+    ports:
+      - "${APP_PORT}:4002"
+    environment:
+      SELF_HOSTED: 1
+      COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984
+      WORKER_URL: http://worker-service:4003
+      MINIO_URL: http://minio-service:9000
+      MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY}
+      MINIO_SECRET_KEY: ${MINIO_SECRET_KEY}
+      INTERNAL_API_KEY: ${INTERNAL_API_KEY}
+      BUDIBASE_ENVIRONMENT: ${BUDIBASE_ENVIRONMENT}
+      PORT: 4002
+      JWT_SECRET: ${JWT_SECRET}
+      LOG_LEVEL: info
+      SENTRY_DSN: https://a34ae347621946bf8acded18e5b7d4b8@o420233.ingest.sentry.io/5338131
+      ENABLE_ANALYTICS: "true"
+      REDIS_URL: redis-service:6379
+      REDIS_PASSWORD: ${REDIS_PASSWORD}
+    volumes:
+      - ./logs:/logs
+    depends_on:
+      - worker-service
+      - redis-service
+
   worker-service:
     restart: always
     image: budibase.docker.scarf.sh/budibase/worker

hosting/scripts/linux/docker-release/budibase-pro.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+tag=$1
+
+if [[ ! "$tag" ]]; then
+	echo "No tag present. You must pass a tag to this script"
+	exit 1
+fi
+
+echo "Tagging images with tag: $tag"
+
+docker tag app-service-pro budibase/apps-pro:$tag
+docker tag worker-service-pro budibase/worker-pro:$tag
+
+docker push --all-tags budibase/apps-pro
+docker push --all-tags budibase/worker-pro

hosting/scripts/pro/download.js

@@ -0,0 +1,52 @@
+const fetch = require('node-fetch')
+const fs = require('fs')
+const util = require('util')
+const streamPipeline = util.promisify(require('stream').pipeline)
+
+async function download (url, opts, path) {
+  const response = await fetch(url, opts)
+  if (!response.ok) throw new Error(`unexpected response ${response.statusText}`)
+  await streamPipeline(response.body, fs.createWriteStream(path))
+}
+
+const install = async () => {
+  // get github token
+  const token = process.env.GITHUB_TOKEN
+  if (!token) {
+    throw new Error("Missing token")
+  }
+  
+  // get version from package.json
+  let packageJson = JSON.parse(fs.readFileSync('package.json'))
+  let version = packageJson.dependencies['@budibase/pro']
+  if (version.startsWith('^')) {
+    version = version.substring(1)
+  }
+  const ghVersion = `v${version}`
+  
+  // get the package url
+  // TODO: Add semver support
+  // TODO: Potentially can use yarn to download directly from github if pro becomes a non monorepo
+  const resp = await fetch(`https://api.github.com/repos/budibase/budibase-pro/releases`, { 
+    headers: { 
+      Authorization: `token ${token}`,
+      Accept: "application/vnd.github.v3.raw"  
+    } 
+  })
+
+  if (!resp.ok) {
+    throw new Error("Failed to get release info. Aborting")
+  }
+  const releases = await resp.json()
+  const release = releases.filter(r => r.name === ghVersion)[0]
+  const url = release.assets.filter(a => a.name === "pro.zip")[0].url
+  
+  console.log("Downloading pro package")
+  // download the package
+  await download(url, { headers: { Authorization: `token ${token}`, Accept: 'application/octet-stream' }}, './pro.zip')
+}
+
+install().catch(e => {
+  console.error(e)
+  process.exit(1)
+})

hosting/scripts/pro/install.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ ! -z "${PRO}" ]]; then
+  echo "Installing pro package"
+  rm -rf node_modules/@budibase/pro
+  unzip pro.zip -d node_modules/@budibase
+  rm pro.zip
+fi

package.json

@@ -19,9 +19,9 @@
   },
   "scripts": {
     "setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
+    "install:pro": "lerna run install:pro --",
     "bootstrap": "lerna link && lerna bootstrap",
     "build": "lerna run build",
-    "publishdev": "lerna run publishdev",
     "publishnpm": "yarn build && lerna publish --force-publish",
     "release": "lerna publish patch --yes --force-publish",
     "release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop",
@@ -30,10 +30,12 @@
     "nuke:packages": "yarn run restore",
     "nuke:docker": "lerna run --parallel dev:stack:nuke",
     "clean": "lerna clean",
-    "kill-port": "kill-port 4001",
-    "dev": "yarn run kill-port && lerna link && lerna run --parallel dev:builder --concurrency 1",
-    "dev:noserver": "lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
-    "dev:server": "lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
+    "kill-builder": "kill-port 3000",
+    "kill-server": "kill-port 4001 4002",
+    "kill-all": "yarn run kill-builder && yarn run kill-server",
+    "dev": "yarn run kill-all && lerna link && lerna run --parallel dev:builder --concurrency 1",
+    "dev:noserver": "yarn run kill-builder && lerna link && lerna run dev:stack:up && lerna run --parallel dev:builder --concurrency 1 --ignore @budibase/server --ignore @budibase/worker",
+    "dev:server": "yarn run kill-server && lerna run --parallel dev:builder --concurrency 1 --scope @budibase/worker --scope @budibase/server",
     "test": "lerna run test",
     "lint:eslint": "eslint packages",
     "lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\"",

packages/builder/src/components/common/bindings/BindingPanel.svelte

@@ -146,7 +146,7 @@
                   <div class="helper__description">
                     {@html helper.description}
                   </div>
-                  <pre class="helper__example">{helper.example || ''}</pre>
+                  <pre class="helper__example">{helper.example || ""}</pre>
                 </div>
               </li>
             {/each}

packages/server/.gitignore

@@ -7,4 +7,6 @@ public/
 db/dev.db/
 dist
 coverage/
-watchtower-hook.json
+watchtower-hook.json
+pro.zip
+scripts/pro/*

packages/server/Dockerfile

@@ -16,10 +16,19 @@ ENV BUDIBASE_ENVIRONMENT=PRODUCTION
 # copy files and install dependencies
 COPY . ./
 RUN yarn
+
+RUN apt-get install unzip
+
+# add the pro package if PRO is set
+ARG PRO
+ENV PRO=$PRO
+RUN yarn install:pro
+
+# build
 RUN yarn build
 
 # Install client for oracle datasource
-RUN apt-get install unzip libaio1
+RUN apt-get install libaio1
 RUN /bin/bash -e scripts/integrations/oracle/instantclient/linux/x86-64/install.sh 
 
 EXPOSE 4001

packages/server/package.json

@@ -13,8 +13,12 @@
     "postbuild": "copyfiles -u 1 src/**/*.svelte dist/ && copyfiles -u 1 src/**/*.hbs dist/ && copyfiles -u 1 src/**/*.json dist/",
     "test": "jest --coverage --maxWorkers=2",
     "test:watch": "jest --watch",
-    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client",
+    "predocker": "copyfiles -f ../client/dist/budibase-client.js ../client/manifest.json client && copyfiles -f ../../hosting/scripts/pro/install.sh scripts/pro",
     "build:docker": "yarn run predocker && docker build . -t app-service --label version=$BUDIBASE_RELEASE_VERSION",
+    "download:pro": "node ../../hosting/scripts/pro/download.js",
+    "predocker:pro": "yarn run predocker && yarn download:pro",
+    "build:docker:pro": "yarn run predocker:pro && docker build --build-arg PRO=1 . -t app-service-pro --label version=$BUDIBASE_RELEASE_VERSION",
+    "install:pro": "./scripts/pro/install.sh",
     "build:docs": "node ./scripts/docs/generate.js open",
     "run:docker": "node dist/index.js",
     "dev:stack:up": "node scripts/dev/manage.js up",
@@ -73,6 +77,7 @@
     "@budibase/backend-core": "^1.0.65",
     "@budibase/client": "^1.0.65",
     "@budibase/string-templates": "^1.0.65",
+    "@budibase/pro": "0.0.2-alpha.27",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",

packages/server/src/app.ts

@@ -2,6 +2,26 @@
 import { ExtendableContext } from "koa"
 
 import * as env from "./environment"
+
+// check we can import pro types no matter the installation method (budibase-pro or budibase)
+import * as pro from "@budibase/pro"
+import { poc, License } from "@budibase/pro"
+
+// check we can use pro types no matter the installation
+const license: License = { id: "123" }
+console.log(license)
+
+// log the difference between how pro is imported
+// in budibase these will be undefined
+// in budibase-pro these will be defined
+console.log(poc)
+console.log(pro)
+
+if (env.PRO) {
+  // check only pro logic runs in pro
+  console.log("PRO")
+}
+
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")

packages/server/src/environment.js

@@ -50,6 +50,7 @@ module.exports = {
   JEST_WORKER_ID: process.env.JEST_WORKER_ID,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
+  PRO: process.env.PRO,
   // minor
   SALT_ROUNDS: process.env.SALT_ROUNDS,
   LOGGER: process.env.LOGGER,