Merge pull request #1429 from Budibase/lab-day/scripting-block

Lab day/scripting block
2024-07-05 22:40:39 +12:00 · 2021-05-04 12:06:00 +01:00 · 2021-05-04 12:06:00 +01:00 · f406fb8ed3
parent ae94ee8f0e 4086d39535
commit f406fb8ed3
26 changed files with 1000 additions and 121 deletions
--- a/packages/bbui/src/Drawer/Drawer.svelte
+++ b/packages/bbui/src/Drawer/Drawer.svelte
@ -38,7 +38,9 @@
      <header>
        <div class="text">
          <Heading size="XS">{title}</Heading>
-          <Body size="XXS"><slot name="description" /></Body>
+          <Body size="XXS">
+            <slot name="description" />
+          </Body>
        </div>
        <div class="buttons">
          <slot name="buttons" />
--- a/packages/bbui/src/Form/Core/TextArea.svelte
+++ b/packages/bbui/src/Form/Core/TextArea.svelte
@ -29,7 +29,8 @@
 >
  {#if error}
    <svg
-      class="spectrum-Icon spectrum-Icon--sizeM spectrum-Textfield-validationIcon"
+      class="spectrum-Icon spectrum-Icon--sizeM
+      spectrum-Textfield-validationIcon"
      focusable="false"
      aria-hidden="true"
    >
--- a/packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte
+++ b/packages/builder/src/components/automation/AutomationPanel/EditAutomationPopover.svelte
@ -24,9 +24,9 @@
  <div slot="control" class="icon">
    <Icon s hoverable name="MoreSmallList" />
  </div>
-  <MenuItem noClose icon="Delete" on:click={confirmDeleteDialog.show}
-    >Delete</MenuItem
-  >
+  <MenuItem noClose icon="Delete" on:click={confirmDeleteDialog.show}>
+    Delete
+  </MenuItem>
 </ActionMenu>

 <ConfirmDialog
--- a/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte
@ -1,12 +1,16 @@
 <script>
  import TableSelector from "./TableSelector.svelte"
  import RowSelector from "./RowSelector.svelte"
+  import QuerySelector from "./QuerySelector.svelte"
  import SchemaSetup from "./SchemaSetup.svelte"
+  import QueryParamSelector from "./QueryParamSelector.svelte"
  import { Button, Input, Select, Label } from "@budibase/bbui"
  import { automationStore } from "builderStore"
  import WebhookDisplay from "../Shared/WebhookDisplay.svelte"
  import DrawerBindableInput from "../../common/DrawerBindableInput.svelte"
  import AutomationBindingPanel from "./AutomationBindingPanel.svelte"
+  import Editor from "components/integration/QueryEditor.svelte"
+  import CodeEditorModal from "./CodeEditorModal.svelte"

  export let block
  export let webhookModal
--- a/packages/builder/src/components/automation/SetupPanel/CodeEditorModal.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/CodeEditorModal.svelte
@ -0,0 +1,35 @@
+<script>
+  import { Button, Modal, ModalContent } from "@budibase/bbui"
+
+  let modal
+
+  export const show = () => {
+    modal.show()
+  }
+  export const hide = () => {
+    modal.hide()
+  }
+</script>
+
+<Modal bind:this={modal} width="60%">
+  <ModalContent
+    title="Edit Code"
+    showConfirmButton={false}
+    showCancelButton={false}
+  >
+    <div class="container">
+      <slot />
+    </div>
+  </ModalContent>
+</Modal>
+<Button primary on:click={show}>Edit Code</Button>
+
+<style>
+  .container :global(section > header) {
+    /* Fix margin defined in BBUI as L rather than XL */
+    margin-bottom: var(--spacing-xl);
+  }
+  .container :global(textarea) {
+    min-height: 60px;
+  }
+</style>
--- a/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/QueryParamSelector.svelte
@ -0,0 +1,55 @@
+<script>
+  import { queries } from "stores/backend"
+  import { Select } from "@budibase/bbui"
+  import DrawerBindableInput from "../../common/DrawerBindableInput.svelte"
+  import AutomationBindingPanel from "./AutomationBindingPanel.svelte"
+
+  export let value
+  export let bindings
+
+  $: query = $queries.list.find(query => query._id === value?.queryId)
+  $: parameters = query?.parameters ?? []
+
+  // Ensure any nullish queryId values get set to empty string so
+  // that the select works
+  $: if (value?.queryId == null) value = { queryId: "" }
+  $: console.log("daValuz", value)
+</script>
+
+<div class="block-field">
+  <Select bind:value={value.queryId} extraThin secondary>
+    <option value="">Choose an option</option>
+    {#each $queries.list as query}
+      <option value={query._id}>{query.name}</option>
+    {/each}
+  </Select>
+</div>
+
+{#if parameters.length}
+  <div class="schema-fields">
+    {#each parameters as field}
+      <DrawerBindableInput
+        panel={AutomationBindingPanel}
+        extraThin
+        value={value[field.name]}
+        on:change={e => {
+          value[field.name] = e.detail
+        }}
+        label={field.name}
+        type="string"
+        {bindings}
+      />
+    {/each}
+  </div>
+{/if}
+
+<style>
+  .schema-fields {
+    display: grid;
+    grid-gap: var(--spacing-xl);
+    margin-top: var(--spacing-xl);
+  }
+  .schema-fields :global(label) {
+    text-transform: capitalize;
+  }
+</style>
--- a/packages/builder/src/components/automation/SetupPanel/QuerySelector.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/QuerySelector.svelte
@ -0,0 +1,15 @@
+<script>
+  import { queries } from "stores/backend"
+  import { Select } from "@budibase/bbui"
+
+  export let value
+</script>
+
+<div class="block-field">
+  <Select bind:value secondary extraThin>
+    <option value="">Choose an option</option>
+    {#each $queries.list as query}
+      <option value={query._id}>{query.name}</option>
+    {/each}
+  </Select>
+</div>
--- a/packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte
+++ b/packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte
@ -54,7 +54,9 @@
 </script>

 <div class="root">
-  <div class="add-field"><i class="ri-add-line" on:click={addField} /></div>
+  <div class="add-field">
+    <i class="ri-add-line" on:click={addField} />
+  </div>
  <div class="spacer" />
  {#each fieldsArray as field}
    <div class="field">
--- a/packages/builder/src/components/common/NavItem.svelte
+++ b/packages/builder/src/components/common/NavItem.svelte
@ -35,7 +35,9 @@

    <slot name="icon" />
    {#if icon}
-      <div class="icon"><Icon size="S" name={icon} /></div>
+      <div class="icon">
+        <Icon size="S" name={icon} />
+      </div>
    {/if}
    <div class="text">{text}</div>
    {#if withActions}
--- a/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte
+++ b/packages/builder/src/components/design/NavigationPanel/ComponentDropdownMenu.svelte
@ -71,36 +71,42 @@
    <Icon size="S" hoverable name="MoreSmallList" />
  </div>
  <MenuItem icon="Delete" on:click={confirmDeleteDialog.show}>Delete</MenuItem>
-  <MenuItem noClose icon="ChevronUp" on:click={moveUpComponent}
-    >Move up</MenuItem
-  >
-  <MenuItem noClose icon="ChevronDown" on:click={moveDownComponent}
-    >Move down</MenuItem
-  >
-  <MenuItem noClose icon="Duplicate" on:click={duplicateComponent}
-    >Duplicate</MenuItem
-  >
-  <MenuItem icon="Cut" on:click={() => storeComponentForCopy(true)}
-    >Cut</MenuItem
-  >
-  <MenuItem icon="Copy" on:click={() => storeComponentForCopy(false)}
-    >Copy</MenuItem
-  >
+  <MenuItem noClose icon="ChevronUp" on:click={moveUpComponent}>
+    Move up
+  </MenuItem>
+  <MenuItem noClose icon="ChevronDown" on:click={moveDownComponent}>
+    Move down
+  </MenuItem>
+  <MenuItem noClose icon="Duplicate" on:click={duplicateComponent}>
+    Duplicate
+  </MenuItem>
+  <MenuItem icon="Cut" on:click={() => storeComponentForCopy(true)}>
+    Cut
+  </MenuItem>
+  <MenuItem icon="Copy" on:click={() => storeComponentForCopy(false)}>
+    Copy
+  </MenuItem>
  <MenuItem
    icon="LayersBringToFront"
    on:click={() => pasteComponent("above")}
-    disabled={noPaste}>Paste above</MenuItem
+    disabled={noPaste}
  >
+    Paste above
+  </MenuItem>
  <MenuItem
    icon="LayersSendToBack"
    on:click={() => pasteComponent("below")}
-    disabled={noPaste}>Paste below</MenuItem
+    disabled={noPaste}
  >
+    Paste below
+  </MenuItem>
  <MenuItem
    icon="ShowOneLayer"
    on:click={() => pasteComponent("inside")}
-    disabled={noPaste || noChildrenAllowed}>Paste inside</MenuItem
+    disabled={noPaste || noChildrenAllowed}
  >
+    Paste inside
+  </MenuItem>
 </ActionMenu>
 <ConfirmDialog
  bind:this={confirmDeleteDialog}
--- a/packages/builder/src/components/integration/KeyValueBuilder.svelte
+++ b/packages/builder/src/components/integration/KeyValueBuilder.svelte
@ -33,7 +33,9 @@
  {/each}
 </div>
 {#if !readOnly}
-  <div><Button secondary thin outline on:click={addEntry}>Add</Button></div>
+  <div>
+    <Button secondary thin outline on:click={addEntry}>Add</Button>
+  </div>
 {/if}

 <style>
--- a/packages/builder/src/components/integration/QueryEditor.svelte
+++ b/packages/builder/src/components/integration/QueryEditor.svelte
@ -132,6 +132,10 @@

    if (destroyed) return

+    CodeMirror.commands.autocomplete = function (cm) {
+      CodeMirror.showHint(cm, CodeMirror.hint.javascript)
+    }
+
    editor = CodeMirror.fromTextArea(refs.editor, opts)

    editor.on("change", instance => {
--- a/packages/builder/src/pages/builder/[application]/data/datasource/[selectedDatasource]/index.svelte
+++ b/packages/builder/src/pages/builder/[application]/data/datasource/[selectedDatasource]/index.svelte
@ -57,9 +57,9 @@
          <Heading size="S">Configuration</Heading>
          <Button secondary on:click={saveDatasource}>Save</Button>
        </div>
-        <Body size="S"
-          >Connect your database to Budibase using the config below.</Body
-        >
+        <Body size="S">
+          Connect your database to Budibase using the config below.
+        </Body>
        <IntegrationConfigForm
          schema={integration.datasource}
          integration={datasource.config}
--- a/packages/builder/yarn.lock
+++ b/packages/builder/yarn.lock
@ -5826,10 +5826,10 @@ svelte-portal@0.1.0:
  resolved "https://registry.yarnpkg.com/svelte-portal/-/svelte-portal-0.1.0.tgz#cc2821cc84b05ed5814e0218dcdfcbebc53c1742"
  integrity sha512-kef+ksXVKun224mRxat+DdO4C+cGHla+fEcZfnBAvoZocwiaceOfhf5azHYOPXSSB1igWVFTEOF3CDENPnuWxg==

-svelte@^3.36.0:
-  version "3.37.0"
-  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.37.0.tgz#dc7cd24bcc275cdb3f8c684ada89e50489144ccd"
-  integrity sha512-TRF30F4W4+d+Jr2KzUUL1j8Mrpns/WM/WacxYlo5MMb2E5Qy2Pk1Guj6GylxsW9OnKQl1tnF8q3hG/hQ3h6VUA==
+svelte@^3.37.0:
+  version "3.38.2"
+  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.38.2.tgz#55e5c681f793ae349b5cc2fe58e5782af4275ef5"
+  integrity sha512-q5Dq0/QHh4BLJyEVWGe7Cej5NWs040LWjMbicBGZ+3qpFWJ1YObRmUDZKbbovddLC9WW7THTj3kYbTOFmU9fbg==

 symbol-observable@^1.1.0:
  version "1.2.0"
--- a/packages/server/package.json
+++ b/packages/server/package.json
@ -90,6 +90,8 @@
    "arangojs": "7.2.0",
    "aws-sdk": "^2.767.0",
    "bcryptjs": "2.4.3",
+    "bull": "^3.22.0",
+    "bull-board": "^1.5.1",
    "chmodr": "1.2.0",
    "csvtojson": "2.0.10",
    "dotenv": "8.2.0",
@ -106,6 +108,7 @@
    "koa": "2.7.0",
    "koa-body": "4.2.0",
    "koa-compress": "4.0.1",
+    "koa-connect": "^2.1.0",
    "koa-pino-logger": "3.0.0",
    "koa-send": "5.0.0",
    "koa-session": "5.12.0",
--- a/packages/server/scripts/dev/manage.js
+++ b/packages/server/scripts/dev/manage.js
@ -37,6 +37,7 @@ async function init() {
    PORT: 4001,
    MINIO_URL: "http://localhost:10000/",
    COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
+    REDIS_URL: "http://localhost:10000/cache/",
    WORKER_URL: "http://localhost:4002",
    JWT_SECRET: "testsecret",
    MINIO_ACCESS_KEY: "budibase",
--- a/packages/server/src/api/controllers/script.js
+++ b/packages/server/src/api/controllers/script.js
@ -0,0 +1,22 @@
+const fetch = require("node-fetch")
+const vm = require("vm")
+
+class ScriptExecutor {
+  constructor(body) {
+    this.script = new vm.Script(body.script)
+    this.context = vm.createContext(body.context)
+    this.context.fetch = fetch
+  }
+
+  execute() {
+    const returnValue = this.script.runInContext(this.context)
+    return returnValue
+  }
+}
+
+exports.execute = async function (ctx) {
+  const executor = new ScriptExecutor(ctx.request.body)
+
+  const result = executor.execute()
+  ctx.body = result
+}
--- a/packages/server/src/api/index.js
+++ b/packages/server/src/api/index.js
@ -5,6 +5,10 @@ const compress = require("koa-compress")
 const zlib = require("zlib")
 const { mainRoutes, staticRoutes } = require("./routes")
 const pkg = require("../../package.json")
+const bullboard = require("bull-board")
+const expressApp = require("express")()
+
+expressApp.use("/bulladmin", bullboard.router)

 const router = new Router()
 const env = require("../environment")
--- a/packages/server/src/api/routes/script.js
+++ b/packages/server/src/api/routes/script.js
@ -0,0 +1,10 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/hosting")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("../../utilities/security/permissions")
+
+const router = Router()
+
+router.post("/api/script", authorized(BUILDER), controller.save)
+
+module.exports = router
--- a/packages/server/src/automations/actions.js
+++ b/packages/server/src/automations/actions.js
@ -3,6 +3,8 @@ const createRow = require("./steps/createRow")
 const updateRow = require("./steps/updateRow")
 const deleteRow = require("./steps/deleteRow")
 const createUser = require("./steps/createUser")
+const executeScript = require("./steps/executeScript")
+const executeQuery = require("./steps/executeQuery")
 const outgoingWebhook = require("./steps/outgoingWebhook")
 const env = require("../environment")
 const Sentry = require("@sentry/node")
@ -18,6 +20,8 @@ const BUILTIN_ACTIONS = {
  DELETE_ROW: deleteRow.run,
  CREATE_USER: createUser.run,
  OUTGOING_WEBHOOK: outgoingWebhook.run,
+  EXECUTE_SCRIPT: executeScript.run,
+  EXECUTE_QUERY: executeQuery.run,
 }
 const BUILTIN_DEFINITIONS = {
  SEND_EMAIL: sendEmail.definition,
@ -26,6 +30,8 @@ const BUILTIN_DEFINITIONS = {
  DELETE_ROW: deleteRow.definition,
  CREATE_USER: createUser.definition,
  OUTGOING_WEBHOOK: outgoingWebhook.definition,
+  EXECUTE_SCRIPT: executeScript.definition,
+  EXECUTE_QUERY: executeQuery.definition,
 }

 let MANIFEST = null
--- a/packages/server/src/automations/steps/executeQuery.js
+++ b/packages/server/src/automations/steps/executeQuery.js
@ -0,0 +1,84 @@
+const queryController = require("../../api/controllers/query")
+
+module.exports.definition = {
+  name: "External Data Connector",
+  tagline: "Execute Data Connector",
+  icon: "ri-database-2-line",
+  description: "Execute a query in an external data connector",
+  type: "ACTION",
+  stepId: "EXECUTE_QUERY",
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        query: {
+          type: "object",
+          properties: {
+            queryId: {
+              type: "string",
+              customType: "query",
+            },
+          },
+          customType: "queryParams",
+          title: "Parameters",
+          required: ["queryId"],
+        },
+      },
+      required: ["query"],
+    },
+    outputs: {
+      properties: {
+        response: {
+          type: "object",
+          description: "The response from the datasource execution",
+        },
+        success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+      },
+    },
+    required: ["response", "success"],
+  },
+}
+
+module.exports.run = async function ({ inputs, appId, emitter }) {
+  if (inputs.query == null) {
+    return {
+      success: false,
+      response: {
+        message: "Invalid inputs",
+      },
+    }
+  }
+
+  const { queryId, ...rest } = inputs.query
+
+  const ctx = {
+    params: {
+      queryId,
+    },
+    request: {
+      body: {
+        parameters: rest,
+      },
+    },
+    appId,
+    eventEmitter: emitter,
+  }
+
+  await queryController.execute(ctx)
+
+  try {
+    return {
+      response: ctx.body,
+      success: ctx.status === 200,
+    }
+  } catch (err) {
+    console.error(err)
+    return {
+      success: false,
+      response: err,
+    }
+  }
+}
--- a/packages/server/src/automations/steps/executeScript.js
+++ b/packages/server/src/automations/steps/executeScript.js
@ -0,0 +1,73 @@
+const scriptController = require("../../api/controllers/script")
+
+module.exports.definition = {
+  name: "Scripting",
+  tagline: "Execute JavaScript Code",
+  icon: "ri-terminal-box-line",
+  description: "Run a piece of JavaScript code in your automation",
+  type: "ACTION",
+  stepId: "EXECUTE_SCRIPT",
+  inputs: {},
+  schema: {
+    inputs: {
+      properties: {
+        code: {
+          type: "string",
+          customType: "code",
+          title: "Code",
+        },
+      },
+      required: ["code"],
+    },
+    outputs: {
+      properties: {
+        value: {
+          type: "string",
+          description:
+            "The result of the last statement of the executed script.",
+        },
+        success: {
+          type: "boolean",
+          description: "Whether the action was successful",
+        },
+      },
+    },
+    required: ["success"],
+  },
+}
+
+module.exports.run = async function ({ inputs, appId, context, emitter }) {
+  if (inputs.code == null) {
+    return {
+      success: false,
+      response: {
+        message: "Invalid inputs",
+      },
+    }
+  }
+
+  const ctx = {
+    request: {
+      body: {
+        script: inputs.code,
+        context,
+      },
+    },
+    user: { appId },
+    eventEmitter: emitter,
+  }
+
+  try {
+    await scriptController.execute(ctx)
+    return {
+      success: ctx.status === 200,
+      value: ctx.body,
+    }
+  } catch (err) {
+    console.error(err)
+    return {
+      success: false,
+      response: err,
+    }
+  }
+}
--- a/packages/server/src/automations/thread.js
+++ b/packages/server/src/automations/thread.js
@ -56,6 +56,7 @@ class Orchestrator {
          appId: this._appId,
          apiKey: automation.apiKey,
          emitter: this._emitter,
+          context: this._context,
        })
        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
          break
--- a/packages/server/src/automations/triggers.js
+++ b/packages/server/src/automations/triggers.js
@ -1,10 +1,14 @@
 const CouchDB = require("../db")
 const emitter = require("../events/index")
-const InMemoryQueue = require("../utilities/queue/inMemoryQueue")
+const Queue = require("bull")
+const { setQueues, BullAdapter } = require("bull-board")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")

-let automationQueue = new InMemoryQueue("automationQueue")
+let automationQueue = new Queue("automationQueue")
+
+// Set up queues for bull board admin
+setQueues([new BullAdapter(automationQueue)])

 const FAKE_STRING = "TEST"
 const FAKE_BOOL = false
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
--- a/packages/standard-components/yarn.lock
+++ b/packages/standard-components/yarn.lock
@ -226,10 +226,10 @@ svelte-hmr@^0.13.3:
  resolved "https://registry.yarnpkg.com/svelte-hmr/-/svelte-hmr-0.13.3.tgz#fba5739b477ea44caf70e542a24a4352bee2b897"
  integrity sha512-gagW62pLQ2lULmvNA3pIZu9pBCYOaGu3rQikUOv6Nokz5VxUgT9/mQLfMxj9phDEKHCg/lgr3i6PkqZDbO9P2Q==

-svelte@^3.35.0:
-  version "3.37.0"
-  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.37.0.tgz#dc7cd24bcc275cdb3f8c684ada89e50489144ccd"
-  integrity sha512-TRF30F4W4+d+Jr2KzUUL1j8Mrpns/WM/WacxYlo5MMb2E5Qy2Pk1Guj6GylxsW9OnKQl1tnF8q3hG/hQ3h6VUA==
+svelte@^3.37.0:
+  version "3.38.2"
+  resolved "https://registry.yarnpkg.com/svelte/-/svelte-3.38.2.tgz#55e5c681f793ae349b5cc2fe58e5782af4275ef5"
+  integrity sha512-q5Dq0/QHh4BLJyEVWGe7Cej5NWs040LWjMbicBGZ+3qpFWJ1YObRmUDZKbbovddLC9WW7THTj3kYbTOFmU9fbg==

 svg.draggable.js@^2.2.2:
  version "2.2.2"