Merge branch 'master' into feature/enable-sqs-in-dev

.github/workflows/pr-labeler.yml

@@ -33,6 +33,6 @@ jobs:
             {
                 "adrinr": "firestorm",
                 "samwho": "firestorm",
-                "pclmnt": "firestorm",
+                "PClmnt": "firestorm",
                 "mike12345567": "firestorm"
             }

lerna.json

@@ -1,5 +1,5 @@
 {
-  "version": "2.28.3",
+  "version": "2.28.4",
   "npmClient": "yarn",
   "packages": [
     "packages/*",

packages/backend-core/src/db/couch/DatabaseImpl.ts

@@ -328,7 +328,14 @@ export class DatabaseImpl implements Database {
   async sqlDiskCleanup(): Promise<void> {
     const dbName = this.name
     const url = `/${dbName}/_cleanup`
-    return await this._sqlQuery<void>(url, "POST")
+    try {
+      await this._sqlQuery<void>(url, "POST")
+    } catch (err: any) {
+      // hack for now - SQS throws a 500 when there is nothing to clean-up
+      if (err.status !== 500) {
+        throw err
+      }
+    }
   }
 
   // removes a document from sqlite
@@ -352,18 +359,15 @@ export class DatabaseImpl implements Database {
   }
 
   async destroy() {
+    if (env.SQS_SEARCH_ENABLE && (await this.exists(SQLITE_DESIGN_DOC_ID))) {
+      // delete the design document, then run the cleanup operation
+      const definition = await this.get<SQLiteDefinition>(SQLITE_DESIGN_DOC_ID)
+      // remove all tables - save the definition then trigger a cleanup
+      definition.sql.tables = {}
+      await this.put(definition)
+      await this.sqlDiskCleanup()
+    }
     try {
-      if (env.SQS_SEARCH_ENABLE) {
-        // delete the design document, then run the cleanup operation
-        try {
-          const definition = await this.get<SQLiteDefinition>(
-            SQLITE_DESIGN_DOC_ID
-          )
-          await this.remove(SQLITE_DESIGN_DOC_ID, definition._rev)
-        } finally {
-          await this.sqlDiskCleanup()
-        }
-      }
       return await this.nano().db.destroy(this.name)
     } catch (err: any) {
       // didn't exist, don't worry

packages/builder/src/components/automation/SetupPanel/AutomationBlockSetup.svelte

@@ -15,6 +15,7 @@
     Checkbox,
     DatePicker,
     DrawerContent,
+    Toggle,
   } from "@budibase/bbui"
   import CreateWebhookModal from "components/automation/Shared/CreateWebhookModal.svelte"
   import { automationStore, selectedAutomation, tables } from "stores/builder"
@@ -118,7 +119,6 @@
         searchableSchema: true,
       }).schema
     }
-
     try {
       if (isTestModal) {
         let newTestData = { schema }
@@ -385,6 +385,16 @@
     return params
   }
 
+  function toggleAttachmentBinding(e, key) {
+    onChange(
+      {
+        detail: "",
+      },
+      key
+    )
+    onChange({ detail: { useAttachmentBinding: e.detail } }, "meta")
+  }
+
   onMount(async () => {
     try {
       await environment.loadVariables()
@@ -462,27 +472,64 @@
               <div class="label-wrapper">
                 <Label>{label}</Label>
               </div>
-              <div class="attachment-field-width">
+              <div class="toggle-container">
-                <KeyValueBuilder
+                <Toggle
-                  on:change={e =>
+                  value={inputData?.meta?.useAttachmentBinding}
-                    onChange(
+                  text={"Use bindings"}
-                      {
+                  size={"XS"}
-                        detail: e.detail.map(({ name, value }) => ({
+                  on:change={e => toggleAttachmentBinding(e, key)}
-                          url: name,
-                          filename: value,
-                        })),
-                      },
-                      key
-                    )}
-                  object={handleAttachmentParams(inputData[key])}
-                  allowJS
-                  {bindings}
-                  keyBindings
-                  customButtonText={"Add attachment"}
-                  keyPlaceholder={"URL"}
-                  valuePlaceholder={"Filename"}
                 />
               </div>
+
+              <div class="attachment-field-width">
+                {#if !inputData?.meta?.useAttachmentBinding}
+                  <KeyValueBuilder
+                    on:change={e =>
+                      onChange(
+                        {
+                          detail: e.detail.map(({ name, value }) => ({
+                            url: name,
+                            filename: value,
+                          })),
+                        },
+                        key
+                      )}
+                    object={handleAttachmentParams(inputData[key])}
+                    allowJS
+                    {bindings}
+                    keyBindings
+                    customButtonText={"Add attachment"}
+                    keyPlaceholder={"URL"}
+                    valuePlaceholder={"Filename"}
+                  />
+                {:else if isTestModal}
+                  <ModalBindableInput
+                    title={value.title || label}
+                    value={inputData[key]}
+                    panel={AutomationBindingPanel}
+                    type={value.customType}
+                    on:change={e => onChange(e, key)}
+                    {bindings}
+                    updateOnChange={false}
+                  />
+                {:else}
+                  <div class="test">
+                    <DrawerBindableInput
+                      title={value.title ?? label}
+                      panel={AutomationBindingPanel}
+                      type={value.customType}
+                      value={inputData[key]}
+                      on:change={e => onChange(e, key)}
+                      {bindings}
+                      updateOnChange={false}
+                      placeholder={value.customType === "queryLimit"
+                        ? queryLimit
+                        : ""}
+                      drawerLeft="260px"
+                    />
+                  </div>
+                {/if}
+              </div>
             </div>
           {:else if value.customType === "filters"}
             <ActionButton on:click={drawer.show}>Define filters</ActionButton>

packages/builder/src/components/automation/SetupPanel/RowSelector.svelte

@@ -10,12 +10,12 @@
   import { TableNames } from "constants"
 
   const dispatch = createEventDispatcher()
-
   export let value
   export let meta
   export let bindings
   export let isTestModal
   export let isUpdateRow
+
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
     clone.icon = "ShareAndroid"
@@ -94,17 +94,22 @@
     dispatch("change", newValue)
   }
 
-  const onChangeSetting = (e, field) => {
+  const onChangeSetting = (field, key, value) => {
-    let fields = {}
+    let newField = {}
-    fields[field] = {
+    newField[field] = {
-      clearRelationships: e.detail,
+      [key]: value,
     }
+
+    let updatedFields = {
+      ...meta?.fields,
+      ...newField,
+    }
+
     dispatch("change", {
       key: "meta",
-      fields,
+      fields: updatedFields,
     })
   }
-
   // Ensure any nullish tableId values get set to empty string so
   // that the select works
   $: if (value?.tableId == null) value = { tableId: "" }
@@ -157,6 +162,9 @@
                 bindings={parsedBindings}
                 {value}
                 {onChange}
+                useAttachmentBinding={meta?.fields?.[field]
+                  ?.useAttachmentBinding}
+                {onChangeSetting}
               />
             </DrawerBindableSlot>
           {/if}
@@ -167,7 +175,8 @@
                 value={meta.fields?.[field]?.clearRelationships}
                 text={"Clear relationships if empty?"}
                 size={"S"}
-                on:change={e => onChangeSetting(e, field)}
+                on:change={e =>
+                  onChangeSetting(field, "clearRelationships", e.detail)}
               />
             </div>
           {/if}

packages/builder/src/components/automation/SetupPanel/RowSelectorTypes.svelte

@@ -1,5 +1,11 @@
 <script>
-  import { Select, DatePicker, Multiselect, TextArea } from "@budibase/bbui"
+  import {
+    Select,
+    DatePicker,
+    Multiselect,
+    TextArea,
+    Toggle,
+  } from "@budibase/bbui"
   import { FieldType } from "@budibase/types"
   import LinkedRowSelector from "components/common/LinkedRowSelector.svelte"
   import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte"
@@ -14,6 +20,8 @@
   export let value
   export let bindings
   export let isTestModal
+  export let useAttachmentBinding
+  export let onChangeSetting
 
   $: parsedBindings = bindings.map(binding => {
     let clone = Object.assign({}, binding)
@@ -27,6 +35,8 @@
     FieldType.SIGNATURE_SINGLE,
   ]
 
+  let previousBindingState = useAttachmentBinding
+
   function schemaHasOptions(schema) {
     return !!schema.constraints?.inclusion?.length
   }
@@ -34,13 +44,6 @@
   function handleAttachmentParams(keyValueObj) {
     let params = {}
 
-    if (
-      (schema.type === FieldType.ATTACHMENT_SINGLE ||
-        schema.type === FieldType.SIGNATURE_SINGLE) &&
-      Object.keys(keyValueObj).length === 0
-    ) {
-      return []
-    }
     if (!Array.isArray(keyValueObj) && keyValueObj) {
       keyValueObj = [keyValueObj]
     }
@@ -52,6 +55,26 @@
     }
     return params
   }
+
+  async function handleToggleChange(toggleField, event) {
+    if (event.detail === true) {
+      value[toggleField] = []
+    } else {
+      value[toggleField] = ""
+    }
+    previousBindingState = event.detail
+    onChangeSetting(toggleField, "useAttachmentBinding", event.detail)
+    onChange({ detail: value[toggleField] }, toggleField)
+  }
+
+  $: if (useAttachmentBinding !== previousBindingState) {
+    if (useAttachmentBinding) {
+      value[field] = []
+    } else {
+      value[field] = ""
+    }
+    previousBindingState = useAttachmentBinding
+  }
 </script>
 
 {#if schemaHasOptions(schema) && schema.type !== "array"}
@@ -108,38 +131,65 @@
     useLabel={false}
   />
 {:else if attachmentTypes.includes(schema.type)}
-  <div class="attachment-field-spacinng">
+  <div class="attachment-field-container">
-    <KeyValueBuilder
+    <div class="toggle-container">
-      on:change={e =>
+      <Toggle
-        onChange(
+        value={useAttachmentBinding}
-          {
+        text={"Use bindings"}
-            detail:
+        size={"XS"}
-              schema.type === FieldType.ATTACHMENT_SINGLE ||
+        on:change={e => handleToggleChange(field, e)}
-              schema.type === FieldType.SIGNATURE_SINGLE
+      />
-                ? e.detail.length > 0
+    </div>
-                  ? {
+    {#if !useAttachmentBinding}
-                      url: e.detail[0].name,
+      <div class="attachment-field-spacing">
-                      filename: e.detail[0].value,
+        <KeyValueBuilder
-                    }
+          on:change={async e => {
-                  : {}
+            onChange(
-                : e.detail.map(({ name, value }) => ({
+              {
-                    url: name,
+                detail:
-                    filename: value,
+                  schema.type === FieldType.ATTACHMENT_SINGLE ||
-                  })),
+                  schema.type === FieldType.SIGNATURE_SINGLE
-          },
+                    ? e.detail.length > 0
-          field
+                      ? {
-        )}
+                          url: e.detail[0].name,
-      object={handleAttachmentParams(value[field])}
+                          filename: e.detail[0].value,
-      allowJS
+                        }
-      {bindings}
+                      : {}
-      keyBindings
+                    : e.detail.map(({ name, value }) => ({
-      customButtonText={"Add attachment"}
+                        url: name,
-      keyPlaceholder={"URL"}
+                        filename: value,
-      valuePlaceholder={"Filename"}
+                      })),
-      actionButtonDisabled={(schema.type === FieldType.ATTACHMENT_SINGLE ||
+              },
-        schema.type === FieldType.SIGNATURE) &&
+              field
-        Object.keys(value[field]).length >= 1}
+            )
-    />
+          }}
+          object={handleAttachmentParams(value[field])}
+          allowJS
+          {bindings}
+          keyBindings
+          customButtonText={"Add attachment"}
+          keyPlaceholder={"URL"}
+          valuePlaceholder={"Filename"}
+          actionButtonDisabled={(schema.type === FieldType.ATTACHMENT_SINGLE ||
+            schema.type === FieldType.SIGNATURE) &&
+            Object.keys(value[field]).length >= 1}
+        />
+      </div>
+    {:else}
+      <div class="json-input-spacing">
+        <svelte:component
+          this={isTestModal ? ModalBindableInput : DrawerBindableInput}
+          panel={AutomationBindingPanel}
+          value={value[field]}
+          on:change={e => onChange(e, field)}
+          type="string"
+          bindings={parsedBindings}
+          allowJS={true}
+          updateOnChange={false}
+          title={schema.name}
+        />
+      </div>
+    {/if}
   </div>
 {:else if ["string", "number", "bigint", "barcodeqr", "array"].includes(schema.type)}
   <svelte:component
@@ -156,7 +206,8 @@
 {/if}
 
 <style>
-  .attachment-field-spacinng {
+  .attachment-field-spacing,
+  .json-input-spacing {
     margin-top: var(--spacing-s);
     margin-bottom: var(--spacing-l);
   }

packages/server/src/api/controllers/row/external.ts

@@ -25,6 +25,7 @@ import {
   outputProcessing,
 } from "../../../utilities/rowProcessor"
 import { cloneDeep } from "lodash"
+import { generateIdForRow } from "./utils"
 
 export async function handleRequest<T extends Operation>(
   operation: T,
@@ -55,11 +56,19 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
     throw { validation: validateResult.errors }
   }
 
+  const beforeRow = await sdk.rows.external.getRow(tableId, _id, {
+    relationships: true,
+  })
+
   const response = await handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(_id),
     row: dataToUpdate,
   })
-  const row = await sdk.rows.external.getRow(tableId, _id, {
+
+  // The id might have been changed, so the refetching would fail. Recalculating the id just in case
+  const updatedId =
+    generateIdForRow({ ...beforeRow, ...dataToUpdate }, table) || _id
+  const row = await sdk.rows.external.getRow(tableId, updatedId, {
     relationships: true,
   })
   const enrichedRow = await outputProcessing(table, row, {

packages/server/src/api/routes/tests/application.spec.ts

@@ -334,6 +334,12 @@ describe("/applications", () => {
       expect(events.app.deleted).toHaveBeenCalledTimes(1)
       expect(events.app.unpublished).toHaveBeenCalledTimes(1)
     })
+
+    it("should be able to delete an app after SQS_SEARCH_ENABLE has been set but app hasn't been migrated", async () => {
+      await config.withCoreEnv({ SQS_SEARCH_ENABLE: "true" }, async () => {
+        await config.api.application.delete(app.appId)
+      })
+    })
   })
 
   describe("POST /api/applications/:appId/duplicate", () => {

packages/server/src/api/routes/tests/row.spec.ts

@@ -38,7 +38,7 @@ describe.each([
   [DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
   [DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
   [DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
-])("/rows (%s)", (__, dsProvider) => {
+])("/rows (%s)", (providerType, dsProvider) => {
   const isInternal = dsProvider === undefined
   const config = setup.getConfig()
 
@@ -693,6 +693,49 @@ describe.each([
       })
       expect(resp.relationship.length).toBe(1)
     })
+
+    !isInternal &&
+      // TODO: SQL is having issues creating composite keys
+      providerType !== DatabaseName.SQL_SERVER &&
+      it("should support updating fields that are part of a composite key", async () => {
+        const tableRequest = saveTableRequest({
+          primary: ["number", "string"],
+          schema: {
+            string: {
+              type: FieldType.STRING,
+              name: "string",
+            },
+            number: {
+              type: FieldType.NUMBER,
+              name: "number",
+            },
+          },
+        })
+
+        delete tableRequest.schema.id
+
+        const table = await config.api.table.save(tableRequest)
+
+        const stringValue = generator.word()
+        const naturalValue = generator.integer({ min: 0, max: 1000 })
+
+        const existing = await config.api.row.save(table._id!, {
+          string: stringValue,
+          number: naturalValue,
+        })
+
+        expect(existing._id).toEqual(`%5B${naturalValue}%2C'${stringValue}'%5D`)
+
+        const row = await config.api.row.patch(table._id!, {
+          _id: existing._id!,
+          _rev: existing._rev!,
+          tableId: table._id!,
+          string: stringValue,
+          number: 1500,
+        })
+
+        expect(row._id).toEqual(`%5B${"1500"}%2C'${stringValue}'%5D`)
+      })
   })
 
   describe("destroy", () => {

packages/server/src/automations/automationUtils.ts

@@ -99,6 +99,15 @@ export function getError(err: any) {
   return typeof err !== "string" ? err.toString() : err
 }
 
+export function guardAttachment(attachmentObject: any) {
+  if (!("url" in attachmentObject) || !("filename" in attachmentObject)) {
+    const providedKeys = Object.keys(attachmentObject).join(", ")
+    throw new Error(
+      `Attachments must have both "url" and "filename" keys. You have provided: ${providedKeys}`
+    )
+  }
+}
+
 export async function sendAutomationAttachmentsToStorage(
   tableId: string,
   row: Row
@@ -116,9 +125,15 @@ export async function sendAutomationAttachmentsToStorage(
       schema?.type === FieldType.ATTACHMENT_SINGLE ||
       schema?.type === FieldType.SIGNATURE_SINGLE
     ) {
+      if (Array.isArray(value)) {
+        value.forEach(item => guardAttachment(item))
+      } else {
+        guardAttachment(value)
+      }
       attachmentRows[prop] = value
     }
   }
+
   for (const [prop, attachments] of Object.entries(attachmentRows)) {
     if (Array.isArray(attachments)) {
       if (attachments.length) {
@@ -133,7 +148,6 @@ export async function sendAutomationAttachmentsToStorage(
 
   return row
 }
-
 async function generateAttachmentRow(attachment: AutomationAttachment) {
   const prodAppId = context.getProdAppId()
 

packages/server/src/automations/steps/createRow.ts

@@ -90,7 +90,6 @@ export async function run({ inputs, appId, emitter }: AutomationStepInput) {
       tableId: inputs.row.tableId,
     },
   })
-
   try {
     inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
     inputs.row = await sendAutomationAttachmentsToStorage(

packages/server/src/automations/steps/sendSmtpEmail.ts

@@ -118,6 +118,14 @@ export async function run({ inputs }: AutomationStepInput) {
   }
   to = to || undefined
 
+  if (attachments) {
+    if (Array.isArray(attachments)) {
+      attachments.forEach(item => automationUtils.guardAttachment(item))
+    } else {
+      automationUtils.guardAttachment(attachments)
+    }
+  }
+
   try {
     let response = await sendSmtpEmail({
       to,

packages/server/src/automations/tests/createRow.spec.ts

@@ -128,4 +128,31 @@ describe("test the create row action", () => {
     expect(objectData).toBeDefined()
     expect(objectData.ContentLength).toBeGreaterThan(0)
   })
+
+  it("should check that attachment without the correct keys throws an error", async () => {
+    let attachmentTable = await config.createTable(
+      basicTableWithAttachmentField()
+    )
+
+    let attachmentRow: any = {
+      tableId: attachmentTable._id,
+    }
+
+    let filename = "test2.txt"
+    let presignedUrl = await uploadTestFile(filename)
+    let attachmentObject = {
+      wrongKey: presignedUrl,
+      anotherWrongKey: filename,
+    }
+
+    attachmentRow.single_file_attachment = attachmentObject
+    const res = await setup.runStep(setup.actions.CREATE_ROW.stepId, {
+      row: attachmentRow,
+    })
+
+    expect(res.success).toEqual(false)
+    expect(res.response).toEqual(
+      'Error: Attachments must have both "url" and "filename" keys. You have provided: wrongKey, anotherWrongKey'
+    )
+  })
 })