From 3b43943f1f148599abaf24fa3a12936cfa260a42 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Fri, 3 Feb 2023 19:29:28 +0000 Subject: [PATCH 1/5] Some quick scripts for loading a lot of rows and apps. --- .../server/scripts/load/create-many-apps.js | 24 +++++++ .../server/scripts/load/create-many-rows.js | 30 +++++++++ packages/server/scripts/load/utils.js | 66 +++++++++++++++++++ .../src/api/routes/public/applications.ts | 1 - 4 files changed, 120 insertions(+), 1 deletion(-) create mode 100755 packages/server/scripts/load/create-many-apps.js create mode 100755 packages/server/scripts/load/create-many-rows.js create mode 100644 packages/server/scripts/load/utils.js diff --git a/packages/server/scripts/load/create-many-apps.js b/packages/server/scripts/load/create-many-apps.js new file mode 100755 index 0000000000..9532313880 --- /dev/null +++ b/packages/server/scripts/load/create-many-apps.js @@ -0,0 +1,24 @@ +#!/bin/node +const { createApp } = require("./utils") + +const APP_COUNT = 100 + +if (!process.argv[2]) { + console.error("Please specify an API key as script argument.") + process.exit(-1) +} + +async function run() { + for (let i = 0; i < APP_COUNT; i++) { + const app = await createApp(process.argv[2]) + console.log(`App created: ${app._id}`) + } +} + +run() + .then(() => { + console.log(`Finished creating ${APP_COUNT} apps.`) + }) + .catch(err => { + console.error(err) + }) diff --git a/packages/server/scripts/load/create-many-rows.js b/packages/server/scripts/load/create-many-rows.js new file mode 100755 index 0000000000..85c0c292a0 --- /dev/null +++ b/packages/server/scripts/load/create-many-rows.js @@ -0,0 +1,30 @@ +#!/bin/node +const { createApp, getTable, createRow } = require("./utils") + +const ROW_COUNT = 1000 + +if (!process.argv[2]) { + console.error("Please specify an API key as script argument.") + process.exit(-1) +} + +async function run() { + const apiKey = process.argv[2] + const app = await createApp(apiKey) + console.log(`App created: ${app._id}`) + const table = await getTable(apiKey, app._id) + console.log(`Table found: ${table.name}`) + const promises = [] + for (let i = 0; i < ROW_COUNT; i++) { + promises.push(await createRow(apiKey, app._id, table)) + } + await Promise.all(promises) +} + +run() + .then(() => { + console.log(`Finished creating ${ROW_COUNT} rows.`) + }) + .catch(err => { + console.error(err) + }) diff --git a/packages/server/scripts/load/utils.js b/packages/server/scripts/load/utils.js new file mode 100644 index 0000000000..97ff8f1e13 --- /dev/null +++ b/packages/server/scripts/load/utils.js @@ -0,0 +1,66 @@ +const fetch = require("node-fetch") +const uuid = require("uuid/v4") + +const URL_APP = "http://localhost:10000/api/public/v1/applications" +const URL_TABLE = "http://localhost:10000/api/public/v1/tables/search" + +async function request(apiKey, url, method, body, appId = undefined) { + const headers = { + "x-budibase-api-key": apiKey, + "Content-Type": "application/json", + } + if (appId) { + headers["x-budibase-app-id"] = appId + } + const res = await fetch(url, { + method, + headers, + body: JSON.stringify(body), + }) + if (res.status !== 200) { + throw new Error(await res.text()) + } + return res +} + +exports.createApp = async apiKey => { + const name = uuid().replace(/-/g, "") + const body = { + name, + url: `/${name}`, + useTemplate: "true", + templateKey: "app/school-admin-panel", + templateName: "School Admin Panel", + } + const res = await request(apiKey, URL_APP, "POST", body) + const json = await res.json() + return json.data +} + +exports.getTable = async (apiKey, appId) => { + const res = await request(apiKey, URL_TABLE, "POST", {}, appId) + const json = await res.json() + return json.data[0] +} + +exports.createRow = async (apiKey, appId, table) => { + const body = {} + for (let [key, schema] of Object.entries(table.schema)) { + let fake + switch (schema.type) { + default: + case "string": + fake = schema.constraints.inclusion + ? schema.constraints.inclusion[0] + : "a" + break + case "number": + fake = 1 + break + } + body[key] = fake + } + const url = `http://localhost:10000/api/public/v1/tables/${table._id}/rows` + const res = await request(apiKey, url, "POST", body, appId) + return (await res.json()).data +} diff --git a/packages/server/src/api/routes/public/applications.ts b/packages/server/src/api/routes/public/applications.ts index 935321cb7b..9d6c380e60 100644 --- a/packages/server/src/api/routes/public/applications.ts +++ b/packages/server/src/api/routes/public/applications.ts @@ -1,7 +1,6 @@ import controller from "../../controllers/public/applications" import Endpoint from "./utils/Endpoint" const { nameValidator, applicationValidator } = require("../utils/validators") -import { db } from "@budibase/backend-core" const read = [], write = [] From 4d86df057be5347e5d262eca3bdfd20d9f338f01 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Feb 2023 17:54:40 +0000 Subject: [PATCH 2/5] Some fixes for #8770 - the CLI was very memory intensive when working with large databases, that contained many revisions. We don't need the revisions for app exports/backups, therefore I've updated our export systems to drop the revision history as it compacts the database significantly and speeds up exports/reduces memory usage. --- packages/backend-core/package.json | 2 +- packages/backend-core/src/db/couch/pouchDB.ts | 2 +- .../backend-core/src/db/couch/pouchDump.ts | 0 packages/backend-core/yarn.lock | 29 ++++++++----------- packages/cli/src/backups/utils.js | 22 +++++++------- .../server/src/sdk/app/backups/exports.ts | 10 +++++-- 6 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 packages/backend-core/src/db/couch/pouchDump.ts diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index 24910c8a52..b058b13001 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -48,7 +48,7 @@ "posthog-node": "1.3.0", "pouchdb": "7.3.0", "pouchdb-find": "7.2.2", - "pouchdb-replication-stream": "1.2.9", + "@budibase/pouchdb-replication-stream": "1.2.10", "redlock": "4.2.0", "sanitize-s3-objectkey": "0.0.1", "semver": "7.3.7", diff --git a/packages/backend-core/src/db/couch/pouchDB.ts b/packages/backend-core/src/db/couch/pouchDB.ts index a6f4323d88..f83127d466 100644 --- a/packages/backend-core/src/db/couch/pouchDB.ts +++ b/packages/backend-core/src/db/couch/pouchDB.ts @@ -39,7 +39,7 @@ export const getPouch = (opts: PouchOptions = {}) => { } if (opts.replication) { - const replicationStream = require("pouchdb-replication-stream") + const replicationStream = require("@budibase/pouchdb-replication-stream") PouchDB.plugin(replicationStream.plugin) // @ts-ignore PouchDB.adapter("writableStream", replicationStream.adapters.writableStream) diff --git a/packages/backend-core/src/db/couch/pouchDump.ts b/packages/backend-core/src/db/couch/pouchDump.ts new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/backend-core/yarn.lock b/packages/backend-core/yarn.lock index c1ec890e92..4575a1c1a4 100644 --- a/packages/backend-core/yarn.lock +++ b/packages/backend-core/yarn.lock @@ -310,10 +310,18 @@ qs "^6.11.0" tough-cookie "^4.1.2" -"@budibase/types@2.2.12-alpha.62": - version "2.2.12-alpha.62" - resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.2.12-alpha.62.tgz#385ef000610d5c00b83cb2eafda2bd63c86b7f3f" - integrity sha512-idlhB4fSyBCEDWsVvQvdmN9Dg9VAEwxZ8TLE9pGnXIRZPg48MKXPNn5AUT9zv6cDlbQdlU2tFFF8st9b6lyLuw== +"@budibase/pouchdb-replication-stream@1.2.10": + version "1.2.10" + resolved "https://registry.yarnpkg.com/@budibase/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.10.tgz#4100df2effd7c823edadddcdbdc380f6827eebf5" + integrity sha512-1zeorOwbelZ7HF5vFB+pKE8Mnh31om8k1M6T3AZXVULYTHLsyJrMTozSv5CJ1P8ZfOIJab09HDzCXDh2icFekg== + dependencies: + argsarray "0.0.1" + inherits "^2.0.3" + lodash.pick "^4.0.0" + ndjson "^1.4.3" + pouch-stream "^0.4.0" + pouchdb-promise "^6.0.4" + through2 "^2.0.0" "@cspotcode/source-map-support@^0.8.0": version "0.8.1" @@ -4726,19 +4734,6 @@ pouchdb-promise@^6.0.4: dependencies: lie "3.1.1" -pouchdb-replication-stream@1.2.9: - version "1.2.9" - resolved "https://registry.yarnpkg.com/pouchdb-replication-stream/-/pouchdb-replication-stream-1.2.9.tgz#aa4fa5d8f52df4825392f18e07c7e11acffc650a" - integrity sha512-hM8XRBfamTTUwRhKwLS/jSNouBhn9R/4ugdHNRD1EvJzwV8iImh6sDYbCU9PGuznjyOjXz6vpFRzKeI2KYfwnQ== - dependencies: - argsarray "0.0.1" - inherits "^2.0.3" - lodash.pick "^4.0.0" - ndjson "^1.4.3" - pouch-stream "^0.4.0" - pouchdb-promise "^6.0.4" - through2 "^2.0.0" - pouchdb-selector-core@7.2.2: version "7.2.2" resolved "https://registry.yarnpkg.com/pouchdb-selector-core/-/pouchdb-selector-core-7.2.2.tgz#264d7436a8c8ac3801f39960e79875ef7f3879a0" diff --git a/packages/cli/src/backups/utils.js b/packages/cli/src/backups/utils.js index 645b90379b..21d3504373 100644 --- a/packages/cli/src/backups/utils.js +++ b/packages/cli/src/backups/utils.js @@ -74,17 +74,17 @@ exports.getConfig = async (envFile = true) => { return config } -exports.replication = (from, to) => { - return new Promise((resolve, reject) => { - from.replicate - .to(to) - .on("complete", () => { - resolve() - }) - .on("error", err => { - reject(err) - }) - }) +exports.replication = async (from, to) => { + const pouch = getPouch() + try { + await pouch.replicate(from, to, { + batch_size: 1000, + batch_limit: 5, + style: "main_only", + }) + } catch (err) { + throw new Error(`Replication failed - ${JSON.stringify(err)}`) + } } exports.getPouches = config => { diff --git a/packages/server/src/sdk/app/backups/exports.ts b/packages/server/src/sdk/app/backups/exports.ts index 9acad1344c..25bd1f4795 100644 --- a/packages/server/src/sdk/app/backups/exports.ts +++ b/packages/server/src/sdk/app/backups/exports.ts @@ -45,12 +45,18 @@ function tarFilesToTmp(tmpDir: string, files: string[]) { * @return {*} either a readable stream or a string */ export async function exportDB(dbName: string, opts: ExportOpts = {}) { + const exportOpts = { + filter: opts?.filter, + batch_size: 1000, + batch_limit: 5, + style: "main_only", + } return dbCore.doWithDB(dbName, async (db: any) => { // Write the dump to file if required if (opts?.exportPath) { const path = opts?.exportPath const writeStream = fs.createWriteStream(path) - await db.dump(writeStream, { filter: opts?.filter }) + await db.dump(writeStream, exportOpts) return path } else { // Stringify the dump in memory if required @@ -59,7 +65,7 @@ export async function exportDB(dbName: string, opts: ExportOpts = {}) { memStream.on("data", (chunk: any) => { appString += chunk.toString() }) - await db.dump(memStream, { filter: opts?.filter }) + await db.dump(memStream, exportOpts) return appString } }) From 9339213910f665d76d0191191769369873e73d26 Mon Sep 17 00:00:00 2001 From: mike12345567 Date: Mon, 6 Feb 2023 18:00:42 +0000 Subject: [PATCH 3/5] Merge. --- charts/budibase/templates/alb-ingress.yaml | 3 + lerna.json | 2 +- packages/backend-core/package.json | 4 +- packages/bbui/package.json | 4 +- packages/bbui/src/Form/Core/DatePicker.svelte | 9 +- packages/bbui/src/Form/Core/RadioGroup.svelte | 21 +- packages/builder/package.json | 10 +- .../SetupPanel/AutomationBlockSetup.svelte | 21 +- .../automation/SetupPanel/RowSelector.svelte | 65 +++- .../automation/SetupPanel/SchemaSetup.svelte | 2 +- .../DataTable/modals/CreateEditColumn.svelte | 312 +++++++++++------- .../common/bindings/ModalBindableInput.svelte | 4 + .../ButtonActionDrawer.svelte | 10 + .../actions/TriggerAutomation.svelte | 8 +- .../data/table/[tableId]/index.svelte | 45 ++- .../settings/ComponentSettingsPanel.svelte | 2 +- packages/cli/package.json | 8 +- packages/client/package.json | 8 +- .../client/src/components/Component.svelte | 17 +- .../src/components/app/DataProvider.svelte | 41 ++- .../src/components/app/forms/Form.svelte | 13 +- .../src/components/app/forms/InnerForm.svelte | 51 ++- .../components/app/forms/OptionsField.svelte | 1 + packages/frontend-core/package.json | 4 +- packages/sdk/package.json | 2 +- packages/server/package.json | 12 +- .../server/src/api/controllers/table/utils.ts | 8 +- .../server/src/api/controllers/view/index.ts | 23 +- .../src/api/controllers/view/viewBuilder.ts | 22 +- .../server/src/automations/automationUtils.ts | 10 + .../server/src/automations/steps/updateRow.ts | 9 +- packages/server/src/integrations/base/sql.ts | 7 +- packages/server/src/integrations/mysql.ts | 3 +- .../server/src/integrations/tests/sql.spec.ts | 42 ++- packages/server/src/integrations/utils.ts | 12 +- packages/server/src/threads/query.ts | 2 +- packages/server/yarn.lock | 30 +- packages/string-templates/package.json | 2 +- packages/types/package.json | 2 +- packages/worker/package.json | 10 +- packages/worker/yarn.lock | 30 +- 41 files changed, 603 insertions(+), 288 deletions(-) diff --git a/charts/budibase/templates/alb-ingress.yaml b/charts/budibase/templates/alb-ingress.yaml index 388bcf1d3e..c128b70843 100644 --- a/charts/budibase/templates/alb-ingress.yaml +++ b/charts/budibase/templates/alb-ingress.yaml @@ -14,6 +14,9 @@ metadata: alb.ingress.kubernetes.io/listen-ports: '[{"HTTP": 80}, {"HTTPS":443}]' alb.ingress.kubernetes.io/certificate-arn: {{ .Values.ingress.certificateArn }} {{- end }} + {{- if .Values.ingress.securityGroups }} + alb.ingress.kubernetes.io/security-groups: {{ .Values.ingress.securityGroups }} + {{- end }} spec: rules: - http: diff --git a/lerna.json b/lerna.json index 8db64ebdca..5f433c0b1f 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.2.12-alpha.71", + "version": "2.3.0", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/package.json b/packages/backend-core/package.json index b058b13001..94b1541499 100644 --- a/packages/backend-core/package.json +++ b/packages/backend-core/package.json @@ -1,6 +1,6 @@ { "name": "@budibase/backend-core", - "version": "2.2.12-alpha.71", + "version": "2.3.0", "description": "Budibase backend core libraries used in server and worker", "main": "dist/src/index.js", "types": "dist/src/index.d.ts", @@ -23,7 +23,7 @@ }, "dependencies": { "@budibase/nano": "10.1.1", - "@budibase/types": "2.2.12-alpha.71", + "@budibase/types": "^2.3.0", "@shopify/jest-koa-mocks": "5.0.1", "@techpass/passport-openidconnect": "0.3.2", "aws-cloudfront-sign": "2.2.0", diff --git a/packages/bbui/package.json b/packages/bbui/package.json index a85e970c43..5456a1fb3a 100644 --- a/packages/bbui/package.json +++ b/packages/bbui/package.json @@ -1,7 +1,7 @@ { "name": "@budibase/bbui", "description": "A UI solution used in the different Budibase projects.", - "version": "2.2.12-alpha.71", + "version": "2.3.0", "license": "MPL-2.0", "svelte": "src/index.js", "module": "dist/bbui.es.js", @@ -38,7 +38,7 @@ ], "dependencies": { "@adobe/spectrum-css-workflow-icons": "1.2.1", - "@budibase/string-templates": "2.2.12-alpha.71", + "@budibase/string-templates": "^2.3.0", "@spectrum-css/accordion": "3.0.24", "@spectrum-css/actionbutton": "1.0.1", "@spectrum-css/actiongroup": "1.0.1", diff --git a/packages/bbui/src/Form/Core/DatePicker.svelte b/packages/bbui/src/Form/Core/DatePicker.svelte index 10aae67ec6..2c89a538a3 100644 --- a/packages/bbui/src/Form/Core/DatePicker.svelte +++ b/packages/bbui/src/Form/Core/DatePicker.svelte @@ -76,13 +76,6 @@ } // If time only set date component to 2000-01-01 if (timeOnly) { - // Classic flackpickr causing issues. - // When selecting a value for the first time for a "time only" field, - // the time is always offset by 1 hour for some reason (regardless of time - // zone) so we need to correct it. - if (!value && newValue) { - newValue = new Date(dates[0].getTime() + 60 * 60 * 1000).toISOString() - } newValue = `2000-01-01T${newValue.split("T")[1]}` } @@ -113,7 +106,7 @@ const clearDateOnBackspace = event => { if (["Backspace", "Clear", "Delete"].includes(event.key)) { - dispatch("change", null) + dispatch("change", "") flatpickr.close() } } diff --git a/packages/bbui/src/Form/Core/RadioGroup.svelte b/packages/bbui/src/Form/Core/RadioGroup.svelte index a3952a9759..f7afc10bbc 100644 --- a/packages/bbui/src/Form/Core/RadioGroup.svelte +++ b/packages/bbui/src/Form/Core/RadioGroup.svelte @@ -11,14 +11,31 @@ export let getOptionLabel = option => option export let getOptionValue = option => option export let getOptionTitle = option => option + export let sort = false const dispatch = createEventDispatcher() const onChange = e => dispatch("change", e.target.value) + + const getSortedOptions = (options, getLabel, sort) => { + if (!options?.length || !Array.isArray(options)) { + return [] + } + if (!sort) { + return options + } + return [...options].sort((a, b) => { + const labelA = getLabel(a) + const labelB = getLabel(b) + return labelA > labelB ? 1 : -1 + }) + } + + $: parsedOptions = getSortedOptions(options, getOptionLabel, sort)
- {#if options && Array.isArray(options)} - {#each options as option} + {#if parsedOptions && Array.isArray(parsedOptions)} + {#each parsedOptions as option}
{ + if (e.detail?.tableId) { + const tableSchema = getSchemaForTable(e.detail.tableId, { + searchableSchema: true, + }).schema + if (isTestModal) { + testData.schema = tableSchema + } else { + block.inputs.schema = tableSchema + } + } try { if (isTestModal) { // Special case for webhook, as it requires a body, but the schema already brings back the body's contents @@ -321,9 +332,17 @@ onChange(e, key)} + meta={inputData["meta"] || {}} + on:change={e => { + if (e.detail?.key) { + onChange(e, e.detail.key) + } else { + onChange(e, key) + } + }} {bindings} {isTestModal} + {isUpdateRow} /> {:else if value.customType === "webhookUrl"} import { tables } from "stores/backend" - import { Select } from "@budibase/bbui" + import { Select, Checkbox } from "@budibase/bbui" import DrawerBindableInput from "../../common/bindings/DrawerBindableInput.svelte" import AutomationBindingPanel from "../../common/bindings/ServerBindingPanel.svelte" import { createEventDispatcher } from "svelte" @@ -10,9 +10,11 @@ const dispatch = createEventDispatcher() export let value + export let meta export let bindings export let block export let isTestModal + export let isUpdateRow $: parsedBindings = bindings.map(binding => { let clone = Object.assign({}, binding) @@ -97,6 +99,17 @@ dispatch("change", value) } + const onChangeSetting = (e, field) => { + let fields = {} + fields[field] = { + clearRelationships: e.detail, + } + dispatch("change", { + key: "meta", + fields, + }) + } + // Ensure any nullish tableId values get set to empty string so // that the select works $: if (value?.tableId == null) value = { tableId: "" } @@ -124,21 +137,33 @@ {onChange} /> {:else} - onChange(e, field, schema.type)} - label={field} - type="string" - bindings={parsedBindings} - fillWidth={true} - allowJS={true} - updateOnChange={false} - /> +
+ onChange(e, field, schema.type)} + label={field} + type="string" + bindings={parsedBindings} + fillWidth={true} + allowJS={true} + updateOnChange={false} + /> + {#if isUpdateRow && schema.type === "link"} +
+ onChangeSetting(e, field)} + /> +
+ {/if} +
{/if} {/if} {/if} @@ -155,4 +180,12 @@ .schema-fields :global(label) { text-transform: capitalize; } + .checkbox-field { + padding-bottom: var(--spacing-s); + padding-left: 1px; + padding-top: var(--spacing-s); + } + .checkbox-field :global(label) { + text-transform: none; + } diff --git a/packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte b/packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte index cb80072694..b5173682d0 100644 --- a/packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte +++ b/packages/builder/src/components/automation/SetupPanel/SchemaSetup.svelte @@ -58,7 +58,7 @@ entries = entries.filter(f => f.name !== originalName) } value = entries.reduce((newVals, current) => { - newVals[current.name] = current.type + newVals[current.name.trim()] = current.type return newVals }, {}) dispatch("change", value) diff --git a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte index 4ed77d55b2..3e5549fcf5 100644 --- a/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte +++ b/packages/builder/src/components/backend/DataTable/modals/CreateEditColumn.svelte @@ -12,7 +12,7 @@ Modal, notifications, } from "@budibase/bbui" - import { createEventDispatcher, onMount } from "svelte" + import { createEventDispatcher } from "svelte" import { cloneDeep } from "lodash/fp" import { tables, datasources } from "stores/backend" import { TableNames, UNEDITABLE_USER_FIELDS } from "constants" @@ -48,7 +48,22 @@ const { hide } = getContext(Context.Modal) let fieldDefinitions = cloneDeep(FIELDS) - export let field = { + export let field + + let originalName + let linkEditDisabled + let primaryDisplay + let indexes = [...($tables.selected.indexes || [])] + let isCreating + + let table = $tables.selected + let confirmDeleteDialog + let deletion + let savingColumn + let deleteColName + let jsonSchemaModal + + let editableColumn = { type: "string", constraints: fieldDefinitions.STRING.constraints, @@ -56,48 +71,80 @@ fieldName: $tables.selected.name, } - let originalName = field.name - const linkEditDisabled = originalName != null - let primaryDisplay = - $tables.selected.primaryDisplay == null || - $tables.selected.primaryDisplay === field.name - let isCreating = originalName == null + $: if (primaryDisplay) { + editableColumn.constraints.presence = { allowEmpty: false } + } - let table = $tables.selected - let indexes = [...($tables.selected.indexes || [])] - let confirmDeleteDialog - let deletion - let deleteColName - let jsonSchemaModal + $: if (field && !savingColumn) { + editableColumn = cloneDeep(field) + originalName = editableColumn.name ? editableColumn.name + "" : null + linkEditDisabled = originalName != null + isCreating = originalName == null + primaryDisplay = + $tables.selected.primaryDisplay == null || + $tables.selected.primaryDisplay === editableColumn.name + } - $: checkConstraints(field) - $: required = !!field?.constraints?.presence || primaryDisplay + $: checkConstraints(editableColumn) + $: required = !!editableColumn?.constraints?.presence || primaryDisplay $: uneditable = $tables.selected?._id === TableNames.USERS && - UNEDITABLE_USER_FIELDS.includes(field.name) + UNEDITABLE_USER_FIELDS.includes(editableColumn.name) $: invalid = - !field.name || - (field.type === LINK_TYPE && !field.tableId) || + !editableColumn?.name || + (editableColumn?.type === LINK_TYPE && !editableColumn?.tableId) || Object.keys(errors).length !== 0 - $: errors = checkErrors(field) + $: errors = checkErrors(editableColumn) $: datasource = $datasources.list.find( source => source._id === table?.sourceId ) + const getTableAutoColumnTypes = table => { + return Object.keys(table?.schema).reduce((acc, key) => { + let fieldSchema = table?.schema[key] + if (fieldSchema.autocolumn) { + acc.push(fieldSchema.subtype) + } + return acc + }, []) + } + + let autoColumnInfo = getAutoColumnInformation() + + $: tableAutoColumnsTypes = getTableAutoColumnTypes($tables?.selected) + $: availableAutoColumns = Object.keys(autoColumnInfo).reduce((acc, key) => { + if (!tableAutoColumnsTypes.includes(key)) { + acc[key] = autoColumnInfo[key] + } + return acc + }, {}) + + $: availableAutoColumnKeys = availableAutoColumns + ? Object.keys(availableAutoColumns) + : [] + + $: autoColumnOptions = editableColumn.autocolumn + ? autoColumnInfo + : availableAutoColumns + // used to select what different options can be displayed for column type $: canBeSearched = - field.type !== LINK_TYPE && - field.type !== JSON_TYPE && - field.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY && - field.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY && - field.type !== FORMULA_TYPE + editableColumn?.type !== LINK_TYPE && + editableColumn?.type !== JSON_TYPE && + editableColumn?.subtype !== AUTO_COLUMN_SUB_TYPES.CREATED_BY && + editableColumn?.subtype !== AUTO_COLUMN_SUB_TYPES.UPDATED_BY && + editableColumn?.type !== FORMULA_TYPE $: canBeDisplay = - field.type !== LINK_TYPE && - field.type !== AUTO_TYPE && - field.type !== JSON_TYPE + editableColumn?.type !== LINK_TYPE && + editableColumn?.type !== AUTO_TYPE && + editableColumn?.type !== JSON_TYPE && + !editableColumn.autocolumn $: canBeRequired = - field.type !== LINK_TYPE && !uneditable && field.type !== AUTO_TYPE - $: relationshipOptions = getRelationshipOptions(field) + editableColumn?.type !== LINK_TYPE && + !uneditable && + editableColumn?.type !== AUTO_TYPE && + !editableColumn.autocolumn + $: relationshipOptions = getRelationshipOptions(editableColumn) $: external = table.type === "external" // in the case of internal tables the sourceId will just be undefined $: tableOptions = $tables.list.filter( @@ -108,76 +155,90 @@ ) $: typeEnabled = !originalName || - (originalName && SWITCHABLE_TYPES.indexOf(field.type) !== -1) + (originalName && + SWITCHABLE_TYPES.indexOf(editableColumn.type) !== -1 && + !editableColumn?.autocolumn) async function saveColumn() { - if (field.type === AUTO_TYPE) { - field = buildAutoColumn($tables.selected.name, field.name, field.subtype) + savingColumn = true + if (errors?.length) { + return } - if (field.type !== LINK_TYPE) { - delete field.fieldName + + let saveColumn = cloneDeep(editableColumn) + + if (saveColumn.type === AUTO_TYPE) { + saveColumn = buildAutoColumn( + $tables.selected.name, + saveColumn.name, + saveColumn.subtype + ) + } + if (saveColumn.type !== LINK_TYPE) { + delete saveColumn.fieldName } try { await tables.saveField({ originalName, - field, + field: saveColumn, primaryDisplay, indexes, }) dispatch("updatecolumns") } catch (err) { - notifications.error("Error saving column") + console.log(err) + notifications.error(`Error saving column: ${err.message}`) } } function cancelEdit() { - field.name = originalName + editableColumn.name = originalName } function deleteColumn() { try { - field.name = deleteColName - if (field.name === $tables.selected.primaryDisplay) { + editableColumn.name = deleteColName + if (editableColumn.name === $tables.selected.primaryDisplay) { notifications.error("You cannot delete the display column") } else { - tables.deleteField(field) - notifications.success(`Column ${field.name} deleted.`) + tables.deleteField(editableColumn) + notifications.success(`Column ${editableColumn.name} deleted.`) confirmDeleteDialog.hide() hide() deletion = false dispatch("updatecolumns") } } catch (error) { - notifications.error("Error deleting column") + notifications.error(`Error deleting column: ${error.message}`) } } function handleTypeChange(event) { // remove any extra fields that may not be related to this type - delete field.autocolumn - delete field.subtype - delete field.tableId - delete field.relationshipType - delete field.formulaType + delete editableColumn.autocolumn + delete editableColumn.subtype + delete editableColumn.tableId + delete editableColumn.relationshipType + delete editableColumn.formulaType // Add in defaults and initial definition const definition = fieldDefinitions[event.detail?.toUpperCase()] if (definition?.constraints) { - field.constraints = definition.constraints + editableColumn.constraints = definition.constraints } // Default relationships many to many - if (field.type === LINK_TYPE) { - field.relationshipType = RelationshipTypes.MANY_TO_MANY + if (editableColumn.type === LINK_TYPE) { + editableColumn.relationshipType = RelationshipTypes.MANY_TO_MANY } - if (field.type === FORMULA_TYPE) { - field.formulaType = "dynamic" + if (editableColumn.type === FORMULA_TYPE) { + editableColumn.formulaType = "dynamic" } } function onChangeRequired(e) { const req = e.detail - field.constraints.presence = req ? { allowEmpty: false } : false + editableColumn.constraints.presence = req ? { allowEmpty: false } : false required = req } @@ -185,17 +246,17 @@ const isPrimary = e.detail // primary display is always required if (isPrimary) { - field.constraints.presence = { allowEmpty: false } + editableColumn.constraints.presence = { allowEmpty: false } } } function onChangePrimaryIndex(e) { - indexes = e.detail ? [field.name] : [] + indexes = e.detail ? [editableColumn.name] : [] } function onChangeSecondaryIndex(e) { if (e.detail) { - indexes[1] = field.name + indexes[1] = editableColumn.name } else { indexes = indexes.slice(0, 1) } @@ -246,11 +307,14 @@ } function getAllowedTypes() { - if (originalName && ALLOWABLE_STRING_TYPES.indexOf(field.type) !== -1) { + if ( + originalName && + ALLOWABLE_STRING_TYPES.indexOf(editableColumn.type) !== -1 + ) { return ALLOWABLE_STRING_OPTIONS } else if ( originalName && - ALLOWABLE_NUMBER_TYPES.indexOf(field.type) !== -1 + ALLOWABLE_NUMBER_TYPES.indexOf(editableColumn.type) !== -1 ) { return ALLOWABLE_NUMBER_OPTIONS } else if (!external) { @@ -275,6 +339,9 @@ } function checkConstraints(fieldToCheck) { + if (!fieldToCheck) { + return + } // most types need this, just make sure its always present if (fieldToCheck && !fieldToCheck.constraints) { fieldToCheck.constraints = {} @@ -296,10 +363,16 @@ } function checkErrors(fieldInfo) { + if (!editableColumn) { + return {} + } function inUse(tbl, column, ogName = null) { - return Object.keys(tbl?.schema || {}).some( - key => key !== ogName && key === column - ) + const parsedColumn = column ? column.toLowerCase().trim() : column + + return Object.keys(tbl?.schema || {}).some(key => { + let lowerKey = key.toLowerCase() + return lowerKey !== ogName?.toLowerCase() && lowerKey === parsedColumn + }) } const newError = {} if (!external && fieldInfo.name?.startsWith("_")) { @@ -313,6 +386,11 @@ } else if (inUse($tables.selected, fieldInfo.name, originalName)) { newError.name = `Column name already in use.` } + + if (fieldInfo.type == "auto" && !fieldInfo.subtype) { + newError.subtype = `Auto Column requires a type` + } + if (fieldInfo.fieldName && fieldInfo.tableId) { const relatedTable = $tables.list.find( tbl => tbl._id === fieldInfo.tableId @@ -323,12 +401,6 @@ } return newError } - - onMount(() => { - if (primaryDisplay) { - field.constraints.presence = { allowEmpty: false } - } - }) - {:else if field.type === "options"} + {:else if editableColumn.type === "options"} - {:else if field.type === "longform"} + {:else if editableColumn.type === "longform"}
- {:else if field.type === "array"} + {:else if editableColumn.type === "array"} - {:else if field.type === "datetime"} + {:else if editableColumn.type === "datetime" && !editableColumn.autocolumn} + - {#if datasource?.source !== "ORACLE" && datasource?.source !== "SQL_SERVER"}
- +
{/if} - {:else if field.type === "number"} + {:else if editableColumn.type === "number" && !editableColumn.autocolumn} - {:else if field.type === "link"} + {:else if editableColumn.type === "link"} - {:else if field.type === FORMULA_TYPE} + {:else if editableColumn.type === FORMULA_TYPE} {#if !table.sql} (field.subtype = e.detail)} - options={Object.entries(getAutoColumnInformation())} - getOptionLabel={option => option[1].name} - getOptionValue={option => option[0]} - /> - {:else if field.type === JSON_TYPE} + {:else if editableColumn.type === JSON_TYPE} {/if} + {#if editableColumn.type === AUTO_TYPE || editableColumn.autocolumn} +