Merge branch 'master' of github.com:budibase/budibase into update-docker-compose-for-sqs
This commit is contained in:
commit
d83248e917
33 changed files with 835 additions and 280 deletions
|
@ -1,5 +1,5 @@
|
||||||
{
|
{
|
||||||
"version": "2.29.1",
|
"version": "2.29.2",
|
||||||
"npmClient": "yarn",
|
"npmClient": "yarn",
|
||||||
"packages": [
|
"packages": [
|
||||||
"packages/*",
|
"packages/*",
|
||||||
|
|
|
@ -72,4 +72,4 @@ export const DEFAULT_JOBS_TABLE_ID = "ta_bb_jobs"
|
||||||
export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
|
export const DEFAULT_INVENTORY_TABLE_ID = "ta_bb_inventory"
|
||||||
export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
|
export const DEFAULT_EXPENSES_TABLE_ID = "ta_bb_expenses"
|
||||||
export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
|
export const DEFAULT_EMPLOYEE_TABLE_ID = "ta_bb_employee"
|
||||||
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
|
export { DEFAULT_BB_DATASOURCE_ID } from "@budibase/shared-core"
|
||||||
|
|
|
@ -449,8 +449,12 @@ class InternalBuilder {
|
||||||
query = query.orderBy(`${aliased}.${key}`, direction, nulls)
|
query = query.orderBy(`${aliased}.${key}`, direction, nulls)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// always add sorting by the primary key - make sure result is deterministic
|
|
||||||
|
// add sorting by the primary key if the result isn't already sorted by it,
|
||||||
|
// to make sure result is deterministic
|
||||||
|
if (!sort || sort[primaryKey[0]] === undefined) {
|
||||||
query = query.orderBy(`${aliased}.${primaryKey[0]}`)
|
query = query.orderBy(`${aliased}.${primaryKey[0]}`)
|
||||||
|
}
|
||||||
return query
|
return query
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -604,7 +608,8 @@ class InternalBuilder {
|
||||||
if (!primary) {
|
if (!primary) {
|
||||||
throw new Error("Primary key is required for upsert")
|
throw new Error("Primary key is required for upsert")
|
||||||
}
|
}
|
||||||
return query.insert(parsedBody).onConflict(primary).merge()
|
const ret = query.insert(parsedBody).onConflict(primary).merge()
|
||||||
|
return ret
|
||||||
} else if (this.client === SqlClient.MS_SQL) {
|
} else if (this.client === SqlClient.MS_SQL) {
|
||||||
// No upsert or onConflict support in MSSQL yet, see:
|
// No upsert or onConflict support in MSSQL yet, see:
|
||||||
// https://github.com/knex/knex/pull/6050
|
// https://github.com/knex/knex/pull/6050
|
||||||
|
|
|
@ -109,8 +109,10 @@ function generateSchema(
|
||||||
const { tableName } = breakExternalTableId(column.tableId)
|
const { tableName } = breakExternalTableId(column.tableId)
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
const relatedTable = tables[tableName]
|
const relatedTable = tables[tableName]
|
||||||
if (!relatedTable) {
|
if (!relatedTable || !relatedTable.primary) {
|
||||||
throw new Error("Referenced table doesn't exist")
|
throw new Error(
|
||||||
|
"Referenced table doesn't exist or has no primary keys"
|
||||||
|
)
|
||||||
}
|
}
|
||||||
const relatedPrimary = relatedTable.primary[0]
|
const relatedPrimary = relatedTable.primary[0]
|
||||||
const externalType = relatedTable.schema[relatedPrimary].externalType
|
const externalType = relatedTable.schema[relatedPrimary].externalType
|
||||||
|
|
|
@ -55,10 +55,7 @@ export function buildExternalTableId(datasourceId: string, tableName: string) {
|
||||||
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
|
return `${datasourceId}${DOUBLE_SEPARATOR}${tableName}`
|
||||||
}
|
}
|
||||||
|
|
||||||
export function breakExternalTableId(tableId: string | undefined) {
|
export function breakExternalTableId(tableId: string) {
|
||||||
if (!tableId) {
|
|
||||||
return {}
|
|
||||||
}
|
|
||||||
const parts = tableId.split(DOUBLE_SEPARATOR)
|
const parts = tableId.split(DOUBLE_SEPARATOR)
|
||||||
let datasourceId = parts.shift()
|
let datasourceId = parts.shift()
|
||||||
// if they need joined
|
// if they need joined
|
||||||
|
@ -67,6 +64,9 @@ export function breakExternalTableId(tableId: string | undefined) {
|
||||||
if (tableName.includes(ENCODED_SPACE)) {
|
if (tableName.includes(ENCODED_SPACE)) {
|
||||||
tableName = decodeURIComponent(tableName)
|
tableName = decodeURIComponent(tableName)
|
||||||
}
|
}
|
||||||
|
if (!datasourceId || !tableName) {
|
||||||
|
throw new Error("Unable to get datasource/table name from table ID")
|
||||||
|
}
|
||||||
return { datasourceId, tableName }
|
return { datasourceId, tableName }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -24,7 +24,6 @@ export const account = (partial: Partial<Account> = {}): Account => {
|
||||||
createdAt: Date.now(),
|
createdAt: Date.now(),
|
||||||
verified: true,
|
verified: true,
|
||||||
verificationSent: true,
|
verificationSent: true,
|
||||||
tier: "FREE", // DEPRECATED
|
|
||||||
authType: AuthType.PASSWORD,
|
authType: AuthType.PASSWORD,
|
||||||
name: generator.name(),
|
name: generator.name(),
|
||||||
size: "10+",
|
size: "10+",
|
||||||
|
|
|
@ -1,9 +1,14 @@
|
||||||
<script>
|
<script>
|
||||||
import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
|
import {
|
||||||
|
FieldType,
|
||||||
|
BBReferenceFieldSubType,
|
||||||
|
SourceName,
|
||||||
|
} from "@budibase/types"
|
||||||
import { Select, Toggle, Multiselect } from "@budibase/bbui"
|
import { Select, Toggle, Multiselect } from "@budibase/bbui"
|
||||||
import { DB_TYPE_INTERNAL } from "constants/backend"
|
import { DB_TYPE_INTERNAL } from "constants/backend"
|
||||||
import { API } from "api"
|
import { API } from "api"
|
||||||
import { parseFile } from "./utils"
|
import { parseFile } from "./utils"
|
||||||
|
import { tables, datasources } from "stores/builder"
|
||||||
|
|
||||||
let error = null
|
let error = null
|
||||||
let fileName = null
|
let fileName = null
|
||||||
|
@ -80,6 +85,9 @@
|
||||||
schema = fetchSchema(tableId)
|
schema = fetchSchema(tableId)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$: table = $tables.list.find(table => table._id === tableId)
|
||||||
|
$: datasource = $datasources.list.find(ds => ds._id === table?.sourceId)
|
||||||
|
|
||||||
async function fetchSchema(tableId) {
|
async function fetchSchema(tableId) {
|
||||||
try {
|
try {
|
||||||
const definition = await API.fetchTableDefinition(tableId)
|
const definition = await API.fetchTableDefinition(tableId)
|
||||||
|
@ -185,20 +193,25 @@
|
||||||
</div>
|
</div>
|
||||||
{/each}
|
{/each}
|
||||||
</div>
|
</div>
|
||||||
{#if tableType === DB_TYPE_INTERNAL}
|
|
||||||
<br />
|
<br />
|
||||||
|
<!-- SQL Server doesn't yet support overwriting rows by existing keys -->
|
||||||
|
{#if datasource?.source !== SourceName.SQL_SERVER}
|
||||||
<Toggle
|
<Toggle
|
||||||
bind:value={updateExistingRows}
|
bind:value={updateExistingRows}
|
||||||
on:change={() => (identifierFields = [])}
|
on:change={() => (identifierFields = [])}
|
||||||
thin
|
thin
|
||||||
text="Update existing rows"
|
text="Update existing rows"
|
||||||
/>
|
/>
|
||||||
|
{/if}
|
||||||
{#if updateExistingRows}
|
{#if updateExistingRows}
|
||||||
|
{#if tableType === DB_TYPE_INTERNAL}
|
||||||
<Multiselect
|
<Multiselect
|
||||||
label="Identifier field(s)"
|
label="Identifier field(s)"
|
||||||
options={Object.keys(validation)}
|
options={Object.keys(validation)}
|
||||||
bind:value={identifierFields}
|
bind:value={identifierFields}
|
||||||
/>
|
/>
|
||||||
|
{:else}
|
||||||
|
<p>Rows will be updated based on the table's primary key.</p>
|
||||||
{/if}
|
{/if}
|
||||||
{/if}
|
{/if}
|
||||||
{#if invalidColumns.length > 0}
|
{#if invalidColumns.length > 0}
|
||||||
|
|
|
@ -233,9 +233,9 @@
|
||||||
response.info = response.info || { code: 200 }
|
response.info = response.info || { code: 200 }
|
||||||
// if existing schema, copy over what it is
|
// if existing schema, copy over what it is
|
||||||
if (schema) {
|
if (schema) {
|
||||||
for (let [name, field] of Object.entries(schema)) {
|
for (let [name, field] of Object.entries(response.schema)) {
|
||||||
if (response.schema[name]) {
|
if (!schema[name]) {
|
||||||
response.schema[name] = field
|
schema[name] = field
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -18,7 +18,7 @@
|
||||||
import FilterUsers from "./FilterUsers.svelte"
|
import FilterUsers from "./FilterUsers.svelte"
|
||||||
import { getFields } from "../utils/searchFields"
|
import { getFields } from "../utils/searchFields"
|
||||||
|
|
||||||
const { OperatorOptions } = Constants
|
const { OperatorOptions, DEFAULT_BB_DATASOURCE_ID } = Constants
|
||||||
|
|
||||||
export let schemaFields
|
export let schemaFields
|
||||||
export let filters = []
|
export let filters = []
|
||||||
|
@ -28,6 +28,23 @@
|
||||||
export let allowBindings = false
|
export let allowBindings = false
|
||||||
export let filtersLabel = "Filters"
|
export let filtersLabel = "Filters"
|
||||||
|
|
||||||
|
$: {
|
||||||
|
if (
|
||||||
|
tables.find(
|
||||||
|
table =>
|
||||||
|
table._id === datasource.tableId &&
|
||||||
|
table.sourceId === DEFAULT_BB_DATASOURCE_ID
|
||||||
|
) &&
|
||||||
|
!schemaFields.some(field => field.name === "_id")
|
||||||
|
) {
|
||||||
|
schemaFields = [
|
||||||
|
...schemaFields,
|
||||||
|
{ name: "_id", type: "string" },
|
||||||
|
{ name: "_rev", type: "string" },
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
$: matchAny = filters?.find(filter => filter.operator === "allOr") != null
|
$: matchAny = filters?.find(filter => filter.operator === "allOr") != null
|
||||||
$: onEmptyFilter =
|
$: onEmptyFilter =
|
||||||
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
|
filters?.find(filter => filter.onEmptyFilter)?.onEmptyFilter ?? "all"
|
||||||
|
@ -35,7 +52,6 @@
|
||||||
$: fieldFilters = filters.filter(
|
$: fieldFilters = filters.filter(
|
||||||
filter => filter.operator !== "allOr" && !filter.onEmptyFilter
|
filter => filter.operator !== "allOr" && !filter.onEmptyFilter
|
||||||
)
|
)
|
||||||
|
|
||||||
const behaviourOptions = [
|
const behaviourOptions = [
|
||||||
{ value: "and", label: "Match all filters" },
|
{ value: "and", label: "Match all filters" },
|
||||||
{ value: "or", label: "Match any filter" },
|
{ value: "or", label: "Match any filter" },
|
||||||
|
@ -44,7 +60,6 @@
|
||||||
{ value: "all", label: "Return all table rows" },
|
{ value: "all", label: "Return all table rows" },
|
||||||
{ value: "none", label: "Return no rows" },
|
{ value: "none", label: "Return no rows" },
|
||||||
]
|
]
|
||||||
|
|
||||||
const context = getContext("context")
|
const context = getContext("context")
|
||||||
|
|
||||||
$: fieldOptions = getFields(tables, schemaFields || [], {
|
$: fieldOptions = getFields(tables, schemaFields || [], {
|
||||||
|
|
|
@ -1,7 +1,11 @@
|
||||||
/**
|
/**
|
||||||
* Operator options for lucene queries
|
* Operator options for lucene queries
|
||||||
*/
|
*/
|
||||||
export { OperatorOptions, SqlNumberTypeRangeMap } from "@budibase/shared-core"
|
export {
|
||||||
|
OperatorOptions,
|
||||||
|
SqlNumberTypeRangeMap,
|
||||||
|
DEFAULT_BB_DATASOURCE_ID,
|
||||||
|
} from "@budibase/shared-core"
|
||||||
export { Feature as Features } from "@budibase/types"
|
export { Feature as Features } from "@budibase/types"
|
||||||
import { BpmCorrelationKey } from "@budibase/shared-core"
|
import { BpmCorrelationKey } from "@budibase/shared-core"
|
||||||
import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
|
import { FieldType, BBReferenceFieldSubType } from "@budibase/types"
|
||||||
|
|
|
@ -1 +1 @@
|
||||||
Subproject commit bf30f47a28292d619cf0837f21d66790ff31c3a6
|
Subproject commit 6c8d0174ca58c578a37022965ddb923fdbf8e32a
|
|
@ -1,4 +1,4 @@
|
||||||
FROM mcr.microsoft.com/mssql/server:2017-latest
|
FROM mcr.microsoft.com/mssql/server:2022-latest
|
||||||
|
|
||||||
ENV ACCEPT_EULA=Y
|
ENV ACCEPT_EULA=Y
|
||||||
ENV SA_PASSWORD=Passw0rd
|
ENV SA_PASSWORD=Passw0rd
|
||||||
|
|
|
@ -311,8 +311,8 @@ export async function preview(
|
||||||
|
|
||||||
// if existing schema, update to include any previous schema keys
|
// if existing schema, update to include any previous schema keys
|
||||||
if (existingSchema) {
|
if (existingSchema) {
|
||||||
for (let key of Object.keys(previewSchema)) {
|
for (let key of Object.keys(existingSchema)) {
|
||||||
if (existingSchema[key]) {
|
if (!previewSchema[key]) {
|
||||||
previewSchema[key] = existingSchema[key]
|
previewSchema[key] = existingSchema[key]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -72,92 +72,6 @@ export type ExternalRequestReturnType<T extends Operation> =
|
||||||
? number
|
? number
|
||||||
: { row: Row; table: Table }
|
: { row: Row; table: Table }
|
||||||
|
|
||||||
function buildFilters(
|
|
||||||
id: string | undefined | string[],
|
|
||||||
filters: SearchFilters,
|
|
||||||
table: Table
|
|
||||||
) {
|
|
||||||
const primary = table.primary
|
|
||||||
// if passed in array need to copy for shifting etc
|
|
||||||
let idCopy: undefined | string | any[] = cloneDeep(id)
|
|
||||||
if (filters) {
|
|
||||||
// need to map over the filters and make sure the _id field isn't present
|
|
||||||
let prefix = 1
|
|
||||||
for (let operator of Object.values(filters)) {
|
|
||||||
for (let field of Object.keys(operator || {})) {
|
|
||||||
if (dbCore.removeKeyNumbering(field) === "_id") {
|
|
||||||
if (primary) {
|
|
||||||
const parts = breakRowIdField(operator[field])
|
|
||||||
for (let field of primary) {
|
|
||||||
operator[`${prefix}:${field}`] = parts.shift()
|
|
||||||
}
|
|
||||||
prefix++
|
|
||||||
}
|
|
||||||
// make sure this field doesn't exist on any filter
|
|
||||||
delete operator[field]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// there is no id, just use the user provided filters
|
|
||||||
if (!idCopy || !table) {
|
|
||||||
return filters
|
|
||||||
}
|
|
||||||
// if used as URL parameter it will have been joined
|
|
||||||
if (!Array.isArray(idCopy)) {
|
|
||||||
idCopy = breakRowIdField(idCopy)
|
|
||||||
}
|
|
||||||
const equal: any = {}
|
|
||||||
if (primary && idCopy) {
|
|
||||||
for (let field of primary) {
|
|
||||||
// work through the ID and get the parts
|
|
||||||
equal[field] = idCopy.shift()
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
equal,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function removeManyToManyRelationships(
|
|
||||||
rowId: string,
|
|
||||||
table: Table,
|
|
||||||
colName: string
|
|
||||||
) {
|
|
||||||
const tableId = table._id!
|
|
||||||
const filters = buildFilters(rowId, {}, table)
|
|
||||||
// safety check, if there are no filters on deletion bad things happen
|
|
||||||
if (Object.keys(filters).length !== 0) {
|
|
||||||
return getDatasourceAndQuery({
|
|
||||||
endpoint: getEndpoint(tableId, Operation.DELETE),
|
|
||||||
body: { [colName]: null },
|
|
||||||
filters,
|
|
||||||
meta: {
|
|
||||||
table,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function removeOneToManyRelationships(rowId: string, table: Table) {
|
|
||||||
const tableId = table._id!
|
|
||||||
const filters = buildFilters(rowId, {}, table)
|
|
||||||
// safety check, if there are no filters on deletion bad things happen
|
|
||||||
if (Object.keys(filters).length !== 0) {
|
|
||||||
return getDatasourceAndQuery({
|
|
||||||
endpoint: getEndpoint(tableId, Operation.UPDATE),
|
|
||||||
filters,
|
|
||||||
meta: {
|
|
||||||
table,
|
|
||||||
},
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
return []
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* This function checks the incoming parameters to make sure all the inputs are
|
* This function checks the incoming parameters to make sure all the inputs are
|
||||||
* valid based on on the table schema. The main thing this is looking for is when a
|
* valid based on on the table schema. The main thing this is looking for is when a
|
||||||
|
@ -212,8 +126,8 @@ function getEndpoint(tableId: string | undefined, operation: string) {
|
||||||
}
|
}
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
return {
|
return {
|
||||||
datasourceId: datasourceId!,
|
datasourceId: datasourceId,
|
||||||
entityId: tableName!,
|
entityId: tableName,
|
||||||
operation: operation as Operation,
|
operation: operation as Operation,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -240,6 +154,7 @@ export class ExternalRequest<T extends Operation> {
|
||||||
private readonly tableId: string
|
private readonly tableId: string
|
||||||
private datasource?: Datasource
|
private datasource?: Datasource
|
||||||
private tables: { [key: string]: Table } = {}
|
private tables: { [key: string]: Table } = {}
|
||||||
|
private tableList: Table[]
|
||||||
|
|
||||||
constructor(operation: T, tableId: string, datasource?: Datasource) {
|
constructor(operation: T, tableId: string, datasource?: Datasource) {
|
||||||
this.operation = operation
|
this.operation = operation
|
||||||
|
@ -248,22 +163,134 @@ export class ExternalRequest<T extends Operation> {
|
||||||
if (datasource && datasource.entities) {
|
if (datasource && datasource.entities) {
|
||||||
this.tables = datasource.entities
|
this.tables = datasource.entities
|
||||||
}
|
}
|
||||||
|
this.tableList = Object.values(this.tables)
|
||||||
|
}
|
||||||
|
|
||||||
|
private prepareFilters(
|
||||||
|
id: string | undefined | string[],
|
||||||
|
filters: SearchFilters,
|
||||||
|
table: Table
|
||||||
|
): SearchFilters {
|
||||||
|
// replace any relationship columns initially, table names and relationship column names are acceptable
|
||||||
|
const relationshipColumns = sdk.rows.filters.getRelationshipColumns(table)
|
||||||
|
filters = sdk.rows.filters.updateFilterKeys(
|
||||||
|
filters,
|
||||||
|
relationshipColumns.map(({ name, definition }) => {
|
||||||
|
const { tableName } = breakExternalTableId(definition.tableId)
|
||||||
|
return {
|
||||||
|
original: name,
|
||||||
|
updated: tableName,
|
||||||
|
}
|
||||||
|
})
|
||||||
|
)
|
||||||
|
const primary = table.primary
|
||||||
|
// if passed in array need to copy for shifting etc
|
||||||
|
let idCopy: undefined | string | any[] = cloneDeep(id)
|
||||||
|
if (filters) {
|
||||||
|
// need to map over the filters and make sure the _id field isn't present
|
||||||
|
let prefix = 1
|
||||||
|
for (let operator of Object.values(filters)) {
|
||||||
|
for (let field of Object.keys(operator || {})) {
|
||||||
|
if (dbCore.removeKeyNumbering(field) === "_id") {
|
||||||
|
if (primary) {
|
||||||
|
const parts = breakRowIdField(operator[field])
|
||||||
|
for (let field of primary) {
|
||||||
|
operator[`${prefix}:${field}`] = parts.shift()
|
||||||
|
}
|
||||||
|
prefix++
|
||||||
|
}
|
||||||
|
// make sure this field doesn't exist on any filter
|
||||||
|
delete operator[field]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// there is no id, just use the user provided filters
|
||||||
|
if (!idCopy || !table) {
|
||||||
|
return filters
|
||||||
|
}
|
||||||
|
// if used as URL parameter it will have been joined
|
||||||
|
if (!Array.isArray(idCopy)) {
|
||||||
|
idCopy = breakRowIdField(idCopy)
|
||||||
|
}
|
||||||
|
const equal: SearchFilters["equal"] = {}
|
||||||
|
if (primary && idCopy) {
|
||||||
|
for (let field of primary) {
|
||||||
|
// work through the ID and get the parts
|
||||||
|
equal[field] = idCopy.shift()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
equal,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async removeManyToManyRelationships(
|
||||||
|
rowId: string,
|
||||||
|
table: Table,
|
||||||
|
colName: string
|
||||||
|
) {
|
||||||
|
const tableId = table._id!
|
||||||
|
const filters = this.prepareFilters(rowId, {}, table)
|
||||||
|
// safety check, if there are no filters on deletion bad things happen
|
||||||
|
if (Object.keys(filters).length !== 0) {
|
||||||
|
return getDatasourceAndQuery({
|
||||||
|
endpoint: getEndpoint(tableId, Operation.DELETE),
|
||||||
|
body: { [colName]: null },
|
||||||
|
filters,
|
||||||
|
meta: {
|
||||||
|
table,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
return []
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private async removeOneToManyRelationships(rowId: string, table: Table) {
|
||||||
|
const tableId = table._id!
|
||||||
|
const filters = this.prepareFilters(rowId, {}, table)
|
||||||
|
// safety check, if there are no filters on deletion bad things happen
|
||||||
|
if (Object.keys(filters).length !== 0) {
|
||||||
|
return getDatasourceAndQuery({
|
||||||
|
endpoint: getEndpoint(tableId, Operation.UPDATE),
|
||||||
|
filters,
|
||||||
|
meta: {
|
||||||
|
table,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
return []
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
getTable(tableId: string | undefined): Table | undefined {
|
getTable(tableId: string | undefined): Table | undefined {
|
||||||
if (!tableId) {
|
if (!tableId) {
|
||||||
throw "Table ID is unknown, cannot find table"
|
throw new Error("Table ID is unknown, cannot find table")
|
||||||
}
|
}
|
||||||
const { tableName } = breakExternalTableId(tableId)
|
const { tableName } = breakExternalTableId(tableId)
|
||||||
if (tableName) {
|
|
||||||
return this.tables[tableName]
|
return this.tables[tableName]
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// seeds the object with table and datasource information
|
||||||
|
async retrieveMetadata(
|
||||||
|
datasourceId: string
|
||||||
|
): Promise<{ tables: Record<string, Table>; datasource: Datasource }> {
|
||||||
|
if (!this.datasource) {
|
||||||
|
this.datasource = await sdk.datasources.get(datasourceId)
|
||||||
|
if (!this.datasource || !this.datasource.entities) {
|
||||||
|
throw "No tables found, fetch tables before query."
|
||||||
|
}
|
||||||
|
this.tables = this.datasource.entities
|
||||||
|
this.tableList = Object.values(this.tables)
|
||||||
|
}
|
||||||
|
return { tables: this.tables, datasource: this.datasource }
|
||||||
}
|
}
|
||||||
|
|
||||||
async getRow(table: Table, rowId: string): Promise<Row> {
|
async getRow(table: Table, rowId: string): Promise<Row> {
|
||||||
const response = await getDatasourceAndQuery({
|
const response = await getDatasourceAndQuery({
|
||||||
endpoint: getEndpoint(table._id!, Operation.READ),
|
endpoint: getEndpoint(table._id!, Operation.READ),
|
||||||
filters: buildFilters(rowId, {}, table),
|
filters: this.prepareFilters(rowId, {}, table),
|
||||||
meta: {
|
meta: {
|
||||||
table,
|
table,
|
||||||
},
|
},
|
||||||
|
@ -289,16 +316,20 @@ export class ExternalRequest<T extends Operation> {
|
||||||
manyRelationships: ManyRelationship[] = []
|
manyRelationships: ManyRelationship[] = []
|
||||||
for (let [key, field] of Object.entries(table.schema)) {
|
for (let [key, field] of Object.entries(table.schema)) {
|
||||||
// if set already, or not set just skip it
|
// if set already, or not set just skip it
|
||||||
if (row[key] === undefined || newRow[key] || !isEditableColumn(field)) {
|
if (row[key] === undefined || newRow[key]) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if (
|
||||||
|
!(this.operation === Operation.BULK_UPSERT) &&
|
||||||
|
!isEditableColumn(field)
|
||||||
|
) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// parse floats/numbers
|
// parse floats/numbers
|
||||||
if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
|
if (field.type === FieldType.NUMBER && !isNaN(parseFloat(row[key]))) {
|
||||||
newRow[key] = parseFloat(row[key])
|
newRow[key] = parseFloat(row[key])
|
||||||
} else if (field.type === FieldType.LINK) {
|
} else if (field.type === FieldType.LINK) {
|
||||||
const { tableName: linkTableName } = breakExternalTableId(
|
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
|
||||||
field?.tableId
|
|
||||||
)
|
|
||||||
// table has to exist for many to many
|
// table has to exist for many to many
|
||||||
if (!linkTableName || !this.tables[linkTableName]) {
|
if (!linkTableName || !this.tables[linkTableName]) {
|
||||||
continue
|
continue
|
||||||
|
@ -379,9 +410,6 @@ export class ExternalRequest<T extends Operation> {
|
||||||
[key: string]: { rows: Row[]; isMany: boolean; tableId: string }
|
[key: string]: { rows: Row[]; isMany: boolean; tableId: string }
|
||||||
} = {}
|
} = {}
|
||||||
const { tableName } = breakExternalTableId(tableId)
|
const { tableName } = breakExternalTableId(tableId)
|
||||||
if (!tableName) {
|
|
||||||
return related
|
|
||||||
}
|
|
||||||
const table = this.tables[tableName]
|
const table = this.tables[tableName]
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
const primaryKey = table.primary[0]
|
const primaryKey = table.primary[0]
|
||||||
|
@ -514,7 +542,7 @@ export class ExternalRequest<T extends Operation> {
|
||||||
endpoint: getEndpoint(tableId, operation),
|
endpoint: getEndpoint(tableId, operation),
|
||||||
// if we're doing many relationships then we're writing, only one response
|
// if we're doing many relationships then we're writing, only one response
|
||||||
body,
|
body,
|
||||||
filters: buildFilters(id, {}, linkTable),
|
filters: this.prepareFilters(id, {}, linkTable),
|
||||||
meta: {
|
meta: {
|
||||||
table: linkTable,
|
table: linkTable,
|
||||||
},
|
},
|
||||||
|
@ -538,8 +566,8 @@ export class ExternalRequest<T extends Operation> {
|
||||||
for (let row of rows) {
|
for (let row of rows) {
|
||||||
const rowId = generateIdForRow(row, table)
|
const rowId = generateIdForRow(row, table)
|
||||||
const promise: Promise<any> = isMany
|
const promise: Promise<any> = isMany
|
||||||
? removeManyToManyRelationships(rowId, table, colName)
|
? this.removeManyToManyRelationships(rowId, table, colName)
|
||||||
: removeOneToManyRelationships(rowId, table)
|
: this.removeOneToManyRelationships(rowId, table)
|
||||||
if (promise) {
|
if (promise) {
|
||||||
promises.push(promise)
|
promises.push(promise)
|
||||||
}
|
}
|
||||||
|
@ -562,12 +590,12 @@ export class ExternalRequest<T extends Operation> {
|
||||||
rows.map(row => {
|
rows.map(row => {
|
||||||
const rowId = generateIdForRow(row, table)
|
const rowId = generateIdForRow(row, table)
|
||||||
return isMany
|
return isMany
|
||||||
? removeManyToManyRelationships(
|
? this.removeManyToManyRelationships(
|
||||||
rowId,
|
rowId,
|
||||||
table,
|
table,
|
||||||
relationshipColumn.fieldName
|
relationshipColumn.fieldName
|
||||||
)
|
)
|
||||||
: removeOneToManyRelationships(rowId, table)
|
: this.removeOneToManyRelationships(rowId, table)
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
@ -575,21 +603,21 @@ export class ExternalRequest<T extends Operation> {
|
||||||
|
|
||||||
async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
|
async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
|
||||||
const { operation, tableId } = this
|
const { operation, tableId } = this
|
||||||
|
if (!tableId) {
|
||||||
|
throw new Error("Unable to run without a table ID")
|
||||||
|
}
|
||||||
let { datasourceId, tableName } = breakExternalTableId(tableId)
|
let { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
if (!tableName) {
|
let datasource = this.datasource
|
||||||
throw "Unable to run without a table name"
|
if (!datasource) {
|
||||||
}
|
const { datasource: ds } = await this.retrieveMetadata(datasourceId)
|
||||||
if (!this.datasource) {
|
datasource = ds
|
||||||
this.datasource = await sdk.datasources.get(datasourceId!)
|
|
||||||
if (!this.datasource || !this.datasource.entities) {
|
|
||||||
throw "No tables found, fetch tables before query."
|
|
||||||
}
|
|
||||||
this.tables = this.datasource.entities
|
|
||||||
}
|
}
|
||||||
const table = this.tables[tableName]
|
const table = this.tables[tableName]
|
||||||
let isSql = isSQL(this.datasource)
|
let isSql = isSQL(datasource)
|
||||||
if (!table) {
|
if (!table) {
|
||||||
throw `Unable to process query, table "${tableName}" not defined.`
|
throw new Error(
|
||||||
|
`Unable to process query, table "${tableName}" not defined.`
|
||||||
|
)
|
||||||
}
|
}
|
||||||
// look for specific components of config which may not be considered acceptable
|
// look for specific components of config which may not be considered acceptable
|
||||||
let { id, row, filters, sort, paginate, rows } = cleanupConfig(
|
let { id, row, filters, sort, paginate, rows } = cleanupConfig(
|
||||||
|
@ -612,7 +640,7 @@ export class ExternalRequest<T extends Operation> {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
filters = buildFilters(id, filters || {}, table)
|
filters = this.prepareFilters(id, filters || {}, table)
|
||||||
const relationships = buildExternalRelationships(table, this.tables)
|
const relationships = buildExternalRelationships(table, this.tables)
|
||||||
|
|
||||||
const incRelationships =
|
const incRelationships =
|
||||||
|
@ -660,10 +688,15 @@ export class ExternalRequest<T extends Operation> {
|
||||||
body: row || rows,
|
body: row || rows,
|
||||||
// pass an id filter into extra, purely for mysql/returning
|
// pass an id filter into extra, purely for mysql/returning
|
||||||
extra: {
|
extra: {
|
||||||
idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
|
idFilter: this.prepareFilters(
|
||||||
|
id || generateIdForRow(row, table),
|
||||||
|
{},
|
||||||
|
table
|
||||||
|
),
|
||||||
},
|
},
|
||||||
meta: {
|
meta: {
|
||||||
table,
|
table,
|
||||||
|
id: config.id,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -136,10 +136,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
|
||||||
const id = ctx.params.rowId
|
const id = ctx.params.rowId
|
||||||
const tableId = utils.getTableId(ctx)
|
const tableId = utils.getTableId(ctx)
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
const datasource: Datasource = await sdk.datasources.get(datasourceId!)
|
const datasource: Datasource = await sdk.datasources.get(datasourceId)
|
||||||
if (!tableName) {
|
|
||||||
ctx.throw(400, "Unable to find table.")
|
|
||||||
}
|
|
||||||
if (!datasource || !datasource.entities) {
|
if (!datasource || !datasource.entities) {
|
||||||
ctx.throw(400, "Datasource has not been configured for plus API.")
|
ctx.throw(400, "Datasource has not been configured for plus API.")
|
||||||
}
|
}
|
||||||
|
@ -163,7 +160,7 @@ export async function fetchEnrichedRow(ctx: UserCtx) {
|
||||||
}
|
}
|
||||||
const links = row[fieldName]
|
const links = row[fieldName]
|
||||||
const linkedTableId = field.tableId
|
const linkedTableId = field.tableId
|
||||||
const linkedTableName = breakExternalTableId(linkedTableId).tableName!
|
const linkedTableName = breakExternalTableId(linkedTableId).tableName
|
||||||
const linkedTable = tables[linkedTableName]
|
const linkedTable = tables[linkedTableName]
|
||||||
// don't support composite keys right now
|
// don't support composite keys right now
|
||||||
const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
|
const linkedIds = links.map((link: Row) => breakRowIdField(link._id!)[0])
|
||||||
|
|
|
@ -99,7 +99,7 @@ export function basicProcessing({
|
||||||
row,
|
row,
|
||||||
tableName: table._id!,
|
tableName: table._id!,
|
||||||
fieldName: internalColumn,
|
fieldName: internalColumn,
|
||||||
isLinked: false,
|
isLinked,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,6 +2,8 @@ import {
|
||||||
DatasourcePlusQueryResponse,
|
DatasourcePlusQueryResponse,
|
||||||
DSPlusOperation,
|
DSPlusOperation,
|
||||||
FieldType,
|
FieldType,
|
||||||
|
isManyToOne,
|
||||||
|
isOneToMany,
|
||||||
ManyToManyRelationshipFieldMetadata,
|
ManyToManyRelationshipFieldMetadata,
|
||||||
RelationshipFieldMetadata,
|
RelationshipFieldMetadata,
|
||||||
RelationshipsJson,
|
RelationshipsJson,
|
||||||
|
@ -93,12 +95,12 @@ export function buildExternalRelationships(
|
||||||
): RelationshipsJson[] {
|
): RelationshipsJson[] {
|
||||||
const relationships = []
|
const relationships = []
|
||||||
for (let [fieldName, field] of Object.entries(table.schema)) {
|
for (let [fieldName, field] of Object.entries(table.schema)) {
|
||||||
if (field.type !== FieldType.LINK) {
|
if (field.type !== FieldType.LINK || !field.tableId) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
|
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
|
||||||
// no table to link to, this is not a valid relationships
|
// no table to link to, this is not a valid relationships
|
||||||
if (!linkTableName || !tables[linkTableName]) {
|
if (!tables[linkTableName]) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
const linkTable = tables[linkTableName]
|
const linkTable = tables[linkTableName]
|
||||||
|
@ -110,7 +112,7 @@ export function buildExternalRelationships(
|
||||||
// need to specify where to put this back into
|
// need to specify where to put this back into
|
||||||
column: fieldName,
|
column: fieldName,
|
||||||
}
|
}
|
||||||
if (isManyToMany(field)) {
|
if (isManyToMany(field) && field.through) {
|
||||||
const { tableName: throughTableName } = breakExternalTableId(
|
const { tableName: throughTableName } = breakExternalTableId(
|
||||||
field.through
|
field.through
|
||||||
)
|
)
|
||||||
|
@ -120,7 +122,7 @@ export function buildExternalRelationships(
|
||||||
definition.to = field.throughFrom || linkTable.primary[0]
|
definition.to = field.throughFrom || linkTable.primary[0]
|
||||||
definition.fromPrimary = table.primary[0]
|
definition.fromPrimary = table.primary[0]
|
||||||
definition.toPrimary = linkTable.primary[0]
|
definition.toPrimary = linkTable.primary[0]
|
||||||
} else {
|
} else if (isManyToOne(field) || isOneToMany(field)) {
|
||||||
// if no foreign key specified then use the name of the field in other table
|
// if no foreign key specified then use the name of the field in other table
|
||||||
definition.from = field.foreignKey || table.primary[0]
|
definition.from = field.foreignKey || table.primary[0]
|
||||||
definition.to = field.fieldName
|
definition.to = field.fieldName
|
||||||
|
@ -180,18 +182,20 @@ export function buildSqlFieldList(
|
||||||
}
|
}
|
||||||
let fields = extractRealFields(table)
|
let fields = extractRealFields(table)
|
||||||
for (let field of Object.values(table.schema)) {
|
for (let field of Object.values(table.schema)) {
|
||||||
if (field.type !== FieldType.LINK || !opts?.relationships) {
|
if (
|
||||||
|
field.type !== FieldType.LINK ||
|
||||||
|
!opts?.relationships ||
|
||||||
|
!field.tableId
|
||||||
|
) {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
|
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
|
||||||
if (linkTableName) {
|
|
||||||
const linkTable = tables[linkTableName]
|
const linkTable = tables[linkTableName]
|
||||||
if (linkTable) {
|
if (linkTable) {
|
||||||
const linkedFields = extractRealFields(linkTable, fields)
|
const linkedFields = extractRealFields(linkTable, fields)
|
||||||
fields = fields.concat(linkedFields)
|
fields = fields.concat(linkedFields)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
|
||||||
return fields
|
return fields
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -16,14 +16,18 @@ import {
|
||||||
import sdk from "../../../sdk"
|
import sdk from "../../../sdk"
|
||||||
import { builderSocket } from "../../../websockets"
|
import { builderSocket } from "../../../websockets"
|
||||||
import { inputProcessing } from "../../../utilities/rowProcessor"
|
import { inputProcessing } from "../../../utilities/rowProcessor"
|
||||||
|
import { isEqual } from "lodash"
|
||||||
|
|
||||||
function getDatasourceId(table: Table) {
|
function getDatasourceId(table: Table) {
|
||||||
if (!table) {
|
if (!table) {
|
||||||
throw "No table supplied"
|
throw new Error("No table supplied")
|
||||||
}
|
}
|
||||||
if (table.sourceId) {
|
if (table.sourceId) {
|
||||||
return table.sourceId
|
return table.sourceId
|
||||||
}
|
}
|
||||||
|
if (!table._id) {
|
||||||
|
throw new Error("No table ID supplied")
|
||||||
|
}
|
||||||
return breakExternalTableId(table._id).datasourceId
|
return breakExternalTableId(table._id).datasourceId
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -82,15 +86,30 @@ export async function bulkImport(
|
||||||
ctx: UserCtx<BulkImportRequest, BulkImportResponse>
|
ctx: UserCtx<BulkImportRequest, BulkImportResponse>
|
||||||
) {
|
) {
|
||||||
let table = await sdk.tables.getTable(ctx.params.tableId)
|
let table = await sdk.tables.getTable(ctx.params.tableId)
|
||||||
const { rows } = ctx.request.body
|
const { rows, identifierFields } = ctx.request.body
|
||||||
const schema = table.schema
|
const schema = table.schema
|
||||||
|
|
||||||
|
if (
|
||||||
|
identifierFields &&
|
||||||
|
identifierFields.length > 0 &&
|
||||||
|
!isEqual(identifierFields, table.primary)
|
||||||
|
) {
|
||||||
|
// This is becuse we make use of the ON CONFLICT functionality in SQL
|
||||||
|
// databases, which only triggers when there's a conflict against a unique
|
||||||
|
// index. The only unique index we can count on atm in Budibase is the
|
||||||
|
// primary key, so this functionality always uses the primary key.
|
||||||
|
ctx.throw(
|
||||||
|
400,
|
||||||
|
"Identifier fields are not supported for bulk import into an external datasource."
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
if (!rows || !isRows(rows) || !isSchema(schema)) {
|
if (!rows || !isRows(rows) || !isSchema(schema)) {
|
||||||
ctx.throw(400, "Provided data import information is invalid.")
|
ctx.throw(400, "Provided data import information is invalid.")
|
||||||
}
|
}
|
||||||
|
|
||||||
const parsedRows = []
|
const parsedRows = []
|
||||||
for (const row of parse(rows, schema)) {
|
for (const row of parse(rows, table)) {
|
||||||
const processed = await inputProcessing(ctx.user?._id, table, row, {
|
const processed = await inputProcessing(ctx.user?._id, table, row, {
|
||||||
noAutoRelationships: true,
|
noAutoRelationships: true,
|
||||||
})
|
})
|
||||||
|
|
|
@ -178,7 +178,7 @@ export async function handleDataImport(
|
||||||
}
|
}
|
||||||
|
|
||||||
const db = context.getAppDB()
|
const db = context.getAppDB()
|
||||||
const data = parse(importRows, schema)
|
const data = parse(importRows, table)
|
||||||
|
|
||||||
let finalData: any = await importToRows(data, table, user)
|
let finalData: any = await importToRows(data, table, user)
|
||||||
|
|
||||||
|
|
|
@ -250,6 +250,67 @@ describe.each(
|
||||||
expect(events.query.previewed).toHaveBeenCalledTimes(1)
|
expect(events.query.previewed).toHaveBeenCalledTimes(1)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("should update schema when column type changes from number to string", async () => {
|
||||||
|
const tableName = "schema_change_test"
|
||||||
|
await client.schema.dropTableIfExists(tableName)
|
||||||
|
|
||||||
|
await client.schema.createTable(tableName, table => {
|
||||||
|
table.increments("id").primary()
|
||||||
|
table.string("name")
|
||||||
|
table.integer("data")
|
||||||
|
})
|
||||||
|
|
||||||
|
await client(tableName).insert({
|
||||||
|
name: "test",
|
||||||
|
data: 123,
|
||||||
|
})
|
||||||
|
|
||||||
|
const firstPreview = await config.api.query.preview({
|
||||||
|
datasourceId: datasource._id!,
|
||||||
|
name: "Test Query",
|
||||||
|
queryVerb: "read",
|
||||||
|
fields: {
|
||||||
|
sql: `SELECT * FROM ${tableName}`,
|
||||||
|
},
|
||||||
|
parameters: [],
|
||||||
|
transformer: "return data",
|
||||||
|
schema: {},
|
||||||
|
readable: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(firstPreview.schema).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
data: { type: "number", name: "data" },
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
await client.schema.alterTable(tableName, table => {
|
||||||
|
table.string("data").alter()
|
||||||
|
})
|
||||||
|
|
||||||
|
await client(tableName).update({
|
||||||
|
data: "string value",
|
||||||
|
})
|
||||||
|
|
||||||
|
const secondPreview = await config.api.query.preview({
|
||||||
|
datasourceId: datasource._id!,
|
||||||
|
name: "Test Query",
|
||||||
|
queryVerb: "read",
|
||||||
|
fields: {
|
||||||
|
sql: `SELECT * FROM ${tableName}`,
|
||||||
|
},
|
||||||
|
parameters: [],
|
||||||
|
transformer: "return data",
|
||||||
|
schema: firstPreview.schema,
|
||||||
|
readable: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(secondPreview.schema).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
data: { type: "string", name: "data" },
|
||||||
|
})
|
||||||
|
)
|
||||||
|
})
|
||||||
it("should work with static variables", async () => {
|
it("should work with static variables", async () => {
|
||||||
await config.api.datasource.update({
|
await config.api.datasource.update({
|
||||||
...datasource,
|
...datasource,
|
||||||
|
|
|
@ -137,6 +137,67 @@ describe("/queries", () => {
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("should update schema when structure changes from object to array", async () => {
|
||||||
|
const name = generator.guid()
|
||||||
|
|
||||||
|
await withCollection(async collection => {
|
||||||
|
await collection.insertOne({ name, field: { subfield: "value" } })
|
||||||
|
})
|
||||||
|
|
||||||
|
const firstPreview = await config.api.query.preview({
|
||||||
|
name: "Test Query",
|
||||||
|
datasourceId: datasource._id!,
|
||||||
|
fields: {
|
||||||
|
json: { name: { $eq: name } },
|
||||||
|
extra: {
|
||||||
|
collection,
|
||||||
|
actionType: "findOne",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
schema: {},
|
||||||
|
queryVerb: "read",
|
||||||
|
parameters: [],
|
||||||
|
transformer: "return data",
|
||||||
|
readable: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(firstPreview.schema).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
field: { type: "json", name: "field" },
|
||||||
|
})
|
||||||
|
)
|
||||||
|
|
||||||
|
await withCollection(async collection => {
|
||||||
|
await collection.updateOne(
|
||||||
|
{ name },
|
||||||
|
{ $set: { field: ["value1", "value2"] } }
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
|
const secondPreview = await config.api.query.preview({
|
||||||
|
name: "Test Query",
|
||||||
|
datasourceId: datasource._id!,
|
||||||
|
fields: {
|
||||||
|
json: { name: { $eq: name } },
|
||||||
|
extra: {
|
||||||
|
collection,
|
||||||
|
actionType: "findOne",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
schema: firstPreview.schema,
|
||||||
|
queryVerb: "read",
|
||||||
|
parameters: [],
|
||||||
|
transformer: "return data",
|
||||||
|
readable: true,
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(secondPreview.schema).toEqual(
|
||||||
|
expect.objectContaining({
|
||||||
|
field: { type: "array", name: "field" },
|
||||||
|
})
|
||||||
|
)
|
||||||
|
})
|
||||||
|
|
||||||
it("should generate a nested schema based on all of the nested items", async () => {
|
it("should generate a nested schema based on all of the nested items", async () => {
|
||||||
const name = generator.guid()
|
const name = generator.guid()
|
||||||
const item = {
|
const item = {
|
||||||
|
|
|
@ -92,6 +92,61 @@ describe("rest", () => {
|
||||||
expect(cached.rows[0].name).toEqual("one")
|
expect(cached.rows[0].name).toEqual("one")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("should update schema when structure changes from JSON to array", async () => {
|
||||||
|
const datasource = await config.api.datasource.create({
|
||||||
|
name: generator.guid(),
|
||||||
|
type: "test",
|
||||||
|
source: SourceName.REST,
|
||||||
|
config: {},
|
||||||
|
})
|
||||||
|
|
||||||
|
nock("http://www.example.com")
|
||||||
|
.get("/")
|
||||||
|
.reply(200, [{ obj: {}, id: "1" }])
|
||||||
|
|
||||||
|
const firstResponse = await config.api.query.preview({
|
||||||
|
datasourceId: datasource._id!,
|
||||||
|
name: "test query",
|
||||||
|
parameters: [],
|
||||||
|
queryVerb: "read",
|
||||||
|
transformer: "",
|
||||||
|
schema: {},
|
||||||
|
readable: true,
|
||||||
|
fields: {
|
||||||
|
path: "www.example.com",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(firstResponse.schema).toEqual({
|
||||||
|
obj: { type: "json", name: "obj" },
|
||||||
|
id: { type: "string", name: "id" },
|
||||||
|
})
|
||||||
|
|
||||||
|
nock.cleanAll()
|
||||||
|
|
||||||
|
nock("http://www.example.com")
|
||||||
|
.get("/")
|
||||||
|
.reply(200, [{ obj: [], id: "1" }])
|
||||||
|
|
||||||
|
const secondResponse = await config.api.query.preview({
|
||||||
|
datasourceId: datasource._id!,
|
||||||
|
name: "test query",
|
||||||
|
parameters: [],
|
||||||
|
queryVerb: "read",
|
||||||
|
transformer: "",
|
||||||
|
schema: firstResponse.schema,
|
||||||
|
readable: true,
|
||||||
|
fields: {
|
||||||
|
path: "www.example.com",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(secondResponse.schema).toEqual({
|
||||||
|
obj: { type: "array", name: "obj" },
|
||||||
|
id: { type: "string", name: "id" },
|
||||||
|
})
|
||||||
|
})
|
||||||
|
|
||||||
it("should parse global and query level header mappings", async () => {
|
it("should parse global and query level header mappings", async () => {
|
||||||
const datasource = await config.api.datasource.create({
|
const datasource = await config.api.datasource.create({
|
||||||
name: generator.guid(),
|
name: generator.guid(),
|
||||||
|
|
|
@ -1,4 +1,8 @@
|
||||||
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
|
import {
|
||||||
|
DatabaseName,
|
||||||
|
getDatasource,
|
||||||
|
knexClient,
|
||||||
|
} from "../../../integrations/tests/utils"
|
||||||
|
|
||||||
import tk from "timekeeper"
|
import tk from "timekeeper"
|
||||||
import emitter from "../../../../src/events"
|
import emitter from "../../../../src/events"
|
||||||
|
@ -31,6 +35,7 @@ import {
|
||||||
import { generator, mocks } from "@budibase/backend-core/tests"
|
import { generator, mocks } from "@budibase/backend-core/tests"
|
||||||
import _, { merge } from "lodash"
|
import _, { merge } from "lodash"
|
||||||
import * as uuid from "uuid"
|
import * as uuid from "uuid"
|
||||||
|
import { Knex } from "knex"
|
||||||
|
|
||||||
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
|
const timestamp = new Date("2023-01-26T11:48:57.597Z").toISOString()
|
||||||
tk.freeze(timestamp)
|
tk.freeze(timestamp)
|
||||||
|
@ -70,13 +75,16 @@ describe.each([
|
||||||
|
|
||||||
let table: Table
|
let table: Table
|
||||||
let datasource: Datasource | undefined
|
let datasource: Datasource | undefined
|
||||||
|
let client: Knex | undefined
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
await config.init()
|
await config.init()
|
||||||
if (dsProvider) {
|
if (dsProvider) {
|
||||||
|
const rawDatasource = await dsProvider
|
||||||
datasource = await config.createDatasource({
|
datasource = await config.createDatasource({
|
||||||
datasource: await dsProvider,
|
datasource: rawDatasource,
|
||||||
})
|
})
|
||||||
|
client = await knexClient(rawDatasource)
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -307,13 +315,13 @@ describe.each([
|
||||||
// as quickly as possible.
|
// as quickly as possible.
|
||||||
await Promise.all(
|
await Promise.all(
|
||||||
sequence.map(async () => {
|
sequence.map(async () => {
|
||||||
const attempts = 20
|
const attempts = 30
|
||||||
for (let attempt = 0; attempt < attempts; attempt++) {
|
for (let attempt = 0; attempt < attempts; attempt++) {
|
||||||
try {
|
try {
|
||||||
await config.api.row.save(table._id!, {})
|
await config.api.row.save(table._id!, {})
|
||||||
return
|
return
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
await new Promise(r => setTimeout(r, Math.random() * 15))
|
await new Promise(r => setTimeout(r, Math.random() * 50))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
throw new Error(`Failed to create row after ${attempts} attempts`)
|
throw new Error(`Failed to create row after ${attempts} attempts`)
|
||||||
|
@ -598,6 +606,35 @@ describe.each([
|
||||||
expect(res.name).toEqual("Updated Name")
|
expect(res.name).toEqual("Updated Name")
|
||||||
await assertRowUsage(rowUsage)
|
await assertRowUsage(rowUsage)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
!isInternal &&
|
||||||
|
it("can update a row on an external table with a primary key", async () => {
|
||||||
|
const tableName = uuid.v4().substring(0, 10)
|
||||||
|
await client!.schema.createTable(tableName, table => {
|
||||||
|
table.increments("id").primary()
|
||||||
|
table.string("name")
|
||||||
|
})
|
||||||
|
|
||||||
|
const res = await config.api.datasource.fetchSchema({
|
||||||
|
datasourceId: datasource!._id!,
|
||||||
|
})
|
||||||
|
const table = res.datasource.entities![tableName]
|
||||||
|
|
||||||
|
const row = await config.api.row.save(table._id!, {
|
||||||
|
id: 1,
|
||||||
|
name: "Row 1",
|
||||||
|
})
|
||||||
|
|
||||||
|
const updatedRow = await config.api.row.save(table._id!, {
|
||||||
|
_id: row._id!,
|
||||||
|
name: "Row 1 Updated",
|
||||||
|
})
|
||||||
|
|
||||||
|
expect(updatedRow.name).toEqual("Row 1 Updated")
|
||||||
|
|
||||||
|
const rows = await config.api.row.fetch(table._id!)
|
||||||
|
expect(rows).toHaveLength(1)
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("patch", () => {
|
describe("patch", () => {
|
||||||
|
@ -667,6 +704,7 @@ describe.each([
|
||||||
expect(event.oldRow.description).toEqual(beforeRow.description)
|
expect(event.oldRow.description).toEqual(beforeRow.description)
|
||||||
expect(event.row.description).toEqual(beforeRow.description)
|
expect(event.row.description).toEqual(beforeRow.description)
|
||||||
})
|
})
|
||||||
|
|
||||||
it("should throw an error when given improper types", async () => {
|
it("should throw an error when given improper types", async () => {
|
||||||
const existing = await config.api.row.save(table._id!, {})
|
const existing = await config.api.row.save(table._id!, {})
|
||||||
const rowUsage = await getRowUsage()
|
const rowUsage = await getRowUsage()
|
||||||
|
@ -758,7 +796,8 @@ describe.each([
|
||||||
})
|
})
|
||||||
|
|
||||||
!isInternal &&
|
!isInternal &&
|
||||||
// TODO: SQL is having issues creating composite keys
|
// MSSQL needs a setting called IDENTITY_INSERT to be set to ON to allow writing
|
||||||
|
// to identity columns. This is not something Budibase does currently.
|
||||||
providerType !== DatabaseName.SQL_SERVER &&
|
providerType !== DatabaseName.SQL_SERVER &&
|
||||||
it("should support updating fields that are part of a composite key", async () => {
|
it("should support updating fields that are part of a composite key", async () => {
|
||||||
const tableRequest = saveTableRequest({
|
const tableRequest = saveTableRequest({
|
||||||
|
@ -911,24 +950,12 @@ describe.each([
|
||||||
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
|
await assertRowUsage(isInternal ? rowUsage - 1 : rowUsage)
|
||||||
})
|
})
|
||||||
|
|
||||||
it("Should ignore malformed/invalid delete requests", async () => {
|
it.each([{ not: "valid" }, { rows: 123 }, "invalid"])(
|
||||||
|
"Should ignore malformed/invalid delete request: %s",
|
||||||
|
async (request: any) => {
|
||||||
const rowUsage = await getRowUsage()
|
const rowUsage = await getRowUsage()
|
||||||
|
|
||||||
await config.api.row.delete(table._id!, { not: "valid" } as any, {
|
await config.api.row.delete(table._id!, request, {
|
||||||
status: 400,
|
|
||||||
body: {
|
|
||||||
message: "Invalid delete rows request",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
await config.api.row.delete(table._id!, { rows: 123 } as any, {
|
|
||||||
status: 400,
|
|
||||||
body: {
|
|
||||||
message: "Invalid delete rows request",
|
|
||||||
},
|
|
||||||
})
|
|
||||||
|
|
||||||
await config.api.row.delete(table._id!, "invalid" as any, {
|
|
||||||
status: 400,
|
status: 400,
|
||||||
body: {
|
body: {
|
||||||
message: "Invalid delete rows request",
|
message: "Invalid delete rows request",
|
||||||
|
@ -936,7 +963,8 @@ describe.each([
|
||||||
})
|
})
|
||||||
|
|
||||||
await assertRowUsage(rowUsage)
|
await assertRowUsage(rowUsage)
|
||||||
})
|
}
|
||||||
|
)
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("bulkImport", () => {
|
describe("bulkImport", () => {
|
||||||
|
@ -1085,6 +1113,121 @@ describe.each([
|
||||||
expect(rows[2].name).toEqual("Row 3")
|
expect(rows[2].name).toEqual("Row 3")
|
||||||
expect(rows[2].description).toEqual("Row 3 description")
|
expect(rows[2].description).toEqual("Row 3 description")
|
||||||
})
|
})
|
||||||
|
|
||||||
|
// Upserting isn't yet supported in MSSQL, see:
|
||||||
|
// https://github.com/knex/knex/pull/6050
|
||||||
|
!isMSSQL &&
|
||||||
|
!isInternal &&
|
||||||
|
it("should be able to update existing rows with composite primary keys with bulkImport", async () => {
|
||||||
|
const tableName = uuid.v4()
|
||||||
|
await client?.schema.createTable(tableName, table => {
|
||||||
|
table.integer("companyId")
|
||||||
|
table.integer("userId")
|
||||||
|
table.string("name")
|
||||||
|
table.string("description")
|
||||||
|
table.primary(["companyId", "userId"])
|
||||||
|
})
|
||||||
|
|
||||||
|
const resp = await config.api.datasource.fetchSchema({
|
||||||
|
datasourceId: datasource!._id!,
|
||||||
|
})
|
||||||
|
const table = resp.datasource.entities![tableName]
|
||||||
|
|
||||||
|
const row1 = await config.api.row.save(table._id!, {
|
||||||
|
companyId: 1,
|
||||||
|
userId: 1,
|
||||||
|
name: "Row 1",
|
||||||
|
description: "Row 1 description",
|
||||||
|
})
|
||||||
|
|
||||||
|
const row2 = await config.api.row.save(table._id!, {
|
||||||
|
companyId: 1,
|
||||||
|
userId: 2,
|
||||||
|
name: "Row 2",
|
||||||
|
description: "Row 2 description",
|
||||||
|
})
|
||||||
|
|
||||||
|
await config.api.row.bulkImport(table._id!, {
|
||||||
|
identifierFields: ["companyId", "userId"],
|
||||||
|
rows: [
|
||||||
|
{
|
||||||
|
companyId: 1,
|
||||||
|
userId: row1.userId,
|
||||||
|
name: "Row 1 updated",
|
||||||
|
description: "Row 1 description updated",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
companyId: 1,
|
||||||
|
userId: row2.userId,
|
||||||
|
name: "Row 2 updated",
|
||||||
|
description: "Row 2 description updated",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
companyId: 1,
|
||||||
|
userId: 3,
|
||||||
|
name: "Row 3",
|
||||||
|
description: "Row 3 description",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
|
||||||
|
const rows = await config.api.row.fetch(table._id!)
|
||||||
|
expect(rows.length).toEqual(3)
|
||||||
|
|
||||||
|
rows.sort((a, b) => a.name.localeCompare(b.name))
|
||||||
|
expect(rows[0].name).toEqual("Row 1 updated")
|
||||||
|
expect(rows[0].description).toEqual("Row 1 description updated")
|
||||||
|
expect(rows[1].name).toEqual("Row 2 updated")
|
||||||
|
expect(rows[1].description).toEqual("Row 2 description updated")
|
||||||
|
expect(rows[2].name).toEqual("Row 3")
|
||||||
|
expect(rows[2].description).toEqual("Row 3 description")
|
||||||
|
})
|
||||||
|
|
||||||
|
// Upserting isn't yet supported in MSSQL, see:
|
||||||
|
// https://github.com/knex/knex/pull/6050
|
||||||
|
!isMSSQL &&
|
||||||
|
!isInternal &&
|
||||||
|
it("should be able to update existing rows an autoID primary key", async () => {
|
||||||
|
const tableName = uuid.v4()
|
||||||
|
await client!.schema.createTable(tableName, table => {
|
||||||
|
table.increments("userId").primary()
|
||||||
|
table.string("name")
|
||||||
|
})
|
||||||
|
|
||||||
|
const resp = await config.api.datasource.fetchSchema({
|
||||||
|
datasourceId: datasource!._id!,
|
||||||
|
})
|
||||||
|
const table = resp.datasource.entities![tableName]
|
||||||
|
|
||||||
|
const row1 = await config.api.row.save(table._id!, {
|
||||||
|
name: "Clare",
|
||||||
|
})
|
||||||
|
|
||||||
|
const row2 = await config.api.row.save(table._id!, {
|
||||||
|
name: "Jeff",
|
||||||
|
})
|
||||||
|
|
||||||
|
await config.api.row.bulkImport(table._id!, {
|
||||||
|
identifierFields: ["userId"],
|
||||||
|
rows: [
|
||||||
|
{
|
||||||
|
userId: row1.userId,
|
||||||
|
name: "Clare updated",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
userId: row2.userId,
|
||||||
|
name: "Jeff updated",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
})
|
||||||
|
|
||||||
|
const rows = await config.api.row.fetch(table._id!)
|
||||||
|
expect(rows.length).toEqual(2)
|
||||||
|
|
||||||
|
rows.sort((a, b) => a.name.localeCompare(b.name))
|
||||||
|
expect(rows[0].name).toEqual("Clare updated")
|
||||||
|
expect(rows[1].name).toEqual("Jeff updated")
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
describe("enrich", () => {
|
describe("enrich", () => {
|
||||||
|
|
|
@ -1,5 +1,9 @@
|
||||||
import { tableForDatasource } from "../../../tests/utilities/structures"
|
import { tableForDatasource } from "../../../tests/utilities/structures"
|
||||||
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
|
import {
|
||||||
|
DatabaseName,
|
||||||
|
getDatasource,
|
||||||
|
knexClient,
|
||||||
|
} from "../../../integrations/tests/utils"
|
||||||
import { db as dbCore, utils } from "@budibase/backend-core"
|
import { db as dbCore, utils } from "@budibase/backend-core"
|
||||||
|
|
||||||
import * as setup from "./utilities"
|
import * as setup from "./utilities"
|
||||||
|
@ -24,6 +28,8 @@ import _ from "lodash"
|
||||||
import tk from "timekeeper"
|
import tk from "timekeeper"
|
||||||
import { encodeJSBinding } from "@budibase/string-templates"
|
import { encodeJSBinding } from "@budibase/string-templates"
|
||||||
import { dataFilters } from "@budibase/shared-core"
|
import { dataFilters } from "@budibase/shared-core"
|
||||||
|
import { Knex } from "knex"
|
||||||
|
import { structures } from "@budibase/backend-core/tests"
|
||||||
|
|
||||||
describe.each([
|
describe.each([
|
||||||
["in-memory", undefined],
|
["in-memory", undefined],
|
||||||
|
@ -42,6 +48,7 @@ describe.each([
|
||||||
|
|
||||||
let envCleanup: (() => void) | undefined
|
let envCleanup: (() => void) | undefined
|
||||||
let datasource: Datasource | undefined
|
let datasource: Datasource | undefined
|
||||||
|
let client: Knex | undefined
|
||||||
let table: Table
|
let table: Table
|
||||||
let rows: Row[]
|
let rows: Row[]
|
||||||
|
|
||||||
|
@ -63,8 +70,10 @@ describe.each([
|
||||||
}
|
}
|
||||||
|
|
||||||
if (dsProvider) {
|
if (dsProvider) {
|
||||||
|
const rawDatasource = await dsProvider
|
||||||
|
client = await knexClient(rawDatasource)
|
||||||
datasource = await config.createDatasource({
|
datasource = await config.createDatasource({
|
||||||
datasource: await dsProvider,
|
datasource: rawDatasource,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
@ -76,9 +85,9 @@ describe.each([
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
async function createTable(schema: TableSchema) {
|
async function createTable(schema: TableSchema, name?: string) {
|
||||||
return await config.api.table.save(
|
return await config.api.table.save(
|
||||||
tableForDatasource(datasource, { schema })
|
tableForDatasource(datasource, { schema, name })
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -909,6 +918,44 @@ describe.each([
|
||||||
}).toMatchExactly([{ name: "foo" }, { name: "bar" }])
|
}).toMatchExactly([{ name: "foo" }, { name: "bar" }])
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
!isInternal &&
|
||||||
|
!isInMemory &&
|
||||||
|
// This test was added because we automatically add in a sort by the
|
||||||
|
// primary key, and we used to do this unconditionally which caused
|
||||||
|
// problems because it was possible for the primary key to appear twice
|
||||||
|
// in the resulting SQL ORDER BY clause, resulting in an SQL error.
|
||||||
|
// We now check first to make sure that the primary key isn't already
|
||||||
|
// in the sort before adding it.
|
||||||
|
describe("sort on primary key", () => {
|
||||||
|
beforeAll(async () => {
|
||||||
|
const tableName = structures.uuid().substring(0, 10)
|
||||||
|
await client!.schema.createTable(tableName, t => {
|
||||||
|
t.string("name").primary()
|
||||||
|
})
|
||||||
|
const resp = await config.api.datasource.fetchSchema({
|
||||||
|
datasourceId: datasource!._id!,
|
||||||
|
})
|
||||||
|
|
||||||
|
table = resp.datasource.entities![tableName]
|
||||||
|
|
||||||
|
await createRows([{ name: "foo" }, { name: "bar" }])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("should be able to sort by a primary key column ascending", async () =>
|
||||||
|
expectSearch({
|
||||||
|
query: {},
|
||||||
|
sort: "name",
|
||||||
|
sortOrder: SortOrder.ASCENDING,
|
||||||
|
}).toMatchExactly([{ name: "bar" }, { name: "foo" }]))
|
||||||
|
|
||||||
|
it("should be able to sort by a primary key column descending", async () =>
|
||||||
|
expectSearch({
|
||||||
|
query: {},
|
||||||
|
sort: "name",
|
||||||
|
sortOrder: SortOrder.DESCENDING,
|
||||||
|
}).toMatchExactly([{ name: "foo" }, { name: "bar" }]))
|
||||||
|
})
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
@ -1956,53 +2003,74 @@ describe.each([
|
||||||
// isn't available.
|
// isn't available.
|
||||||
!isInMemory &&
|
!isInMemory &&
|
||||||
describe("relations", () => {
|
describe("relations", () => {
|
||||||
let otherTable: Table
|
let productCategoryTable: Table, productCatRows: Row[]
|
||||||
let otherRows: Row[]
|
|
||||||
|
|
||||||
beforeAll(async () => {
|
beforeAll(async () => {
|
||||||
otherTable = await createTable({
|
productCategoryTable = await createTable(
|
||||||
one: { name: "one", type: FieldType.STRING },
|
{
|
||||||
})
|
name: { name: "name", type: FieldType.STRING },
|
||||||
table = await createTable({
|
},
|
||||||
two: { name: "two", type: FieldType.STRING },
|
"productCategory"
|
||||||
other: {
|
)
|
||||||
|
table = await createTable(
|
||||||
|
{
|
||||||
|
name: { name: "name", type: FieldType.STRING },
|
||||||
|
productCat: {
|
||||||
type: FieldType.LINK,
|
type: FieldType.LINK,
|
||||||
relationshipType: RelationshipType.ONE_TO_MANY,
|
relationshipType: RelationshipType.ONE_TO_MANY,
|
||||||
name: "other",
|
name: "productCat",
|
||||||
fieldName: "other",
|
fieldName: "product",
|
||||||
tableId: otherTable._id!,
|
tableId: productCategoryTable._id!,
|
||||||
constraints: {
|
constraints: {
|
||||||
type: "array",
|
type: "array",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
})
|
},
|
||||||
|
"product"
|
||||||
|
)
|
||||||
|
|
||||||
otherRows = await Promise.all([
|
productCatRows = await Promise.all([
|
||||||
config.api.row.save(otherTable._id!, { one: "foo" }),
|
config.api.row.save(productCategoryTable._id!, { name: "foo" }),
|
||||||
config.api.row.save(otherTable._id!, { one: "bar" }),
|
config.api.row.save(productCategoryTable._id!, { name: "bar" }),
|
||||||
])
|
])
|
||||||
|
|
||||||
await Promise.all([
|
await Promise.all([
|
||||||
config.api.row.save(table._id!, {
|
config.api.row.save(table._id!, {
|
||||||
two: "foo",
|
name: "foo",
|
||||||
other: [otherRows[0]._id],
|
productCat: [productCatRows[0]._id],
|
||||||
}),
|
}),
|
||||||
config.api.row.save(table._id!, {
|
config.api.row.save(table._id!, {
|
||||||
two: "bar",
|
name: "bar",
|
||||||
other: [otherRows[1]._id],
|
productCat: [productCatRows[1]._id],
|
||||||
|
}),
|
||||||
|
config.api.row.save(table._id!, {
|
||||||
|
name: "baz",
|
||||||
|
productCat: [],
|
||||||
}),
|
}),
|
||||||
])
|
])
|
||||||
|
|
||||||
rows = await config.api.row.fetch(table._id!)
|
|
||||||
})
|
})
|
||||||
|
|
||||||
it("can search through relations", async () => {
|
it("should be able to filter by relationship using column name", async () => {
|
||||||
await expectQuery({
|
await expectQuery({
|
||||||
equal: { [`${otherTable.name}.one`]: "foo" },
|
equal: { ["productCat.name"]: "foo" },
|
||||||
}).toContainExactly([
|
}).toContainExactly([
|
||||||
{ two: "foo", other: [{ _id: otherRows[0]._id }] },
|
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
|
||||||
])
|
])
|
||||||
})
|
})
|
||||||
|
|
||||||
|
it("should be able to filter by relationship using table name", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
equal: { ["productCategory.name"]: "foo" },
|
||||||
|
}).toContainExactly([
|
||||||
|
{ name: "foo", productCat: [{ _id: productCatRows[0]._id }] },
|
||||||
|
])
|
||||||
|
})
|
||||||
|
|
||||||
|
it("shouldn't return any relationship for last row", async () => {
|
||||||
|
await expectQuery({
|
||||||
|
equal: { ["name"]: "baz" },
|
||||||
|
}).toContainExactly([{ name: "baz", productCat: undefined }])
|
||||||
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
// lucene can't count the total rows
|
// lucene can't count the total rows
|
||||||
|
|
|
@ -3,12 +3,14 @@ import * as rows from "./rows"
|
||||||
import * as search from "./search"
|
import * as search from "./search"
|
||||||
import * as utils from "./utils"
|
import * as utils from "./utils"
|
||||||
import * as external from "./external"
|
import * as external from "./external"
|
||||||
|
import * as filters from "./search/filters"
|
||||||
import AliasTables from "./sqlAlias"
|
import AliasTables from "./sqlAlias"
|
||||||
|
|
||||||
export default {
|
export default {
|
||||||
...attachments,
|
...attachments,
|
||||||
...rows,
|
...rows,
|
||||||
...search,
|
...search,
|
||||||
|
filters,
|
||||||
utils,
|
utils,
|
||||||
external,
|
external,
|
||||||
AliasTables,
|
AliasTables,
|
||||||
|
|
|
@ -145,6 +145,10 @@ export async function exportRows(
|
||||||
delimiter,
|
delimiter,
|
||||||
customHeaders,
|
customHeaders,
|
||||||
} = options
|
} = options
|
||||||
|
|
||||||
|
if (!tableId) {
|
||||||
|
throw new HTTPError("No table ID for search provided.", 400)
|
||||||
|
}
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
|
|
||||||
let requestQuery: SearchFilters = {}
|
let requestQuery: SearchFilters = {}
|
||||||
|
@ -167,7 +171,7 @@ export async function exportRows(
|
||||||
requestQuery = query || {}
|
requestQuery = query || {}
|
||||||
}
|
}
|
||||||
|
|
||||||
const datasource = await sdk.datasources.get(datasourceId!)
|
const datasource = await sdk.datasources.get(datasourceId)
|
||||||
const table = await sdk.tables.getTable(tableId)
|
const table = await sdk.tables.getTable(tableId)
|
||||||
if (!datasource || !datasource.entities) {
|
if (!datasource || !datasource.entities) {
|
||||||
throw new HTTPError("Datasource has not been configured for plus API.", 400)
|
throw new HTTPError("Datasource has not been configured for plus API.", 400)
|
||||||
|
@ -180,10 +184,6 @@ export async function exportRows(
|
||||||
let rows: Row[] = []
|
let rows: Row[] = []
|
||||||
let headers
|
let headers
|
||||||
|
|
||||||
if (!tableName) {
|
|
||||||
throw new HTTPError("Could not find table name.", 400)
|
|
||||||
}
|
|
||||||
|
|
||||||
// Filter data to only specified columns if required
|
// Filter data to only specified columns if required
|
||||||
if (columns && columns.length) {
|
if (columns && columns.length) {
|
||||||
for (let i = 0; i < result.rows.length; i++) {
|
for (let i = 0; i < result.rows.length; i++) {
|
||||||
|
|
62
packages/server/src/sdk/app/rows/search/filters.ts
Normal file
62
packages/server/src/sdk/app/rows/search/filters.ts
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
import {
|
||||||
|
FieldType,
|
||||||
|
RelationshipFieldMetadata,
|
||||||
|
SearchFilters,
|
||||||
|
Table,
|
||||||
|
} from "@budibase/types"
|
||||||
|
import { isPlainObject } from "lodash"
|
||||||
|
|
||||||
|
export function getRelationshipColumns(table: Table): {
|
||||||
|
name: string
|
||||||
|
definition: RelationshipFieldMetadata
|
||||||
|
}[] {
|
||||||
|
// performing this with a for loop rather than an array filter improves
|
||||||
|
// type guarding, as no casts are required
|
||||||
|
const linkEntries: [string, RelationshipFieldMetadata][] = []
|
||||||
|
for (let entry of Object.entries(table.schema)) {
|
||||||
|
if (entry[1].type === FieldType.LINK) {
|
||||||
|
const linkColumn: RelationshipFieldMetadata = entry[1]
|
||||||
|
linkEntries.push([entry[0], linkColumn])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return linkEntries.map(entry => ({
|
||||||
|
name: entry[0],
|
||||||
|
definition: entry[1],
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
export function getTableIDList(
|
||||||
|
tables: Table[]
|
||||||
|
): { name: string; id: string }[] {
|
||||||
|
return tables
|
||||||
|
.filter(table => table.originalName && table._id)
|
||||||
|
.map(table => ({ id: table._id!, name: table.originalName! }))
|
||||||
|
}
|
||||||
|
|
||||||
|
export function updateFilterKeys(
|
||||||
|
filters: SearchFilters,
|
||||||
|
updates: { original: string; updated: string }[]
|
||||||
|
): SearchFilters {
|
||||||
|
const makeFilterKeyRegex = (str: string) =>
|
||||||
|
new RegExp(`^${str}\\.|:${str}\\.`)
|
||||||
|
for (let filter of Object.values(filters)) {
|
||||||
|
if (!isPlainObject(filter)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for (let [key, keyFilter] of Object.entries(filter)) {
|
||||||
|
if (keyFilter === "") {
|
||||||
|
delete filter[key]
|
||||||
|
}
|
||||||
|
const possibleKey = updates.find(({ original }) =>
|
||||||
|
key.match(makeFilterKeyRegex(original))
|
||||||
|
)
|
||||||
|
if (possibleKey && possibleKey.original !== possibleKey.updated) {
|
||||||
|
// only replace the first, not replaceAll
|
||||||
|
filter[key.replace(possibleKey.original, possibleKey.updated)] =
|
||||||
|
filter[key]
|
||||||
|
delete filter[key]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return filters
|
||||||
|
}
|
|
@ -1,4 +1,5 @@
|
||||||
import {
|
import {
|
||||||
|
Datasource,
|
||||||
DocumentType,
|
DocumentType,
|
||||||
FieldType,
|
FieldType,
|
||||||
Operation,
|
Operation,
|
||||||
|
@ -12,7 +13,6 @@ import {
|
||||||
SortType,
|
SortType,
|
||||||
SqlClient,
|
SqlClient,
|
||||||
Table,
|
Table,
|
||||||
Datasource,
|
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import {
|
import {
|
||||||
buildInternalRelationships,
|
buildInternalRelationships,
|
||||||
|
@ -30,6 +30,11 @@ import AliasTables from "../sqlAlias"
|
||||||
import { outputProcessing } from "../../../../utilities/rowProcessor"
|
import { outputProcessing } from "../../../../utilities/rowProcessor"
|
||||||
import pick from "lodash/pick"
|
import pick from "lodash/pick"
|
||||||
import { processRowCountResponse } from "../utils"
|
import { processRowCountResponse } from "../utils"
|
||||||
|
import {
|
||||||
|
updateFilterKeys,
|
||||||
|
getRelationshipColumns,
|
||||||
|
getTableIDList,
|
||||||
|
} from "./filters"
|
||||||
|
|
||||||
const builder = new sql.Sql(SqlClient.SQL_LITE)
|
const builder = new sql.Sql(SqlClient.SQL_LITE)
|
||||||
|
|
||||||
|
@ -60,34 +65,31 @@ function buildInternalFieldList(
|
||||||
return fieldList
|
return fieldList
|
||||||
}
|
}
|
||||||
|
|
||||||
function tableNameInFieldRegex(tableName: string) {
|
function cleanupFilters(
|
||||||
return new RegExp(`^${tableName}.|:${tableName}.`, "g")
|
filters: SearchFilters,
|
||||||
}
|
table: Table,
|
||||||
|
allTables: Table[]
|
||||||
function cleanupFilters(filters: SearchFilters, tables: Table[]) {
|
) {
|
||||||
for (let filter of Object.values(filters)) {
|
// get a list of all relationship columns in the table for updating
|
||||||
if (typeof filter !== "object") {
|
const relationshipColumns = getRelationshipColumns(table)
|
||||||
continue
|
// get table names to ID map for relationships
|
||||||
}
|
const tableNameToID = getTableIDList(allTables)
|
||||||
for (let [key, keyFilter] of Object.entries(filter)) {
|
// all should be applied at once
|
||||||
if (keyFilter === "") {
|
filters = updateFilterKeys(
|
||||||
delete filter[key]
|
filters,
|
||||||
}
|
relationshipColumns
|
||||||
|
.map(({ name, definition }) => ({
|
||||||
// relationship, switch to table ID
|
original: name,
|
||||||
const tableRelated = tables.find(
|
updated: definition.tableId,
|
||||||
table =>
|
}))
|
||||||
table.originalName &&
|
.concat(
|
||||||
key.match(tableNameInFieldRegex(table.originalName))
|
tableNameToID.map(({ name, id }) => ({
|
||||||
|
original: name,
|
||||||
|
updated: id,
|
||||||
|
}))
|
||||||
)
|
)
|
||||||
if (tableRelated && tableRelated.originalName) {
|
)
|
||||||
// only replace the first, not replaceAll
|
|
||||||
filter[key.replace(tableRelated.originalName, tableRelated._id!)] =
|
|
||||||
filter[key]
|
|
||||||
delete filter[key]
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return filters
|
return filters
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -176,7 +178,7 @@ export async function search(
|
||||||
operation: Operation.READ,
|
operation: Operation.READ,
|
||||||
},
|
},
|
||||||
filters: {
|
filters: {
|
||||||
...cleanupFilters(query, allTables),
|
...cleanupFilters(query, table, allTables),
|
||||||
documentType: DocumentType.ROW,
|
documentType: DocumentType.ROW,
|
||||||
},
|
},
|
||||||
table,
|
table,
|
||||||
|
|
|
@ -90,10 +90,10 @@ export async function getExternalTable(
|
||||||
export async function getTable(tableId: string): Promise<Table> {
|
export async function getTable(tableId: string): Promise<Table> {
|
||||||
const db = context.getAppDB()
|
const db = context.getAppDB()
|
||||||
let output: Table
|
let output: Table
|
||||||
if (isExternalTableID(tableId)) {
|
if (tableId && isExternalTableID(tableId)) {
|
||||||
let { datasourceId, tableName } = breakExternalTableId(tableId)
|
let { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
const datasource = await datasources.get(datasourceId!)
|
const datasource = await datasources.get(datasourceId)
|
||||||
const table = await getExternalTable(datasourceId!, tableName!)
|
const table = await getExternalTable(datasourceId, tableName)
|
||||||
output = { ...table, sql: isSQL(datasource) }
|
output = { ...table, sql: isSQL(datasource) }
|
||||||
} else {
|
} else {
|
||||||
output = await db.get<Table>(tableId)
|
output = await db.get<Table>(tableId)
|
||||||
|
|
|
@ -10,9 +10,9 @@ export async function get(viewId: string): Promise<ViewV2> {
|
||||||
const { tableId } = utils.extractViewInfoFromID(viewId)
|
const { tableId } = utils.extractViewInfoFromID(viewId)
|
||||||
|
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
const ds = await sdk.datasources.get(datasourceId!)
|
const ds = await sdk.datasources.get(datasourceId)
|
||||||
|
|
||||||
const table = ds.entities![tableName!]
|
const table = ds.entities![tableName]
|
||||||
const views = Object.values(table.views!).filter(isV2)
|
const views = Object.values(table.views!).filter(isV2)
|
||||||
const found = views.find(v => v.id === viewId)
|
const found = views.find(v => v.id === viewId)
|
||||||
if (!found) {
|
if (!found) {
|
||||||
|
@ -25,9 +25,9 @@ export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
|
||||||
const { tableId } = utils.extractViewInfoFromID(viewId)
|
const { tableId } = utils.extractViewInfoFromID(viewId)
|
||||||
|
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
const ds = await sdk.datasources.get(datasourceId!)
|
const ds = await sdk.datasources.get(datasourceId)
|
||||||
|
|
||||||
const table = ds.entities![tableName!]
|
const table = ds.entities![tableName]
|
||||||
const views = Object.values(table.views!).filter(isV2)
|
const views = Object.values(table.views!).filter(isV2)
|
||||||
const found = views.find(v => v.id === viewId)
|
const found = views.find(v => v.id === viewId)
|
||||||
if (!found) {
|
if (!found) {
|
||||||
|
@ -49,9 +49,9 @@ export async function create(
|
||||||
const db = context.getAppDB()
|
const db = context.getAppDB()
|
||||||
|
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
const ds = await sdk.datasources.get(datasourceId!)
|
const ds = await sdk.datasources.get(datasourceId)
|
||||||
ds.entities![tableName!].views ??= {}
|
ds.entities![tableName].views ??= {}
|
||||||
ds.entities![tableName!].views![view.name] = view
|
ds.entities![tableName].views![view.name] = view
|
||||||
await db.put(ds)
|
await db.put(ds)
|
||||||
return view
|
return view
|
||||||
}
|
}
|
||||||
|
@ -60,9 +60,9 @@ export async function update(tableId: string, view: ViewV2): Promise<ViewV2> {
|
||||||
const db = context.getAppDB()
|
const db = context.getAppDB()
|
||||||
|
|
||||||
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
const { datasourceId, tableName } = breakExternalTableId(tableId)
|
||||||
const ds = await sdk.datasources.get(datasourceId!)
|
const ds = await sdk.datasources.get(datasourceId)
|
||||||
ds.entities![tableName!].views ??= {}
|
ds.entities![tableName].views ??= {}
|
||||||
const views = ds.entities![tableName!].views!
|
const views = ds.entities![tableName].views!
|
||||||
|
|
||||||
const existingView = Object.values(views).find(
|
const existingView = Object.values(views).find(
|
||||||
v => isV2(v) && v.id === view.id
|
v => isV2(v) && v.id === view.id
|
||||||
|
@ -87,9 +87,9 @@ export async function remove(viewId: string): Promise<ViewV2> {
|
||||||
}
|
}
|
||||||
|
|
||||||
const { datasourceId, tableName } = breakExternalTableId(view.tableId)
|
const { datasourceId, tableName } = breakExternalTableId(view.tableId)
|
||||||
const ds = await sdk.datasources.get(datasourceId!)
|
const ds = await sdk.datasources.get(datasourceId)
|
||||||
|
|
||||||
delete ds.entities![tableName!].views![view?.name]
|
delete ds.entities![tableName].views![view?.name]
|
||||||
await db.put(ds)
|
await db.put(ds)
|
||||||
return view
|
return view
|
||||||
}
|
}
|
||||||
|
|
|
@ -4,6 +4,7 @@ import {
|
||||||
TableSchema,
|
TableSchema,
|
||||||
FieldSchema,
|
FieldSchema,
|
||||||
Row,
|
Row,
|
||||||
|
Table,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import { ValidColumnNameRegex, helpers, utils } from "@budibase/shared-core"
|
import { ValidColumnNameRegex, helpers, utils } from "@budibase/shared-core"
|
||||||
import { db } from "@budibase/backend-core"
|
import { db } from "@budibase/backend-core"
|
||||||
|
@ -118,16 +119,26 @@ export function validate(rows: Rows, schema: TableSchema): ValidationResults {
|
||||||
return results
|
return results
|
||||||
}
|
}
|
||||||
|
|
||||||
export function parse(rows: Rows, schema: TableSchema): Rows {
|
export function parse(rows: Rows, table: Table): Rows {
|
||||||
return rows.map(row => {
|
return rows.map(row => {
|
||||||
const parsedRow: Row = {}
|
const parsedRow: Row = {}
|
||||||
|
|
||||||
Object.entries(row).forEach(([columnName, columnData]) => {
|
Object.entries(row).forEach(([columnName, columnData]) => {
|
||||||
if (!(columnName in schema) || schema[columnName]?.autocolumn) {
|
const schema = table.schema
|
||||||
|
if (!(columnName in schema)) {
|
||||||
// Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
|
// Objects can be present in the row data but not in the schema, so make sure we don't proceed in such a case
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
schema[columnName].autocolumn &&
|
||||||
|
!table.primary?.includes(columnName)
|
||||||
|
) {
|
||||||
|
// Don't want the user specifying values for autocolumns unless they're updating
|
||||||
|
// a row through its primary key.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
const columnSchema = schema[columnName]
|
const columnSchema = schema[columnName]
|
||||||
const { type: columnType } = columnSchema
|
const { type: columnType } = columnSchema
|
||||||
if (columnType === FieldType.NUMBER) {
|
if (columnType === FieldType.NUMBER) {
|
||||||
|
|
|
@ -180,3 +180,5 @@ export enum BpmStatusValue {
|
||||||
VERIFYING_EMAIL = "verifying_email",
|
VERIFYING_EMAIL = "verifying_email",
|
||||||
COMPLETED = "completed",
|
COMPLETED = "completed",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_BB_DATASOURCE_ID = "datasource_internal_bb_default"
|
||||||
|
|
|
@ -42,10 +42,7 @@ export interface Account extends CreateAccount {
|
||||||
verified: boolean
|
verified: boolean
|
||||||
verificationSent: boolean
|
verificationSent: boolean
|
||||||
// licensing
|
// licensing
|
||||||
tier: string // deprecated
|
|
||||||
planType?: PlanType
|
planType?: PlanType
|
||||||
/** @deprecated */
|
|
||||||
planTier?: number
|
|
||||||
license?: License
|
license?: License
|
||||||
installId?: string
|
installId?: string
|
||||||
installTenantId?: string
|
installTenantId?: string
|
||||||
|
|
Loading…
Reference in a new issue