Stage {index + 1}
diff --git a/packages/builder/src/constants/backend/index.js b/packages/builder/src/constants/backend/index.js
index eb47ac97fe..f1e3e1e2c2 100644
--- a/packages/builder/src/constants/backend/index.js
+++ b/packages/builder/src/constants/backend/index.js
@@ -310,6 +310,7 @@ export const BannedSearchTypes = [
"formula",
"json",
"jsonarray",
+ "queryarray",
]
export const DatasourceTypes = {
diff --git a/packages/builder/src/dataBinding.js b/packages/builder/src/dataBinding.js
index 8c71631b09..0442a67da9 100644
--- a/packages/builder/src/dataBinding.js
+++ b/packages/builder/src/dataBinding.js
@@ -425,7 +425,7 @@ const generateComponentContextBindings = (asset, componentContext) => {
table = info.table
// Determine what to prefix bindings with
- if (datasource.type === "jsonarray") {
+ if (datasource.type === "jsonarray" || datasource.type === "queryarray") {
// For JSON arrays, use the array name as the readable prefix
const split = datasource.label.split(".")
readablePrefix = split[split.length - 1]
@@ -904,6 +904,19 @@ export const getSchemaForDatasource = (asset, datasource, options) => {
schema = JSONUtils.getJSONArrayDatasourceSchema(tableSchema, datasource)
}
+ // "queryarray" datasources are arrays inside JSON responses
+ else if (type === "queryarray") {
+ const queries = get(queriesStores).list
+ table = queries.find(query => query._id === datasource.tableId)
+ let tableSchema = table?.schema
+ let nestedSchemaFields = table?.nestedSchemaFields
+ schema = JSONUtils.generateQueryArraySchemas(
+ tableSchema,
+ nestedSchemaFields
+ )
+ schema = JSONUtils.getJSONArrayDatasourceSchema(schema, datasource)
+ }
+
// Otherwise we assume we're targeting an internal table or a plus
// datasource, and we can treat it as a table with a schema
else {
diff --git a/packages/client/src/components/app/forms/Form.svelte b/packages/client/src/components/app/forms/Form.svelte
index 464ca95829..5522bd4b46 100644
--- a/packages/client/src/components/app/forms/Form.svelte
+++ b/packages/client/src/components/app/forms/Form.svelte
@@ -84,7 +84,7 @@
// Fetches the form schema from this form's dataSource
const fetchSchema = async dataSource => {
- if (dataSource?.tableId && dataSource?.type !== "query") {
+ if (dataSource?.tableId && !dataSource?.type?.startsWith("query")) {
try {
table = await API.fetchTableDefinition(dataSource.tableId)
} catch (error) {
diff --git a/packages/client/src/utils/schema.js b/packages/client/src/utils/schema.js
index f20e724a6e..e2399e8738 100644
--- a/packages/client/src/utils/schema.js
+++ b/packages/client/src/utils/schema.js
@@ -7,6 +7,7 @@ import NestedProviderFetch from "@budibase/frontend-core/src/fetch/NestedProvide
import FieldFetch from "@budibase/frontend-core/src/fetch/FieldFetch.js"
import JSONArrayFetch from "@budibase/frontend-core/src/fetch/JSONArrayFetch.js"
import ViewV2Fetch from "@budibase/frontend-core/src/fetch/ViewV2Fetch.js"
+import QueryArrayFetch from "@budibase/frontend-core/src/fetch/QueryArrayFetch"
/**
* Fetches the schema of any kind of datasource.
@@ -28,6 +29,7 @@ export const fetchDatasourceSchema = async (
provider: NestedProviderFetch,
field: FieldFetch,
jsonarray: JSONArrayFetch,
+ queryarray: QueryArrayFetch,
}[datasource?.type]
if (!handler) {
return null
diff --git a/packages/frontend-core/src/fetch/QueryArrayFetch.js b/packages/frontend-core/src/fetch/QueryArrayFetch.js
new file mode 100644
index 0000000000..0b36b640a6
--- /dev/null
+++ b/packages/frontend-core/src/fetch/QueryArrayFetch.js
@@ -0,0 +1,25 @@
+import FieldFetch from "./FieldFetch.js"
+import {
+ getJSONArrayDatasourceSchema,
+ generateQueryArraySchemas,
+} from "../utils/json"
+
+export default class QueryArrayFetch extends FieldFetch {
+ async getDefinition(datasource) {
+ if (!datasource?.tableId) {
+ return null
+ }
+ // JSON arrays need their table definitions fetched.
+ // We can then extract their schema as a subset of the table schema.
+ try {
+ const table = await this.API.fetchQueryDefinition(datasource.tableId)
+ const schema = generateQueryArraySchemas(
+ table?.schema,
+ table?.nestedSchemaFields
+ )
+ return { schema: getJSONArrayDatasourceSchema(schema, datasource) }
+ } catch (error) {
+ return null
+ }
+ }
+}
diff --git a/packages/frontend-core/src/fetch/index.js b/packages/frontend-core/src/fetch/index.js
index a41a859351..903810ac25 100644
--- a/packages/frontend-core/src/fetch/index.js
+++ b/packages/frontend-core/src/fetch/index.js
@@ -9,6 +9,7 @@ import JSONArrayFetch from "./JSONArrayFetch.js"
import UserFetch from "./UserFetch.js"
import GroupUserFetch from "./GroupUserFetch.js"
import CustomFetch from "./CustomFetch.js"
+import QueryArrayFetch from "./QueryArrayFetch.js"
const DataFetchMap = {
table: TableFetch,
@@ -24,6 +25,7 @@ const DataFetchMap = {
provider: NestedProviderFetch,
field: FieldFetch,
jsonarray: JSONArrayFetch,
+ queryarray: QueryArrayFetch,
}
// Constructs a new fetch model for a certain datasource
diff --git a/packages/frontend-core/src/utils/json.js b/packages/frontend-core/src/utils/json.js
index 29bf2df34e..8cd37f9ad1 100644
--- a/packages/frontend-core/src/utils/json.js
+++ b/packages/frontend-core/src/utils/json.js
@@ -1,3 +1,5 @@
+import { utils } from "@budibase/shared-core"
+
/**
* Gets the schema for a datasource which is targeting a JSON array, including
* nested JSON arrays. The returned schema is a squashed, table-like schema
@@ -119,3 +121,33 @@ const extractJSONSchemaKeys = (jsonSchema, squashObjects = false) => {
})
return keys
}
+
+export const generateQueryArraySchemas = (schema, nestedSchemaFields) => {
+ for (let key in schema) {
+ if (
+ schema[key]?.type === "json" &&
+ schema[key]?.subtype === "array" &&
+ utils.hasSchema(nestedSchemaFields[key])
+ ) {
+ schema[key] = {
+ schema: {
+ schema: Object.entries(nestedSchemaFields[key] || {}).reduce(
+ (acc, [nestedKey, fieldSchema]) => {
+ acc[nestedKey] = {
+ name: nestedKey,
+ type: fieldSchema.type,
+ subtype: fieldSchema.subtype,
+ }
+ return acc
+ },
+ {}
+ ),
+ type: "json",
+ },
+ type: "json",
+ subtype: "array",
+ }
+ }
+ }
+ return schema
+}
diff --git a/packages/server/src/api/controllers/query/index.ts b/packages/server/src/api/controllers/query/index.ts
index 8dabe5b3cc..89330f3216 100644
--- a/packages/server/src/api/controllers/query/index.ts
+++ b/packages/server/src/api/controllers/query/index.ts
@@ -1,5 +1,4 @@
import { generateQueryID } from "../../../db/utils"
-import { BaseQueryVerbs } from "../../../constants"
import { Thread, ThreadType } from "../../../threads"
import { save as saveDatasource } from "../datasource"
import { RestImporter } from "./import"
@@ -7,36 +6,27 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
import env from "../../../environment"
import { events, context, utils, constants } from "@budibase/backend-core"
import sdk from "../../../sdk"
-import { QueryEvent, QueryResponse } from "../../../threads/definitions"
+import { QueryEvent } from "../../../threads/definitions"
import {
ConfigType,
Query,
UserCtx,
SessionCookie,
+ JsonFieldSubType,
+ QueryResponse,
+ QueryPreview,
QuerySchema,
FieldType,
type ExecuteQueryRequest,
type ExecuteQueryResponse,
type Row,
} from "@budibase/types"
-import { ValidQueryNameRegex } from "@budibase/shared-core"
+import { ValidQueryNameRegex, utils as JsonUtils } from "@budibase/shared-core"
const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: env.QUERY_THREAD_TIMEOUT,
})
-// simple function to append "readable" to all read queries
-function enrichQueries(input: any) {
- const wasArray = Array.isArray(input)
- const queries = wasArray ? input : [input]
- for (let query of queries) {
- if (query.queryVerb === BaseQueryVerbs.READ) {
- query.readable = true
- }
- }
- return wasArray ? queries : queries[0]
-}
-
export async function fetch(ctx: UserCtx) {
ctx.body = await sdk.queries.fetch()
}
@@ -84,7 +74,7 @@ export { _import as import }
export async function save(ctx: UserCtx) {
const db = context.getAppDB()
- const query = ctx.request.body
+ const query: Query = ctx.request.body
// Validate query name
if (!query?.name.match(ValidQueryNameRegex)) {
@@ -100,7 +90,6 @@ export async function save(ctx: UserCtx) {
} else {
eventFn = () => events.query.updated(datasource, query)
}
-
const response = await db.put(query)
await eventFn()
query._rev = response.rev
@@ -133,7 +122,7 @@ export async function preview(ctx: UserCtx) {
const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
ctx.request.body.datasourceId
)
- const query = ctx.request.body
+ const query: QueryPreview = ctx.request.body
// preview may not have a queryId as it hasn't been saved, but if it does
// this stops dynamic variables from calling the same query
const { fields, parameters, queryVerb, transformer, queryId, schema } = query
@@ -153,6 +142,69 @@ export async function preview(ctx: UserCtx) {
const authConfigCtx: any = getAuthConfig(ctx)
+ function getSchemaFields(
+ rows: any[],
+ keys: string[]
+ ): {
+ previewSchema: Record
+ nestedSchemaFields: {
+ [key: string]: Record
+ }
+ } {
+ const previewSchema: Record = {}
+ const nestedSchemaFields: {
+ [key: string]: Record
+ } = {}
+ const makeQuerySchema = (
+ type: FieldType,
+ name: string,
+ subtype?: string
+ ): QuerySchema => ({
+ type,
+ name,
+ subtype,
+ })
+ if (rows?.length > 0) {
+ for (let key of [...new Set(keys)] as string[]) {
+ const field = rows[0][key]
+ let type = typeof field,
+ fieldMetadata = makeQuerySchema(FieldType.STRING, key)
+ if (field)
+ switch (type) {
+ case "boolean":
+ fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
+ break
+ case "object":
+ if (field instanceof Date) {
+ fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
+ } else if (Array.isArray(field)) {
+ if (JsonUtils.hasSchema(field[0])) {
+ fieldMetadata = makeQuerySchema(
+ FieldType.JSON,
+ key,
+ JsonFieldSubType.ARRAY
+ )
+ } else {
+ fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
+ }
+ nestedSchemaFields[key] = getSchemaFields(
+ field,
+ Object.keys(field[0])
+ ).previewSchema
+ } else {
+ fieldMetadata = makeQuerySchema(FieldType.JSON, key)
+ }
+ break
+ case "number":
+ fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
+ break
+ }
+ previewSchema[key] = fieldMetadata
+ }
+ }
+ return { previewSchema, nestedSchemaFields }
+ }
+
try {
const inputs: QueryEvent = {
appId: ctx.appId,
@@ -171,38 +223,11 @@ export async function preview(ctx: UserCtx) {
},
}
- const { rows, keys, info, extra } = await Runner.run(inputs)
- const previewSchema: Record = {}
- const makeQuerySchema = (type: FieldType, name: string): QuerySchema => ({
- type,
- name,
- })
- if (rows?.length > 0) {
- for (let key of [...new Set(keys)] as string[]) {
- const field = rows[0][key]
- let type = typeof field,
- fieldMetadata = makeQuerySchema(FieldType.STRING, key)
- if (field)
- switch (type) {
- case "boolean":
- fieldMetadata = makeQuerySchema(FieldType.BOOLEAN, key)
- break
- case "object":
- if (field instanceof Date) {
- fieldMetadata = makeQuerySchema(FieldType.DATETIME, key)
- } else if (Array.isArray(field)) {
- fieldMetadata = makeQuerySchema(FieldType.ARRAY, key)
- } else {
- fieldMetadata = makeQuerySchema(FieldType.JSON, key)
- }
- break
- case "number":
- fieldMetadata = makeQuerySchema(FieldType.NUMBER, key)
- break
- }
- previewSchema[key] = fieldMetadata
- }
- }
+ const { rows, keys, info, extra } = (await Runner.run(
+ inputs
+ )) as QueryResponse
+ const { previewSchema, nestedSchemaFields } = getSchemaFields(rows, keys)
+
// if existing schema, update to include any previous schema keys
if (existingSchema) {
for (let key of Object.keys(previewSchema)) {
@@ -216,6 +241,7 @@ export async function preview(ctx: UserCtx) {
await events.query.previewed(datasource, query)
ctx.body = {
rows,
+ nestedSchemaFields,
schema: previewSchema,
info,
extra,
diff --git a/packages/shared-core/src/utils.ts b/packages/shared-core/src/utils.ts
index c775010909..cc5646a00e 100644
--- a/packages/shared-core/src/utils.ts
+++ b/packages/shared-core/src/utils.ts
@@ -57,3 +57,13 @@ export function filterValueToLabel() {
{}
)
}
+
+export function hasSchema(test: any) {
+ return (
+ typeof test === "object" &&
+ !Array.isArray(test) &&
+ test !== null &&
+ !(test instanceof Date) &&
+ Object.keys(test).length > 0
+ )
+}
diff --git a/packages/types/src/documents/app/query.ts b/packages/types/src/documents/app/query.ts
index 81aa90b807..f4547b9774 100644
--- a/packages/types/src/documents/app/query.ts
+++ b/packages/types/src/documents/app/query.ts
@@ -4,6 +4,7 @@ import type { Row } from "./row"
export interface QuerySchema {
name?: string
type: string
+ subtype?: string
}
export interface Query extends Document {
@@ -17,11 +18,23 @@ export interface Query extends Document {
queryVerb: string
}
+export interface QueryPreview extends Omit {
+ queryId: string
+}
+
export interface QueryParameter {
name: string
default: string
}
+export interface QueryResponse {
+ rows: any[]
+ keys: string[]
+ info: any
+ extra: any
+ pagination: any
+}
+
export interface RestQueryFields {
path: string
queryString?: string
diff --git a/packages/types/src/documents/app/table/constants.ts b/packages/types/src/documents/app/table/constants.ts
index fc831e7e7c..1d9d14695a 100644
--- a/packages/types/src/documents/app/table/constants.ts
+++ b/packages/types/src/documents/app/table/constants.ts
@@ -16,6 +16,10 @@ export enum AutoFieldSubType {
AUTO_ID = "autoID",
}
+export enum JsonFieldSubType {
+ ARRAY = "array",
+}
+
export enum FormulaType {
STATIC = "static",
DYNAMIC = "dynamic",
diff --git a/packages/types/src/documents/app/table/schema.ts b/packages/types/src/documents/app/table/schema.ts
index 47ec303b66..17abf747b2 100644
--- a/packages/types/src/documents/app/table/schema.ts
+++ b/packages/types/src/documents/app/table/schema.ts
@@ -5,6 +5,7 @@ import {
AutoFieldSubType,
AutoReason,
FormulaType,
+ JsonFieldSubType,
RelationshipType,
} from "./constants"
@@ -81,6 +82,11 @@ export interface NumberFieldMetadata extends Omit {
}
}
+export interface JsonFieldMetadata extends Omit {
+ type: FieldType.JSON
+ subtype?: JsonFieldSubType.ARRAY
+}
+
export interface DateFieldMetadata extends Omit {
type: FieldType.DATETIME
ignoreTimezones?: boolean
@@ -162,6 +168,7 @@ export type FieldSchema =
| NumberFieldMetadata
| LongFormFieldMetadata
| BBReferenceFieldMetadata
+ | JsonFieldMetadata
export interface TableSchema {
[key: string]: FieldSchema
diff --git a/yarn.lock b/yarn.lock
index 1937482837..c0a11b9bf7 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -5572,9 +5572,9 @@
integrity sha512-7GgtHCs/QZrBrDzgIJnQtuSvhFSwhyYSI2uafSwZoNt1iOGhEN5fwNrQMjtONyHm9+/LoA4453jH0CMYcr06Pg==
"@types/node@^18.11.18":
- version "18.19.10"
- resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.10.tgz#4de314ab66faf6bc8ba691021a091ddcdf13a158"
- integrity sha512-IZD8kAM02AW1HRDTPOlz3npFava678pr8Ie9Vp8uRhBROXAv8MXT2pCnGZZAKYdromsNQLHQcfWQ6EOatVLtqA==
+ version "18.19.13"
+ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.13.tgz#c3e989ca967b862a1f6c8c4148fe31865eedaf1a"
+ integrity sha512-kgnbRDj8ioDyGxoiaXsiu1Ybm/K14ajCgMOkwiqpHrnF7d7QiYRoRqHIpglMMs3DwXinlK4qJ8TZGlj4hfleJg==
dependencies:
undici-types "~5.26.4"
@@ -10763,7 +10763,7 @@ fetch-cookie@0.11.0:
dependencies:
tough-cookie "^2.3.3 || ^3.0.1 || ^4.0.0"
-fflate@^0.4.1:
+fflate@^0.4.1, fflate@^0.4.8:
version "0.4.8"
resolved "https://registry.yarnpkg.com/fflate/-/fflate-0.4.8.tgz#f90b82aefbd8ac174213abb338bd7ef848f0f5ae"
integrity sha512-FJqqoDBR00Mdj9ppamLa/Y7vxm+PRmNWA67N846RvsoYVMKB4q3y/de5PA7gUmRMYK/8CMz2GDZQmCRN1wBcWA==