CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-10-04 03:54:37 +13:00
Code Issues Projects Releases Wiki Activity

Merge pull request #11239 from Budibase/BUDI-7189/extract_search_functionality

Browse source 
Moving files and functions
This commit is contained in:
Adria Navarro 2023-07-17 14:17:26 +01:00 committed by GitHub
commit 832483442f
20 changed files with 824 additions and 690 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.tool-versions
View file

@ -1,2 +1,2 @@
nodejs 14.20.1
nodejs 14.21.3
python 3.10.0

3
packages/server/src/api/controllers/datasource.ts
View file

@ -9,7 +9,6 @@ import {
import { destroy as tableDestroy } from "./table/internal"
import { BuildSchemaErrors, InvalidColumns } from "../../constants"
import { getIntegration } from "../../integrations"
import { getDatasourceAndQuery } from "./row/utils"
import { invalidateDynamicVariables } from "../../threads/utils"
import { db as dbCore, context, events } from "@budibase/backend-core"
import {
@ -442,7 +441,7 @@ export async function find(ctx: UserCtx) {
export async function query(ctx: UserCtx) {
const queryJson = ctx.request.body
try {
ctx.body = await getDatasourceAndQuery(queryJson)
ctx.body = await sdk.rows.utils.getDatasourceAndQuery(queryJson)
} catch (err: any) {
ctx.throw(400, err)
}

3
packages/server/src/api/controllers/row/ExternalRequest.ts
View file

@ -23,14 +23,13 @@ import {
isRowId,
isSQL,
} from "../../../integrations/utils"
import { getDatasourceAndQuery } from "./utils"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { FieldTypes } from "../../../constants"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { processDates, processFormulas } from "../../../utilities/rowProcessor"
import { db as dbCore } from "@budibase/backend-core"
import sdk from "../../../sdk"
import { isEditableColumn } from "../../../sdk/app/tables/validation"
export interface ManyRelationship {
tableId?: string

150
packages/server/src/api/controllers/row/external.ts
View file

@ -1,30 +1,20 @@
import {
FieldTypes,
NoEmptyFilterStrings,
SortDirection,
} from "../../../constants"
import { FieldTypes, NoEmptyFilterStrings } from "../../../constants"
import {
breakExternalTableId,
breakRowIdField,
} from "../../../integrations/utils"
import { ExternalRequest, RunConfig } from "./ExternalRequest"
import * as exporters from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
Datasource,
IncludeRelationship,
Operation,
PaginationJson,
Row,
SortJson,
Table,
UserCtx,
} from "@budibase/types"
import sdk from "../../../sdk"
import * as utils from "./utils"
const { cleanExportRows } = require("./utils")
async function getRow(
tableId: string,
rowId: string,
@ -114,21 +104,6 @@ export async function save(ctx: UserCtx) {
}
}
export async function fetchView(ctx: UserCtx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
ctx.params.tableId = split[1] ? split[1] : split[0]
return fetch(ctx)
}
export async function fetch(ctx: UserCtx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function find(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId
@ -161,129 +136,6 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: responses.map(resp => resp.row) }
}
export async function search(ctx: UserCtx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
if (!bookmark && paginate) {
bookmark = 1
}
let paginateObj = {}
if (paginate) {
paginateObj = {
// add one so we can track if there is another page
limit: limit,
page: bookmark,
}
} else if (params && limit) {
paginateObj = {
limit: limit,
}
}
let sort: SortJson | undefined
if (params.sort) {
const direction =
params.sortOrder === "descending"
? SortDirection.DESCENDING
: SortDirection.ASCENDING
sort = {
[params.sort]: { direction },
}
}
try {
const rows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: paginateObj as PaginationJson,
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
let hasNextPage = false
if (paginate && rows.length === limit) {
const nextRows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: {
limit: 1,
page: bookmark * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
)
} else {
throw err
}
}
}
export async function exportRows(ctx: UserCtx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format
const { columns } = ctx.request.body
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
ctx.throw(400, "Export data does not support composite keys.")
}
return ids[0]
}),
},
},
}
}
let result = await search(ctx)
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result.rows[i][column]
}
}
} else {
rows = result.rows
}
if (!tableName) {
ctx.throw(400, "Could not find table name.")
}
let schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
// @ts-ignore
const exporter = exporters[format]
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(exporter(headers, exportRows))
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const id = ctx.params.rowId
const tableId = ctx.params.tableId

10
packages/server/src/api/controllers/row/index.ts
View file

@ -5,6 +5,7 @@ import { isExternalTable } from "../../../integrations/utils"
import { Ctx } from "@budibase/types"
import * as utils from "./utils"
import { gridSocket } from "../../../websockets"
import sdk from "../../../sdk"
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
@ -64,14 +65,14 @@ export const save = async (ctx: any) => {
}
export async function fetchView(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.fetchView(tableId, ctx), {
datasourceId: tableId,
})
}
export async function fetch(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.fetch(tableId, ctx), {
datasourceId: tableId,
})
}
@ -120,7 +121,8 @@ export async function destroy(ctx: any) {
export async function search(ctx: any) {
const tableId = utils.getTableId(ctx)
ctx.status = 200
ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.search(tableId, ctx), {
datasourceId: tableId,
})
}
@ -150,7 +152,7 @@ export async function fetchEnrichedRow(ctx: any) {
export const exportRows = async (ctx: any) => {
const tableId = utils.getTableId(ctx)
ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
ctx.body = await quotas.addQuery(() => sdk.rows.exportRows(tableId, ctx), {
datasourceId: tableId,
})
}

234
packages/server/src/api/controllers/row/internal.ts
View file

@ -1,7 +1,6 @@
import * as linkRows from "../../../db/linkedRows"
import {
generateRowID,
getRowParams,
getTableIDFromRowID,
DocumentType,
InternalTables,
@ -14,8 +13,6 @@ import {
} from "../../../utilities/rowProcessor"
import { FieldTypes } from "../../../constants"
import * as utils from "./utils"
import { fullSearch, paginatedSearch } from "./internalSearch"
import { getGlobalUsersFromMetadata } from "../../../utilities/global"
import * as inMemoryViews from "../../../db/inMemoryView"
import env from "../../../environment"
import {
@ -27,65 +24,7 @@ import {
import { cloneDeep } from "lodash/fp"
import { context, db as dbCore } from "@budibase/backend-core"
import { finaliseRow, updateRelatedFormula } from "./staticFormula"
import { csv, json, jsonWithSchema, Format } from "../view/exporters"
import { apiFileReturn } from "../../../utilities/fileSystem"
import {
UserCtx,
Database,
LinkDocumentValue,
Row,
Table,
} from "@budibase/types"
import { cleanExportRows } from "./utils"
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
STATS: "stats",
}
async function getView(db: Database, viewName: string) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err: any) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
return viewInfo
}
async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows as Row[]
}
import { UserCtx, LinkDocumentValue, Row, Table } from "@budibase/types"
export async function patch(ctx: UserCtx) {
const db = context.getAppDB()
@ -195,82 +134,6 @@ export async function save(ctx: UserCtx) {
})
}
export async function fetchView(ctx: UserCtx) {
const viewName = decodeURIComponent(ctx.params.viewName)
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
ctx.params.tableId = viewName
return fetch(ctx)
}
const db = context.getAppDB()
const { calculation, group, field } = ctx.query
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {
include_docs: !calculation,
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await getRawTableData(ctx, db, tableId)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
!!group,
data
)
}
let rows
if (!calculation) {
response.rows = response.rows.map(row => row.doc)
let table
try {
table = await db.get(viewInfo.meta.tableId)
} catch (err) {
/* istanbul ignore next */
table = {
schema: {},
}
}
rows = await outputProcessing(table, response.rows)
}
if (calculation === CALCULATION_TYPES.STATS) {
response.rows = response.rows.map(row => ({
group: row.key,
field,
...row.value,
avg: row.value.sum / row.value.count,
}))
rows = response.rows
}
if (
calculation === CALCULATION_TYPES.COUNT ||
calculation === CALCULATION_TYPES.SUM
) {
rows = response.rows.map(row => ({
group: row.key,
field,
value: row.value,
}))
}
return rows
}
export async function fetch(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
let table = await db.get(tableId)
let rows = await getRawTableData(ctx, db, tableId)
return outputProcessing(table, rows)
}
export async function find(ctx: UserCtx) {
const db = dbCore.getDB(ctx.appId)
const table = await db.get(ctx.params.tableId)
@ -354,101 +217,6 @@ export async function bulkDestroy(ctx: UserCtx) {
return { response: { ok: true }, rows: processedRows }
}
export async function search(ctx: UserCtx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
}
const { tableId } = ctx.params
const db = context.getAppDB()
const { paginate, query, ...params } = ctx.request.body
params.version = ctx.version
params.tableId = tableId
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
table = table || (await db.get(tableId))
response.rows = await outputProcessing(table, response.rows)
}
return response
}
export async function exportRows(ctx: UserCtx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search(ctx)
result = searchResponse.rows
}
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result[i][column]
}
}
} else {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}
}
export async function fetchEnrichedRow(ctx: UserCtx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId

101
packages/server/src/api/controllers/row/utils.ts
View file

@ -1,14 +1,19 @@
import { InternalTables } from "../../../db/utils"
import * as userController from "../user"
import { FieldTypes } from "../../../constants"
import { context } from "@budibase/backend-core"
import { makeExternalQuery } from "../../../integrations/base/query"
import { FieldType, Row, Table, UserCtx } from "@budibase/types"
import { Format } from "../view/exporters"
import { Ctx, FieldType, Row, Table, UserCtx } from "@budibase/types"
import { FieldTypes } from "../../../constants"
import sdk from "../../../sdk"
const validateJs = require("validate.js")
const { cloneDeep } = require("lodash/fp")
import validateJs from "validate.js"
import { cloneDeep } from "lodash/fp"
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(relationship => relationship.foreignKey === key)
}
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
@ -20,19 +25,6 @@ validateJs.extend(validateJs.validators.datetime, {
},
})
function isForeignKey(key: string, table: Table) {
const relationships = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
return relationships.some(relationship => relationship.foreignKey === key)
}
export async function getDatasourceAndQuery(json: any) {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)
}
export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
const db = context.getAppDB()
let row
@ -52,6 +44,18 @@ export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
return row
}
export function getTableId(ctx: Ctx) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
if (ctx.params && ctx.params.tableId) {
return ctx.params.tableId
}
if (ctx.params && ctx.params.viewName) {
return ctx.params.viewName
}
}
export async function validate({
tableId,
row,
@ -81,8 +85,8 @@ export async function validate({
continue
}
// special case for options, need to always allow unselected (empty)
if (type === FieldTypes.OPTIONS && constraints.inclusion) {
constraints.inclusion.push(null, "")
if (type === FieldTypes.OPTIONS && constraints?.inclusion) {
constraints.inclusion.push(null as any, "")
}
let res
@ -94,13 +98,13 @@ export async function validate({
}
row[fieldName].map((val: any) => {
if (
!constraints.inclusion.includes(val) &&
constraints.inclusion.length !== 0
!constraints?.inclusion?.includes(val) &&
constraints?.inclusion?.length !== 0
) {
errors[fieldName] = "Field not in list"
}
})
} else if (constraints.presence && row[fieldName].length === 0) {
} else if (constraints?.presence && row[fieldName].length === 0) {
// non required MultiSelect creates an empty array, which should not throw errors
errors[fieldName] = [`${fieldName} is required`]
}
@ -128,52 +132,3 @@ export async function validate({
}
return { valid: Object.keys(errors).length === 0, errors }
}
export function cleanExportRows(
rows: any[],
schema: any,
format: string,
columns: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = undefined
}
}
}
}
return cleanRows
}
export function getTableId(ctx: any) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
if (ctx.params && ctx.params.tableId) {
return ctx.params.tableId
}
if (ctx.params && ctx.params.viewName) {
return ctx.params.viewName
}
}

4
packages/server/src/api/controllers/user.ts
View file

@ -3,10 +3,10 @@ import { InternalTables } from "../../db/utils"
import { getGlobalUsers } from "../../utilities/global"
import { getFullUser } from "../../utilities/users"
import { context } from "@budibase/backend-core"
import { UserCtx } from "@budibase/types"
import { Ctx, UserCtx } from "@budibase/types"
import sdk from "../../sdk"
export async function fetchMetadata(ctx: UserCtx) {
export async function fetchMetadata(ctx: Ctx) {
const global = await getGlobalUsers()
const metadata = await sdk.users.rawUserMetadata()
const users = []

3
packages/server/src/api/controllers/view/index.ts
View file

@ -15,7 +15,6 @@ import {
TableSchema,
View,
} from "@budibase/types"
import { cleanExportRows } from "../row/utils"
import { builderSocket } from "../../../websockets"
const { cloneDeep, isEqual } = require("lodash")
@ -169,7 +168,7 @@ export async function exportView(ctx: Ctx) {
schema = table.schema
}
let exportRows = cleanExportRows(rows, schema, format, [])
let exportRows = sdk.rows.utils.cleanExportRows(rows, schema, format, [])
if (format === Format.CSV) {
ctx.attachment(`${viewName}.csv`)

197
packages/server/src/api/routes/tests/internalSearch.spec.js
View file

@ -1,197 +0,0 @@
const fetch = require("node-fetch")
fetch.mockSearch()
const search = require("../../controllers/row/internalSearch")
// this will be mocked out for _search endpoint
const PARAMS = {
tableId: "ta_12345679abcdef",
version: "1",
bookmark: null,
sort: null,
sortOrder: "ascending",
sortType: "string",
}
function checkLucene(resp, expected, params = PARAMS) {
const query = resp.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
expect(json.limit).toBe(params.limit || 50)
}
describe("internal search", () => {
it("default query", async () => {
const response = await search.paginatedSearch({
}, PARAMS)
checkLucene(response, `*:*`)
})
it("test equal query", async () => {
const response = await search.paginatedSearch({
equal: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `*:* AND column:"1"`)
})
it("test notEqual query", async () => {
const response = await search.paginatedSearch({
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `*:* AND !column:"1"`)
})
it("test OR query", async () => {
const response = await search.paginatedSearch({
allOr: true,
equal: {
"column": "2",
},
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `(column:"2" OR !column:"1")`)
})
it("test AND query", async () => {
const response = await search.paginatedSearch({
equal: {
"column": "2",
},
notEqual: {
"column": "1",
}
}, PARAMS)
checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
})
it("test pagination query", async () => {
const updatedParams = {
...PARAMS,
limit: 100,
bookmark: "awd",
sort: "column",
}
const response = await search.paginatedSearch({
string: {
"column": "2",
},
}, updatedParams)
checkLucene(response, `*:* AND column:2*`, updatedParams)
})
it("test range query", async () => {
const response = await search.paginatedSearch({
range: {
"column": { low: 1, high: 2 },
},
}, PARAMS)
checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
})
it("test empty query", async () => {
const response = await search.paginatedSearch({
empty: {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {
const response = await search.paginatedSearch({
notEmpty: {
"column": "",
},
}, PARAMS)
checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
})
it("test oneOf query", async () => {
const response = await search.paginatedSearch({
oneOf: {
"column": ["a", "b"],
},
}, PARAMS)
checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
})
it("test contains query", async () => {
const response = await search.paginatedSearch({
contains: {
"column": "a",
"colArr": [1, 2, 3],
},
}, PARAMS)
checkLucene(response, `(*:* AND column:a AND colArr:(1 AND 2 AND 3))`, PARAMS)
})
it("test multiple of same column", async () => {
const response = await search.paginatedSearch({
allOr: true,
equal: {
"1:column": "a",
"2:column": "b",
"3:column": "c",
},
}, PARAMS)
checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
})
it("check a weird case for lucene building", async () => {
const response = await search.paginatedSearch({
equal: {
"1:1:column": "a",
},
}, PARAMS)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch({
containsAny: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch({
notContains: {
"column": ["a", "b", "c"]
},
}, PARAMS)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
it("test equal without version query", async () => {
PARAMS.version = null
const response = await search.paginatedSearch({
equal: {
"column": "1",
}
}, PARAMS)
const query = response.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
})
})

248
packages/server/src/api/routes/tests/internalSearch.spec.ts Normal file
View file

@ -0,0 +1,248 @@
const nodeFetch = require("node-fetch")
nodeFetch.mockSearch()
import { SearchParams } from "@budibase/backend-core"
import * as search from "../../../sdk/app/rows/search/internalSearch"
import { Row } from "@budibase/types"
// this will be mocked out for _search endpoint
const PARAMS: SearchParams<Row> = {
tableId: "ta_12345679abcdef",
version: "1",
bookmark: undefined,
sort: undefined,
sortOrder: "ascending",
sortType: "string",
}
function checkLucene(resp: any, expected: any, params = PARAMS) {
const query = resp.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`${expected} AND tableId:"${params.tableId}"`)
expect(json.limit).toBe(params.limit || 50)
}
describe("internal search", () => {
it("default query", async () => {
const response = await search.paginatedSearch({}, PARAMS)
checkLucene(response, `*:*`)
})
it("test equal query", async () => {
const response = await search.paginatedSearch(
{
equal: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `*:* AND column:"1"`)
})
it("test notEqual query", async () => {
const response = await search.paginatedSearch(
{
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `*:* AND !column:"1"`)
})
it("test OR query", async () => {
const response = await search.paginatedSearch(
{
allOr: true,
equal: {
column: "2",
},
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `(column:"2" OR !column:"1")`)
})
it("test AND query", async () => {
const response = await search.paginatedSearch(
{
equal: {
column: "2",
},
notEqual: {
column: "1",
},
},
PARAMS
)
checkLucene(response, `(*:* AND column:"2" AND !column:"1")`)
})
it("test pagination query", async () => {
const updatedParams = {
...PARAMS,
limit: 100,
bookmark: "awd",
sort: "column",
}
const response = await search.paginatedSearch(
{
string: {
column: "2",
},
},
updatedParams
)
checkLucene(response, `*:* AND column:2*`, updatedParams)
})
it("test range query", async () => {
const response = await search.paginatedSearch(
{
range: {
column: { low: 1, high: 2 },
},
},
PARAMS
)
checkLucene(response, `*:* AND column:[1 TO 2]`, PARAMS)
})
it("test empty query", async () => {
const response = await search.paginatedSearch(
{
empty: {
column: "",
},
},
PARAMS
)
checkLucene(response, `*:* AND (*:* -column:["" TO *])`, PARAMS)
})
it("test notEmpty query", async () => {
const response = await search.paginatedSearch(
{
notEmpty: {
column: "",
},
},
PARAMS
)
checkLucene(response, `*:* AND column:["" TO *]`, PARAMS)
})
it("test oneOf query", async () => {
const response = await search.paginatedSearch(
{
oneOf: {
column: ["a", "b"],
},
},
PARAMS
)
checkLucene(response, `*:* AND column:("a" OR "b")`, PARAMS)
})
it("test contains query", async () => {
const response = await search.paginatedSearch(
{
contains: {
column: "a",
colArr: [1, 2, 3],
},
},
PARAMS
)
checkLucene(
response,
`(*:* AND column:a AND colArr:(1 AND 2 AND 3))`,
PARAMS
)
})
it("test multiple of same column", async () => {
const response = await search.paginatedSearch(
{
allOr: true,
equal: {
"1:column": "a",
"2:column": "b",
"3:column": "c",
},
},
PARAMS
)
checkLucene(response, `(column:"a" OR column:"b" OR column:"c")`, PARAMS)
})
it("check a weird case for lucene building", async () => {
const response = await search.paginatedSearch(
{
equal: {
"1:1:column": "a",
},
},
PARAMS
)
checkLucene(response, `*:* AND 1\\:column:"a"`, PARAMS)
})
it("test containsAny query", async () => {
const response = await search.paginatedSearch(
{
containsAny: {
column: ["a", "b", "c"],
},
},
PARAMS
)
checkLucene(response, `*:* AND column:(a OR b OR c)`, PARAMS)
})
it("test notContains query", async () => {
const response = await search.paginatedSearch(
{
notContains: {
column: ["a", "b", "c"],
},
},
PARAMS
)
checkLucene(response, `*:* AND NOT column:(a AND b AND c)`, PARAMS)
})
it("test equal without version query", async () => {
PARAMS.version = undefined
const response = await search.paginatedSearch(
{
equal: {
column: "1",
},
},
PARAMS
)
const query = response.rows[0].query
const json = JSON.parse(query)
if (PARAMS.sort) {
expect(json.sort).toBe(`${PARAMS.sort}<${PARAMS.sortType}>`)
}
if (PARAMS.bookmark) {
expect(json.bookmark).toBe(PARAMS.bookmark)
}
expect(json.include_docs).toBe(true)
expect(json.q).toBe(`*:* AND column:"1" AND tableId:${PARAMS.tableId}`)
})
})

13
packages/server/src/automations/tests/queryRows.spec.js → packages/server/src/automations/tests/queryRows.spec.ts
View file

@ -1,6 +1,6 @@
// lucene searching not supported in test due to use of PouchDB
let rows = []
jest.mock("../../api/controllers/row/internalSearch", () => ({
let rows: Row[] = []
jest.mock("../../sdk/app/rows/search/internalSearch", () => ({
fullSearch: jest.fn(() => {
return {
rows,
@ -8,12 +8,13 @@ jest.mock("../../api/controllers/row/internalSearch", () => ({
}),
paginatedSearch: jest.fn(),
}))
const setup = require("./utilities")
import { Row, Table } from "@budibase/types"
import * as setup from "./utilities"
const NAME = "Test"
describe("Test a query step automation", () => {
let table
let table: Table
let config = setup.getConfig()
beforeAll(async () => {
@ -87,8 +88,8 @@ describe("Test a query step automation", () => {
filters: {},
"filters-def": [
{
value: null
}
value: null,
},
],
sortColumn: "name",
sortOrder: "ascending",

4
packages/server/src/sdk/app/rows/index.ts
View file

@ -1,7 +1,11 @@
import * as attachments from "./attachments"
import * as rows from "./rows"
import * as search from "./search"
import * as utils from "./utils"
export default {
...attachments,
...rows,
...search,
utils: utils,
}

38
packages/server/src/sdk/app/rows/search.ts Normal file
View file

@ -0,0 +1,38 @@
import { Ctx, SearchFilters } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
export interface SearchParams {
tableId: string
paginate: boolean
query?: SearchFilters
bookmark?: number
limit: number
sort?: string
sortOrder?: string
sortType?: string
}
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
return external
}
return internal
}
export async function search(tableId: string, ctx: Ctx) {
return pickApi(tableId).search(ctx)
}
export async function exportRows(tableId: string, ctx: Ctx) {
return pickApi(tableId).exportRows(ctx)
}
export async function fetch(tableId: string, ctx: Ctx) {
return pickApi(tableId).fetch(ctx)
}
export async function fetchView(tableId: string, ctx: Ctx) {
return pickApi(tableId).fetchView(ctx)
}

177
packages/server/src/sdk/app/rows/search/external.ts Normal file
View file

@ -0,0 +1,177 @@
import {
SortJson,
SortDirection,
Operation,
PaginationJson,
IncludeRelationship,
Row,
Ctx,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { apiFileReturn } from "../../../../utilities/fileSystem"
import { utils } from "@budibase/shared-core"
export async function search(ctx: Ctx) {
const tableId = ctx.params.tableId
const { paginate, query, ...params } = ctx.request.body
let { bookmark, limit } = params
if (!bookmark && paginate) {
bookmark = 1
}
let paginateObj = {}
if (paginate) {
paginateObj = {
// add one so we can track if there is another page
limit: limit,
page: bookmark,
}
} else if (params && limit) {
paginateObj = {
limit: limit,
}
}
let sort: SortJson | undefined
if (params.sort) {
const direction =
params.sortOrder === "descending"
? SortDirection.DESCENDING
: SortDirection.ASCENDING
sort = {
[params.sort]: { direction },
}
}
try {
const rows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: paginateObj as PaginationJson,
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
let hasNextPage = false
if (paginate && rows.length === limit) {
const nextRows = (await handleRequest(Operation.READ, tableId, {
filters: query,
sort,
paginate: {
limit: 1,
page: bookmark * limit + 1,
},
includeSqlRelationships: IncludeRelationship.INCLUDE,
})) as Row[]
hasNextPage = nextRows.length > 0
}
// need wrapper object for bookmarks etc when paginating
return { rows, hasNextPage, bookmark: bookmark + 1 }
} catch (err: any) {
if (err.message && err.message.includes("does not exist")) {
throw new Error(
`Table updated externally, please re-fetch - ${err.message}`
)
} else {
throw err
}
}
}
export async function exportRows(ctx: Ctx) {
const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
const format = ctx.query.format as string
const { columns } = ctx.request.body
const datasource = await sdk.datasources.get(datasourceId!)
if (!datasource || !datasource.entities) {
ctx.throw(400, "Datasource has not been configured for plus API.")
}
if (!exporters.isFormat(format)) {
ctx.throw(
400,
`Format ${format} not valid. Valid values: ${Object.values(
exporters.Format
)}`
)
}
if (ctx.request.body.rows) {
ctx.request.body = {
query: {
oneOf: {
_id: ctx.request.body.rows.map((row: string) => {
const ids = JSON.parse(
decodeURI(row).replace(/'/g, `"`).replace(/%2C/g, ",")
)
if (ids.length > 1) {
ctx.throw(400, "Export data does not support composite keys.")
}
return ids[0]
}),
},
},
}
}
let result = await search(ctx)
let rows: Row[] = []
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.rows.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result.rows[i][column]
}
}
} else {
rows = result.rows
}
if (!tableName) {
ctx.throw(400, "Could not find table name.")
}
let schema = datasource.entities[tableName].schema
let exportRows = cleanExportRows(rows, schema, format, columns)
let headers = Object.keys(schema)
let content
switch (format) {
case exporters.Format.CSV:
content = exporters.csv(headers, exportRows)
break
case exporters.Format.JSON:
content = exporters.json(exportRows)
break
case exporters.Format.JSON_WITH_SCHEMA:
content = exporters.jsonWithSchema(schema, exportRows)
break
default:
utils.unreachable(format)
break
}
const filename = `export.${format}`
// send down the file
ctx.attachment(filename)
return apiFileReturn(content)
}
export async function fetch(ctx: Ctx) {
const tableId = ctx.params.tableId
return handleRequest(Operation.READ, tableId, {
includeSqlRelationships: IncludeRelationship.INCLUDE,
})
}
export async function fetchView(ctx: Ctx) {
// there are no views in external datasources, shouldn't ever be called
// for now just fetch
const split = ctx.params.viewName.split("all_")
ctx.params.tableId = split[1] ? split[1] : split[0]
return fetch(ctx)
}

246
packages/server/src/sdk/app/rows/search/internal.ts Normal file
View file

@ -0,0 +1,246 @@
import { context } from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./internalSearch"
import {
InternalTables,
getRowParams,
DocumentType,
} from "../../../../db/utils"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Ctx, Database, Row, UserCtx } from "@budibase/types"
import { cleanExportRows } from "../utils"
import {
Format,
csv,
json,
jsonWithSchema,
} from "../../../../api/controllers/view/exporters"
import { apiFileReturn } from "../../../../utilities/fileSystem"
import * as userController from "../../../../api/controllers/user"
import * as inMemoryViews from "../../../../db/inMemoryView"
import {
migrateToInMemoryView,
migrateToDesignView,
getFromDesignDoc,
getFromMemoryDoc,
} from "../../../../api/controllers/view/utils"
export async function search(ctx: Ctx) {
// Fetch the whole table when running in cypress, as search doesn't work
if (!env.COUCH_DB_URL && env.isCypress()) {
return { rows: await fetch(ctx) }
}
const { tableId } = ctx.params
const db = context.getAppDB()
const { paginate, query, ...params } = ctx.request.body
params.version = ctx.version
params.tableId = tableId
let table
if (params.sort && !params.sortType) {
table = await db.get(tableId)
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type == "number" ? "number" : "string"
}
let response
if (paginate) {
response = await paginatedSearch(query, params)
} else {
response = await fullSearch(query, params)
}
// Enrich search results with relationships
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
}
table = table || (await db.get(tableId))
response.rows = await outputProcessing(table, response.rows)
}
return response
}
export async function exportRows(ctx: Ctx) {
const db = context.getAppDB()
const table = await db.get(ctx.params.tableId)
const rowIds = ctx.request.body.rows
let format = ctx.query.format
if (typeof format !== "string") {
ctx.throw(400, "Format parameter is not valid")
}
const { columns, query } = ctx.request.body
let result
if (rowIds) {
let response = (
await db.allDocs({
include_docs: true,
keys: rowIds,
})
).rows.map(row => row.doc)
result = await outputProcessing(table, response)
} else if (query) {
let searchResponse = await search(ctx)
result = searchResponse.rows
}
let rows: Row[] = []
let schema = table.schema
// Filter data to only specified columns if required
if (columns && columns.length) {
for (let i = 0; i < result.length; i++) {
rows[i] = {}
for (let column of columns) {
rows[i][column] = result[i][column]
}
}
} else {
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns)
if (format === Format.CSV) {
ctx.attachment("export.csv")
return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
} else if (format === Format.JSON) {
ctx.attachment("export.json")
return apiFileReturn(json(exportRows))
} else if (format === Format.JSON_WITH_SCHEMA) {
ctx.attachment("export.json")
return apiFileReturn(jsonWithSchema(schema, exportRows))
} else {
throw "Format not recognised"
}
}
export async function fetch(ctx: Ctx) {
const db = context.getAppDB()
const tableId = ctx.params.tableId
let table = await db.get(tableId)
let rows = await getRawTableData(ctx, db, tableId)
return outputProcessing(table, rows)
}
async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
let rows
if (tableId === InternalTables.USER_METADATA) {
await userController.fetchMetadata(ctx)
rows = ctx.body
} else {
const response = await db.allDocs(
getRowParams(tableId, null, {
include_docs: true,
})
)
rows = response.rows.map(row => row.doc)
}
return rows as Row[]
}
export async function fetchView(ctx: Ctx) {
const viewName = decodeURIComponent(ctx.params.viewName)
// if this is a table view being looked for just transfer to that
if (viewName.startsWith(DocumentType.TABLE)) {
ctx.params.tableId = viewName
return fetch(ctx)
}
const db = context.getAppDB()
const { calculation, group, field } = ctx.query
const viewInfo = await getView(db, viewName)
let response
if (env.SELF_HOSTED) {
response = await db.query(`database/${viewName}`, {
include_docs: !calculation,
group: !!group,
})
} else {
const tableId = viewInfo.meta.tableId
const data = await getRawTableData(ctx, db, tableId)
response = await inMemoryViews.runView(
viewInfo,
calculation as string,
!!group,
data
)
}
let rows
if (!calculation) {
response.rows = response.rows.map(row => row.doc)
let table
try {
table = await db.get(viewInfo.meta.tableId)
} catch (err) {
/* istanbul ignore next */
table = {
schema: {},
}
}
rows = await outputProcessing(table, response.rows)
}
if (calculation === CALCULATION_TYPES.STATS) {
response.rows = response.rows.map(row => ({
group: row.key,
field,
...row.value,
avg: row.value.sum / row.value.count,
}))
rows = response.rows
}
if (
calculation === CALCULATION_TYPES.COUNT ||
calculation === CALCULATION_TYPES.SUM
) {
rows = response.rows.map(row => ({
group: row.key,
field,
value: row.value,
}))
}
return rows
}
const CALCULATION_TYPES = {
SUM: "sum",
COUNT: "count",
STATS: "stats",
}
async function getView(db: Database, viewName: string) {
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err: any) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."
}
return viewInfo
}

0
packages/server/src/api/controllers/row/internalSearch.ts → packages/server/src/sdk/app/rows/search/internalSearch.ts
View file

47
packages/server/src/sdk/app/rows/utils.ts Normal file
View file

@ -0,0 +1,47 @@
import { FieldTypes } from "../../../constants"
import { makeExternalQuery } from "../../../integrations/base/query"
import { Format } from "../../../api/controllers/view/exporters"
import sdk from "../.."
export async function getDatasourceAndQuery(json: any) {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)
}
export function cleanExportRows(
rows: any[],
schema: any,
format: string,
columns: string[]
) {
let cleanRows = [...rows]
const relationships = Object.entries(schema)
.filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
.map(entry => entry[0])
relationships.forEach(column => {
cleanRows.forEach(row => {
delete row[column]
})
delete schema[column]
})
if (format === Format.CSV) {
// Intended to append empty values in export
const schemaKeys = Object.keys(schema)
for (let key of schemaKeys) {
if (columns?.length && columns.indexOf(key) > 0) {
continue
}
for (let row of cleanRows) {
if (row[key] == null) {
row[key] = undefined
}
}
}
}
return cleanRows
}

32
packages/server/src/api/controllers/tests/row.spec.ts → packages/server/src/sdk/tests/rows/row.spec.ts
View file

@ -1,14 +1,14 @@
import { exportRows } from "../row/external"
import sdk from "../../../sdk"
import { ExternalRequest } from "../row/ExternalRequest"
import { exportRows } from "../../app/rows/search/external"
import sdk from "../.."
import { ExternalRequest } from "../../../api/controllers/row/ExternalRequest"
// @ts-ignore
sdk.datasources = {
get: jest.fn(),
}
const mockDatasourcesGet = jest.fn()
sdk.datasources.get = mockDatasourcesGet
jest.mock("../row/ExternalRequest")
jest.mock("../view/exporters", () => ({
jest.mock("../../../api/controllers/row/ExternalRequest")
jest.mock("../../../api/controllers/view/exporters", () => ({
...jest.requireActual("../../../api/controllers/view/exporters"),
csv: jest.fn(),
Format: {
CSV: "csv",
@ -31,14 +31,15 @@ function getUserCtx() {
throw "Err"
}),
attachment: jest.fn(),
}
} as any
}
describe("external row controller", () => {
describe("exportRows", () => {
beforeAll(() => {
//@ts-ignore
jest.spyOn(ExternalRequest.prototype, "run").mockImplementation(() => [])
jest
.spyOn(ExternalRequest.prototype, "run")
.mockImplementation(() => Promise.resolve([]))
})
afterEach(() => {
@ -48,7 +49,6 @@ describe("external row controller", () => {
it("should throw a 400 if no datasource entities are present", async () => {
let userCtx = getUserCtx()
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
@ -59,8 +59,7 @@ describe("external row controller", () => {
})
it("should handle single quotes from a row ID", async () => {
//@ts-ignore
sdk.datasources.get.mockImplementation(() => ({
mockDatasourcesGet.mockImplementation(async () => ({
entities: {
tablename: {
schema: {},
@ -72,7 +71,6 @@ describe("external row controller", () => {
rows: ["['d001']"],
}
//@ts-ignore
await exportRows(userCtx)
expect(userCtx.request.body).toEqual({
@ -90,7 +88,6 @@ describe("external row controller", () => {
rows: ["[123]", "['d001'%2C'10111']"],
}
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(
@ -107,7 +104,6 @@ describe("external row controller", () => {
rows: ["[123]"],
}
try {
//@ts-ignore
await exportRows(userCtx)
} catch (e) {
expect(userCtx.throw).toHaveBeenCalledWith(

2
packages/types/src/sdk/search.ts
View file

@ -32,7 +32,7 @@ export interface SearchFilters {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
[key: string]: any[] | any
}
notContains?: {
[key: string]: any[]