Bit more work towards row counting, as well as moving external SQL to use row + 1 for working out pagination.
This commit is contained in:
parent
2c6262844b
commit
77556820bf
8 changed files with 49 additions and 65 deletions
|
@ -571,15 +571,10 @@ class InternalBuilder {
|
||||||
return query.insert(parsedBody)
|
return query.insert(parsedBody)
|
||||||
}
|
}
|
||||||
|
|
||||||
read(
|
read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder {
|
||||||
knex: Knex,
|
|
||||||
json: QueryJson,
|
|
||||||
limit: number,
|
|
||||||
opts?: { counting?: boolean }
|
|
||||||
): Knex.QueryBuilder {
|
|
||||||
let { endpoint, resource, filters, paginate, relationships, tableAliases } =
|
let { endpoint, resource, filters, paginate, relationships, tableAliases } =
|
||||||
json
|
json
|
||||||
const counting = opts?.counting
|
const counting = endpoint.operation === Operation.COUNT
|
||||||
|
|
||||||
const tableName = endpoint.entityId
|
const tableName = endpoint.entityId
|
||||||
// select all if not specified
|
// select all if not specified
|
||||||
|
@ -730,6 +725,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
|
||||||
query = builder.create(client, json, opts)
|
query = builder.create(client, json, opts)
|
||||||
break
|
break
|
||||||
case Operation.READ:
|
case Operation.READ:
|
||||||
|
case Operation.COUNT:
|
||||||
query = builder.read(client, json, this.limit)
|
query = builder.read(client, json, this.limit)
|
||||||
break
|
break
|
||||||
case Operation.UPDATE:
|
case Operation.UPDATE:
|
||||||
|
@ -752,20 +748,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
|
||||||
return this.convertToNative(query, opts)
|
return this.convertToNative(query, opts)
|
||||||
}
|
}
|
||||||
|
|
||||||
_count(json: QueryJson, opts: QueryOptions = {}) {
|
|
||||||
const sqlClient = this.getSqlClient()
|
|
||||||
const config: Knex.Config = {
|
|
||||||
client: sqlClient,
|
|
||||||
}
|
|
||||||
if (sqlClient === SqlClient.SQL_LITE) {
|
|
||||||
config.useNullAsDefault = true
|
|
||||||
}
|
|
||||||
const client = knex(config)
|
|
||||||
const builder = new InternalBuilder(sqlClient)
|
|
||||||
const query = builder.read(client, json, this.limit, { counting: true })
|
|
||||||
return this.convertToNative(query, opts)
|
|
||||||
}
|
|
||||||
|
|
||||||
async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
|
async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
|
||||||
if (!json.extra || !json.extra.idFilter) {
|
if (!json.extra || !json.extra.idFilter) {
|
||||||
return {}
|
return {}
|
||||||
|
|
|
@ -39,6 +39,7 @@ import { cloneDeep } from "lodash/fp"
|
||||||
import { db as dbCore } from "@budibase/backend-core"
|
import { db as dbCore } from "@budibase/backend-core"
|
||||||
import sdk from "../../../sdk"
|
import sdk from "../../../sdk"
|
||||||
import env from "../../../environment"
|
import env from "../../../environment"
|
||||||
|
import { makeExternalQuery } from "../../../integrations/base/query"
|
||||||
|
|
||||||
export interface ManyRelationship {
|
export interface ManyRelationship {
|
||||||
tableId?: string
|
tableId?: string
|
||||||
|
@ -517,7 +518,7 @@ export class ExternalRequest<T extends Operation> {
|
||||||
// finally cleanup anything that needs to be removed
|
// finally cleanup anything that needs to be removed
|
||||||
for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
|
for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
|
||||||
const table: Table | undefined = this.getTable(tableId)
|
const table: Table | undefined = this.getTable(tableId)
|
||||||
// if its not the foreign key skip it, nothing to do
|
// if it's not the foreign key skip it, nothing to do
|
||||||
if (
|
if (
|
||||||
!table ||
|
!table ||
|
||||||
(!isMany && table.primary && table.primary.indexOf(colName) !== -1)
|
(!isMany && table.primary && table.primary.indexOf(colName) !== -1)
|
||||||
|
@ -667,7 +668,7 @@ export class ExternalRequest<T extends Operation> {
|
||||||
response = await getDatasourceAndQuery(json)
|
response = await getDatasourceAndQuery(json)
|
||||||
} else {
|
} else {
|
||||||
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
|
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
|
||||||
response = await aliasing.queryWithAliasing(json)
|
response = await aliasing.queryWithAliasing(json, makeExternalQuery)
|
||||||
}
|
}
|
||||||
|
|
||||||
const responseRows = Array.isArray(response) ? response : []
|
const responseRows = Array.isArray(response) ? response : []
|
||||||
|
|
|
@ -8,8 +8,8 @@ import { getIntegration } from "../index"
|
||||||
import sdk from "../../sdk"
|
import sdk from "../../sdk"
|
||||||
|
|
||||||
export async function makeExternalQuery(
|
export async function makeExternalQuery(
|
||||||
datasource: Datasource,
|
json: QueryJson,
|
||||||
json: QueryJson
|
datasource?: Datasource
|
||||||
): Promise<DatasourcePlusQueryResponse> {
|
): Promise<DatasourcePlusQueryResponse> {
|
||||||
const entityId = json.endpoint.entityId,
|
const entityId = json.endpoint.entityId,
|
||||||
tableName = json.meta.table.name,
|
tableName = json.meta.table.name,
|
||||||
|
@ -22,6 +22,9 @@ export async function makeExternalQuery(
|
||||||
) {
|
) {
|
||||||
throw new Error("Entity ID and table metadata do not align")
|
throw new Error("Entity ID and table metadata do not align")
|
||||||
}
|
}
|
||||||
|
if (!datasource) {
|
||||||
|
throw new Error("No datasource provided for external query")
|
||||||
|
}
|
||||||
datasource = await sdk.datasources.enrich(datasource)
|
datasource = await sdk.datasources.enrich(datasource)
|
||||||
const Integration = await getIntegration(datasource.source)
|
const Integration = await getIntegration(datasource.source)
|
||||||
// query is the opinionated function
|
// query is the opinionated function
|
||||||
|
|
|
@ -28,7 +28,7 @@ export async function search(
|
||||||
table: Table
|
table: Table
|
||||||
): Promise<SearchResponse<Row>> {
|
): Promise<SearchResponse<Row>> {
|
||||||
const { tableId } = options
|
const { tableId } = options
|
||||||
const { paginate, query, ...params } = options
|
const { countRows, paginate, query, ...params } = options
|
||||||
const { limit } = params
|
const { limit } = params
|
||||||
let bookmark =
|
let bookmark =
|
||||||
(params.bookmark && parseInt(params.bookmark as string)) || undefined
|
(params.bookmark && parseInt(params.bookmark as string)) || undefined
|
||||||
|
@ -37,10 +37,14 @@ export async function search(
|
||||||
}
|
}
|
||||||
let paginateObj = {}
|
let paginateObj = {}
|
||||||
|
|
||||||
if (paginate) {
|
if (paginate && !limit) {
|
||||||
|
throw new Error("Cannot paginate query without a limit")
|
||||||
|
}
|
||||||
|
|
||||||
|
if (paginate && limit) {
|
||||||
paginateObj = {
|
paginateObj = {
|
||||||
// add one so we can track if there is another page
|
// add one so we can track if there is another page
|
||||||
limit: limit,
|
limit: limit + 1,
|
||||||
page: bookmark,
|
page: bookmark,
|
||||||
}
|
}
|
||||||
} else if (params && limit) {
|
} else if (params && limit) {
|
||||||
|
@ -76,17 +80,10 @@ export async function search(
|
||||||
includeSqlRelationships: IncludeRelationship.INCLUDE,
|
includeSqlRelationships: IncludeRelationship.INCLUDE,
|
||||||
})
|
})
|
||||||
let hasNextPage = false
|
let hasNextPage = false
|
||||||
if (paginate && rows.length === limit) {
|
// remove the extra row if it's there
|
||||||
const nextRows = await handleRequest(Operation.READ, tableId, {
|
if (paginate && limit && rows.length > limit) {
|
||||||
filters: query,
|
rows.pop()
|
||||||
sort,
|
hasNextPage = true
|
||||||
paginate: {
|
|
||||||
limit: 1,
|
|
||||||
page: bookmark! * limit + 1,
|
|
||||||
},
|
|
||||||
includeSqlRelationships: IncludeRelationship.INCLUDE,
|
|
||||||
})
|
|
||||||
hasNextPage = nextRows.length > 0
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if (options.fields) {
|
if (options.fields) {
|
||||||
|
|
|
@ -12,7 +12,6 @@ import {
|
||||||
SortOrder,
|
SortOrder,
|
||||||
SortType,
|
SortType,
|
||||||
SqlClient,
|
SqlClient,
|
||||||
SqlQuery,
|
|
||||||
Table,
|
Table,
|
||||||
} from "@budibase/types"
|
} from "@budibase/types"
|
||||||
import {
|
import {
|
||||||
|
@ -114,17 +113,13 @@ async function runSqlQuery(
|
||||||
opts?: { countTotalRows?: boolean }
|
opts?: { countTotalRows?: boolean }
|
||||||
) {
|
) {
|
||||||
const alias = new AliasTables(tables.map(table => table.name))
|
const alias = new AliasTables(tables.map(table => table.name))
|
||||||
|
if (opts?.countTotalRows) {
|
||||||
|
json.endpoint.operation = Operation.COUNT
|
||||||
|
}
|
||||||
const processSQLQuery = async (json: QueryJson) => {
|
const processSQLQuery = async (json: QueryJson) => {
|
||||||
let query: SqlQuery | SqlQuery[]
|
const query = builder._query(json, {
|
||||||
if (opts?.countTotalRows) {
|
disableReturning: true,
|
||||||
query = builder._count(json, {
|
})
|
||||||
disableReturning: true,
|
|
||||||
})
|
|
||||||
} else {
|
|
||||||
query = builder._query(json, {
|
|
||||||
disableReturning: true,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|
||||||
if (Array.isArray(query)) {
|
if (Array.isArray(query)) {
|
||||||
throw new Error("SQS cannot currently handle multiple queries")
|
throw new Error("SQS cannot currently handle multiple queries")
|
||||||
|
@ -227,14 +222,18 @@ export async function search(
|
||||||
)
|
)
|
||||||
|
|
||||||
// check for pagination final row
|
// check for pagination final row
|
||||||
let nextRow: Row | undefined, rowCount: number | undefined
|
let nextRow: Row | undefined
|
||||||
if (paginate && params.limit && processed.length > params.limit) {
|
if (paginate && params.limit && processed.length > params.limit) {
|
||||||
// get the total count of rows
|
|
||||||
rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
|
|
||||||
// remove the extra row that confirmed if there is another row to move to
|
// remove the extra row that confirmed if there is another row to move to
|
||||||
nextRow = processed.pop()
|
nextRow = processed.pop()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let rowCount: number | undefined
|
||||||
|
if (options.countRows) {
|
||||||
|
// get the total count of rows
|
||||||
|
rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
|
||||||
|
}
|
||||||
|
|
||||||
// get the rows
|
// get the rows
|
||||||
let finalRows = await outputProcessing<Row[]>(table, processed, {
|
let finalRows = await outputProcessing<Row[]>(table, processed, {
|
||||||
preserveLinks: true,
|
preserveLinks: true,
|
||||||
|
@ -255,9 +254,11 @@ export async function search(
|
||||||
const hasNextPage = !!nextRow
|
const hasNextPage = !!nextRow
|
||||||
response.hasNextPage = hasNextPage
|
response.hasNextPage = hasNextPage
|
||||||
if (hasNextPage) {
|
if (hasNextPage) {
|
||||||
response.totalRows = rowCount
|
|
||||||
response.bookmark = bookmark + 1
|
response.bookmark = bookmark + 1
|
||||||
}
|
}
|
||||||
|
if (rowCount != null) {
|
||||||
|
response.totalRows = rowCount
|
||||||
|
}
|
||||||
return response
|
return response
|
||||||
} else {
|
} else {
|
||||||
return {
|
return {
|
||||||
|
|
|
@ -11,7 +11,11 @@ import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
|
||||||
import { getSQLClient } from "./utils"
|
import { getSQLClient } from "./utils"
|
||||||
import { cloneDeep } from "lodash"
|
import { cloneDeep } from "lodash"
|
||||||
import datasources from "../datasources"
|
import datasources from "../datasources"
|
||||||
import { makeExternalQuery } from "../../../integrations/base/query"
|
|
||||||
|
type PerformQueryFunction = (
|
||||||
|
json: QueryJson,
|
||||||
|
datasource?: Datasource
|
||||||
|
) => Promise<DatasourcePlusQueryResponse>
|
||||||
|
|
||||||
const WRITE_OPERATIONS: Operation[] = [
|
const WRITE_OPERATIONS: Operation[] = [
|
||||||
Operation.CREATE,
|
Operation.CREATE,
|
||||||
|
@ -171,7 +175,7 @@ export default class AliasTables {
|
||||||
|
|
||||||
async queryWithAliasing(
|
async queryWithAliasing(
|
||||||
json: QueryJson,
|
json: QueryJson,
|
||||||
queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
|
queryFn: PerformQueryFunction
|
||||||
): Promise<DatasourcePlusQueryResponse> {
|
): Promise<DatasourcePlusQueryResponse> {
|
||||||
const datasourceId = json.endpoint.datasourceId
|
const datasourceId = json.endpoint.datasourceId
|
||||||
const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
|
const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
|
||||||
|
@ -229,14 +233,7 @@ export default class AliasTables {
|
||||||
json.tableAliases = invertedTableAliases
|
json.tableAliases = invertedTableAliases
|
||||||
}
|
}
|
||||||
|
|
||||||
let response: DatasourcePlusQueryResponse
|
let response: DatasourcePlusQueryResponse = await queryFn(json, datasource)
|
||||||
if (datasource && !isSqs) {
|
|
||||||
response = await makeExternalQuery(datasource, json)
|
|
||||||
} else if (queryFn) {
|
|
||||||
response = await queryFn(json)
|
|
||||||
} else {
|
|
||||||
throw new Error("No supplied method to perform aliased query")
|
|
||||||
}
|
|
||||||
if (Array.isArray(response) && aliasingEnabled) {
|
if (Array.isArray(response) && aliasingEnabled) {
|
||||||
return this.reverse(response)
|
return this.reverse(response)
|
||||||
} else {
|
} else {
|
||||||
|
@ -247,8 +244,9 @@ export default class AliasTables {
|
||||||
// handles getting the count out of the query
|
// handles getting the count out of the query
|
||||||
async countWithAliasing(
|
async countWithAliasing(
|
||||||
json: QueryJson,
|
json: QueryJson,
|
||||||
queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
|
queryFn: PerformQueryFunction
|
||||||
): Promise<number> {
|
): Promise<number> {
|
||||||
|
json.endpoint.operation = Operation.COUNT
|
||||||
let response = await this.queryWithAliasing(json, queryFn)
|
let response = await this.queryWithAliasing(json, queryFn)
|
||||||
if (response && response.length === 1 && "total" in response[0]) {
|
if (response && response.length === 1 && "total" in response[0]) {
|
||||||
return response[0].total
|
return response[0].total
|
||||||
|
|
|
@ -25,6 +25,7 @@ export interface SearchViewRowRequest
|
||||||
| "bookmark"
|
| "bookmark"
|
||||||
| "paginate"
|
| "paginate"
|
||||||
| "query"
|
| "query"
|
||||||
|
| "countRows"
|
||||||
> {}
|
> {}
|
||||||
|
|
||||||
export interface SearchRowResponse {
|
export interface SearchRowResponse {
|
||||||
|
|
|
@ -17,6 +17,7 @@ export interface SearchParams {
|
||||||
fields?: string[]
|
fields?: string[]
|
||||||
indexer?: () => Promise<any>
|
indexer?: () => Promise<any>
|
||||||
rows?: Row[]
|
rows?: Row[]
|
||||||
|
countRows?: boolean
|
||||||
}
|
}
|
||||||
|
|
||||||
// when searching for rows we want a more extensive search type that requires certain properties
|
// when searching for rows we want a more extensive search type that requires certain properties
|
||||||
|
|
Loading…
Reference in a new issue