Bit more work towards row counting, as well as moving external SQL to use row + 1 for working out pagination.
This commit is contained in:
parent
2c6262844b
commit
77556820bf
8 changed files with 49 additions and 65 deletions
|
@ -571,15 +571,10 @@ class InternalBuilder {
|
|||
return query.insert(parsedBody)
|
||||
}
|
||||
|
||||
read(
|
||||
knex: Knex,
|
||||
json: QueryJson,
|
||||
limit: number,
|
||||
opts?: { counting?: boolean }
|
||||
): Knex.QueryBuilder {
|
||||
read(knex: Knex, json: QueryJson, limit: number): Knex.QueryBuilder {
|
||||
let { endpoint, resource, filters, paginate, relationships, tableAliases } =
|
||||
json
|
||||
const counting = opts?.counting
|
||||
const counting = endpoint.operation === Operation.COUNT
|
||||
|
||||
const tableName = endpoint.entityId
|
||||
// select all if not specified
|
||||
|
@ -730,6 +725,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
|
|||
query = builder.create(client, json, opts)
|
||||
break
|
||||
case Operation.READ:
|
||||
case Operation.COUNT:
|
||||
query = builder.read(client, json, this.limit)
|
||||
break
|
||||
case Operation.UPDATE:
|
||||
|
@ -752,20 +748,6 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
|
|||
return this.convertToNative(query, opts)
|
||||
}
|
||||
|
||||
_count(json: QueryJson, opts: QueryOptions = {}) {
|
||||
const sqlClient = this.getSqlClient()
|
||||
const config: Knex.Config = {
|
||||
client: sqlClient,
|
||||
}
|
||||
if (sqlClient === SqlClient.SQL_LITE) {
|
||||
config.useNullAsDefault = true
|
||||
}
|
||||
const client = knex(config)
|
||||
const builder = new InternalBuilder(sqlClient)
|
||||
const query = builder.read(client, json, this.limit, { counting: true })
|
||||
return this.convertToNative(query, opts)
|
||||
}
|
||||
|
||||
async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
|
||||
if (!json.extra || !json.extra.idFilter) {
|
||||
return {}
|
||||
|
|
|
@ -39,6 +39,7 @@ import { cloneDeep } from "lodash/fp"
|
|||
import { db as dbCore } from "@budibase/backend-core"
|
||||
import sdk from "../../../sdk"
|
||||
import env from "../../../environment"
|
||||
import { makeExternalQuery } from "../../../integrations/base/query"
|
||||
|
||||
export interface ManyRelationship {
|
||||
tableId?: string
|
||||
|
@ -517,7 +518,7 @@ export class ExternalRequest<T extends Operation> {
|
|||
// finally cleanup anything that needs to be removed
|
||||
for (let [colName, { isMany, rows, tableId }] of Object.entries(related)) {
|
||||
const table: Table | undefined = this.getTable(tableId)
|
||||
// if its not the foreign key skip it, nothing to do
|
||||
// if it's not the foreign key skip it, nothing to do
|
||||
if (
|
||||
!table ||
|
||||
(!isMany && table.primary && table.primary.indexOf(colName) !== -1)
|
||||
|
@ -667,7 +668,7 @@ export class ExternalRequest<T extends Operation> {
|
|||
response = await getDatasourceAndQuery(json)
|
||||
} else {
|
||||
const aliasing = new sdk.rows.AliasTables(Object.keys(this.tables))
|
||||
response = await aliasing.queryWithAliasing(json)
|
||||
response = await aliasing.queryWithAliasing(json, makeExternalQuery)
|
||||
}
|
||||
|
||||
const responseRows = Array.isArray(response) ? response : []
|
||||
|
|
|
@ -8,8 +8,8 @@ import { getIntegration } from "../index"
|
|||
import sdk from "../../sdk"
|
||||
|
||||
export async function makeExternalQuery(
|
||||
datasource: Datasource,
|
||||
json: QueryJson
|
||||
json: QueryJson,
|
||||
datasource?: Datasource
|
||||
): Promise<DatasourcePlusQueryResponse> {
|
||||
const entityId = json.endpoint.entityId,
|
||||
tableName = json.meta.table.name,
|
||||
|
@ -22,6 +22,9 @@ export async function makeExternalQuery(
|
|||
) {
|
||||
throw new Error("Entity ID and table metadata do not align")
|
||||
}
|
||||
if (!datasource) {
|
||||
throw new Error("No datasource provided for external query")
|
||||
}
|
||||
datasource = await sdk.datasources.enrich(datasource)
|
||||
const Integration = await getIntegration(datasource.source)
|
||||
// query is the opinionated function
|
||||
|
|
|
@ -28,7 +28,7 @@ export async function search(
|
|||
table: Table
|
||||
): Promise<SearchResponse<Row>> {
|
||||
const { tableId } = options
|
||||
const { paginate, query, ...params } = options
|
||||
const { countRows, paginate, query, ...params } = options
|
||||
const { limit } = params
|
||||
let bookmark =
|
||||
(params.bookmark && parseInt(params.bookmark as string)) || undefined
|
||||
|
@ -37,10 +37,14 @@ export async function search(
|
|||
}
|
||||
let paginateObj = {}
|
||||
|
||||
if (paginate) {
|
||||
if (paginate && !limit) {
|
||||
throw new Error("Cannot paginate query without a limit")
|
||||
}
|
||||
|
||||
if (paginate && limit) {
|
||||
paginateObj = {
|
||||
// add one so we can track if there is another page
|
||||
limit: limit,
|
||||
limit: limit + 1,
|
||||
page: bookmark,
|
||||
}
|
||||
} else if (params && limit) {
|
||||
|
@ -76,17 +80,10 @@ export async function search(
|
|||
includeSqlRelationships: IncludeRelationship.INCLUDE,
|
||||
})
|
||||
let hasNextPage = false
|
||||
if (paginate && rows.length === limit) {
|
||||
const nextRows = await handleRequest(Operation.READ, tableId, {
|
||||
filters: query,
|
||||
sort,
|
||||
paginate: {
|
||||
limit: 1,
|
||||
page: bookmark! * limit + 1,
|
||||
},
|
||||
includeSqlRelationships: IncludeRelationship.INCLUDE,
|
||||
})
|
||||
hasNextPage = nextRows.length > 0
|
||||
// remove the extra row if it's there
|
||||
if (paginate && limit && rows.length > limit) {
|
||||
rows.pop()
|
||||
hasNextPage = true
|
||||
}
|
||||
|
||||
if (options.fields) {
|
||||
|
|
|
@ -12,7 +12,6 @@ import {
|
|||
SortOrder,
|
||||
SortType,
|
||||
SqlClient,
|
||||
SqlQuery,
|
||||
Table,
|
||||
} from "@budibase/types"
|
||||
import {
|
||||
|
@ -114,17 +113,13 @@ async function runSqlQuery(
|
|||
opts?: { countTotalRows?: boolean }
|
||||
) {
|
||||
const alias = new AliasTables(tables.map(table => table.name))
|
||||
const processSQLQuery = async (json: QueryJson) => {
|
||||
let query: SqlQuery | SqlQuery[]
|
||||
if (opts?.countTotalRows) {
|
||||
query = builder._count(json, {
|
||||
disableReturning: true,
|
||||
})
|
||||
} else {
|
||||
query = builder._query(json, {
|
||||
disableReturning: true,
|
||||
})
|
||||
json.endpoint.operation = Operation.COUNT
|
||||
}
|
||||
const processSQLQuery = async (json: QueryJson) => {
|
||||
const query = builder._query(json, {
|
||||
disableReturning: true,
|
||||
})
|
||||
|
||||
if (Array.isArray(query)) {
|
||||
throw new Error("SQS cannot currently handle multiple queries")
|
||||
|
@ -227,14 +222,18 @@ export async function search(
|
|||
)
|
||||
|
||||
// check for pagination final row
|
||||
let nextRow: Row | undefined, rowCount: number | undefined
|
||||
let nextRow: Row | undefined
|
||||
if (paginate && params.limit && processed.length > params.limit) {
|
||||
// get the total count of rows
|
||||
rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
|
||||
// remove the extra row that confirmed if there is another row to move to
|
||||
nextRow = processed.pop()
|
||||
}
|
||||
|
||||
let rowCount: number | undefined
|
||||
if (options.countRows) {
|
||||
// get the total count of rows
|
||||
rowCount = await runSqlQuery(request, allTables, { countTotalRows: true })
|
||||
}
|
||||
|
||||
// get the rows
|
||||
let finalRows = await outputProcessing<Row[]>(table, processed, {
|
||||
preserveLinks: true,
|
||||
|
@ -255,9 +254,11 @@ export async function search(
|
|||
const hasNextPage = !!nextRow
|
||||
response.hasNextPage = hasNextPage
|
||||
if (hasNextPage) {
|
||||
response.totalRows = rowCount
|
||||
response.bookmark = bookmark + 1
|
||||
}
|
||||
if (rowCount != null) {
|
||||
response.totalRows = rowCount
|
||||
}
|
||||
return response
|
||||
} else {
|
||||
return {
|
||||
|
|
|
@ -11,7 +11,11 @@ import { SQS_DATASOURCE_INTERNAL } from "@budibase/backend-core"
|
|||
import { getSQLClient } from "./utils"
|
||||
import { cloneDeep } from "lodash"
|
||||
import datasources from "../datasources"
|
||||
import { makeExternalQuery } from "../../../integrations/base/query"
|
||||
|
||||
type PerformQueryFunction = (
|
||||
json: QueryJson,
|
||||
datasource?: Datasource
|
||||
) => Promise<DatasourcePlusQueryResponse>
|
||||
|
||||
const WRITE_OPERATIONS: Operation[] = [
|
||||
Operation.CREATE,
|
||||
|
@ -171,7 +175,7 @@ export default class AliasTables {
|
|||
|
||||
async queryWithAliasing(
|
||||
json: QueryJson,
|
||||
queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
|
||||
queryFn: PerformQueryFunction
|
||||
): Promise<DatasourcePlusQueryResponse> {
|
||||
const datasourceId = json.endpoint.datasourceId
|
||||
const isSqs = datasourceId === SQS_DATASOURCE_INTERNAL
|
||||
|
@ -229,14 +233,7 @@ export default class AliasTables {
|
|||
json.tableAliases = invertedTableAliases
|
||||
}
|
||||
|
||||
let response: DatasourcePlusQueryResponse
|
||||
if (datasource && !isSqs) {
|
||||
response = await makeExternalQuery(datasource, json)
|
||||
} else if (queryFn) {
|
||||
response = await queryFn(json)
|
||||
} else {
|
||||
throw new Error("No supplied method to perform aliased query")
|
||||
}
|
||||
let response: DatasourcePlusQueryResponse = await queryFn(json, datasource)
|
||||
if (Array.isArray(response) && aliasingEnabled) {
|
||||
return this.reverse(response)
|
||||
} else {
|
||||
|
@ -247,8 +244,9 @@ export default class AliasTables {
|
|||
// handles getting the count out of the query
|
||||
async countWithAliasing(
|
||||
json: QueryJson,
|
||||
queryFn?: (json: QueryJson) => Promise<DatasourcePlusQueryResponse>
|
||||
queryFn: PerformQueryFunction
|
||||
): Promise<number> {
|
||||
json.endpoint.operation = Operation.COUNT
|
||||
let response = await this.queryWithAliasing(json, queryFn)
|
||||
if (response && response.length === 1 && "total" in response[0]) {
|
||||
return response[0].total
|
||||
|
|
|
@ -25,6 +25,7 @@ export interface SearchViewRowRequest
|
|||
| "bookmark"
|
||||
| "paginate"
|
||||
| "query"
|
||||
| "countRows"
|
||||
> {}
|
||||
|
||||
export interface SearchRowResponse {
|
||||
|
|
|
@ -17,6 +17,7 @@ export interface SearchParams {
|
|||
fields?: string[]
|
||||
indexer?: () => Promise<any>
|
||||
rows?: Row[]
|
||||
countRows?: boolean
|
||||
}
|
||||
|
||||
// when searching for rows we want a more extensive search type that requires certain properties
|
||||
|
|
Loading…
Reference in a new issue