1
0
Fork 0
mirror of synced 2024-10-06 04:54:52 +13:00

Merge remote-tracking branch 'origin/develop' into feature/form-block-ux-updates

This commit is contained in:
Dean 2023-08-08 09:14:16 +01:00
commit 211d8cdda6
40 changed files with 633 additions and 544 deletions

View file

@ -25,13 +25,13 @@ jobs:
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
- name: Checkout repo only
uses: actions/checkout@v3
if: github.repository != github.event.pull_request.head.repo.full_name
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
- name: Use Node.js 14.x
uses: actions/setup-node@v3
@ -46,13 +46,13 @@ jobs:
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
- name: Checkout repo only
uses: actions/checkout@v3
if: github.repository != github.event.pull_request.head.repo.full_name
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
- name: Use Node.js 14.x
uses: actions/setup-node@v3
@ -70,13 +70,13 @@ jobs:
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
- name: Checkout repo only
uses: actions/checkout@v3
if: github.repository != github.event.pull_request.head.repo.full_name
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
- name: Use Node.js 14.x
uses: actions/setup-node@v3
@ -96,13 +96,13 @@ jobs:
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
- name: Checkout repo only
uses: actions/checkout@v3
if: github.repository != github.event.pull_request.head.repo.full_name
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
- name: Use Node.js 14.x
uses: actions/setup-node@v3
@ -119,7 +119,7 @@ jobs:
test-pro:
runs-on: ubuntu-latest
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
@ -140,13 +140,13 @@ jobs:
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN || github.token }}
- name: Checkout repo only
uses: actions/checkout@v3
if: github.repository != github.event.pull_request.head.repo.full_name
if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != 'Budibase/budibase'
- name: Use Node.js 14.x
uses: actions/setup-node@v3
@ -166,7 +166,7 @@ jobs:
check-pro-submodule:
runs-on: ubuntu-latest
if: github.repository == github.event.pull_request.head.repo.full_name
if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'Budibase/budibase'
steps:
- name: Checkout repo and submodules
uses: actions/checkout@v3
@ -190,6 +190,8 @@ jobs:
base_commit=$(git rev-parse origin/develop)
fi
echo "target_branch=$branch"
echo "target_branch=$branch" >> "$GITHUB_OUTPUT"
echo "pro_commit=$pro_commit"
echo "pro_commit=$pro_commit" >> "$GITHUB_OUTPUT"
echo "base_commit=$base_commit"
@ -204,7 +206,7 @@ jobs:
const baseCommit = '${{ steps.get_pro_commits.outputs.base_commit }}';
if (submoduleCommit !== baseCommit) {
console.error('Submodule commit does not match the latest commit on the develop branch.');
console.error('Submodule commit does not match the latest commit on the "${{ steps.get_pro_commits.outputs.target_branch }}"" branch.');
console.error('Refer to the pro repo to merge your changes: https://github.com/Budibase/budibase-pro/blob/develop/docs/getting_started.md')
process.exit(1);
} else {

View file

@ -0,0 +1,29 @@
name: check_unreleased_changes
on:
pull_request:
branches:
- master
jobs:
check_unreleased:
runs-on: ubuntu-latest
steps:
- name: Check for unreleased changes
env:
REPO: "Budibase/budibase"
TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
RELEASE_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
"https://api.github.com/repos/$REPO/releases/latest" | \
jq -r .published_at)
COMMIT_TIMESTAMP=$(curl -s -H "Authorization: token $TOKEN" \
"https://api.github.com/repos/$REPO/commits/master" | \
jq -r .commit.committer.date)
RELEASE_SECONDS=$(date --date="$RELEASE_TIMESTAMP" "+%s")
COMMIT_SECONDS=$(date --date="$COMMIT_TIMESTAMP" "+%s")
if (( COMMIT_SECONDS > RELEASE_SECONDS )); then
echo "There are unreleased changes. Please release these changes before merging."
exit 1
fi
echo "No unreleased changes detected."

View file

@ -60,9 +60,9 @@ jobs:
- name: "Get Current tag"
id: currenttag
run: |
version=v$(./scripts/getCurrentVersion.sh)
echo 'Using tag $version'
echo "::set-output name=tag::$resversionult"
version=$(./scripts/getCurrentVersion.sh)
echo "Using tag $version"
echo "version=$version" >> "$GITHUB_OUTPUT"
- name: Build/release Docker images
run: |
@ -71,7 +71,7 @@ jobs:
env:
DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.tag }}
BUDIBASE_RELEASE_VERSION: ${{ steps.currenttag.outputs.version }}
release-helm-chart:
needs: [release-images]

View file

@ -8,7 +8,7 @@ env:
PERSONAL_ACCESS_TOKEN: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
REGISTRY_URL: registry.hub.docker.com
jobs:
build-amd64:
build-amd64-arm64:
name: "build-amd64"
runs-on: ubuntu-latest
strategy:
@ -68,81 +68,7 @@ jobs:
with:
context: .
push: true
platforms: linux/amd64
tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
file: ./hosting/single/Dockerfile
- name: Tag and release Budibase Azure App Service docker image
uses: docker/build-push-action@v2
with:
context: .
push: true
platforms: linux/amd64
build-args: TARGETBUILD=aas
tags: budibase/budibase-aas,budibase/budibase-aas:v${{ env.RELEASE_VERSION }}
file: ./hosting/single/Dockerfile
build-arm64:
name: "build-arm64"
runs-on: ubuntu-latest
strategy:
matrix:
node-version: [14.x]
steps:
- name: Fail if not a tag
run: |
if [[ $GITHUB_REF != refs/tags/* ]]; then
echo "Workflow Dispatch can only be run on tags"
exit 1
fi
- name: "Checkout"
uses: actions/checkout@v2
with:
submodules: true
token: ${{ secrets.PERSONAL_ACCESS_TOKEN }}
fetch-depth: 0
- name: Fail if tag is not in master
run: |
if ! git merge-base --is-ancestor ${{ github.sha }} origin/master; then
echo "Tag is not in master. This pipeline can only execute tags that are present on the master branch"
exit 1
fi
- name: Use Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v1
with:
node-version: ${{ matrix.node-version }}
- name: Setup QEMU
uses: docker/setup-qemu-action@v1
- name: Setup Docker Buildx
id: buildx
uses: docker/setup-buildx-action@v1
- name: Run Yarn
run: yarn
- name: Update versions
run: ./scripts/updateVersions.sh
- name: Runt Yarn Lint
run: yarn lint
- name: Update versions
run: ./scripts/updateVersions.sh
- name: Run Yarn Build
run: yarn build:docker:pre
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_API_KEY }}
- name: Get the latest release version
id: version
run: |
release_version=$(cat lerna.json | jq -r '.version')
echo $release_version
echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
- name: Tag and release Budibase service docker image
uses: docker/build-push-action@v2
with:
context: .
push: true
platforms: linux/arm64
platforms: linux/amd64,linux/arm64
tags: budibase/budibase,budibase/budibase:v${{ env.RELEASE_VERSION }}
file: ./hosting/single/Dockerfile

View file

@ -120,6 +120,8 @@ spec:
{{ end }}
- name: MULTI_TENANCY
value: {{ .Values.globals.multiTenancy | quote }}
- name: OFFLINE_MODE
value: {{ .Values.globals.offlineMode | quote }}
- name: LOG_LEVEL
value: {{ .Values.services.apps.logLevel | quote }}
- name: REDIS_PASSWORD

View file

@ -116,6 +116,8 @@ spec:
value: {{ .Values.services.worker.port | quote }}
- name: MULTI_TENANCY
value: {{ .Values.globals.multiTenancy | quote }}
- name: OFFLINE_MODE
value: {{ .Values.globals.offlineMode | quote }}
- name: LOG_LEVEL
value: {{ .Values.services.worker.logLevel | quote }}
- name: REDIS_PASSWORD

View file

@ -82,6 +82,7 @@ globals:
posthogToken: "phc_bIjZL7oh2GEUd2vqvTBH8WvrX0fWTFQMs6H5KQxiUxU"
selfHosted: "1" # set to 0 for budibase cloud environment, set to 1 for self-hosted setup
multiTenancy: "0" # set to 0 to disable multiple orgs, set to 1 to enable multiple orgs
offlineMode: "0" # set to 1 to enable offline mode
accountPortalUrl: ""
accountPortalApiKey: ""
cookieDomain: ""

View file

@ -1,5 +1,5 @@
{
"version": "2.8.29-alpha.17",
"version": "2.9.8-alpha.0",
"npmClient": "yarn",
"packages": [
"packages/*"

View file

@ -108,7 +108,13 @@
/****************************************************/
const getInputData = (testData, blockInputs) => {
let newInputData = cloneDeep(testData || blockInputs)
// Test data is not cloned for reactivity
let newInputData = testData || cloneDeep(blockInputs)
// Ensures the app action fields are populated
if (block.event === "app:trigger" && !newInputData?.fields) {
newInputData = cloneDeep(blockInputs)
}
/**
* TODO - Remove after November 2023

View file

@ -2,14 +2,15 @@
import { Button, Layout } from "@budibase/bbui"
import DatasourceNavigator from "components/backend/DatasourceNavigator/DatasourceNavigator.svelte"
import Panel from "components/design/Panel.svelte"
import { isActive, goto, redirect } from "@roxi/routify"
import { isActive, redirect, goto, params } from "@roxi/routify"
import BetaButton from "./_components/BetaButton.svelte"
import { datasources } from "stores/backend"
$: {
// If we ever don't have any data other than the users table, prompt the
// user to add some
if (!$datasources.hasData) {
// Don't redirect if setting up google sheets, or we lose the query parameter
if (!$datasources.hasData && !$params["?continue_google_setup"]) {
$redirect("./new")
}
}

View file

@ -2,16 +2,16 @@
"name": "@budibase/cli",
"version": "0.0.0",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "dist/src/index.js",
"main": "dist/index.js",
"bin": {
"budi": "dist/src/index.js"
"budi": "dist/index.js"
},
"author": "Budibase",
"license": "GPL-3.0",
"scripts": {
"prebuild": "rm -rf prebuilds 2> /dev/null && cp -r ../../node_modules/leveldown/prebuilds prebuilds",
"rename": "renamer --find .node --replace .fake 'prebuilds/**'",
"tsc": "tsc -p tsconfig.build.json",
"tsc": "node ../../scripts/build.js",
"pkg": "pkg . --out-path build --no-bytecode --public --public-packages \"*\" -C GZip",
"build": "yarn prebuild && yarn rename && yarn tsc && yarn pkg && yarn postbuild",
"check:types": "tsc -p tsconfig.json --noEmit --paths null",
@ -19,12 +19,11 @@
},
"pkg": {
"targets": [
"node16-linux",
"node16-win",
"node16-macos"
"node18-linux",
"node18-win",
"node18-macos"
],
"assets": [
"node_modules/@budibase/backend-core/dist/**/*",
"prebuilds/**/*"
],
"outputPath": "build"

View file

@ -1,5 +1,6 @@
import util from "util"
const runCommand = util.promisify(require("child_process").exec)
import childProcess from "child_process"
const runCommand = util.promisify(childProcess.exec)
export async function exec(command: string, dir = "./") {
const { stdout } = await runCommand(command, { cwd: dir })
@ -16,12 +17,12 @@ export async function utilityInstalled(utilName: string) {
}
export async function runPkgCommand(command: string, dir = "./") {
const yarn = await exports.utilityInstalled("yarn")
const npm = await exports.utilityInstalled("npm")
const yarn = await utilityInstalled("yarn")
const npm = await utilityInstalled("npm")
if (!yarn && !npm) {
throw new Error("Must have yarn or npm installed to run build.")
}
const npmCmd = command === "install" ? `npm ${command}` : `npm run ${command}`
const cmd = yarn ? `yarn ${command} --ignore-engines` : npmCmd
await exports.exec(cmd, dir)
await exec(cmd, dir)
}

View file

@ -5,7 +5,7 @@ import { error } from "./utils"
const PREBUILDS = "prebuilds"
const ARCH = `${os.platform()}-${os.arch()}`
const PREBUILD_DIR = join(process.execPath, "..", PREBUILDS, ARCH)
const PREBUILD_DIR = join(process.execPath, "..", "cli", PREBUILDS, ARCH)
// running as built CLI pkg bundle
if (!process.argv[0].includes("node")) {
@ -13,17 +13,19 @@ if (!process.argv[0].includes("node")) {
}
function checkForBinaries() {
const readDir = join(__filename, "..", "..", "..", PREBUILDS, ARCH)
const readDir = join(__filename, "..", "..", "..", "cli", PREBUILDS, ARCH)
if (fs.existsSync(PREBUILD_DIR) || !fs.existsSync(readDir)) {
return
}
const natives = fs.readdirSync(readDir)
if (fs.existsSync(readDir)) {
fs.mkdirSync(PREBUILD_DIR, { recursive: true })
const writePath = join(process.execPath, PREBUILDS, ARCH)
fs.mkdirSync(writePath, { recursive: true })
for (let native of natives) {
const filename = `${native.split(".fake")[0]}.node`
fs.cpSync(join(readDir, native), join(PREBUILD_DIR, filename))
fs.cpSync(join(readDir, native), join(writePath, filename))
}
console.log("copied something")
}
}
@ -39,8 +41,9 @@ function cleanup(evt?: number) {
)
console.error(error(evt))
}
if (fs.existsSync(PREBUILD_DIR)) {
fs.rmSync(PREBUILD_DIR, { recursive: true })
const path = join(process.execPath, PREBUILDS)
if (fs.existsSync(path)) {
fs.rmSync(path, { recursive: true })
}
}

View file

@ -10,7 +10,12 @@
"incremental": true,
"types": [ "node", "jest" ],
"outDir": "dist",
"skipLibCheck": true
"skipLibCheck": true,
"paths": {
"@budibase/types": ["../types/src"],
"@budibase/backend-core": ["../backend-core/src"],
"@budibase/backend-core/*": ["../backend-core/*"]
}
},
"include": [
"src/**/*"

View file

@ -5,12 +5,7 @@
"declaration": true,
"sourceMap": true,
"baseUrl": ".",
"resolveJsonModule": true,
"paths": {
"@budibase/types": ["../types/src"],
"@budibase/backend-core": ["../backend-core/src"],
"@budibase/backend-core/*": ["../backend-core/*"]
}
"resolveJsonModule": true
},
"ts-node": {
"require": ["tsconfig-paths/register"],

View file

@ -27,7 +27,6 @@
"array",
"attachment",
"boolean",
"formula",
"json",
]

@ -1 +1 @@
Subproject commit cf3bef2aad9c739111b306fd0712397adc363f81
Subproject commit ecee8071ebe0f98a5bb19646954e373264be210d

View file

@ -317,6 +317,11 @@ async function performAppCreate(ctx: UserCtx) {
}
})
// Keep existing validation setting
if (!existing.features?.componentValidation) {
newApplication.features!.componentValidation = false
}
// Migrate navigation settings and screens if required
if (existing) {
const navigation = await migrateAppNavigation()

View file

@ -44,7 +44,7 @@ export async function handleRequest(
export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
const tableId = ctx.params.tableId
const { id, ...rowData } = ctx.request.body
const { _id, ...rowData } = ctx.request.body
const validateResult = await sdk.rows.utils.validate({
row: rowData,
@ -54,10 +54,10 @@ export async function patch(ctx: UserCtx<PatchRowRequest, PatchRowResponse>) {
throw { validation: validateResult.errors }
}
const response = await handleRequest(Operation.UPDATE, tableId, {
id: breakRowIdField(id),
id: breakRowIdField(_id),
row: rowData,
})
const row = await sdk.rows.external.getRow(tableId, id, {
const row = await sdk.rows.external.getRow(tableId, _id, {
relationships: true,
})
const table = await sdk.tables.getTable(tableId)
@ -104,9 +104,9 @@ export async function find(ctx: UserCtx) {
export async function destroy(ctx: UserCtx) {
const tableId = ctx.params.tableId
const id = ctx.request.body._id
const _id = ctx.request.body._id
const { row } = (await handleRequest(Operation.DELETE, tableId, {
id: breakRowIdField(id),
id: breakRowIdField(_id),
includeSqlRelationships: IncludeRelationship.EXCLUDE,
})) as { row: Row }
return { response: { ok: true }, row }

View file

@ -11,6 +11,9 @@ import {
Row,
PatchRowRequest,
PatchRowResponse,
SearchRowResponse,
SearchRowRequest,
SearchParams,
} from "@budibase/types"
import * as utils from "./utils"
import { gridSocket } from "../../../websockets"
@ -197,10 +200,10 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
ctx.body = response
}
export async function search(ctx: any) {
export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
const tableId = utils.getTableId(ctx)
const searchParams = {
const searchParams: SearchParams = {
...ctx.request.body,
tableId,
}

View file

@ -1,14 +1,18 @@
import { quotas } from "@budibase/pro"
import {
UserCtx,
SearchResponse,
SortOrder,
SortType,
ViewV2,
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
SearchParams,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk"
export async function searchView(ctx: UserCtx<void, SearchResponse>) {
export async function searchView(
ctx: UserCtx<SearchViewRowRequest, SearchRowResponse>
) {
const { viewId } = ctx.params
const view = await sdk.views.get(viewId)
@ -29,49 +33,35 @@ export async function searchView(ctx: UserCtx<void, SearchResponse>) {
undefined
ctx.status = 200
const result = await quotas.addQuery(
() =>
sdk.rows.search({
tableId: view.tableId,
query: view.query || {},
fields: viewFields,
...getSortOptions(ctx, view),
}),
{
datasourceId: view.tableId,
}
)
const { body } = ctx.request
const query = dataFilters.buildLuceneQuery(view.query || [])
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
tableId: view.tableId,
query,
fields: viewFields,
...getSortOptions(body, view),
limit: body.limit,
bookmark: body.bookmark,
paginate: body.paginate,
}
const result = await quotas.addQuery(() => sdk.rows.search(searchOptions), {
datasourceId: view.tableId,
})
result.rows.forEach(r => (r._viewId = view.id))
ctx.body = result
}
function getSortOptions(
ctx: UserCtx,
view: ViewV2
):
| {
sort: string
sortOrder?: SortOrder
sortType?: SortType
}
| undefined {
const { sort_column, sort_order, sort_type } = ctx.query
if (Array.isArray(sort_column)) {
ctx.throw(400, "sort_column cannot be an array")
}
if (Array.isArray(sort_order)) {
ctx.throw(400, "sort_order cannot be an array")
}
if (Array.isArray(sort_type)) {
ctx.throw(400, "sort_type cannot be an array")
}
if (sort_column) {
function getSortOptions(request: SearchViewRowRequest, view: ViewV2) {
if (request.sort) {
return {
sort: sort_column,
sortOrder: sort_order as SortOrder,
sortType: sort_type as SortType,
sort: request.sort,
sortOrder: request.sortOrder,
sortType: request.sortType,
}
}
if (view.sort) {
@ -82,5 +72,9 @@ function getSortOptions(
}
}
return
return {
sort: undefined,
sortOrder: undefined,
sortType: undefined,
}
}

View file

@ -269,7 +269,7 @@ router
)
router
.get(
.post(
"/api/v2/views/:viewId/search",
authorized(PermissionType.VIEW, PermissionLevel.READ),
rowController.views.searchView

View file

@ -813,252 +813,6 @@ describe("/rows", () => {
})
})
describe("view search", () => {
function userTable(): Table {
return {
name: "user",
type: "user",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: { type: "string" },
},
age: {
type: FieldType.NUMBER,
name: "age",
constraints: {},
},
},
}
}
it("returns table rows from view", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body).toEqual({
rows: expect.arrayContaining(rows.map(expect.objectContaining)),
})
})
it("searching respects the view filters", async () => {
const table = await config.createTable(userTable())
const expectedRows = []
for (let i = 0; i < 10; i++)
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.integer({ min: 10, max: 30 }),
})
for (let i = 0; i < 5; i++)
expectedRows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: 40,
})
)
const createViewResponse = await config.api.viewV2.create({
query: { equal: { age: 40 } },
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(5)
expect(response.body).toEqual({
rows: expect.arrayContaining(expectedRows.map(expect.objectContaining)),
})
})
const sortTestOptions: [
{
field: string
order?: SortOrder
type?: SortType
},
string[]
][] = [
[
{
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
type: SortType.STRING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
type: SortType.number,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
},
["Bob", "Charly", "Alice", "Danny"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
type: SortType.number,
},
["Bob", "Charly", "Alice", "Danny"],
],
]
it.each(sortTestOptions)(
"allow sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: sortParams,
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it.each(sortTestOptions)(
"allow override the default view sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: {
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
})
const response = await config.api.viewV2.search(createViewResponse.id, {
sort: {
column: sortParams.field,
order: sortParams.order,
type: sortParams.type,
},
})
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it("when schema is defined, defined columns and row attributes are returned", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.age(),
})
)
}
const view = await config.api.viewV2.create({
schema: { name: {} },
})
const response = await config.api.viewV2.search(view.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body.rows).toEqual(
expect.arrayContaining(
rows.map(r => ({
...expectAnyInternalColsAttributes,
_viewId: view.id,
name: r.name,
}))
)
)
})
it("views without data can be returned", async () => {
const table = await config.createTable(userTable())
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(0)
})
})
describe("view 2.0", () => {
function userTable(): Table {
return {
@ -1261,5 +1015,327 @@ describe("/rows", () => {
await config.api.row.get(tableId, rows[1]._id!, { expectStatus: 200 })
})
})
describe("view search", () => {
function userTable(): Table {
return {
name: "user",
type: "user",
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: { type: "string" },
},
age: {
type: FieldType.NUMBER,
name: "age",
constraints: {},
},
},
}
}
it("returns table rows from view", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body).toEqual({
rows: expect.arrayContaining(rows.map(expect.objectContaining)),
})
})
it("searching respects the view filters", async () => {
const table = await config.createTable(userTable())
const expectedRows = []
for (let i = 0; i < 10; i++)
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.integer({ min: 10, max: 30 }),
})
for (let i = 0; i < 5; i++)
expectedRows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: 40,
})
)
const createViewResponse = await config.api.viewV2.create({
query: [{ operator: "equal", field: "age", value: 40 }],
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(5)
expect(response.body).toEqual({
rows: expect.arrayContaining(
expectedRows.map(expect.objectContaining)
),
})
})
const sortTestOptions: [
{
field: string
order?: SortOrder
type?: SortType
},
string[]
][] = [
[
{
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
},
["Alice", "Bob", "Charly", "Danny"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "name",
order: SortOrder.DESCENDING,
type: SortType.STRING,
},
["Danny", "Charly", "Bob", "Alice"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
type: SortType.number,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.ASCENDING,
},
["Danny", "Alice", "Charly", "Bob"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
},
["Bob", "Charly", "Alice", "Danny"],
],
[
{
field: "age",
order: SortOrder.DESCENDING,
type: SortType.number,
},
["Bob", "Charly", "Alice", "Danny"],
],
]
it.each(sortTestOptions)(
"allow sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: sortParams,
})
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it.each(sortTestOptions)(
"allow override the default view sorting (%s)",
async (sortParams, expected) => {
await config.createTable(userTable())
const users = [
{ name: "Alice", age: 25 },
{ name: "Bob", age: 30 },
{ name: "Charly", age: 27 },
{ name: "Danny", age: 15 },
]
for (const user of users) {
await config.createRow({
tableId: config.table!._id,
...user,
})
}
const createViewResponse = await config.api.viewV2.create({
sort: {
field: "name",
order: SortOrder.ASCENDING,
type: SortType.STRING,
},
})
const response = await config.api.viewV2.search(
createViewResponse.id,
{
sort: sortParams.field,
sortOrder: sortParams.order,
sortType: sortParams.type,
}
)
expect(response.body.rows).toHaveLength(4)
expect(response.body).toEqual({
rows: expected.map(name => expect.objectContaining({ name })),
})
}
)
it("when schema is defined, defined columns and row attributes are returned", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(
await config.createRow({
tableId: table._id,
name: generator.name(),
age: generator.age(),
})
)
}
const view = await config.api.viewV2.create({
schema: { name: {} },
})
const response = await config.api.viewV2.search(view.id)
expect(response.body.rows).toHaveLength(10)
expect(response.body.rows).toEqual(
expect.arrayContaining(
rows.map(r => ({
...expectAnyInternalColsAttributes,
_viewId: view.id,
name: r.name,
}))
)
)
})
it("views without data can be returned", async () => {
const table = await config.createTable(userTable())
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id)
expect(response.body.rows).toHaveLength(0)
})
it("respects the limit parameter", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
const limit = generator.integer({ min: 1, max: 8 })
const createViewResponse = await config.api.viewV2.create()
const response = await config.api.viewV2.search(createViewResponse.id, {
limit,
})
expect(response.body.rows).toHaveLength(limit)
})
it("can handle pagination", async () => {
const table = await config.createTable(userTable())
const rows = []
for (let i = 0; i < 10; i++) {
rows.push(await config.createRow({ tableId: table._id }))
}
// rows.sort((a, b) => (a._id! > b._id! ? 1 : -1))
const createViewResponse = await config.api.viewV2.create()
const allRows = (await config.api.viewV2.search(createViewResponse.id))
.body.rows
const firstPageResponse = await config.api.viewV2.search(
createViewResponse.id,
{
paginate: true,
limit: 4,
}
)
expect(firstPageResponse.body).toEqual({
rows: expect.arrayContaining(allRows.slice(0, 4)),
totalRows: 10,
hasNextPage: true,
bookmark: expect.any(String),
})
const secondPageResponse = await config.api.viewV2.search(
createViewResponse.id,
{
paginate: true,
limit: 4,
bookmark: firstPageResponse.body.bookmark,
}
)
expect(secondPageResponse.body).toEqual({
rows: expect.arrayContaining(allRows.slice(4, 8)),
totalRows: 10,
hasNextPage: true,
bookmark: expect.any(String),
})
const lastPageResponse = await config.api.viewV2.search(
createViewResponse.id,
{
paginate: true,
limit: 4,
bookmark: secondPageResponse.body.bookmark,
}
)
expect(lastPageResponse.body).toEqual({
rows: expect.arrayContaining(allRows.slice(8)),
totalRows: 10,
hasNextPage: false,
bookmark: expect.any(String),
})
})
})
})
})

View file

@ -62,7 +62,7 @@ describe("/v2/views", () => {
name: generator.name(),
tableId: config.table!._id!,
primaryDisplay: generator.word(),
query: { allOr: false, equal: { field: "value" } },
query: [{ operator: "equal", field: "field", value: "value" }],
sort: {
field: "fieldToSort",
order: SortOrder.DESCENDING,
@ -190,7 +190,7 @@ describe("/v2/views", () => {
const tableId = config.table!._id!
await config.api.viewV2.update({
...view,
query: { equal: { newField: "thatValue" } },
query: [{ operator: "equal", field: "newField", value: "thatValue" }],
})
expect(await config.api.table.get(tableId)).toEqual({
@ -198,7 +198,9 @@ describe("/v2/views", () => {
views: {
[view.name]: {
...view,
query: { equal: { newField: "thatValue" } },
query: [
{ operator: "equal", field: "newField", value: "thatValue" },
],
schema: expect.anything(),
},
},
@ -216,7 +218,13 @@ describe("/v2/views", () => {
tableId,
name: view.name,
primaryDisplay: generator.word(),
query: { equal: { [generator.word()]: generator.word() } },
query: [
{
operator: "equal",
field: generator.word(),
value: generator.word(),
},
],
sort: {
field: generator.word(),
order: SortOrder.DESCENDING,
@ -285,7 +293,7 @@ describe("/v2/views", () => {
{
...view,
tableId: generator.guid(),
query: { equal: { newField: "thatValue" } },
query: [{ operator: "equal", field: "newField", value: "thatValue" }],
},
{ expectStatus: 404 }
)

View file

@ -315,7 +315,7 @@ class InternalBuilder {
addSorting(query: KnexQuery, json: QueryJson): KnexQuery {
let { sort, paginate } = json
const table = json.meta?.table
if (sort) {
if (sort && Object.keys(sort || {}).length > 0) {
for (let [key, value] of Object.entries(sort)) {
const direction =
value.direction === SortDirection.ASCENDING ? "asc" : "desc"

View file

@ -26,6 +26,12 @@ function generateReadJson({
filters: filters || {},
sort: sort || {},
paginate: paginate || {},
meta: {
table: {
name: table || TABLE_NAME,
primary: ["id"],
},
},
}
}
@ -636,4 +642,19 @@ describe("SQL query builder", () => {
sql: `select * from (select * from (select * from \"test\" where LOWER(\"test\".\"name\") LIKE :1) where rownum <= :2) \"test\"`,
})
})
it("should sort SQL Server tables by the primary key if no sort data is provided", () => {
let query = new Sql(SqlClient.MS_SQL, limit)._query(
generateReadJson({
sort: {},
paginate: {
limit: 10,
},
})
)
expect(query).toEqual({
bindings: [10],
sql: `select * from (select top (@p0) * from [test] order by [test].[id] asc) as [test]`,
})
})
})

View file

@ -1,23 +1,9 @@
import { SearchFilters, SortOrder, SortType } from "@budibase/types"
import { SearchFilters, SearchParams } from "@budibase/types"
import { isExternalTable } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
import { Format } from "../../../api/controllers/view/exporters"
export interface SearchParams {
tableId: string
paginate?: boolean
query: SearchFilters
bookmark?: string
limit?: number
sort?: string
sortOrder?: SortOrder
sortType?: SortType
version?: string
disableEscaping?: boolean
fields?: string[]
}
export interface ViewParams {
calculation: string
group: string

View file

@ -6,6 +6,7 @@ import {
IncludeRelationship,
Row,
SearchFilters,
SearchParams,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
@ -13,7 +14,7 @@ import { handleRequest } from "../../../../api/controllers/row/external"
import { breakExternalTableId } from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
import { ExportRowsParams, ExportRowsResult } from "../search"
import { HTTPError, db } from "@budibase/backend-core"
import pick from "lodash/pick"

View file

@ -12,7 +12,7 @@ import {
} from "../../../../db/utils"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Database, Row, Table } from "@budibase/types"
import { Database, Row, Table, SearchParams } from "@budibase/types"
import { cleanExportRows } from "../utils"
import {
Format,
@ -28,7 +28,7 @@ import {
getFromMemoryDoc,
} from "../../../../api/controllers/view/utils"
import sdk from "../../../../sdk"
import { ExportRowsParams, ExportRowsResult, SearchParams } from "../search"
import { ExportRowsParams, ExportRowsResult } from "../search"
import pick from "lodash/pick"
export async function search(options: SearchParams) {

View file

@ -1,8 +1,15 @@
import { GenericContainer } from "testcontainers"
import { Datasource, FieldType, Row, SourceName, Table } from "@budibase/types"
import {
Datasource,
FieldType,
Row,
SourceName,
Table,
SearchParams,
} from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { SearchParams } from "../../search"
import { search } from "../external"
import {
expectAnyExternalColsAttributes,

View file

@ -1,6 +1,5 @@
import { FieldType, Row, Table } from "@budibase/types"
import { FieldType, Row, Table, SearchParams } from "@budibase/types"
import TestConfiguration from "../../../../../tests/utilities/TestConfiguration"
import { SearchParams } from "../../search"
import { search } from "../internal"
import {
expectAnyInternalColsAttributes,

View file

@ -1,13 +1,12 @@
import {
CreateViewRequest,
SortOrder,
SortType,
UpdateViewRequest,
DeleteRowRequest,
PatchRowRequest,
PatchRowResponse,
Row,
ViewV2,
SearchViewRowRequest,
} from "@budibase/types"
import TestConfiguration from "../TestConfiguration"
import { TestAPI } from "./base"
@ -81,31 +80,12 @@ export class ViewV2API extends TestAPI {
search = async (
viewId: string,
options?: {
sort: {
column: string
order?: SortOrder
type?: SortType
}
},
params?: SearchViewRowRequest,
{ expectStatus } = { expectStatus: 200 }
) => {
const qs: [string, any][] = []
if (options?.sort.column) {
qs.push(["sort_column", options.sort.column])
}
if (options?.sort.order) {
qs.push(["sort_order", options.sort.order])
}
if (options?.sort.type) {
qs.push(["sort_type", options.sort.type])
}
let url = `/api/v2/views/${viewId}/search`
if (qs.length) {
url += "?" + qs.map(q => q.join("=")).join("&")
}
return this.request
.get(url)
.post(`/api/v2/views/${viewId}/search`)
.send(params)
.set(this.config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(expectStatus)

View file

@ -486,10 +486,13 @@ class Orchestrator {
const end = performance.now()
const executionTime = end - start
console.info(`Execution time: ${executionTime} milliseconds`, {
_logKey: "automation",
executionTime,
})
console.info(
`Automation ID: ${automation._id} Execution time: ${executionTime} milliseconds`,
{
_logKey: "automation",
executionTime,
}
)
// store the logs for the automation run
try {

View file

@ -1,4 +1,12 @@
import { Datasource, FieldType, SortDirection, SortType } from "@budibase/types"
import {
Datasource,
FieldType,
SearchFilter,
SearchQuery,
SearchQueryFields,
SortDirection,
SortType,
} from "@budibase/types"
import { OperatorOptions, SqlNumberTypeRangeMap } from "./constants"
import { deepGet } from "./helpers"
@ -73,13 +81,13 @@ export const NoEmptyFilterStrings = [
OperatorOptions.NotEquals.value,
OperatorOptions.Contains.value,
OperatorOptions.NotContains.value,
] as (keyof QueryFields)[]
] as (keyof SearchQueryFields)[]
/**
* Removes any fields that contain empty strings that would cause inconsistent
* behaviour with how backend tables are filtered (no value means no filter).
*/
const cleanupQuery = (query: Query) => {
const cleanupQuery = (query: SearchQuery) => {
if (!query) {
return query
}
@ -110,66 +118,12 @@ const removeKeyNumbering = (key: string) => {
}
}
type Filter = {
operator: keyof Query
field: string
type: any
value: any
externalType: keyof typeof SqlNumberTypeRangeMap
}
type Query = QueryFields & QueryConfig
type QueryFields = {
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
type QueryConfig = {
allOr?: boolean
}
type QueryFieldsType = keyof QueryFields
/**
* Builds a lucene JSON query from the filter structure generated in the builder
* @param filter the builder filter structure
*/
export const buildLuceneQuery = (filter: Filter[]) => {
let query: Query = {
export const buildLuceneQuery = (filter: SearchFilter[]) => {
let query: SearchQuery = {
string: {},
fuzzy: {},
range: {},
@ -227,9 +181,13 @@ export const buildLuceneQuery = (filter: Filter[]) => {
}
if (operator.startsWith("range") && query.range) {
const minint =
SqlNumberTypeRangeMap[externalType]?.min || Number.MIN_SAFE_INTEGER
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.min || Number.MIN_SAFE_INTEGER
const maxint =
SqlNumberTypeRangeMap[externalType]?.max || Number.MAX_SAFE_INTEGER
SqlNumberTypeRangeMap[
externalType as keyof typeof SqlNumberTypeRangeMap
]?.max || Number.MAX_SAFE_INTEGER
if (!query.range[field]) {
query.range[field] = {
low: type === "number" ? minint : "0000-00-00T00:00:00.000Z",
@ -275,7 +233,7 @@ export const buildLuceneQuery = (filter: Filter[]) => {
* @param docs the data
* @param query the JSON lucene query
*/
export const runLuceneQuery = (docs: any[], query?: Query) => {
export const runLuceneQuery = (docs: any[], query?: SearchQuery) => {
if (!docs || !Array.isArray(docs)) {
return []
}
@ -289,7 +247,7 @@ export const runLuceneQuery = (docs: any[], query?: Query) => {
// Iterates over a set of filters and evaluates a fail function against a doc
const match =
(
type: QueryFieldsType,
type: keyof SearchQueryFields,
failFn: (docValue: any, testValue: any) => boolean
) =>
(doc: any) => {
@ -456,7 +414,7 @@ export const luceneLimit = (docs: any[], limit: string) => {
return docs.slice(0, numLimit)
}
export const hasFilters = (query?: Query) => {
export const hasFilters = (query?: SearchQuery) => {
if (!query) {
return false
}

View file

@ -1,3 +1,4 @@
import { SearchParams } from "../../../sdk"
import { Row } from "../../../documents"
export interface PatchRowRequest extends Row {
@ -8,6 +9,14 @@ export interface PatchRowRequest extends Row {
export interface PatchRowResponse extends Row {}
export interface SearchResponse {
export interface SearchRowRequest extends Omit<SearchParams, "tableId"> {}
export interface SearchViewRowRequest
extends Pick<
SearchRowRequest,
"sort" | "sortOrder" | "sortType" | "limit" | "bookmark" | "paginate"
> {}
export interface SearchRowResponse {
rows: any[]
}

View file

@ -8,3 +8,4 @@ export * from "./system"
export * from "./app"
export * from "./global"
export * from "./pagination"
export * from "./searchFilter"

View file

@ -0,0 +1,51 @@
import { FieldType } from "../../documents"
export type SearchFilter = {
operator: keyof SearchQuery
field: string
type?: FieldType
value: any
externalType?: string
}
export type SearchQuery = {
allOr?: boolean
string?: {
[key: string]: string
}
fuzzy?: {
[key: string]: string
}
range?: {
[key: string]: {
high: number | string
low: number | string
}
}
equal?: {
[key: string]: any
}
notEqual?: {
[key: string]: any
}
empty?: {
[key: string]: any
}
notEmpty?: {
[key: string]: any
}
oneOf?: {
[key: string]: any[]
}
contains?: {
[key: string]: any[]
}
notContains?: {
[key: string]: any[]
}
containsAny?: {
[key: string]: any[]
}
}
export type SearchQueryFields = Omit<SearchQuery, "allOr">

View file

@ -1,6 +1,5 @@
import { SortOrder, SortType } from "../../api"
import { SearchFilters } from "../../sdk"
import { TableSchema, UIFieldMetadata } from "./table"
import { SearchFilter, SortOrder, SortType } from "../../api"
import { UIFieldMetadata } from "./table"
export interface View {
name: string
@ -20,7 +19,7 @@ export interface ViewV2 {
name: string
primaryDisplay?: string
tableId: string
query?: SearchFilters
query?: SearchFilter[]
sort?: {
field: string
order?: SortOrder

View file

@ -19,3 +19,4 @@ export * from "./user"
export * from "./cli"
export * from "./websocket"
export * from "./permissions"
export * from "./row"

View file

@ -0,0 +1,16 @@
import { SortOrder, SortType } from "../api"
import { SearchFilters } from "./search"
export interface SearchParams {
tableId: string
paginate?: boolean
query: SearchFilters
bookmark?: string
limit?: number
sort?: string
sortOrder?: SortOrder
sortType?: SortType
version?: string
disableEscaping?: boolean
fields?: string[]
}