CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-03 13:30:46 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'master' into revert-13398-revert-13356-BUDI-8122/single-attachment-column-type

Browse source
This commit is contained in:
Adria Navarro 2024-04-10 09:01:36 +02:00
parent 28d10ec086 c753ffede9
commit 5b6c3d0c96
121 changed files with 2577 additions and 1358 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.github/workflows/budibase_ci.yml vendored
View file

@ -92,7 +92,6 @@ jobs:
test-libraries:
runs-on: ubuntu-latest
env:
DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
REUSE_CONTAINERS: true
steps:
- name: Checkout repo
@ -110,7 +109,7 @@ jobs:
- name: Pull testcontainers images
run: |
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb &
docker pull budibase/couchdb:v3.2.1-sql &
docker pull redis &
wait $(jobs -p)
@ -151,7 +150,6 @@ jobs:
test-server:
runs-on: budi-tubby-tornado-quad-core-150gb
env:
DEBUG: testcontainers,testcontainers:exec,testcontainers:build,testcontainers:pull
REUSE_CONTAINERS: true
steps:
- name: Checkout repo
@ -175,7 +173,7 @@ jobs:
docker pull mongo:7.0-jammy &
docker pull mariadb:lts &
docker pull testcontainers/ryuk:0.5.1 &
docker pull budibase/couchdb &
docker pull budibase/couchdb:v3.2.1-sql &
docker pull redis &
wait $(jobs -p)

4
globalSetup.ts
View file

@ -13,8 +13,8 @@ export default async function setup() {
}
try {
let couchdb = new GenericContainer("budibase/couchdb")
.withExposedPorts(5984)
let couchdb = new GenericContainer("budibase/couchdb:v3.2.1-sqs")
.withExposedPorts(5984, 4984)
.withEnvironment({
COUCHDB_PASSWORD: "budibase",
COUCHDB_USER: "budibase",

2
hosting/couchdb/Dockerfile
View file

@ -128,4 +128,4 @@ ADD couch/vm.args couch/local.ini ./etc/
WORKDIR /
ADD runner.sh ./bbcouch-runner.sh
RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau
CMD ["./bbcouch-runner.sh"]
CMD ["./bbcouch-runner.sh"]

135
hosting/couchdb/Dockerfile.v2 Normal file
View file

@ -0,0 +1,135 @@
# Modified from https://github.com/apache/couchdb-docker/blob/main/3.3.3/Dockerfile
#
# Everything in this `base` image is adapted from the official `couchdb` image's
# Dockerfile. Only modifications related to upgrading from Debian bullseye to
# bookworm have been included. The `runner` image contains Budibase's
# customisations to the image, e.g. adding Clouseau.
FROM node:20-slim AS base
# Add CouchDB user account to make sure the IDs are assigned consistently
RUN groupadd -g 5984 -r couchdb && useradd -u 5984 -d /opt/couchdb -g couchdb couchdb
# be sure GPG and apt-transport-https are available and functional
RUN set -ex; \
apt-get update; \
apt-get install -y --no-install-recommends \
apt-transport-https \
ca-certificates \
dirmngr \
gnupg \
; \
rm -rf /var/lib/apt/lists/*
# grab tini for signal handling and zombie reaping
# see https://github.com/apache/couchdb-docker/pull/28#discussion_r141112407
RUN set -eux; \
apt-get update; \
apt-get install -y --no-install-recommends tini; \
rm -rf /var/lib/apt/lists/*; \
tini --version
# http://docs.couchdb.org/en/latest/install/unix.html#installing-the-apache-couchdb-packages
ENV GPG_COUCH_KEY \
# gpg: rsa8192 205-01-19 The Apache Software Foundation (Package repository signing key) <root@apache.org>
390EF70BB1EA12B2773962950EE62FB37A00258D
RUN set -eux; \
apt-get update; \
apt-get install -y curl; \
export GNUPGHOME="$(mktemp -d)"; \
curl -fL -o keys.asc https://couchdb.apache.org/repo/keys.asc; \
gpg --batch --import keys.asc; \
gpg --batch --export "${GPG_COUCH_KEY}" > /usr/share/keyrings/couchdb-archive-keyring.gpg; \
command -v gpgconf && gpgconf --kill all || :; \
rm -rf "$GNUPGHOME"; \
apt-key list; \
apt purge -y --autoremove curl; \
rm -rf /var/lib/apt/lists/*
ENV COUCHDB_VERSION 3.3.3
RUN . /etc/os-release; \
echo "deb [signed-by=/usr/share/keyrings/couchdb-archive-keyring.gpg] https://apache.jfrog.io/artifactory/couchdb-deb/ ${VERSION_CODENAME} main" | \
tee /etc/apt/sources.list.d/couchdb.list >/dev/null
# https://github.com/apache/couchdb-pkg/blob/master/debian/README.Debian
RUN set -eux; \
apt-get update; \
\
echo "couchdb couchdb/mode select none" | debconf-set-selections; \
# we DO want recommends this time
DEBIAN_FRONTEND=noninteractive apt-get install -y --allow-downgrades --allow-remove-essential --allow-change-held-packages \
couchdb="$COUCHDB_VERSION"~bookworm \
; \
# Undo symlinks to /var/log and /var/lib
rmdir /var/lib/couchdb /var/log/couchdb; \
rm /opt/couchdb/data /opt/couchdb/var/log; \
mkdir -p /opt/couchdb/data /opt/couchdb/var/log; \
chown couchdb:couchdb /opt/couchdb/data /opt/couchdb/var/log; \
chmod 777 /opt/couchdb/data /opt/couchdb/var/log; \
# Remove file that sets logging to a file
rm /opt/couchdb/etc/default.d/10-filelog.ini; \
# Check we own everything in /opt/couchdb. Matches the command in dockerfile_entrypoint.sh
find /opt/couchdb \! \( -user couchdb -group couchdb \) -exec chown -f couchdb:couchdb '{}' +; \
# Setup directories and permissions for config. Technically these could be 555 and 444 respectively
# but we keep them as 755 and 644 for consistency with CouchDB defaults and the dockerfile_entrypoint.sh.
find /opt/couchdb/etc -type d ! -perm 0755 -exec chmod -f 0755 '{}' +; \
find /opt/couchdb/etc -type f ! -perm 0644 -exec chmod -f 0644 '{}' +; \
# only local.d needs to be writable for the docker_entrypoint.sh
chmod -f 0777 /opt/couchdb/etc/local.d; \
# apt clean-up
rm -rf /var/lib/apt/lists/*;
# Add configuration
COPY --chown=couchdb:couchdb couch/10-docker-default.ini /opt/couchdb/etc/default.d/
# COPY --chown=couchdb:couchdb vm.args /opt/couchdb/etc/
COPY docker-entrypoint.sh /usr/local/bin
RUN ln -s usr/local/bin/docker-entrypoint.sh /docker-entrypoint.sh # backwards compat
ENTRYPOINT ["tini", "--", "/docker-entrypoint.sh"]
VOLUME /opt/couchdb/data
# 5984: Main CouchDB endpoint
# 4369: Erlang portmap daemon (epmd)
# 9100: CouchDB cluster communication port
EXPOSE 5984 4369 9100
CMD ["/opt/couchdb/bin/couchdb"]
FROM base as runner
ENV COUCHDB_USER admin
ENV COUCHDB_PASSWORD admin
EXPOSE 5984
EXPOSE 4984
RUN apt-get update && apt-get install -y --no-install-recommends software-properties-common wget unzip curl && \
wget -O - https://packages.adoptium.net/artifactory/api/gpg/key/public | apt-key add - && \
apt-add-repository 'deb http://security.debian.org/debian-security bookworm-security/updates main' && \
apt-add-repository 'deb http://archive.debian.org/debian stretch-backports main' && \
apt-add-repository 'deb https://packages.adoptium.net/artifactory/deb bookworm main' && \
apt-get update && apt-get install -y --no-install-recommends temurin-8-jdk && \
rm -rf /var/lib/apt/lists/
# setup clouseau
WORKDIR /
RUN wget https://github.com/cloudant-labs/clouseau/releases/download/2.21.0/clouseau-2.21.0-dist.zip && \
unzip clouseau-2.21.0-dist.zip && \
mv clouseau-2.21.0 /opt/clouseau && \
rm clouseau-2.21.0-dist.zip
WORKDIR /opt/clouseau
RUN mkdir ./bin
ADD clouseau/clouseau ./bin/
ADD clouseau/log4j.properties clouseau/clouseau.ini ./
# setup CouchDB
WORKDIR /opt/couchdb
ADD couch/vm.args couch/local.ini ./etc/
WORKDIR /opt/sqs
ADD sqs/sqs sqs/better_sqlite3.node ./
WORKDIR /
ADD runner.v2.sh ./bbcouch-runner.sh
RUN chmod +x ./bbcouch-runner.sh /opt/clouseau/bin/clouseau /opt/sqs/sqs
CMD ["./bbcouch-runner.sh"]

88
hosting/couchdb/runner.v2.sh Normal file
View file

@ -0,0 +1,88 @@
#!/bin/bash
DATA_DIR=${DATA_DIR:-/data}
COUCHDB_ERLANG_COOKIE=${COUCHDB_ERLANG_COOKIE:-B9CFC32C-3458-4A86-8448-B3C753991CA7}
mkdir -p ${DATA_DIR}
mkdir -p ${DATA_DIR}/couch/{dbs,views}
mkdir -p ${DATA_DIR}/search
chown -R couchdb:couchdb ${DATA_DIR}/couch
echo ${TARGETBUILD} > /buildtarget.txt
if [[ "${TARGETBUILD}" = "aas" ]]; then
# Azure AppService uses /home for persistent data & SSH on port 2222
DATA_DIR="${DATA_DIR:-/home}"
WEBSITES_ENABLE_APP_SERVICE_STORAGE=true
mkdir -p $DATA_DIR/{search,minio,couch}
mkdir -p $DATA_DIR/couch/{dbs,views}
chown -R couchdb:couchdb $DATA_DIR/couch/
apt update
apt-get install -y openssh-server
echo "root:Docker!" | chpasswd
mkdir -p /tmp
chmod +x /tmp/ssh_setup.sh \
&& (sleep 1;/tmp/ssh_setup.sh 2>&1 > /dev/null)
cp /etc/sshd_config /etc/ssh/sshd_config
/etc/init.d/ssh restart
sed -i "s#DATA_DIR#/home#g" /opt/clouseau/clouseau.ini
sed -i "s#DATA_DIR#/home#g" /opt/couchdb/etc/local.ini
elif [[ "${TARGETBUILD}" = "single" ]]; then
# In the single image build, the Dockerfile specifies /data as a volume
# mount, so we use that for all persistent data.
sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
elif [[ "${TARGETBUILD}" = "docker-compose" ]]; then
# We remove the database_dir and view_index_dir settings from the local.ini
# in docker-compose because it will default to /opt/couchdb/data which is what
# our docker-compose was using prior to us switching to using our own CouchDB
# image.
sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
elif [[ -n $KUBERNETES_SERVICE_HOST ]]; then
# In Kubernetes the directory /opt/couchdb/data has a persistent volume
# mount for storing database data.
sed -i "s#^dir=.*\$#dir=/opt/couchdb/data#g" /opt/clouseau/clouseau.ini
# We remove the database_dir and view_index_dir settings from the local.ini
# in Kubernetes because it will default to /opt/couchdb/data which is what
# our Helm chart was using prior to us switching to using our own CouchDB
# image.
sed -i "s#^database_dir.*\$##g" /opt/couchdb/etc/local.ini
sed -i "s#^view_index_dir.*\$##g" /opt/couchdb/etc/local.ini
# We remove the -name setting from the vm.args file in Kubernetes because
# it will default to the pod FQDN, which is what's required for clustering
# to work.
sed -i "s/^-name .*$//g" /opt/couchdb/etc/vm.args
else
# For all other builds, we use /data for persistent data.
sed -i "s#DATA_DIR#/data#g" /opt/clouseau/clouseau.ini
sed -i "s#DATA_DIR#/data#g" /opt/couchdb/etc/local.ini
fi
sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/couchdb/etc/vm.args
sed -i "s#COUCHDB_ERLANG_COOKIE#${COUCHDB_ERLANG_COOKIE}#g" /opt/clouseau/clouseau.ini
# Start Clouseau. Budibase won't function correctly without Clouseau running, it
# powers the search API endpoints which are used to do all sorts, including
# populating app grids.
/opt/clouseau/bin/clouseau > /dev/stdout 2>&1 &
# Start CouchDB.
/docker-entrypoint.sh /opt/couchdb/bin/couchdb &
# Start SQS.
/opt/sqs/sqs --server "http://localhost:5984" --data-dir ${DATA_DIR}/sqs --bind-address=0.0.0.0 &
# Wait for CouchDB to start up.
while [[ $(curl -s -w "%{http_code}\n" http://localhost:5984/_up -o /dev/null) -ne 200 ]]; do
echo 'Waiting for CouchDB to start...';
sleep 5;
done
# CouchDB needs the `_users` and `_replicator` databases to exist before it will
# function correctly, so we create them here.
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_users
curl -X PUT -u "${COUCHDB_USER}:${COUCHDB_PASSWORD}" http://localhost:5984/_replicator
sleep infinity

BIN
hosting/couchdb/sqs/better_sqlite3.node Executable file
View file

Binary file not shown.

BIN
hosting/couchdb/sqs/sqs Executable file
View file

Binary file not shown.

1
hosting/docker-compose.dev.yaml
View file

@ -40,7 +40,6 @@ services:
- PROXY_ADDRESS=host.docker.internal
couchdb-service:
# platform: linux/amd64
container_name: budi-couchdb3-dev
restart: on-failure
image: budibase/couchdb

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.22.16",
"version": "2.22.19",
"npmClient": "yarn",
"packages": [
"packages/*",

3
package.json
View file

@ -59,7 +59,7 @@
"dev:all": "yarn run kill-all && lerna run --stream dev",
"dev:built": "yarn run kill-all && cd packages/server && yarn dev:stack:up && cd ../../ && lerna run --stream dev:built",
"dev:docker": "yarn build --scope @budibase/server --scope @budibase/worker && docker-compose -f hosting/docker-compose.build.yaml -f hosting/docker-compose.dev.yaml --env-file hosting/.env up --build --scale proxy-service=0",
"test": "lerna run --stream test --stream",
"test": "REUSE_CONTAINERS=1 lerna run --concurrency 1 --stream test --stream",
"lint:eslint": "eslint packages --max-warnings=0",
"lint:prettier": "prettier --check \"packages/**/*.{js,ts,svelte}\" && prettier --write \"examples/**/*.{js,ts,svelte}\"",
"lint": "yarn run lint:eslint && yarn run lint:prettier",
@ -74,6 +74,7 @@
"build:docker:single": "./scripts/build-single-image.sh",
"build:docker:dependencies": "docker build -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest ./hosting",
"publish:docker:couch": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile -t budibase/couchdb:latest -t budibase/couchdb:v3.2.1 --push ./hosting/couchdb",
"publish:docker:couch-sqs": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/couchdb/Dockerfile.v2 -t budibase/couchdb:v3.2.1-sqs --push ./hosting/couchdb",
"publish:docker:dependencies": "docker buildx build --platform linux/arm64,linux/amd64 -f hosting/dependencies/Dockerfile -t budibase/dependencies:latest -t budibase/dependencies:v3.2.1 --push ./hosting",
"release:helm": "node scripts/releaseHelmChart",
"env:multi:enable": "lerna run --stream env:multi:enable",

2
packages/account-portal

@ -1 +1 @@
Subproject commit 532c4db35cecd346b5c24f0b89ab7b397a122a36
Subproject commit a0ee9cad8cefb8f9f40228705711be174f018fa9

1
packages/backend-core/src/constants/db.ts
View file

@ -66,3 +66,4 @@ export const APP_PREFIX = prefixed(DocumentType.APP)
export const APP_DEV = prefixed(DocumentType.APP_DEV)
export const APP_DEV_PREFIX = APP_DEV
export const BUDIBASE_DATASOURCE_TYPE = "budibase"
export const SQLITE_DESIGN_DOC_ID = "_design/sqlite"

16
packages/backend-core/src/db/couch/DatabaseImpl.ts
View file

@ -18,6 +18,7 @@ import { directCouchUrlCall } from "./utils"
import { getPouchDB } from "./pouchDB"
import { WriteStream, ReadStream } from "fs"
import { newid } from "../../docIds/newid"
import { SQLITE_DESIGN_DOC_ID } from "../../constants"
import { DDInstrumentedDatabase } from "../instrumentation"
const DATABASE_NOT_FOUND = "Database does not exist."
@ -247,6 +248,21 @@ export class DatabaseImpl implements Database {
})
}
async sql<T extends Document>(sql: string): Promise<T[]> {
const dbName = this.name
const url = `/${dbName}/${SQLITE_DESIGN_DOC_ID}`
const response = await directCouchUrlCall({
url: `${this.couchInfo.sqlUrl}/${url}`,
method: "POST",
cookie: this.couchInfo.cookie,
body: sql,
})
if (response.status > 300) {
throw new Error(await response.text())
}
return (await response.json()) as T[]
}
async query<T extends Document>(
viewName: string,
params: DatabaseQueryOpts

1
packages/backend-core/src/db/couch/connections.ts
View file

@ -25,6 +25,7 @@ export const getCouchInfo = (connection?: string) => {
const authCookie = Buffer.from(`${username}:${password}`).toString("base64")
return {
url: urlInfo.url!,
sqlUrl: env.COUCH_DB_SQL_URL,
auth: {
username: username,
password: password,

9
packages/backend-core/src/db/couch/utils.ts
View file

@ -30,8 +30,13 @@ export async function directCouchUrlCall({
},
}
if (body && method !== "GET") {
params.body = JSON.stringify(body)
params.headers["Content-Type"] = "application/json"
if (typeof body === "string") {
params.body = body
params.headers["Content-Type"] = "text/plain"
} else {
params.body = JSON.stringify(body)
params.headers["Content-Type"] = "application/json"
}
}
return await fetch(checkSlashesInUrl(encodeURI(url)), params)
}

7
packages/backend-core/src/db/instrumentation.ts
View file

@ -149,4 +149,11 @@ export class DDInstrumentedDatabase implements Database {
return this.db.getIndexes(...args)
})
}
sql<T extends Document>(sql: string): Promise<T[]> {
return tracer.trace("db.sql", span => {
span?.addTags({ db_name: this.name })
return this.db.sql(sql)
})
}
}

121
packages/backend-core/src/db/lucene.ts
View file

@ -1,28 +1,16 @@
import fetch from "node-fetch"
import { getCouchInfo } from "./couch"
import { SearchFilters, Row, EmptyFilterOption } from "@budibase/types"
import {
SearchFilters,
Row,
EmptyFilterOption,
SearchResponse,
SearchParams,
WithRequired,
} from "@budibase/types"
const QUERY_START_REGEX = /\d[0-9]*:/g
interface SearchResponse<T> {
rows: T[] | any[]
bookmark?: string
totalRows: number
}
export type SearchParams<T> = {
tableId?: string
sort?: string
sortOrder?: string
sortType?: string
limit?: number
bookmark?: string
version?: string
indexer?: () => Promise<any>
disableEscaping?: boolean
rows?: T | Row[]
}
export function removeKeyNumbering(key: any): string {
if (typeof key === "string" && key.match(QUERY_START_REGEX) != null) {
const parts = key.split(":")
@ -44,7 +32,7 @@ export class QueryBuilder<T> {
#query: SearchFilters
#limit: number
#sort?: string
#bookmark?: string
#bookmark?: string | number
#sortOrder: string
#sortType: string
#includeDocs: boolean
@ -130,7 +118,7 @@ export class QueryBuilder<T> {
return this
}
setBookmark(bookmark?: string) {
setBookmark(bookmark?: string | number) {
if (bookmark != null) {
this.#bookmark = bookmark
}
@ -226,14 +214,20 @@ export class QueryBuilder<T> {
}
}
/**
* Preprocesses a value before going into a lucene search.
* Transforms strings to lowercase and wraps strings and bools in quotes.
* @param value The value to process
* @param options The preprocess options
* @returns {string|*}
*/
preprocess(value: any, { escape, lowercase, wrap, type }: any = {}) {
preprocess(
value: any,
{
escape,
lowercase,
wrap,
type,
}: {
escape?: boolean
lowercase?: boolean
wrap?: boolean
type?: string
} = {}
): string | any {
const hasVersion = !!this.#version
// Determine if type needs wrapped
const originalType = typeof value
@ -561,7 +555,7 @@ async function runQuery<T>(
url: string,
body: any,
cookie: string
): Promise<SearchResponse<T>> {
): Promise<WithRequired<SearchResponse<T>, "totalRows">> {
const response = await fetch(url, {
body: JSON.stringify(body),
method: "POST",
@ -575,7 +569,7 @@ async function runQuery<T>(
}
const json = await response.json()
let output: SearchResponse<T> = {
let output: WithRequired<SearchResponse<T>, "totalRows"> = {
rows: [],
totalRows: 0,
}
@ -613,63 +607,51 @@ async function recursiveSearch<T>(
dbName: string,
index: string,
query: any,
params: any
params: SearchParams
): Promise<any> {
const bookmark = params.bookmark
const rows = params.rows || []
if (rows.length >= params.limit) {
if (params.limit && rows.length >= params.limit) {
return rows
}
let pageSize = QueryBuilder.maxLimit
if (rows.length > params.limit - QueryBuilder.maxLimit) {
if (params.limit && rows.length > params.limit - QueryBuilder.maxLimit) {
pageSize = params.limit - rows.length
}
const page = await new QueryBuilder<T>(dbName, index, query)
const queryBuilder = new QueryBuilder<T>(dbName, index, query)
queryBuilder
.setVersion(params.version)
.setTable(params.tableId)
.setBookmark(bookmark)
.setLimit(pageSize)
.setSort(params.sort)
.setSortOrder(params.sortOrder)
.setSortType(params.sortType)
.run()
if (params.tableId) {
queryBuilder.setTable(params.tableId)
}
const page = await queryBuilder.run()
if (!page.rows.length) {
return rows
}
if (page.rows.length < QueryBuilder.maxLimit) {
return [...rows, ...page.rows]
}
const newParams = {
const newParams: SearchParams = {
...params,
bookmark: page.bookmark,
rows: [...rows, ...page.rows],
rows: [...rows, ...page.rows] as Row[],
}
return await recursiveSearch(dbName, index, query, newParams)
}
/**
* Performs a paginated search. A bookmark will be returned to allow the next
* page to be fetched. There is a max limit off 200 results per page in a
* paginated search.
* @param dbName Which database to run a lucene query on
* @param index Which search index to utilise
* @param query The JSON query structure
* @param params The search params including:
* tableId {string} The table ID to search
* sort {string} The sort column
* sortOrder {string} The sort order ("ascending" or "descending")
* sortType {string} Whether to treat sortable values as strings or
* numbers. ("string" or "number")
* limit {number} The desired page size
* bookmark {string} The bookmark to resume from
* @returns {Promise<{hasNextPage: boolean, rows: *[]}>}
*/
export async function paginatedSearch<T>(
dbName: string,
index: string,
query: SearchFilters,
params: SearchParams<T>
) {
params: SearchParams
): Promise<SearchResponse<T>> {
let limit = params.limit
if (limit == null || isNaN(limit) || limit < 0) {
limit = 50
@ -713,29 +695,12 @@ export async function paginatedSearch<T>(
}
}
/**
* Performs a full search, fetching multiple pages if required to return the
* desired amount of results. There is a limit of 1000 results to avoid
* heavy performance hits, and to avoid client components breaking from
* handling too much data.
* @param dbName Which database to run a lucene query on
* @param index Which search index to utilise
* @param query The JSON query structure
* @param params The search params including:
* tableId {string} The table ID to search
* sort {string} The sort column
* sortOrder {string} The sort order ("ascending" or "descending")
* sortType {string} Whether to treat sortable values as strings or
* numbers. ("string" or "number")
* limit {number} The desired number of results
* @returns {Promise<{rows: *}>}
*/
export async function fullSearch<T>(
dbName: string,
index: string,
query: SearchFilters,
params: SearchParams<T>
) {
params: SearchParams
): Promise<{ rows: Row[] }> {
let limit = params.limit
if (limit == null || isNaN(limit) || limit < 0) {
limit = 1000

62
packages/backend-core/src/db/tests/lucene.spec.ts
View file

@ -1,23 +1,39 @@
import { newid } from "../../docIds/newid"
import { getDB } from "../db"
import { Database, EmptyFilterOption } from "@budibase/types"
import { QueryBuilder, paginatedSearch, fullSearch } from "../lucene"
import {
Database,
EmptyFilterOption,
SortOrder,
SortType,
DocumentType,
SEPARATOR,
} from "@budibase/types"
import { fullSearch, paginatedSearch, QueryBuilder } from "../lucene"
const INDEX_NAME = "main"
const TABLE_ID = DocumentType.TABLE + SEPARATOR + newid()
const index = `function(doc) {
let props = ["property", "number", "array"]
for (let key of props) {
if (Array.isArray(doc[key])) {
for (let val of doc[key]) {
if (!doc._id.startsWith("ro_")) {
return
}
let keys = Object.keys(doc).filter(key => !key.startsWith("_"))
for (let key of keys) {
const value = doc[key]
if (Array.isArray(value)) {
for (let val of value) {
index(key, val)
}
} else if (doc[key]) {
index(key, doc[key])
} else if (value) {
index(key, value)
}
}
}`
function rowId(id?: string) {
return DocumentType.ROW + SEPARATOR + (id || newid())
}
describe("lucene", () => {
let db: Database, dbName: string
@ -25,10 +41,21 @@ describe("lucene", () => {
dbName = `db-${newid()}`
// create the DB for testing
db = getDB(dbName)
await db.put({ _id: newid(), property: "word", array: ["1", "4"] })
await db.put({ _id: newid(), property: "word2", array: ["3", "1"] })
await db.put({
_id: newid(),
_id: rowId(),
tableId: TABLE_ID,
property: "word",
array: ["1", "4"],
})
await db.put({
_id: rowId(),
tableId: TABLE_ID,
property: "word2",
array: ["3", "1"],
})
await db.put({
_id: rowId(),
tableId: TABLE_ID,
property: "word3",
number: 1,
array: ["1", "2"],
@ -240,7 +267,8 @@ describe("lucene", () => {
docs = Array(QueryBuilder.maxLimit * 2.5)
.fill(0)
.map((_, i) => ({
_id: i.toString().padStart(3, "0"),
_id: rowId(i.toString().padStart(3, "0")),
tableId: TABLE_ID,
property: `value_${i.toString().padStart(3, "0")}`,
array: [],
}))
@ -338,10 +366,11 @@ describe("lucene", () => {
},
},
{
tableId: TABLE_ID,
limit: 1,
sort: "property",
sortType: "string",
sortOrder: "desc",
sortType: SortType.STRING,
sortOrder: SortOrder.DESCENDING,
}
)
expect(page.rows.length).toBe(1)
@ -360,7 +389,10 @@ describe("lucene", () => {
property: "wo",
},
},
{}
{
tableId: TABLE_ID,
query: {},
}
)
expect(page.rows.length).toBe(3)
})

1
packages/backend-core/src/index.ts
View file

@ -32,7 +32,6 @@ export { default as env } from "./environment"
export * as blacklist from "./blacklist"
export * as docUpdates from "./docUpdates"
export * from "./utils/Duration"
export { SearchParams } from "./db"
export * as docIds from "./docIds"
export * as security from "./security"
// Add context to tenancy for backwards compatibility

3
packages/backend-core/src/users/db.ts
View file

@ -45,6 +45,7 @@ type GroupFns = {
getGroupBuilderAppIds: GroupBuildersFn
}
type CreateAdminUserOpts = {
password?: string
ssoId?: string
hashPassword?: boolean
requirePassword?: boolean
@ -501,9 +502,9 @@ export class UserDB {
static async createAdminUser(
email: string,
tenantId: string,
password?: string,
opts?: CreateAdminUserOpts
) {
const password = opts?.password
const user: User = {
email: email,
password,

6
packages/backend-core/tests/core/utilities/testContainerUtils.ts
View file

@ -77,9 +77,15 @@ export function setupEnv(...envs: any[]) {
throw new Error("CouchDB port not found")
}
const couchSqlPort = getExposedV4Port(couch, 4984)
if (!couchSqlPort) {
throw new Error("CouchDB SQL port not found")
}
const configs = [
{ key: "COUCH_DB_PORT", value: `${couchPort}` },
{ key: "COUCH_DB_URL", value: `http://127.0.0.1:${couchPort}` },
{ key: "COUCH_DB_SQL_URL", value: `http://127.0.0.1:${couchSqlPort}` },
]
for (const config of configs.filter(x => !!x.value)) {

1
packages/bbui/src/Form/Core/Dropzone.svelte
View file

@ -37,6 +37,7 @@
"svg",
"bmp",
"jfif",
"webp",
]
const fieldId = id || uuid()

2
packages/builder/package.json
View file

@ -72,7 +72,7 @@
"fast-json-patch": "^3.1.1",
"json-format-highlight": "^1.0.4",
"lodash": "4.17.21",
"posthog-js": "^1.116.6",
"posthog-js": "^1.118.0",
"remixicon": "2.5.0",
"sanitize-html": "^2.7.0",
"shortid": "2.2.15",

95
packages/builder/src/analytics/IntercomClient.js
View file

@ -1,95 +0,0 @@
export default class IntercomClient {
constructor(token) {
this.token = token
}
//
/**
* Instantiate intercom using their provided script.
*/
init() {
if (!this.token) return
const token = this.token
var w = window
var ic = w.Intercom
if (typeof ic === "function") {
ic("reattach_activator")
ic("update", w.intercomSettings)
} else {
var d = document
var i = function () {
i.c(arguments)
}
i.q = []
i.c = function (args) {
i.q.push(args)
}
w.Intercom = i
var l = function () {
var s = d.createElement("script")
s.type = "text/javascript"
s.async = true
s.src = "https://widget.intercom.io/widget/" + token
var x = d.getElementsByTagName("script")[0]
x.parentNode.insertBefore(s, x)
}
if (document.readyState === "complete") {
l()
} else if (w.attachEvent) {
w.attachEvent("onload", l)
} else {
w.addEventListener("load", l, false)
}
this.initialised = true
}
}
/**
* Show the intercom chat bubble.
* @param {Object} user - user to identify
* @returns Intercom global object
*/
show(user = {}, enabled) {
if (!this.initialised || !enabled) return
return window.Intercom("boot", {
app_id: this.token,
...user,
})
}
/**
* Update intercom user details and messages.
* @returns Intercom global object
*/
update() {
if (!this.initialised) return
return window.Intercom("update")
}
/**
* Capture analytics events and send them to intercom.
* @param {String} event - event identifier
* @param {Object} props - properties for the event
* @returns Intercom global object
*/
captureEvent(event, props = {}) {
if (!this.initialised) return
return window.Intercom("trackEvent", event, props)
}
/**
* Disassociate the user from the current session.
* @returns Intercom global object
*/
logout() {
if (!this.initialised) return
return window.Intercom("shutdown")
}
}

10
packages/builder/src/analytics/index.js
View file

@ -1,14 +1,12 @@
import { API } from "api"
import PosthogClient from "./PosthogClient"
import IntercomClient from "./IntercomClient"
import { Events, EventSource } from "./constants"
const posthog = new PosthogClient(process.env.POSTHOG_TOKEN)
const intercom = new IntercomClient(process.env.INTERCOM_TOKEN)
class AnalyticsHub {
constructor() {
this.clients = [posthog, intercom]
this.clients = [posthog]
this.initialised = false
}
@ -31,16 +29,10 @@ class AnalyticsHub {
captureEvent(eventName, props = {}) {
posthog.captureEvent(eventName, props)
intercom.captureEvent(eventName, props)
}
showChat(user) {
intercom.show(user)
}
async logout() {
posthog.logout()
intercom.logout()
}
}

8
packages/builder/src/components/deploy/AppActions.svelte
View file

@ -32,14 +32,10 @@
import TourWrap from "components/portal/onboarding/TourWrap.svelte"
import { TOUR_STEP_KEYS } from "components/portal/onboarding/tours.js"
import { goto } from "@roxi/routify"
import { onMount } from "svelte"
import PosthogClient from "../../analytics/PosthogClient"
export let application
export let loaded
const posthog = new PosthogClient(process.env.POSTHOG_TOKEN)
let unpublishModal
let updateAppModal
let revertModal
@ -154,10 +150,6 @@
notifications.error("Error refreshing app")
}
}
onMount(() => {
posthog.init()
})
</script>
<!-- svelte-ignore a11y-click-events-have-key-events -->

12
packages/builder/src/components/design/settings/componentSettings.js
View file

@ -1,4 +1,6 @@
import { Checkbox, Select, RadioGroup, Stepper, Input } from "@budibase/bbui"
import { licensing } from "stores/portal"
import { get } from "svelte/store"
import DataSourceSelect from "./controls/DataSourceSelect/DataSourceSelect.svelte"
import S3DataSourceSelect from "./controls/S3DataSourceSelect.svelte"
import DataProviderSelect from "./controls/DataProviderSelect.svelte"
@ -26,7 +28,8 @@ import FieldConfiguration from "./controls/FieldConfiguration/FieldConfiguration
import ButtonConfiguration from "./controls/ButtonConfiguration/ButtonConfiguration.svelte"
import RelationshipFilterEditor from "./controls/RelationshipFilterEditor.svelte"
import FormStepConfiguration from "./controls/FormStepConfiguration.svelte"
import FormStepControls from "components/design/settings/controls/FormStepControls.svelte"
import FormStepControls from "./controls/FormStepControls.svelte"
import PaywalledSetting from "./controls/PaywalledSetting.svelte"
const componentMap = {
text: DrawerBindableInput,
@ -86,11 +89,16 @@ const componentMap = {
}
export const getComponentForSetting = setting => {
const { type, showInBar, barStyle } = setting || {}
const { type, showInBar, barStyle, license } = setting || {}
if (!type) {
return null
}
// Check for paywalled settings
if (license && get(licensing).isFreePlan) {
return PaywalledSetting
}
// We can show a clone of the bar settings for certain select settings
if (showInBar && type === "select" && barStyle === "buttons") {
return BarButtonList

23
packages/builder/src/components/design/settings/controls/PaywalledSetting.svelte Normal file
View file

@ -0,0 +1,23 @@
<script>
import { Tag, Tags } from "@budibase/bbui"
import { getFormattedPlanName } from "helpers/planTitle"
export let license
$: title = getFormattedPlanName(license)
</script>
<div>
<Tags>
<Tag icon="LockClosed">{title}</Tag>
</Tags>
</div>
<style>
div {
height: 32px;
display: flex;
flex-direction: column;
justify-content: center;
}
</style>

1
packages/builder/src/pages/builder/app/[application]/design/[screenId]/[componentId]/_components/Component/ComponentSettingsSection.svelte
View file

@ -183,6 +183,7 @@
props={{
// Generic settings
placeholder: setting.placeholder || null,
license: setting.license,
// Select settings
options: setting.options || [],

15
packages/builder/src/stores/portal/auth.js
View file

@ -2,7 +2,6 @@ import { derived, writable, get } from "svelte/store"
import { API } from "api"
import { admin } from "stores/portal"
import analytics from "analytics"
import { sdk } from "@budibase/shared-core"
export function createAuthStore() {
const auth = writable({
@ -42,20 +41,6 @@ export function createAuthStore() {
.activate()
.then(() => {
analytics.identify(user._id)
analytics.showChat(
{
email: user.email,
created_at: (user.createdAt || Date.now()) / 1000,
name: user.account?.name,
user_id: user._id,
tenant: user.tenantId,
admin: sdk.users.isAdmin(user),
builder: sdk.users.isBuilder(user),
"Company size": user.account?.size,
"Job role": user.account?.profession,
},
!!user?.account
)
})
.catch(() => {
// This request may fail due to browser extensions blocking requests

3
packages/builder/vite.config.js
View file

@ -77,9 +77,6 @@ export default defineConfig(({ mode }) => {
isProduction ? "production" : "development"
),
"process.env.POSTHOG_TOKEN": JSON.stringify(process.env.POSTHOG_TOKEN),
"process.env.INTERCOM_TOKEN": JSON.stringify(
process.env.INTERCOM_TOKEN
),
}),
copyFonts("fonts"),
...(isProduction ? [] : devOnlyPlugins),

174
packages/client/manifest.json
View file

@ -4707,6 +4707,35 @@
"key": "dataSource",
"required": true
},
{
"type": "select",
"label": "Auto-refresh",
"key": "autoRefresh",
"license": "premium",
"placeholder": "Never",
"options": [
{
"label": "10 seconds",
"value": 10
},
{
"label": "30 seconds",
"value": 30
},
{
"label": "1 minute",
"value": 60
},
{
"label": "5 minutes",
"value": 300
},
{
"label": "10 minutes",
"value": 600
}
]
},
{
"type": "filter",
"label": "Filtering",
@ -5074,6 +5103,35 @@
"key": "dataSource",
"required": true
},
{
"type": "select",
"label": "Auto-refresh",
"key": "autoRefresh",
"license": "premium",
"placeholder": "Never",
"options": [
{
"label": "10 seconds",
"value": 10
},
{
"label": "30 seconds",
"value": 30
},
{
"label": "1 minute",
"value": 60
},
{
"label": "5 minutes",
"value": 300
},
{
"label": "10 minutes",
"value": 600
}
]
},
{
"type": "text",
"label": "Title",
@ -5542,6 +5600,35 @@
"key": "dataSource",
"required": true
},
{
"type": "select",
"label": "Auto-refresh",
"key": "autoRefresh",
"license": "premium",
"placeholder": "Never",
"options": [
{
"label": "10 seconds",
"value": 10
},
{
"label": "30 seconds",
"value": 30
},
{
"label": "1 minute",
"value": 60
},
{
"label": "5 minutes",
"value": 300
},
{
"label": "10 minutes",
"value": 600
}
]
},
{
"type": "columns",
"label": "Columns",
@ -5828,6 +5915,35 @@
"key": "dataSource",
"required": true
},
{
"type": "select",
"label": "Auto-refresh",
"key": "autoRefresh",
"license": "premium",
"placeholder": "Never",
"options": [
{
"label": "10 seconds",
"value": 10
},
{
"label": "30 seconds",
"value": 30
},
{
"label": "1 minute",
"value": 60
},
{
"label": "5 minutes",
"value": 300
},
{
"label": "10 minutes",
"value": 600
}
]
},
{
"type": "searchfield",
"label": "Search columns",
@ -6005,6 +6121,35 @@
"key": "dataSource",
"required": true
},
{
"type": "select",
"label": "Auto-refresh",
"key": "autoRefresh",
"license": "premium",
"placeholder": "Never",
"options": [
{
"label": "10 seconds",
"value": 10
},
{
"label": "30 seconds",
"value": 30
},
{
"label": "1 minute",
"value": 60
},
{
"label": "5 minutes",
"value": 300
},
{
"label": "10 minutes",
"value": 600
}
]
},
{
"type": "filter",
"label": "Filtering",
@ -6601,6 +6746,35 @@
"key": "dataSource",
"required": true
},
{
"type": "select",
"label": "Auto-refresh",
"key": "autoRefresh",
"license": "premium",
"placeholder": "Never",
"options": [
{
"label": "10 seconds",
"value": 10
},
{
"label": "30 seconds",
"value": 30
},
{
"label": "1 minute",
"value": 60
},
{
"label": "5 minutes",
"value": 300
},
{
"label": "10 minutes",
"value": 600
}
]
},
{
"type": "text",
"label": "Height",

18
packages/client/src/components/Screen.svelte
View file

@ -5,29 +5,29 @@
import Provider from "./context/Provider.svelte"
import { onMount, getContext } from "svelte"
import { enrichButtonActions } from "../utils/buttonActions.js"
import { memo } from "@budibase/frontend-core"
export let params = {}
const context = getContext("context")
const onLoadActions = memo()
// Get the screen definition for the current route
$: screenDefinition = $screenStore.activeScreen?.props
$: runOnLoadActions(params)
$: onLoadActions.set($screenStore.activeScreen?.onLoad)
$: runOnLoadActions($onLoadActions, params)
// Enrich and execute any on load actions.
// We manually construct the full context here as this component is the
// one that provides the url context, so it is not available in $context yet
const runOnLoadActions = params => {
const screenState = get(screenStore)
if (screenState.activeScreen?.onLoad && !get(builderStore).inBuilder) {
const actions = enrichButtonActions(screenState.activeScreen.onLoad, {
const runOnLoadActions = (actions, params) => {
if (actions?.length && !get(builderStore).inBuilder) {
const enrichedActions = enrichButtonActions(actions, {
...get(context),
url: params,
})
if (actions != null) {
actions()
if (enrichedActions != null) {
enrichedActions()
}
}
}

21
packages/client/src/components/app/DataProvider.svelte
View file

@ -9,17 +9,18 @@
export let sortOrder
export let limit
export let paginate
export let autoRefresh
const { styleable, Provider, ActionTypes, API } = getContext("sdk")
const component = getContext("component")
let interval
let queryExtensions = {}
// We need to manage our lucene query manually as we want to allow components
// to extend it
let queryExtensions = {}
$: defaultQuery = LuceneUtils.buildLuceneQuery(filter)
$: query = extendQuery(defaultQuery, queryExtensions)
// Fetch data and refresh when needed
$: fetch = createFetch(dataSource)
$: fetch.update({
query,
@ -28,11 +29,8 @@
limit,
paginate,
})
// Sanitize schema to remove hidden fields
$: schema = sanitizeSchema($fetch.schema)
// Build our action context
$: setUpAutoRefresh(autoRefresh)
$: actions = [
{
type: ActionTypes.RefreshDatasource,
@ -63,8 +61,6 @@
},
},
]
// Build our data context
$: dataContext = {
rows: $fetch.rows,
info: $fetch.info,
@ -140,6 +136,13 @@
})
return extendedQuery
}
const setUpAutoRefresh = autoRefresh => {
clearInterval(interval)
if (autoRefresh) {
interval = setInterval(fetch.refresh, Math.max(10000, autoRefresh * 1000))
}
}
</script>
<div use:styleable={$component.styles} class="container">

2
packages/client/src/components/app/GridBlock.svelte
View file

@ -18,6 +18,7 @@
export let columns = null
export let onRowClick = null
export let buttons = null
export let repeat = null
const context = getContext("context")
const component = getContext("component")
@ -122,6 +123,7 @@
{fixedRowHeight}
{columnWhitelist}
{schemaOverrides}
{repeat}
canAddRows={allowAddRows}
canEditRows={allowEditRows}
canDeleteRows={allowDeleteRows}

16
packages/client/src/components/app/Layout.svelte
View file

@ -10,13 +10,11 @@
const {
routeStore,
roleStore,
styleable,
linkable,
builderStore,
sidePanelStore,
appStore,
} = sdk
const component = getContext("component")
const context = getContext("context")
// Legacy props which must remain unchanged for backwards compatibility
@ -169,15 +167,14 @@
<!-- svelte-ignore a11y-no-static-element-interactions -->
<!-- svelte-ignore a11y-click-events-have-key-events -->
<div
class="component {screenId} layout layout--{typeClass}"
use:styleable={$component.styles}
class="component layout layout--{typeClass}"
class:desktop={!mobile}
class:mobile={!!mobile}
data-id={screenId}
data-name="Screen"
data-icon="WebPage"
>
<div class="{screenId}-dom screen-wrapper layout-body">
<div class="screen-wrapper layout-body">
{#if typeClass !== "none"}
<div
class="interactive component {navigationId}"
@ -282,7 +279,14 @@
</div>
</div>
{/if}
<div class="main-wrapper">
<div
class="main-wrapper"
on:click={() => {
if ($builderStore.inBuilder) {
builderStore.actions.selectComponent(screenId)
}
}}
>
<div class="main size--{pageWidthClass}">
<slot />
</div>

2
packages/client/src/components/app/blocks/CardsBlock.svelte
View file

@ -31,6 +31,7 @@
export let cardButtonOnClick
export let linkColumn
export let noRowsMessage
export let autoRefresh
const context = getContext("context")
const { fetchDatasourceSchema, generateGoldenSample } = getContext("sdk")
@ -184,6 +185,7 @@
sortOrder,
paginate,
limit,
autoRefresh,
}}
order={1}
>

2
packages/client/src/components/app/blocks/ChartBlock.svelte
View file

@ -8,6 +8,7 @@
export let sortColumn
export let sortOrder
export let limit
export let autoRefresh
// Block
export let chartTitle
@ -65,6 +66,7 @@
sortColumn,
sortOrder,
limit,
autoRefresh,
}}
>
{#if dataProviderId && chartType}

2
packages/client/src/components/app/blocks/RepeaterBlock.svelte
View file

@ -17,6 +17,7 @@
export let hAlign
export let vAlign
export let gap
export let autoRefresh
const component = getContext("component")
const context = getContext("context")
@ -47,6 +48,7 @@
sortOrder,
limit,
paginate,
autoRefresh,
}}
>
{#if $component.empty}

2
packages/client/src/components/app/blocks/RowExplorer.svelte
View file

@ -16,6 +16,7 @@
export let detailFields
export let detailTitle
export let noRowsMessage
export let autoRefresh
const stateKey = generate()
const context = getContext("context")
@ -66,6 +67,7 @@
noValue: false,
},
],
autoRefresh,
}}
styles={{
custom: `

2
packages/client/src/components/app/deprecated/TableBlock.svelte
View file

@ -33,6 +33,7 @@
export let sidePanelSaveLabel
export let sidePanelDeleteLabel
export let notificationOverride
export let autoRefresh
const { fetchDatasourceSchema, API, generateGoldenSample } = getContext("sdk")
const component = getContext("component")
@ -243,6 +244,7 @@
sortOrder,
paginate,
limit: rowCount,
autoRefresh,
}}
context="provider"
order={1}

4
packages/server/package.json
View file

@ -83,7 +83,7 @@
"joi": "17.6.0",
"js-yaml": "4.1.0",
"jsonschema": "1.4.0",
"knex": "2.4.0",
"knex": "2.4.2",
"koa": "2.13.4",
"koa-body": "4.2.0",
"koa-compress": "4.0.1",
@ -109,6 +109,8 @@
"server-destroy": "1.0.1",
"snowflake-promise": "^4.5.0",
"socket.io": "4.6.1",
"sqlite3": "5.1.6",
"swagger-parser": "10.0.3",
"tar": "6.1.15",
"to-json-schema": "0.2.5",
"undici": "^6.0.1",

349
packages/server/src/api/controllers/row/ExternalRequest.ts
View file

@ -6,12 +6,10 @@ import {
FieldType,
FilterType,
IncludeRelationship,
ManyToManyRelationshipFieldMetadata,
OneToManyRelationshipFieldMetadata,
Operation,
PaginationJson,
RelationshipFieldMetadata,
RelationshipsJson,
Row,
SearchFilters,
SortJson,
@ -23,14 +21,20 @@ import {
breakExternalTableId,
breakRowIdField,
convertRowId,
generateRowIdField,
isRowId,
isSQL,
generateRowIdField,
} from "../../../integrations/utils"
import {
buildExternalRelationships,
buildSqlFieldList,
generateIdForRow,
sqlOutputProcessing,
isManyToMany,
} from "./utils"
import { getDatasourceAndQuery } from "../../../sdk/app/rows/utils"
import { processObjectSync } from "@budibase/string-templates"
import { cloneDeep } from "lodash/fp"
import { processDates, processFormulas } from "../../../utilities/rowProcessor"
import { db as dbCore } from "@budibase/backend-core"
import AliasTables from "./alias"
import sdk from "../../../sdk"
@ -154,7 +158,8 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
// filter out fields which cannot be keys
const fieldNames = Object.entries(table.schema)
.filter(schema => primaryOptions.find(val => val === schema[1].type))
.map(([fieldName]) => fieldName)
// map to fieldName
.map(entry => entry[0])
const iterateObject = (obj: { [key: string]: any }) => {
for (let [field, value] of Object.entries(obj)) {
if (fieldNames.find(name => name === field) && isRowId(value)) {
@ -183,34 +188,6 @@ function cleanupConfig(config: RunConfig, table: Table): RunConfig {
return config
}
function generateIdForRow(
row: Row | undefined,
table: Table,
isLinked: boolean = false
): string {
const primary = table.primary
if (!row || !primary) {
return ""
}
// build id array
let idParts = []
for (let field of primary) {
let fieldValue = extractFieldValue({
row,
tableName: table.name,
fieldName: field,
isLinked,
})
if (fieldValue != null) {
idParts.push(fieldValue)
}
}
if (idParts.length === 0) {
return ""
}
return generateRowIdField(idParts)
}
function getEndpoint(tableId: string | undefined, operation: string) {
if (!tableId) {
throw new Error("Cannot get endpoint information - no table ID specified")
@ -223,71 +200,6 @@ function getEndpoint(tableId: string | undefined, operation: string) {
}
}
// need to handle table name + field or just field, depending on if relationships used
function extractFieldValue({
row,
tableName,
fieldName,
isLinked,
}: {
row: Row
tableName: string
fieldName: string
isLinked: boolean
}) {
let value = row[`${tableName}.${fieldName}`]
if (value == null && !isLinked) {
value = row[fieldName]
}
return value
}
function basicProcessing({
row,
table,
isLinked,
}: {
row: Row
table: Table
isLinked: boolean
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
for (let field of Object.values(table.schema)) {
const fieldName = field.name
const value = extractFieldValue({
row,
tableName: table.name,
fieldName,
isLinked,
})
// all responses include "select col as table.col" so that overlaps are handled
if (value != null) {
thisRow[fieldName] = value
}
}
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
return thisRow
}
function fixArrayTypes(row: Row, table: Table) {
for (let [fieldName, schema] of Object.entries(table.schema)) {
if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
try {
row[fieldName] = JSON.parse(row[fieldName])
} catch (err) {
// couldn't convert back to array, ignore
delete row[fieldName]
}
}
}
return row
}
function isOneSide(
field: RelationshipFieldMetadata
): field is OneToManyRelationshipFieldMetadata {
@ -296,12 +208,6 @@ function isOneSide(
)
}
function isManyToMany(
field: RelationshipFieldMetadata
): field is ManyToManyRelationshipFieldMetadata {
return !!(field as ManyToManyRelationshipFieldMetadata).through
}
function isEditableColumn(column: FieldSchema) {
const isExternalAutoColumn =
column.autocolumn &&
@ -435,187 +341,6 @@ export class ExternalRequest<T extends Operation> {
return { row: newRow, manyRelationships }
}
async processRelationshipFields(
table: Table,
row: Row,
relationships: RelationshipsJson[]
): Promise<Row> {
for (let relationship of relationships) {
const linkedTable = this.tables[relationship.tableName]
if (!linkedTable || !row[relationship.column]) {
continue
}
for (let key of Object.keys(row[relationship.column])) {
let relatedRow: Row = row[relationship.column][key]
// add this row as context for the relationship
for (let col of Object.values(linkedTable.schema)) {
if (col.type === FieldType.LINK && col.tableId === table._id) {
relatedRow[col.name] = [row]
}
}
// process additional types
relatedRow = processDates(table, relatedRow)
relatedRow = await processFormulas(linkedTable, relatedRow)
row[relationship.column][key] = relatedRow
}
}
return row
}
/**
* This iterates through the returned rows and works out what elements of the rows
* actually match up to another row (based on primary keys) - this is pretty specific
* to SQL and the way that SQL relationships are returned based on joins.
* This is complicated, but the idea is that when a SQL query returns all the relations
* will be separate rows, with all of the data in each row. We have to decipher what comes
* from where (which tables) and how to convert that into budibase columns.
*/
updateRelationshipColumns(
table: Table,
row: Row,
rows: { [key: string]: Row },
relationships: RelationshipsJson[]
) {
const columns: { [key: string]: any } = {}
for (let relationship of relationships) {
const linkedTable = this.tables[relationship.tableName]
if (!linkedTable) {
continue
}
const fromColumn = `${table.name}.${relationship.from}`
const toColumn = `${linkedTable.name}.${relationship.to}`
// this is important when working with multiple relationships
// between the same tables, don't want to overlap/multiply the relations
if (
!relationship.through &&
row[fromColumn]?.toString() !== row[toColumn]?.toString()
) {
continue
}
let linked = basicProcessing({ row, table: linkedTable, isLinked: true })
if (!linked._id) {
continue
}
columns[relationship.column] = linked
}
for (let [column, related] of Object.entries(columns)) {
if (!row._id) {
continue
}
const rowId: string = row._id
if (!Array.isArray(rows[rowId][column])) {
rows[rowId][column] = []
}
// make sure relationship hasn't been found already
if (
!rows[rowId][column].find(
(relation: Row) => relation._id === related._id
)
) {
rows[rowId][column].push(related)
}
}
return rows
}
async outputProcessing(
rows: Row[] = [],
table: Table,
relationships: RelationshipsJson[]
) {
if (!rows || rows.length === 0 || rows[0].read === true) {
return []
}
let finalRows: { [key: string]: Row } = {}
for (let row of rows) {
const rowId = generateIdForRow(row, table)
row._id = rowId
// this is a relationship of some sort
if (finalRows[rowId]) {
finalRows = this.updateRelationshipColumns(
table,
row,
finalRows,
relationships
)
continue
}
const thisRow = fixArrayTypes(
basicProcessing({ row, table, isLinked: false }),
table
)
if (thisRow._id == null) {
throw "Unable to generate row ID for SQL rows"
}
finalRows[thisRow._id] = thisRow
// do this at end once its been added to the final rows
finalRows = this.updateRelationshipColumns(
table,
row,
finalRows,
relationships
)
}
// make sure all related rows are correct
let finalRowArray = []
for (let row of Object.values(finalRows)) {
finalRowArray.push(
await this.processRelationshipFields(table, row, relationships)
)
}
// process some additional types
finalRowArray = processDates(table, finalRowArray)
return finalRowArray
}
/**
* Gets the list of relationship JSON structures based on the columns in the table,
* this will be used by the underlying library to build whatever relationship mechanism
* it has (e.g. SQL joins).
*/
buildRelationships(table: Table): RelationshipsJson[] {
const relationships = []
for (let [fieldName, field] of Object.entries(table.schema)) {
if (field.type !== FieldType.LINK) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
// no table to link to, this is not a valid relationships
if (!linkTableName || !this.tables[linkTableName]) {
continue
}
const linkTable = this.tables[linkTableName]
if (!table.primary || !linkTable.primary) {
continue
}
const definition: RelationshipsJson = {
tableName: linkTableName,
// need to specify where to put this back into
column: fieldName,
}
if (isManyToMany(field)) {
const { tableName: throughTableName } = breakExternalTableId(
field.through
)
definition.through = throughTableName
// don't support composite keys for relationships
definition.from = field.throughTo || table.primary[0]
definition.to = field.throughFrom || linkTable.primary[0]
definition.fromPrimary = table.primary[0]
definition.toPrimary = linkTable.primary[0]
} else {
// if no foreign key specified then use the name of the field in other table
definition.from = field.foreignKey || table.primary[0]
definition.to = field.fieldName
}
relationships.push(definition)
}
return relationships
}
/**
* This is a cached lookup, of relationship records, this is mainly for creating/deleting junction
* information.
@ -801,41 +526,6 @@ export class ExternalRequest<T extends Operation> {
}
}
/**
* This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
* you have column overlap in relationships, e.g. we join a few different tables and they all have the
* concept of an ID, but for some of them it will be null (if they say don't have a relationship).
* Creating the specific list of fields that we desire, and excluding the ones that are no use to us
* is more performant and has the added benefit of protecting against this scenario.
*/
buildFields(table: Table, includeRelations: boolean) {
function extractRealFields(table: Table, existing: string[] = []) {
return Object.entries(table.schema)
.filter(
column =>
column[1].type !== FieldType.LINK &&
column[1].type !== FieldType.FORMULA &&
!existing.find((field: string) => field === column[0])
)
.map(column => `${table.name}.${column[0]}`)
}
let fields = extractRealFields(table)
for (let field of Object.values(table.schema)) {
if (field.type !== FieldType.LINK || !includeRelations) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
if (linkTableName) {
const linkTable = this.tables[linkTableName]
if (linkTable) {
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
}
}
}
return fields
}
async run(config: RunConfig): Promise<ExternalRequestReturnType<T>> {
const { operation, tableId } = this
let { datasourceId, tableName } = breakExternalTableId(tableId)
@ -869,14 +559,16 @@ export class ExternalRequest<T extends Operation> {
delete sort?.[sortColumn]
break
case FieldType.NUMBER:
sort[sortColumn].type = SortType.number
if (sort && sort[sortColumn]) {
sort[sortColumn].type = SortType.NUMBER
}
break
}
}
filters = buildFilters(id, filters || {}, table)
const relationships = this.buildRelationships(table)
const relationships = buildExternalRelationships(table, this.tables)
const includeSqlRelationships =
const incRelationships =
config.includeSqlRelationships === IncludeRelationship.INCLUDE
// clean up row on ingress using schema
@ -896,7 +588,11 @@ export class ExternalRequest<T extends Operation> {
},
resource: {
// have to specify the fields to avoid column overlap (for SQL)
fields: isSql ? this.buildFields(table, includeSqlRelationships) : [],
fields: isSql
? buildSqlFieldList(table, this.tables, {
relationships: incRelationships,
})
: [],
},
filters,
sort,
@ -935,9 +631,10 @@ export class ExternalRequest<T extends Operation> {
processed.manyRelationships
)
}
const output = await this.outputProcessing(
responseRows,
const output = await sqlOutputProcessing(
response,
table,
this.tables,
relationships
)
// if reading it'll just be an array of rows, return whole thing

4
packages/server/src/api/controllers/row/alias.ts
View file

@ -155,7 +155,9 @@ export default class AliasTables {
return map
}
async queryWithAliasing(json: QueryJson): DatasourcePlusQueryResponse {
async queryWithAliasing(
json: QueryJson
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)

4
packages/server/src/api/controllers/row/index.ts
View file

@ -13,7 +13,7 @@ import {
PatchRowRequest,
PatchRowResponse,
Row,
SearchParams,
RowSearchParams,
SearchRowRequest,
SearchRowResponse,
UserCtx,
@ -192,7 +192,7 @@ export async function destroy(ctx: UserCtx<DeleteRowRequest>) {
export async function search(ctx: Ctx<SearchRowRequest, SearchRowResponse>) {
const tableId = utils.getTableId(ctx)
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
...ctx.request.body,
tableId,
}

59
packages/server/src/api/controllers/row/utils.ts
View file

@ -1,59 +0,0 @@
import { InternalTables } from "../../../db/utils"
import * as userController from "../user"
import { context } from "@budibase/backend-core"
import { Ctx, Row, UserCtx } from "@budibase/types"
import validateJs from "validate.js"
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
return new Date(value).getTime()
},
// Input is a unix timestamp
format: function (value: string) {
return new Date(value).toISOString()
},
})
export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
const db = context.getAppDB()
let row: Row
// TODO remove special user case in future
if (tableId === InternalTables.USER_METADATA) {
ctx.params = {
id: rowId,
}
await userController.findMetadata(ctx)
row = ctx.body
} else {
row = await db.get(rowId)
}
if (row.tableId !== tableId) {
throw "Supplied tableId does not match the rows tableId"
}
return row
}
export function getTableId(ctx: Ctx): string {
// top priority, use the URL first
if (ctx.params?.sourceId) {
return ctx.params.sourceId
}
// now check for old way of specifying table ID
if (ctx.params?.tableId) {
return ctx.params.tableId
}
// check body for a table ID
if (ctx.request.body?.tableId) {
return ctx.request.body.tableId
}
// now check if a specific view name
if (ctx.params?.viewName) {
return ctx.params.viewName
}
throw new Error("Unable to find table ID in request")
}
export function isUserMetadataTable(tableId: string) {
return tableId === InternalTables.USER_METADATA
}

116
packages/server/src/api/controllers/row/utils/basic.ts Normal file
View file

@ -0,0 +1,116 @@
// need to handle table name + field or just field, depending on if relationships used
import { FieldType, Row, Table } from "@budibase/types"
import { generateRowIdField } from "../../../../integrations/utils"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
function extractFieldValue({
row,
tableName,
fieldName,
isLinked,
}: {
row: Row
tableName: string
fieldName: string
isLinked: boolean
}) {
let value = row[`${tableName}.${fieldName}`]
if (value == null && !isLinked) {
value = row[fieldName]
}
return value
}
export function getInternalRowId(row: Row, table: Table): string {
return extractFieldValue({
row,
tableName: table._id!,
fieldName: "_id",
isLinked: false,
})
}
export function generateIdForRow(
row: Row | undefined,
table: Table,
isLinked: boolean = false
): string {
const primary = table.primary
if (!row || !primary) {
return ""
}
// build id array
let idParts = []
for (let field of primary) {
let fieldValue = extractFieldValue({
row,
tableName: table.name,
fieldName: field,
isLinked,
})
if (fieldValue != null) {
idParts.push(fieldValue)
}
}
if (idParts.length === 0) {
return ""
}
return generateRowIdField(idParts)
}
export function basicProcessing({
row,
table,
isLinked,
internal,
}: {
row: Row
table: Table
isLinked: boolean
internal?: boolean
}): Row {
const thisRow: Row = {}
// filter the row down to what is actually the row (not joined)
for (let field of Object.values(table.schema)) {
const fieldName = field.name
const value = extractFieldValue({
row,
tableName: table.name,
fieldName,
isLinked,
})
// all responses include "select col as table.col" so that overlaps are handled
if (value != null) {
thisRow[fieldName] = value
}
}
if (!internal) {
thisRow._id = generateIdForRow(row, table, isLinked)
thisRow.tableId = table._id
thisRow._rev = "rev"
} else {
for (let internalColumn of CONSTANT_INTERNAL_ROW_COLS) {
thisRow[internalColumn] = extractFieldValue({
row,
tableName: table._id!,
fieldName: internalColumn,
isLinked: false,
})
}
}
return thisRow
}
export function fixArrayTypes(row: Row, table: Table) {
for (let [fieldName, schema] of Object.entries(table.schema)) {
if (schema.type === FieldType.ARRAY && typeof row[fieldName] === "string") {
try {
row[fieldName] = JSON.parse(row[fieldName])
} catch (err) {
// couldn't convert back to array, ignore
delete row[fieldName]
}
}
}
return row
}

3
packages/server/src/api/controllers/row/utils/index.ts Normal file
View file

@ -0,0 +1,3 @@
export * from "./basic"
export * from "./sqlUtils"
export * from "./utils"

194
packages/server/src/api/controllers/row/utils/sqlUtils.ts Normal file
View file

@ -0,0 +1,194 @@
import {
FieldType,
ManyToManyRelationshipFieldMetadata,
RelationshipFieldMetadata,
RelationshipsJson,
Row,
Table,
} from "@budibase/types"
import { breakExternalTableId } from "../../../../integrations/utils"
import { basicProcessing } from "./basic"
import { generateJunctionTableID } from "../../../../db/utils"
type TableMap = Record<string, Table>
export function isManyToMany(
field: RelationshipFieldMetadata
): field is ManyToManyRelationshipFieldMetadata {
return !!(field as ManyToManyRelationshipFieldMetadata).through
}
/**
* This iterates through the returned rows and works out what elements of the rows
* actually match up to another row (based on primary keys) - this is pretty specific
* to SQL and the way that SQL relationships are returned based on joins.
* This is complicated, but the idea is that when a SQL query returns all the relations
* will be separate rows, with all of the data in each row. We have to decipher what comes
* from where (which tables) and how to convert that into budibase columns.
*/
export async function updateRelationshipColumns(
table: Table,
tables: TableMap,
row: Row,
rows: { [key: string]: Row },
relationships: RelationshipsJson[],
opts?: { sqs?: boolean }
) {
const columns: { [key: string]: any } = {}
for (let relationship of relationships) {
const linkedTable = tables[relationship.tableName]
if (!linkedTable) {
continue
}
const fromColumn = `${table.name}.${relationship.from}`
const toColumn = `${linkedTable.name}.${relationship.to}`
// this is important when working with multiple relationships
// between the same tables, don't want to overlap/multiply the relations
if (
!relationship.through &&
row[fromColumn]?.toString() !== row[toColumn]?.toString()
) {
continue
}
let linked = await basicProcessing({
row,
table: linkedTable,
isLinked: true,
internal: opts?.sqs,
})
if (!linked._id) {
continue
}
columns[relationship.column] = linked
}
for (let [column, related] of Object.entries(columns)) {
if (!row._id) {
continue
}
const rowId: string = row._id
if (!Array.isArray(rows[rowId][column])) {
rows[rowId][column] = []
}
// make sure relationship hasn't been found already
if (
!rows[rowId][column].find((relation: Row) => relation._id === related._id)
) {
rows[rowId][column].push(related)
}
}
return rows
}
/**
* Gets the list of relationship JSON structures based on the columns in the table,
* this will be used by the underlying library to build whatever relationship mechanism
* it has (e.g. SQL joins).
*/
export function buildExternalRelationships(
table: Table,
tables: TableMap
): RelationshipsJson[] {
const relationships = []
for (let [fieldName, field] of Object.entries(table.schema)) {
if (field.type !== FieldType.LINK) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
// no table to link to, this is not a valid relationships
if (!linkTableName || !tables[linkTableName]) {
continue
}
const linkTable = tables[linkTableName]
if (!table.primary || !linkTable.primary) {
continue
}
const definition: RelationshipsJson = {
tableName: linkTableName,
// need to specify where to put this back into
column: fieldName,
}
if (isManyToMany(field)) {
const { tableName: throughTableName } = breakExternalTableId(
field.through
)
definition.through = throughTableName
// don't support composite keys for relationships
definition.from = field.throughTo || table.primary[0]
definition.to = field.throughFrom || linkTable.primary[0]
definition.fromPrimary = table.primary[0]
definition.toPrimary = linkTable.primary[0]
} else {
// if no foreign key specified then use the name of the field in other table
definition.from = field.foreignKey || table.primary[0]
definition.to = field.fieldName
}
relationships.push(definition)
}
return relationships
}
export function buildInternalRelationships(table: Table): RelationshipsJson[] {
const relationships: RelationshipsJson[] = []
const links = Object.values(table.schema).filter(
column => column.type === FieldType.LINK
)
const tableId = table._id!
for (let link of links) {
if (link.type !== FieldType.LINK) {
continue
}
const linkTableId = link.tableId!
const junctionTableId = generateJunctionTableID(tableId, linkTableId)
const isFirstTable = tableId > linkTableId
relationships.push({
through: junctionTableId,
column: link.name,
tableName: linkTableId,
fromPrimary: "_id",
to: isFirstTable ? "doc2.rowId" : "doc1.rowId",
from: isFirstTable ? "doc1.rowId" : "doc2.rowId",
toPrimary: "_id",
})
}
return relationships
}
/**
* This function is a bit crazy, but the exact purpose of it is to protect against the scenario in which
* you have column overlap in relationships, e.g. we join a few different tables and they all have the
* concept of an ID, but for some of them it will be null (if they say don't have a relationship).
* Creating the specific list of fields that we desire, and excluding the ones that are no use to us
* is more performant and has the added benefit of protecting against this scenario.
*/
export function buildSqlFieldList(
table: Table,
tables: TableMap,
opts?: { relationships: boolean }
) {
function extractRealFields(table: Table, existing: string[] = []) {
return Object.entries(table.schema)
.filter(
column =>
column[1].type !== FieldType.LINK &&
column[1].type !== FieldType.FORMULA &&
!existing.find((field: string) => field === column[0])
)
.map(column => `${table.name}.${column[0]}`)
}
let fields = extractRealFields(table)
for (let field of Object.values(table.schema)) {
if (field.type !== FieldType.LINK || !opts?.relationships) {
continue
}
const { tableName: linkTableName } = breakExternalTableId(field.tableId)
if (linkTableName) {
const linkTable = tables[linkTableName]
if (linkTable) {
const linkedFields = extractRealFields(linkTable, fields)
fields = fields.concat(linkedFields)
}
}
}
return fields
}

189
packages/server/src/api/controllers/row/utils/utils.ts Normal file
View file

@ -0,0 +1,189 @@
import { InternalTables } from "../../../../db/utils"
import * as userController from "../../user"
import { context } from "@budibase/backend-core"
import {
Ctx,
DatasourcePlusQueryResponse,
FieldType,
RelationshipsJson,
Row,
Table,
UserCtx,
} from "@budibase/types"
import {
processDates,
processFormulas,
} from "../../../../utilities/rowProcessor"
import { updateRelationshipColumns } from "./sqlUtils"
import {
basicProcessing,
generateIdForRow,
fixArrayTypes,
getInternalRowId,
} from "./basic"
import sdk from "../../../../sdk"
import validateJs from "validate.js"
validateJs.extend(validateJs.validators.datetime, {
parse: function (value: string) {
return new Date(value).getTime()
},
// Input is a unix timestamp
format: function (value: string) {
return new Date(value).toISOString()
},
})
export async function processRelationshipFields(
table: Table,
tables: Record<string, Table>,
row: Row,
relationships: RelationshipsJson[]
): Promise<Row> {
for (let relationship of relationships) {
const linkedTable = tables[relationship.tableName]
if (!linkedTable || !row[relationship.column]) {
continue
}
for (let key of Object.keys(row[relationship.column])) {
let relatedRow: Row = row[relationship.column][key]
// add this row as context for the relationship
for (let col of Object.values(linkedTable.schema)) {
if (col.type === FieldType.LINK && col.tableId === table._id) {
relatedRow[col.name] = [row]
}
}
// process additional types
relatedRow = processDates(table, relatedRow)
relatedRow = await processFormulas(linkedTable, relatedRow)
row[relationship.column][key] = relatedRow
}
}
return row
}
export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
const db = context.getAppDB()
let row: Row
// TODO remove special user case in future
if (tableId === InternalTables.USER_METADATA) {
ctx.params = {
id: rowId,
}
await userController.findMetadata(ctx)
row = ctx.body
} else {
row = await db.get(rowId)
}
if (row.tableId !== tableId) {
throw "Supplied tableId does not match the rows tableId"
}
return row
}
export function getTableId(ctx: Ctx): string {
// top priority, use the URL first
if (ctx.params?.sourceId) {
return ctx.params.sourceId
}
// now check for old way of specifying table ID
if (ctx.params?.tableId) {
return ctx.params.tableId
}
// check body for a table ID
if (ctx.request.body?.tableId) {
return ctx.request.body.tableId
}
// now check if a specific view name
if (ctx.params?.viewName) {
return ctx.params.viewName
}
throw new Error("Unable to find table ID in request")
}
export async function validate(
opts: { row: Row } & ({ tableId: string } | { table: Table })
) {
let fetchedTable: Table
if ("tableId" in opts) {
fetchedTable = await sdk.tables.getTable(opts.tableId)
} else {
fetchedTable = opts.table
}
return sdk.rows.utils.validate({
...opts,
table: fetchedTable,
})
}
export async function sqlOutputProcessing(
rows: DatasourcePlusQueryResponse,
table: Table,
tables: Record<string, Table>,
relationships: RelationshipsJson[],
opts?: { sqs?: boolean }
): Promise<Row[]> {
if (!Array.isArray(rows) || rows.length === 0 || rows[0].read === true) {
return []
}
let finalRows: { [key: string]: Row } = {}
for (let row of rows as Row[]) {
let rowId = row._id
if (opts?.sqs) {
rowId = getInternalRowId(row, table)
} else if (!rowId) {
rowId = generateIdForRow(row, table)
row._id = rowId
}
// this is a relationship of some sort
if (finalRows[rowId]) {
finalRows = await updateRelationshipColumns(
table,
tables,
row,
finalRows,
relationships,
opts
)
continue
}
const thisRow = fixArrayTypes(
basicProcessing({
row,
table,
isLinked: false,
internal: opts?.sqs,
}),
table
)
if (thisRow._id == null) {
throw new Error("Unable to generate row ID for SQL rows")
}
finalRows[thisRow._id] = thisRow
// do this at end once its been added to the final rows
finalRows = await updateRelationshipColumns(
table,
tables,
row,
finalRows,
relationships
)
}
// make sure all related rows are correct
let finalRowArray = []
for (let row of Object.values(finalRows)) {
finalRowArray.push(
await processRelationshipFields(table, tables, row, relationships)
)
}
// process some additional types
finalRowArray = processDates(table, finalRowArray)
return finalRowArray
}
export function isUserMetadataTable(tableId: string) {
return tableId === InternalTables.USER_METADATA
}

4
packages/server/src/api/controllers/row/views.ts
View file

@ -4,8 +4,8 @@ import {
SearchRowResponse,
SearchViewRowRequest,
RequiredKeys,
SearchParams,
SearchFilters,
RowSearchParams,
} from "@budibase/types"
import { dataFilters } from "@budibase/shared-core"
import sdk from "../../../sdk"
@ -57,7 +57,7 @@ export async function searchView(
}
const searchOptions: RequiredKeys<SearchViewRowRequest> &
RequiredKeys<Pick<SearchParams, "tableId" | "query" | "fields">> = {
RequiredKeys<Pick<RowSearchParams, "tableId" | "query" | "fields">> = {
tableId: view.tableId,
query,
fields: viewFields,

15
packages/server/src/api/controllers/static/index.ts
View file

@ -2,7 +2,7 @@ import { InvalidFileExtensions } from "@budibase/shared-core"
import AppComponent from "./templates/BudibaseApp.svelte"
import { join } from "../../../utilities/centralPath"
import * as uuid from "uuid"
import { ObjectStoreBuckets, devClientVersion } from "../../../constants"
import { devClientVersion, ObjectStoreBuckets } from "../../../constants"
import { processString } from "@budibase/string-templates"
import {
loadHandlebarsFile,
@ -10,24 +10,24 @@ import {
TOP_LEVEL_PATH,
} from "../../../utilities/fileSystem"
import env from "../../../environment"
import { DocumentType } from "../../../db/utils"
import {
BadRequestError,
configs,
context,
objectStore,
utils,
configs,
BadRequestError,
} from "@budibase/backend-core"
import AWS from "aws-sdk"
import fs from "fs"
import sdk from "../../../sdk"
import * as pro from "@budibase/pro"
import {
UserCtx,
App,
Ctx,
ProcessAttachmentResponse,
DocumentType,
Feature,
ProcessAttachmentResponse,
UserCtx,
} from "@budibase/types"
import {
getAppMigrationVersion,
@ -147,8 +147,7 @@ const requiresMigration = async (ctx: Ctx) => {
const latestMigrationApplied = await getAppMigrationVersion(appId)
const requiresMigrations = latestMigrationApplied !== latestMigration
return requiresMigrations
return latestMigrationApplied !== latestMigration
}
export const serveApp = async function (ctx: UserCtx) {

2
packages/server/src/api/controllers/table/index.ts
View file

@ -84,8 +84,8 @@ export async function save(ctx: UserCtx<SaveTableRequest, SaveTableResponse>) {
}
let savedTable = await api.save(ctx, renaming)
if (!table._id) {
await events.table.created(savedTable)
savedTable = sdk.tables.enrichViewSchemas(savedTable)
await events.table.created(savedTable)
} else {
await events.table.updated(savedTable)
}

5
packages/server/src/api/controllers/table/utils.ts
View file

@ -31,6 +31,8 @@ import {
RelationshipFieldMetadata,
FieldType,
} from "@budibase/types"
import sdk from "../../../sdk"
import env from "../../../environment"
export async function clearColumns(table: Table, columnNames: string[]) {
const db = context.getAppDB()
@ -320,6 +322,9 @@ class TableSaveFunctions {
importRows: this.importRows,
user: this.user,
})
if (env.SQS_SEARCH_ENABLE) {
await sdk.tables.sqs.addTableToSqlite(table)
}
return table
}

9
packages/server/src/api/controllers/view/utils.ts
View file

@ -2,13 +2,18 @@ import {
ViewName,
generateMemoryViewID,
getMemoryViewParams,
DocumentType,
SEPARATOR,
} from "../../../db/utils"
import env from "../../../environment"
import { context } from "@budibase/backend-core"
import viewBuilder from "./viewBuilder"
import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types"
import {
Database,
DBView,
DocumentType,
DesignDocument,
InMemoryView,
} from "@budibase/types"
export async function getView(viewName: string) {
const db = context.getAppDB()

2
packages/server/src/api/controllers/view/views.ts
View file

@ -4,7 +4,6 @@ import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
import { deleteView, getView, getViews, saveView } from "./utils"
import { fetchView } from "../row"
import { context, events } from "@budibase/backend-core"
import { DocumentType } from "../../../db/utils"
import sdk from "../../../sdk"
import {
FieldType,
@ -14,6 +13,7 @@ import {
TableExportFormat,
TableSchema,
View,
DocumentType,
} from "@budibase/types"
import { builderSocket } from "../../../websockets"

5
packages/server/src/api/controllers/view/viewsV2.ts
View file

@ -6,6 +6,7 @@ import {
UIFieldMetadata,
UpdateViewRequest,
ViewResponse,
ViewResponseEnriched,
ViewV2,
} from "@budibase/types"
import { builderSocket, gridSocket } from "../../../websockets"
@ -39,9 +40,9 @@ async function parseSchema(view: CreateViewRequest) {
return finalViewSchema
}
export async function get(ctx: Ctx<void, ViewResponse>) {
export async function get(ctx: Ctx<void, ViewResponseEnriched>) {
ctx.body = {
data: await sdk.views.get(ctx.params.viewId, { enriched: true }),
data: await sdk.views.getEnriched(ctx.params.viewId),
}
}

34
packages/server/src/api/routes/tests/row.spec.ts
View file

@ -722,6 +722,39 @@ describe.each([
})
})
describe("bulkImport", () => {
isInternal &&
it("should update Auto ID field after bulk import", async () => {
const table = await config.api.table.save(
saveTableRequest({
primary: ["autoId"],
schema: {
autoId: {
name: "autoId",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
constraints: {
type: "number",
presence: false,
},
},
},
})
)
let row = await config.api.row.save(table._id!, {})
expect(row.autoId).toEqual(1)
await config.api.row.bulkImport(table._id!, {
rows: [{ autoId: 2 }],
})
row = await config.api.row.save(table._id!, {})
expect(row.autoId).toEqual(3)
})
})
describe("enrich", () => {
beforeAll(async () => {
table = await config.api.table.save(defaultTable())
@ -1239,7 +1272,6 @@ describe.each([
? {}
: {
hasNextPage: false,
bookmark: null,
}),
})
})

74
packages/server/src/api/routes/tests/search.spec.ts Normal file
View file

@ -0,0 +1,74 @@
import { tableForDatasource } from "../../../tests/utilities/structures"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import * as setup from "./utilities"
import { Datasource, FieldType, Table } from "@budibase/types"
jest.unmock("mssql")
describe.each([
["internal", undefined],
["internal-sqs", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/api/:sourceId/search (%s)", (name, dsProvider) => {
const isSqs = name === "internal-sqs"
const config = setup.getConfig()
let envCleanup: (() => void) | undefined
let table: Table
let datasource: Datasource | undefined
beforeAll(async () => {
if (isSqs) {
envCleanup = config.setEnv({ SQS_SEARCH_ENABLE: "true" })
}
await config.init()
if (dsProvider) {
datasource = await config.createDatasource({
datasource: await dsProvider,
})
}
})
afterAll(async () => {
setup.afterAll()
if (envCleanup) {
envCleanup()
}
})
beforeEach(async () => {
table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
name: {
name: "name",
type: FieldType.STRING,
},
},
})
)
})
it("should return rows", async () => {
const rows = await Promise.all([
config.api.row.save(table._id!, { name: "foo" }),
config.api.row.save(table._id!, { name: "bar" }),
])
const result = await config.api.row.search(table._id!, {
tableId: table._id!,
query: {},
})
expect(result.rows).toEqual(
expect.arrayContaining([
expect.objectContaining({ _id: rows[0]._id }),
expect.objectContaining({ _id: rows[1]._id }),
])
)
})
})

628
packages/server/src/api/routes/tests/table.spec.ts
View file

@ -1,11 +1,11 @@
import { context, events } from "@budibase/backend-core"
import {
AutoFieldSubType,
Datasource,
FieldSubtype,
FieldType,
INTERNAL_TABLE_SOURCE_ID,
InternalTable,
NumberFieldMetadata,
RelationshipType,
Row,
SaveTableRequest,
@ -13,31 +13,41 @@ import {
TableSourceType,
User,
ViewCalculation,
ViewV2Enriched,
} from "@budibase/types"
import { checkBuilderEndpoint } from "./utilities/TestFunctions"
import * as setup from "./utilities"
import sdk from "../../../sdk"
import * as uuid from "uuid"
import tk from "timekeeper"
import { generator, mocks } from "@budibase/backend-core/tests"
import { TableToBuild } from "../../../tests/utilities/TestConfiguration"
tk.freeze(mocks.date.MOCK_DATE)
import { generator } from "@budibase/backend-core/tests"
import { DatabaseName, getDatasource } from "../../../integrations/tests/utils"
import { tableForDatasource } from "../../../tests/utilities/structures"
import timekeeper from "timekeeper"
const { basicTable } = setup.structures
const ISO_REGEX_PATTERN = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/
describe("/tables", () => {
let request = setup.getRequest()
describe.each([
["internal", undefined],
[DatabaseName.POSTGRES, getDatasource(DatabaseName.POSTGRES)],
[DatabaseName.MYSQL, getDatasource(DatabaseName.MYSQL)],
[DatabaseName.SQL_SERVER, getDatasource(DatabaseName.SQL_SERVER)],
[DatabaseName.MARIADB, getDatasource(DatabaseName.MARIADB)],
])("/tables (%s)", (_, dsProvider) => {
let isInternal: boolean
let datasource: Datasource | undefined
let config = setup.getConfig()
let appId: string
afterAll(setup.afterAll)
beforeAll(async () => {
const app = await config.init()
appId = app.appId
await config.init()
if (dsProvider) {
datasource = await config.api.datasource.create(await dsProvider)
isInternal = false
} else {
isInternal = true
}
})
describe("create", () => {
@ -45,102 +55,28 @@ describe("/tables", () => {
jest.clearAllMocks()
})
const createTable = (table?: Table) => {
if (!table) {
table = basicTable()
}
return request
.post(`/api/tables`)
.send(table)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
}
it("returns a success message when the table is successfully created", async () => {
const res = await createTable()
expect((res as any).res.statusMessage).toEqual(
"Table TestTable saved successfully."
it("creates a table successfully", async () => {
const name = generator.guid()
const table = await config.api.table.save(
tableForDatasource(datasource, { name })
)
expect(res.body.name).toEqual("TestTable")
expect(table.name).toEqual(name)
expect(events.table.created).toHaveBeenCalledTimes(1)
expect(events.table.created).toHaveBeenCalledWith(res.body)
})
it("creates all the passed fields", async () => {
const tableData: TableToBuild = {
name: "TestTable",
type: "table",
schema: {
autoId: {
name: "id",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
constraints: {
type: "number",
presence: false,
},
},
},
views: {
"table view": {
id: "viewId",
version: 2,
name: "table view",
tableId: "tableId",
},
},
}
const testTable = await config.createTable(tableData)
const expected: Table = {
...tableData,
type: "table",
views: {
"table view": {
...tableData.views!["table view"],
schema: {
autoId: {
autocolumn: true,
constraints: {
presence: false,
type: "number",
},
name: "id",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
visible: false,
} as NumberFieldMetadata,
},
},
},
sourceType: TableSourceType.INTERNAL,
sourceId: expect.any(String),
_rev: expect.stringMatching(/^1-.+/),
_id: expect.any(String),
createdAt: mocks.date.MOCK_DATE.toISOString(),
updatedAt: mocks.date.MOCK_DATE.toISOString(),
}
expect(testTable).toEqual(expected)
const persistedTable = await config.api.table.get(testTable._id!)
expect(persistedTable).toEqual(expected)
expect(events.table.created).toHaveBeenCalledWith(table)
})
it("creates a table via data import", async () => {
const table: SaveTableRequest = basicTable()
table.rows = [{ name: "test-name", description: "test-desc" }]
const res = await createTable(table)
const res = await config.api.table.save(table)
expect(events.table.created).toHaveBeenCalledTimes(1)
expect(events.table.created).toHaveBeenCalledWith(res.body)
expect(events.table.created).toHaveBeenCalledWith(res)
expect(events.table.imported).toHaveBeenCalledTimes(1)
expect(events.table.imported).toHaveBeenCalledWith(res.body)
expect(events.table.imported).toHaveBeenCalledWith(res)
expect(events.rows.imported).toHaveBeenCalledTimes(1)
expect(events.rows.imported).toHaveBeenCalledWith(res.body, 1)
expect(events.rows.imported).toHaveBeenCalledWith(res, 1)
})
it("should apply authorization to endpoint", async () => {
@ -155,21 +91,31 @@ describe("/tables", () => {
describe("update", () => {
it("updates a table", async () => {
const testTable = await config.createTable()
const table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
name: {
type: FieldType.STRING,
name: "name",
constraints: {
type: "string",
},
},
},
})
)
const res = await request
.post(`/api/tables`)
.send(testTable)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
const updatedTable = await config.api.table.save({
...table,
name: generator.guid(),
})
expect(events.table.updated).toHaveBeenCalledTimes(1)
expect(events.table.updated).toHaveBeenCalledWith(res.body)
expect(events.table.updated).toHaveBeenCalledWith(updatedTable)
})
it("updates all the row fields for a table when a schema key is renamed", async () => {
const testTable = await config.createTable()
const testTable = await config.api.table.save(basicTable(datasource))
await config.createLegacyView({
name: "TestView",
field: "Price",
@ -179,112 +125,96 @@ describe("/tables", () => {
filters: [],
})
const testRow = await request
.post(`/api/${testTable._id}/rows`)
.send({
name: "test",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
const testRow = await config.api.row.save(testTable._id!, {
name: "test",
})
const updatedTable = await request
.post(`/api/tables`)
.send({
_id: testTable._id,
_rev: testTable._rev,
name: "TestTable",
key: "name",
_rename: {
old: "name",
updated: "updatedName",
const { name, ...otherColumns } = testTable.schema
const updatedTable = await config.api.table.save({
...testTable,
_rename: {
old: "name",
updated: "updatedName",
},
schema: {
...otherColumns,
updatedName: {
...name,
name: "updatedName",
},
schema: {
updatedName: { type: "string" },
},
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect((updatedTable as any).res.statusMessage).toEqual(
"Table TestTable saved successfully."
)
expect(updatedTable.body.name).toEqual("TestTable")
},
})
const res = await request
.get(`/api/${testTable._id}/rows/${testRow.body._id}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(updatedTable.name).toEqual(testTable.name)
expect(res.body.updatedName).toEqual("test")
expect(res.body.name).toBeUndefined()
const res = await config.api.row.get(testTable._id!, testRow._id!)
expect(res.updatedName).toEqual("test")
expect(res.name).toBeUndefined()
})
it("updates only the passed fields", async () => {
const testTable = await config.createTable({
name: "TestTable",
type: "table",
schema: {
autoId: {
name: "id",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
constraints: {
type: "number",
presence: false,
await timekeeper.withFreeze(new Date(2021, 1, 1), async () => {
const table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
autoId: {
name: "id",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
constraints: {
type: "number",
presence: false,
},
},
},
},
},
views: {
view1: {
id: "viewId",
version: 2,
name: "table view",
tableId: "tableId",
},
},
})
})
)
const response = await request
.post(`/api/tables`)
.send({
...testTable,
name: "UpdatedName",
const newName = generator.guid()
const updatedTable = await config.api.table.save({
...table,
name: newName,
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(response.body).toEqual({
...testTable,
name: "UpdatedName",
_rev: expect.stringMatching(/^2-.+/),
})
let expected: Table = {
...table,
name: newName,
_id: expect.any(String),
}
if (isInternal) {
expected._rev = expect.stringMatching(/^2-.+/)
}
const persistedTable = await config.api.table.get(testTable._id!)
expect(persistedTable).toEqual({
...testTable,
name: "UpdatedName",
_rev: expect.stringMatching(/^2-.+/),
expect(updatedTable).toEqual(expected)
const persistedTable = await config.api.table.get(updatedTable._id!)
expected = {
...table,
name: newName,
_id: updatedTable._id,
}
if (datasource?.isSQL) {
expected.sql = true
}
if (isInternal) {
expected._rev = expect.stringMatching(/^2-.+/)
}
expect(persistedTable).toEqual(expected)
})
})
describe("user table", () => {
it("should add roleId and email field when adjusting user table schema", async () => {
const res = await request
.post(`/api/tables`)
.send({
...basicTable(),
isInternal &&
it("should add roleId and email field when adjusting user table schema", async () => {
const table = await config.api.table.save({
...basicTable(datasource),
_id: "ta_users",
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.schema.email).toBeDefined()
expect(res.body.schema.roleId).toBeDefined()
})
expect(table.schema.email).toBeDefined()
expect(table.schema.roleId).toBeDefined()
})
})
it("should add a new column for an internal DB table", async () => {
@ -295,12 +225,7 @@ describe("/tables", () => {
...basicTable(),
}
const response = await request
.post(`/api/tables`)
.send(saveTableRequest)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
const response = await config.api.table.save(saveTableRequest)
const expectedResponse = {
...saveTableRequest,
@ -311,15 +236,16 @@ describe("/tables", () => {
views: {},
}
delete expectedResponse._add
expect(response.status).toBe(200)
expect(response.body).toEqual(expectedResponse)
expect(response).toEqual(expectedResponse)
})
})
describe("import", () => {
it("imports rows successfully", async () => {
const table = await config.createTable()
const name = generator.guid()
const table = await config.api.table.save(
basicTable(datasource, { name })
)
const importRequest = {
schema: table.schema,
rows: [{ name: "test-name", description: "test-desc" }],
@ -327,83 +253,36 @@ describe("/tables", () => {
jest.clearAllMocks()
await request
.post(`/api/tables/${table._id}/import`)
.send(importRequest)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
await config.api.table.import(table._id!, importRequest)
expect(events.table.created).not.toHaveBeenCalled()
expect(events.rows.imported).toHaveBeenCalledTimes(1)
expect(events.rows.imported).toHaveBeenCalledWith(
expect.objectContaining({
name: "TestTable",
name,
_id: table._id,
}),
1
)
})
it("should update Auto ID field after bulk import", async () => {
const table = await config.createTable({
name: "TestTable",
type: "table",
schema: {
autoId: {
name: "id",
type: FieldType.NUMBER,
subtype: AutoFieldSubType.AUTO_ID,
autocolumn: true,
constraints: {
type: "number",
presence: false,
},
},
},
})
let row = await config.api.row.save(table._id!, {})
expect(row.autoId).toEqual(1)
await config.api.row.bulkImport(table._id!, {
rows: [{ autoId: 2 }],
identifierFields: [],
})
row = await config.api.row.save(table._id!, {})
expect(row.autoId).toEqual(3)
})
})
describe("fetch", () => {
let testTable: Table
const enrichViewSchemasMock = jest.spyOn(sdk.tables, "enrichViewSchemas")
beforeEach(async () => {
testTable = await config.createTable(testTable)
testTable = await config.api.table.save(
basicTable(datasource, { name: generator.guid() })
)
})
afterEach(() => {
delete testTable._rev
})
afterAll(() => {
enrichViewSchemasMock.mockRestore()
})
it("returns all the tables for that instance in the response body", async () => {
const res = await request
.get(`/api/tables`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
const table = res.body.find((t: Table) => t._id === testTable._id)
it("returns all tables", async () => {
const res = await config.api.table.fetch()
const table = res.find(t => t._id === testTable._id)
expect(table).toBeDefined()
expect(table.name).toEqual(testTable.name)
expect(table.type).toEqual("table")
expect(table.sourceType).toEqual("internal")
expect(table!.name).toEqual(testTable.name)
expect(table!.type).toEqual("table")
expect(table!.sourceType).toEqual(testTable.sourceType)
})
it("should apply authorization to endpoint", async () => {
@ -414,99 +293,110 @@ describe("/tables", () => {
})
})
it("should fetch views", async () => {
const tableId = config.table!._id!
const views = [
await config.api.viewV2.create({ tableId, name: generator.guid() }),
await config.api.viewV2.create({ tableId, name: generator.guid() }),
]
const res = await request
.get(`/api/tables`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body).toEqual(
expect.arrayContaining([
expect.objectContaining({
_id: tableId,
views: views.reduce((p, c) => {
p[c.name] = { ...c, schema: expect.anything() }
return p
}, {} as any),
}),
])
)
})
it("should enrich the view schemas for viewsV2", async () => {
const tableId = config.table!._id!
enrichViewSchemasMock.mockImplementation(t => ({
...t,
views: {
view1: {
version: 2,
name: "view1",
schema: {},
id: "new_view_id",
tableId: t._id!,
},
},
}))
await config.api.viewV2.create({ tableId, name: generator.guid() })
await config.createLegacyView()
it("should enrich the view schemas", async () => {
const viewV2 = await config.api.viewV2.create({
tableId: testTable._id!,
name: generator.guid(),
})
const legacyView = await config.api.legacyView.save({
tableId: testTable._id!,
name: generator.guid(),
filters: [],
schema: {},
})
const res = await config.api.table.fetch()
expect(res).toEqual(
expect.arrayContaining([
expect.objectContaining({
_id: tableId,
views: {
view1: {
version: 2,
name: "view1",
schema: {},
id: "new_view_id",
tableId,
},
const table = res.find(t => t._id === testTable._id)
expect(table).toBeDefined()
expect(table!.views![viewV2.name]).toBeDefined()
const expectedViewV2: ViewV2Enriched = {
...viewV2,
schema: {
description: {
constraints: {
type: "string",
},
}),
])
name: "description",
type: FieldType.STRING,
visible: false,
},
name: {
constraints: {
type: "string",
},
name: "name",
type: FieldType.STRING,
visible: false,
},
},
}
if (!isInternal) {
expectedViewV2.schema!.id = {
name: "id",
type: FieldType.NUMBER,
visible: false,
autocolumn: true,
}
}
expect(table!.views![viewV2.name!]).toEqual(expectedViewV2)
if (isInternal) {
expect(table!.views![legacyView.name!]).toBeDefined()
expect(table!.views![legacyView.name!]).toEqual({
...legacyView,
schema: {
description: {
constraints: {
type: "string",
},
name: "description",
type: "string",
},
name: {
constraints: {
type: "string",
},
name: "name",
type: "string",
},
},
})
}
})
})
describe("get", () => {
it("returns a table", async () => {
const table = await config.api.table.save(
basicTable(datasource, { name: generator.guid() })
)
const res = await config.api.table.get(table._id!)
expect(res).toEqual(table)
})
})
describe("indexing", () => {
it("should be able to create a table with indexes", async () => {
await context.doInAppContext(appId, async () => {
await context.doInAppContext(config.getAppId(), async () => {
const db = context.getAppDB()
const indexCount = (await db.getIndexes()).total_rows
const table = basicTable()
table.indexes = ["name"]
const res = await request
.post(`/api/tables`)
.send(table)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body._id).toBeDefined()
expect(res.body._rev).toBeDefined()
const res = await config.api.table.save(table)
expect(res._id).toBeDefined()
expect(res._rev).toBeDefined()
expect((await db.getIndexes()).total_rows).toEqual(indexCount + 1)
// update index to see what happens
table.indexes = ["name", "description"]
await request
.post(`/api/tables`)
.send({
...table,
_id: res.body._id,
_rev: res.body._rev,
})
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
await config.api.table.save({
...table,
_id: res._id,
_rev: res._rev,
})
// shouldn't have created a new index
expect((await db.getIndexes()).total_rows).toEqual(indexCount + 1)
})
@ -521,12 +411,9 @@ describe("/tables", () => {
})
it("returns a success response when a table is deleted.", async () => {
const res = await request
.delete(`/api/tables/${testTable._id}/${testTable._rev}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.message).toEqual(`Table ${testTable._id} deleted.`)
await config.api.table.destroy(testTable._id!, testTable._rev!, {
body: { message: `Table ${testTable._id} deleted.` },
})
expect(events.table.deleted).toHaveBeenCalledTimes(1)
expect(events.table.deleted).toHaveBeenCalledWith({
...testTable,
@ -559,12 +446,9 @@ describe("/tables", () => {
},
})
const res = await request
.delete(`/api/tables/${testTable._id}/${testTable._rev}`)
.set(config.defaultHeaders())
.expect("Content-Type", /json/)
.expect(200)
expect(res.body.message).toEqual(`Table ${testTable._id} deleted.`)
await config.api.table.destroy(testTable._id!, testTable._rev!, {
body: { message: `Table ${testTable._id} deleted.` },
})
const dependentTable = await config.api.table.get(linkedTable._id!)
expect(dependentTable.schema.TestTable).not.toBeDefined()
})
@ -816,33 +700,31 @@ describe("/tables", () => {
describe("unhappy paths", () => {
let table: Table
beforeAll(async () => {
table = await config.api.table.save({
name: "table",
type: "table",
sourceId: INTERNAL_TABLE_SOURCE_ID,
sourceType: TableSourceType.INTERNAL,
schema: {
"user relationship": {
type: FieldType.LINK,
fieldName: "test",
name: "user relationship",
constraints: {
type: "array",
presence: false,
table = await config.api.table.save(
tableForDatasource(datasource, {
schema: {
"user relationship": {
type: FieldType.LINK,
fieldName: "test",
name: "user relationship",
constraints: {
type: "array",
presence: false,
},
relationshipType: RelationshipType.MANY_TO_ONE,
tableId: InternalTable.USER_METADATA,
},
relationshipType: RelationshipType.MANY_TO_ONE,
tableId: InternalTable.USER_METADATA,
},
num: {
type: FieldType.NUMBER,
name: "num",
constraints: {
type: "number",
presence: false,
num: {
type: FieldType.NUMBER,
name: "num",
constraints: {
type: "number",
presence: false,
},
},
},
},
})
})
)
})
it("should fail if the new column name is blank", async () => {

20
packages/server/src/api/routes/tests/viewV2.spec.ts
View file

@ -181,7 +181,7 @@ describe.each([
const createdView = await config.api.viewV2.create(newView)
expect(await config.api.viewV2.get(createdView.id)).toEqual({
expect(createdView).toEqual({
...newView,
schema: {
Price: {
@ -398,7 +398,7 @@ describe.each([
})
it("updates only UI schema overrides", async () => {
await config.api.viewV2.update({
const updatedView = await config.api.viewV2.update({
...view,
schema: {
Price: {
@ -417,7 +417,7 @@ describe.each([
} as Record<string, FieldSchema>,
})
expect(await config.api.viewV2.get(view.id)).toEqual({
expect(updatedView).toEqual({
...view,
schema: {
Price: {
@ -479,17 +479,17 @@ describe.each([
describe("fetch view (through table)", () => {
it("should be able to fetch a view V2", async () => {
const newView: CreateViewRequest = {
const res = await config.api.viewV2.create({
name: generator.name(),
tableId: table._id!,
schema: {
Price: { visible: false },
Category: { visible: true },
},
}
const res = await config.api.viewV2.create(newView)
})
expect(res.schema?.Price).toBeUndefined()
const view = await config.api.viewV2.get(res.id)
expect(view!.schema?.Price).toBeUndefined()
const updatedTable = await config.api.table.get(table._id!)
const viewSchema = updatedTable.views![view!.name!].schema as Record<
string,
@ -652,7 +652,6 @@ describe.each([
? {}
: {
hasNextPage: false,
bookmark: null,
}),
})
})
@ -705,7 +704,6 @@ describe.each([
? {}
: {
hasNextPage: false,
bookmark: null,
}),
})
})
@ -813,7 +811,7 @@ describe.each([
{
field: "age",
order: SortOrder.ASCENDING,
type: SortType.number,
type: SortType.NUMBER,
},
["Danny", "Alice", "Charly", "Bob"],
],
@ -835,7 +833,7 @@ describe.each([
{
field: "age",
order: SortOrder.DESCENDING,
type: SortType.number,
type: SortType.NUMBER,
},
["Bob", "Charly", "Alice", "Danny"],
],

2
packages/server/src/app.ts
View file

@ -21,7 +21,7 @@ async function start() {
app = koa.app
server = koa.server
// startup includes automation runner - if enabled
await startup(app, server)
await startup({ app, server })
}
start().catch(err => {

11
packages/server/src/db/linkedRows/LinkDocument.ts
View file

@ -1,4 +1,4 @@
import { generateLinkID } from "../utils"
import { generateLinkID, generateJunctionTableID } from "../utils"
import { FieldType, LinkDocument } from "@budibase/types"
/**
@ -16,6 +16,7 @@ import { FieldType, LinkDocument } from "@budibase/types"
class LinkDocumentImpl implements LinkDocument {
_id: string
type: string
tableId: string
doc1: {
rowId: string
fieldName: string
@ -43,16 +44,20 @@ class LinkDocumentImpl implements LinkDocument {
fieldName2
)
this.type = FieldType.LINK
this.doc1 = {
this.tableId = generateJunctionTableID(tableId1, tableId2)
const docA = {
tableId: tableId1,
fieldName: fieldName1,
rowId: rowId1,
}
this.doc2 = {
const docB = {
tableId: tableId2,
fieldName: fieldName2,
rowId: rowId2,
}
// have to determine which one will be doc1 - very important for SQL linking
this.doc1 = docA.tableId > docB.tableId ? docA : docB
this.doc2 = docA.tableId > docB.tableId ? docB : docA
}
}

14
packages/server/src/db/utils.ts
View file

@ -55,6 +55,14 @@ export const getUserMetadataParams = dbCore.getUserMetadataParams
export const generateUserMetadataID = dbCore.generateUserMetadataID
export const getGlobalIDFromUserMetadataID =
dbCore.getGlobalIDFromUserMetadataID
export const CONSTANT_INTERNAL_ROW_COLS = [
"_id",
"_rev",
"type",
"createdAt",
"updatedAt",
"tableId",
]
/**
* Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
@ -286,6 +294,12 @@ export function generatePluginID(name: string) {
return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
}
export function generateJunctionTableID(tableId1: string, tableId2: string) {
const first = tableId1 > tableId2 ? tableId1 : tableId2
const second = tableId1 > tableId2 ? tableId2 : tableId1
return `${first}${SEPARATOR}${second}`
}
/**
* Generates a new view ID.
* @returns The new view ID which the view doc can be stored under.

4
packages/server/src/db/views/staticViews.ts
View file

@ -1,6 +1,6 @@
import { context } from "@budibase/backend-core"
import { DocumentType, SEPARATOR, ViewName } from "../utils"
import { LinkDocument, Row, SearchIndex } from "@budibase/types"
import { SEPARATOR, ViewName } from "../utils"
import { DocumentType, LinkDocument, Row, SearchIndex } from "@budibase/types"
const SCREEN_PREFIX = DocumentType.SCREEN + SEPARATOR

1
packages/server/src/definitions/datasource.ts
View file

@ -6,4 +6,5 @@
export interface QueryOptions {
disableReturning?: boolean
disableBindings?: boolean
}

1
packages/server/src/environment.ts
View file

@ -86,6 +86,7 @@ const environment = {
SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
SQL_LOGGING_ENABLE: process.env.SQL_LOGGING_ENABLE,
SQL_ALIASING_DISABLE: process.env.SQL_ALIASING_DISABLE,
SQS_SEARCH_ENABLE: process.env.SQS_SEARCH_ENABLE,
// flags
ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
DISABLE_THREADING: process.env.DISABLE_THREADING,

3
packages/server/src/integration-test/postgres.spec.ts
View file

@ -685,7 +685,6 @@ describe("postgres integrations", () => {
expect(res.body).toEqual({
rows: [],
bookmark: null,
hasNextPage: false,
})
})
@ -710,7 +709,6 @@ describe("postgres integrations", () => {
rows: expect.arrayContaining(
rows.map(r => expect.objectContaining(r.rowData))
),
bookmark: null,
hasNextPage: false,
})
expect(res.body.rows).toHaveLength(rowsCount)
@ -772,7 +770,6 @@ describe("postgres integrations", () => {
expect(res.body).toEqual({
rows: expect.arrayContaining(rowsToFilter.map(expect.objectContaining)),
bookmark: null,
hasNextPage: false,
})
expect(res.body.rows).toHaveLength(4)

2
packages/server/src/integrations/base/query.ts
View file

@ -9,7 +9,7 @@ import sdk from "../../sdk"
export async function makeExternalQuery(
datasource: Datasource,
json: QueryJson
): DatasourcePlusQueryResponse {
): Promise<DatasourcePlusQueryResponse> {
datasource = await sdk.datasources.enrich(datasource)
const Integration = await getIntegration(datasource.source)
// query is the opinionated function

35
packages/server/src/integrations/base/sql.ts
View file

@ -1,7 +1,12 @@
import { Knex, knex } from "knex"
import { db as dbCore } from "@budibase/backend-core"
import { QueryOptions } from "../../definitions/datasource"
import { isIsoDateString, SqlClient, isValidFilter } from "../utils"
import {
isIsoDateString,
SqlClient,
isValidFilter,
getNativeSql,
} from "../utils"
import SqlTableQueryBuilder from "./sqlTable"
import {
BBReferenceFieldMetadata,
@ -11,14 +16,16 @@ import {
JsonFieldMetadata,
Operation,
QueryJson,
SqlQuery,
RelationshipsJson,
SearchFilters,
SortDirection,
SqlQueryBinding,
Table,
} from "@budibase/types"
import environment from "../../environment"
type QueryFunction = (query: Knex.SqlNative, operation: Operation) => any
type QueryFunction = (query: SqlQuery | SqlQuery[], operation: Operation) => any
const envLimit = environment.SQL_MAX_ROWS
? parseInt(environment.SQL_MAX_ROWS)
@ -43,8 +50,11 @@ function likeKey(client: string, key: string): string {
start = "["
end = "]"
break
case SqlClient.SQL_LITE:
start = end = "'"
break
default:
throw "Unknown client"
throw new Error("Unknown client generating like key")
}
const parts = key.split(".")
key = parts.map(part => `${start}${part}${end}`).join(".")
@ -587,9 +597,15 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
* which for the sake of mySQL stops adding the returning statement to inserts, updates and deletes.
* @return the query ready to be passed to the driver.
*/
_query(json: QueryJson, opts: QueryOptions = {}): Knex.SqlNative | Knex.Sql {
_query(json: QueryJson, opts: QueryOptions = {}): SqlQuery | SqlQuery[] {
const sqlClient = this.getSqlClient()
const client = knex({ client: sqlClient })
const config: { client: string; useNullAsDefault?: boolean } = {
client: sqlClient,
}
if (sqlClient === SqlClient.SQL_LITE) {
config.useNullAsDefault = true
}
const client = knex(config)
let query: Knex.QueryBuilder
const builder = new InternalBuilder(sqlClient)
switch (this._operation(json)) {
@ -615,7 +631,12 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
default:
throw `Operation type is not supported by SQL query builder`
}
return query.toSQL().toNative()
if (opts?.disableBindings) {
return { sql: query.toString() }
} else {
return getNativeSql(query)
}
}
async getReturningRow(queryFn: QueryFunction, json: QueryJson) {
@ -730,7 +751,7 @@ class SqlQueryBuilder extends SqlTableQueryBuilder {
)
}
log(query: string, values?: any[]) {
log(query: string, values?: SqlQueryBinding) {
if (!environment.SQL_LOGGING_ENABLE) {
return
}

32
packages/server/src/integrations/base/sqlTable.ts
View file

@ -8,8 +8,9 @@ import {
RenameColumn,
Table,
FieldType,
SqlQuery,
} from "@budibase/types"
import { breakExternalTableId, SqlClient } from "../utils"
import { breakExternalTableId, getNativeSql, SqlClient } from "../utils"
import SchemaBuilder = Knex.SchemaBuilder
import CreateTableBuilder = Knex.CreateTableBuilder
import { utils } from "@budibase/shared-core"
@ -199,7 +200,7 @@ class SqlTableQueryBuilder {
return json.endpoint.operation
}
_tableQuery(json: QueryJson): Knex.Sql | Knex.SqlNative {
_tableQuery(json: QueryJson): SqlQuery | SqlQuery[] {
let client = knex({ client: this.sqlClient }).schema
let schemaName = json?.endpoint?.schema
if (schemaName) {
@ -224,12 +225,12 @@ class SqlTableQueryBuilder {
const tableName = schemaName
? `\`${schemaName}\`.\`${json.table.name}\``
: `\`${json.table.name}\``
const externalType = json.table.schema[updatedColumn].externalType!
return {
sql: `alter table ${tableName} change column \`${json.meta.renamed.old}\` \`${updatedColumn}\` ${externalType};`,
sql: `alter table ${tableName} rename column \`${json.meta.renamed.old}\` to \`${updatedColumn}\`;`,
bindings: [],
}
}
query = buildUpdateTable(
client,
json.table,
@ -237,6 +238,27 @@ class SqlTableQueryBuilder {
json.meta.table,
json.meta.renamed!
)
// renameColumn for SQL Server returns a parameterised `sp_rename` query,
// which is not supported by SQL Server and gives a syntax error.
if (this.sqlClient === SqlClient.MS_SQL && json.meta.renamed) {
const oldColumn = json.meta.renamed.old
const updatedColumn = json.meta.renamed.updated
const tableName = schemaName
? `${schemaName}.${json.table.name}`
: `${json.table.name}`
const sql = getNativeSql(query)
if (Array.isArray(sql)) {
for (const query of sql) {
if (query.sql.startsWith("exec sp_rename")) {
query.sql = `exec sp_rename '${tableName}.${oldColumn}', '${updatedColumn}', 'COLUMN'`
query.bindings = []
}
}
}
return sql
}
break
case Operation.DELETE_TABLE:
query = buildDeleteTable(client, json.table)
@ -244,7 +266,7 @@ class SqlTableQueryBuilder {
default:
throw "Table operation is of unknown type"
}
return query.toSQL()
return getNativeSql(query)
}
}

2
packages/server/src/integrations/googlesheets.ts
View file

@ -336,7 +336,7 @@ class GoogleSheetsIntegration implements DatasourcePlus {
return { tables: externalTables, errors }
}
async query(json: QueryJson): DatasourcePlusQueryResponse {
async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
const sheet = json.endpoint.entityId
switch (json.endpoint.operation) {
case Operation.CREATE:

2
packages/server/src/integrations/microsoftSqlServer.ts
View file

@ -496,7 +496,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
return response.recordset || [{ deleted: true }]
}
async query(json: QueryJson): DatasourcePlusQueryResponse {
async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
const schema = this.config.schema
await this.connect()
if (schema && schema !== DEFAULT_SCHEMA && json?.endpoint) {

7
packages/server/src/integrations/mysql.ts
View file

@ -13,6 +13,7 @@ import {
Schema,
TableSourceType,
DatasourcePlusQueryResponse,
SqlQueryBinding,
} from "@budibase/types"
import {
getSqlQuery,
@ -113,7 +114,7 @@ const defaultTypeCasting = function (field: any, next: any) {
return next()
}
export function bindingTypeCoerce(bindings: any[]) {
export function bindingTypeCoerce(bindings: SqlQueryBinding) {
for (let i = 0; i < bindings.length; i++) {
const binding = bindings[i]
if (typeof binding !== "string") {
@ -143,7 +144,7 @@ export function bindingTypeCoerce(bindings: any[]) {
}
class MySQLIntegration extends Sql implements DatasourcePlus {
private config: MySQLConfig
private readonly config: MySQLConfig
private client?: mysql.Connection
constructor(config: MySQLConfig) {
@ -382,7 +383,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
return results.length ? results : [{ deleted: true }]
}
async query(json: QueryJson): DatasourcePlusQueryResponse {
async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
await this.connect()
try {
const queryFn = (query: any) =>

2
packages/server/src/integrations/oracle.ts
View file

@ -423,7 +423,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
: [{ deleted: true }]
}
async query(json: QueryJson): DatasourcePlusQueryResponse {
async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
const operation = this._operation(json)
const input = this._query(json, { disableReturning: true }) as SqlQuery
if (Array.isArray(input)) {

2
packages/server/src/integrations/postgres.ts
View file

@ -421,7 +421,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
return response.rows.length ? response.rows : [{ deleted: true }]
}
async query(json: QueryJson): DatasourcePlusQueryResponse {
async query(json: QueryJson): Promise<DatasourcePlusQueryResponse> {
const operation = this._operation(json).toLowerCase()
const input = this._query(json) as SqlQuery
if (Array.isArray(input)) {

2
packages/server/src/integrations/tests/sql.spec.ts
View file

@ -722,7 +722,7 @@ describe("SQL query builder", () => {
})
expect(query).toEqual({
bindings: [],
sql: `alter table \`${TABLE_NAME}\` change column \`name\` \`first_name\` varchar(45);`,
sql: `alter table \`${TABLE_NAME}\` rename column \`name\` to \`first_name\`;`,
})
})

13
packages/server/src/integrations/tests/sqlAlias.spec.ts
View file

@ -1,10 +1,15 @@
import { Datasource, Operation, QueryJson, SourceName } from "@budibase/types"
import {
Datasource,
Operation,
QueryJson,
SourceName,
SqlQuery,
} from "@budibase/types"
import { join } from "path"
import Sql from "../base/sql"
import { SqlClient } from "../utils"
import AliasTables from "../../api/controllers/row/alias"
import { generator } from "@budibase/backend-core/tests"
import { Knex } from "knex"
function multiline(sql: string) {
return sql.replace(/\n/g, "").replace(/ +/g, " ")
@ -172,8 +177,8 @@ describe("Captures of real examples", () => {
})
// now check returning
let returningQuery: Knex.SqlNative = { sql: "", bindings: [] }
SQL.getReturningRow((input: Knex.SqlNative) => {
let returningQuery: SqlQuery | SqlQuery[] = { sql: "", bindings: [] }
SQL.getReturningRow((input: SqlQuery | SqlQuery[]) => {
returningQuery = input
}, queryJson)
expect(returningQuery).toEqual({

55
packages/server/src/integrations/utils.ts
View file

@ -1,19 +1,15 @@
import {
SqlQuery,
Table,
SearchFilters,
Datasource,
FieldType,
TableSourceType,
} from "@budibase/types"
import { DocumentType, SEPARATOR } from "../db/utils"
import {
InvalidColumns,
NoEmptyFilterStrings,
DEFAULT_BB_DATASOURCE_ID,
} from "../constants"
import { InvalidColumns, DEFAULT_BB_DATASOURCE_ID } from "../constants"
import { helpers } from "@budibase/shared-core"
import env from "../environment"
import { Knex } from "knex"
const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
const ROW_ID_REGEX = /^\[.*]$/g
@ -91,6 +87,7 @@ export enum SqlClient {
POSTGRES = "pg",
MY_SQL = "mysql2",
ORACLE = "oracledb",
SQL_LITE = "sqlite3",
}
const isCloud = env.isProd() && !env.SELF_HOSTED
@ -109,6 +106,23 @@ export function isInternalTableID(tableId: string) {
return !isExternalTableID(tableId)
}
export function getNativeSql(
query: Knex.SchemaBuilder | Knex.QueryBuilder
): SqlQuery | SqlQuery[] {
let sql = query.toSQL()
if (Array.isArray(sql)) {
return sql as SqlQuery[]
}
let native: Knex.SqlNative | undefined
if (sql.toNative) {
native = sql.toNative()
}
return {
sql: native?.sql || sql.sql,
bindings: native?.bindings || sql.bindings,
} as SqlQuery
}
export function isExternalTable(table: Table) {
if (
table?.sourceId &&
@ -420,32 +434,3 @@ export function getPrimaryDisplay(testValue: unknown): string | undefined {
export function isValidFilter(value: any) {
return value != null && value !== ""
}
// don't do a pure falsy check, as 0 is included
// https://github.com/Budibase/budibase/issues/10118
export function removeEmptyFilters(filters: SearchFilters) {
for (let filterField of NoEmptyFilterStrings) {
if (!filters[filterField]) {
continue
}
for (let filterType of Object.keys(filters)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = filters[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(
filters[filterType] as object
)) {
if (value == null || value === "") {
// @ts-ignore
delete filters[filterField][key]
}
}
}
}
}
return filters
}

8
packages/server/src/middleware/builder.ts
View file

@ -1,8 +1,4 @@
import {
APP_DEV_PREFIX,
DocumentType,
getGlobalIDFromUserMetadataID,
} from "../db/utils"
import { APP_DEV_PREFIX, getGlobalIDFromUserMetadataID } from "../db/utils"
import {
doesUserHaveLock,
updateLock,
@ -10,7 +6,7 @@ import {
setDebounce,
} from "../utilities/redis"
import { db as dbCore, cache } from "@budibase/backend-core"
import { UserCtx, Database } from "@budibase/types"
import { DocumentType, UserCtx, Database } from "@budibase/types"
const DEBOUNCE_TIME_SEC = 30

73
packages/server/src/sdk/app/rows/search.ts
View file

@ -1,10 +1,18 @@
import { Row, SearchFilters, SearchParams, SortOrder } from "@budibase/types"
import {
Row,
RowSearchParams,
SearchFilters,
SearchResponse,
} from "@budibase/types"
import { isExternalTableID } from "../../../integrations/utils"
import * as internal from "./search/internal"
import * as external from "./search/external"
import { Format } from "../../../api/controllers/view/exporters"
import { NoEmptyFilterStrings } from "../../../constants"
import * as sqs from "./search/sqs"
import env from "../../../environment"
import { ExportRowsParams, ExportRowsResult } from "./search/types"
export { isValidFilter, removeEmptyFilters } from "../../../integrations/utils"
export { isValidFilter } from "../../../integrations/utils"
export interface ViewParams {
calculation: string
@ -19,29 +27,46 @@ function pickApi(tableId: any) {
return internal
}
export async function search(options: SearchParams): Promise<{
rows: any[]
hasNextPage?: boolean
bookmark?: number | null
}> {
return pickApi(options.tableId).search(options)
// don't do a pure falsy check, as 0 is included
// https://github.com/Budibase/budibase/issues/10118
export function removeEmptyFilters(filters: SearchFilters) {
for (let filterField of NoEmptyFilterStrings) {
if (!filters[filterField]) {
continue
}
for (let filterType of Object.keys(filters)) {
if (filterType !== filterField) {
continue
}
// don't know which one we're checking, type could be anything
const value = filters[filterType] as unknown
if (typeof value === "object") {
for (let [key, value] of Object.entries(
filters[filterType] as object
)) {
if (value == null || value === "") {
// @ts-ignore
delete filters[filterField][key]
}
}
}
}
}
return filters
}
export interface ExportRowsParams {
tableId: string
format: Format
delimiter?: string
rowIds?: string[]
columns?: string[]
query?: SearchFilters
sort?: string
sortOrder?: SortOrder
customHeaders?: { [key: string]: string }
}
export interface ExportRowsResult {
fileName: string
content: string
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const isExternalTable = isExternalTableID(options.tableId)
if (isExternalTable) {
return external.search(options)
} else if (env.SQS_SEARCH_ENABLE) {
return sqs.search(options)
} else {
return internal.search(options)
}
}
export async function exportRows(

27
packages/server/src/sdk/app/rows/search/external.ts
View file

@ -6,28 +6,31 @@ import {
IncludeRelationship,
Row,
SearchFilters,
SearchParams,
RowSearchParams,
SearchResponse,
} from "@budibase/types"
import * as exporters from "../../../../api/controllers/view/exporters"
import sdk from "../../../../sdk"
import { handleRequest } from "../../../../api/controllers/row/external"
import {
breakExternalTableId,
breakRowIdField,
} from "../../../../integrations/utils"
import { cleanExportRows } from "../utils"
import { utils } from "@budibase/shared-core"
import { ExportRowsParams, ExportRowsResult } from "../search"
import { ExportRowsParams, ExportRowsResult } from "./types"
import { HTTPError, db } from "@budibase/backend-core"
import { searchInputMapping } from "./utils"
import pick from "lodash/pick"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import sdk from "../../../"
export async function search(options: SearchParams) {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query, ...params } = options
const { limit } = params
let bookmark = (params.bookmark && parseInt(params.bookmark)) || null
let bookmark =
(params.bookmark && parseInt(params.bookmark as string)) || undefined
if (paginate && !bookmark) {
bookmark = 1
}
@ -92,7 +95,7 @@ export async function search(options: SearchParams) {
rows = rows.map((r: any) => pick(r, fields))
}
rows = await outputProcessing(table, rows, {
rows = await outputProcessing<Row[]>(table, rows, {
preserveLinks: true,
squash: true,
})
@ -158,7 +161,6 @@ export async function exportRows(
if (!tableName) {
throw new HTTPError("Could not find table name.", 400)
}
const schema = datasource.entities[tableName].schema
// Filter data to only specified columns if required
if (columns && columns.length) {
@ -173,7 +175,14 @@ export async function exportRows(
rows = result.rows
}
let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
const schema = datasource.entities[tableName].schema
let exportRows = sdk.rows.utils.cleanExportRows(
rows,
schema,
format,
columns,
customHeaders
)
let content: string
switch (format) {

54
packages/server/src/sdk/app/rows/search/internal.ts
View file

@ -1,20 +1,19 @@
import {
context,
db,
HTTPError,
SearchParams as InternalSearchParams,
} from "@budibase/backend-core"
import { context, db, HTTPError } from "@budibase/backend-core"
import env from "../../../../environment"
import { fullSearch, paginatedSearch } from "./internalSearch"
import { fullSearch, paginatedSearch, searchInputMapping } from "./utils"
import { getRowParams, InternalTables } from "../../../../db/utils"
import {
Database,
DocumentType,
getRowParams,
InternalTables,
} from "../../../../db/utils"
Row,
RowSearchParams,
SearchResponse,
SortType,
Table,
User,
} from "@budibase/types"
import { getGlobalUsersFromMetadata } from "../../../../utilities/global"
import { outputProcessing } from "../../../../utilities/rowProcessor"
import { Database, Row, SearchParams, Table } from "@budibase/types"
import { cleanExportRows } from "../utils"
import {
csv,
Format,
@ -29,17 +28,18 @@ import {
migrateToInMemoryView,
} from "../../../../api/controllers/view/utils"
import sdk from "../../../../sdk"
import { ExportRowsParams, ExportRowsResult } from "../search"
import { searchInputMapping } from "./utils"
import { ExportRowsParams, ExportRowsResult } from "./types"
import pick from "lodash/pick"
import { breakRowIdField } from "../../../../integrations/utils"
export async function search(options: SearchParams) {
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const { tableId } = options
const { paginate, query } = options
const params: InternalSearchParams<any> = {
const params: RowSearchParams = {
tableId: options.tableId,
sort: options.sort,
sortOrder: options.sortOrder,
@ -48,6 +48,7 @@ export async function search(options: SearchParams) {
bookmark: options.bookmark,
version: options.version,
disableEscaping: options.disableEscaping,
query: {},
}
let table = await sdk.tables.getTable(tableId)
@ -55,7 +56,8 @@ export async function search(options: SearchParams) {
if (params.sort && !params.sortType) {
const schema = table.schema
const sortField = schema[params.sort]
params.sortType = sortField.type === "number" ? "number" : "string"
params.sortType =
sortField.type === "number" ? SortType.NUMBER : SortType.STRING
}
let response
@ -69,7 +71,7 @@ export async function search(options: SearchParams) {
if (response.rows && response.rows.length) {
// enrich with global users if from users table
if (tableId === InternalTables.USER_METADATA) {
response.rows = await getGlobalUsersFromMetadata(response.rows)
response.rows = await getGlobalUsersFromMetadata(response.rows as User[])
}
if (options.fields) {
@ -100,10 +102,10 @@ export async function exportRows(
const db = context.getAppDB()
const table = await sdk.tables.getTable(tableId)
let result
let result: Row[] = []
if (rowIds) {
let response = (
await db.allDocs({
await db.allDocs<Row>({
include_docs: true,
keys: rowIds.map((row: string) => {
const ids = breakRowIdField(row)
@ -116,9 +118,9 @@ export async function exportRows(
return ids[0]
}),
})
).rows.map(row => row.doc)
).rows.map(row => row.doc!)
result = await outputProcessing(table, response)
result = await outputProcessing<Row[]>(table, response)
} else if (query) {
let searchResponse = await search({
tableId,
@ -145,7 +147,13 @@ export async function exportRows(
rows = result
}
let exportRows = cleanExportRows(rows, schema, format, columns, customHeaders)
let exportRows = sdk.rows.utils.cleanExportRows(
rows,
schema,
format,
columns,
customHeaders
)
if (format === Format.CSV) {
return {
fileName: "export.csv",

18
packages/server/src/sdk/app/rows/search/internalSearch.ts
View file

@ -1,18 +0,0 @@
import { db as dbCore, context, SearchParams } from "@budibase/backend-core"
import { SearchFilters, Row, SearchIndex } from "@budibase/types"
export async function paginatedSearch(
query: SearchFilters,
params: SearchParams<Row>
) {
const appId = context.getAppId()
return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
}
export async function fullSearch(
query: SearchFilters,
params: SearchParams<Row>
) {
const appId = context.getAppId()
return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
}

190
packages/server/src/sdk/app/rows/search/sqs.ts Normal file
View file

@ -0,0 +1,190 @@
import {
FieldType,
Operation,
QueryJson,
RelationshipFieldMetadata,
Row,
SearchFilters,
RowSearchParams,
SearchResponse,
SortDirection,
SortOrder,
SortType,
Table,
} from "@budibase/types"
import SqlQueryBuilder from "../../../../integrations/base/sql"
import { SqlClient } from "../../../../integrations/utils"
import {
buildInternalRelationships,
sqlOutputProcessing,
} from "../../../../api/controllers/row/utils"
import sdk from "../../../index"
import { context } from "@budibase/backend-core"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
function buildInternalFieldList(
table: Table,
tables: Table[],
opts: { relationships: boolean } = { relationships: true }
) {
let fieldList: string[] = []
fieldList = fieldList.concat(
CONSTANT_INTERNAL_ROW_COLS.map(col => `${table._id}.${col}`)
)
if (opts.relationships) {
for (let col of Object.values(table.schema)) {
if (col.type === FieldType.LINK) {
const linkCol = col as RelationshipFieldMetadata
const relatedTable = tables.find(
table => table._id === linkCol.tableId
)!
fieldList = fieldList.concat(
buildInternalFieldList(relatedTable, tables, { relationships: false })
)
} else {
fieldList.push(`${table._id}.${col.name}`)
}
}
}
return fieldList
}
function tableInFilter(name: string) {
return `:${name}.`
}
function cleanupFilters(filters: SearchFilters, tables: Table[]) {
for (let filter of Object.values(filters)) {
if (typeof filter !== "object") {
continue
}
for (let [key, keyFilter] of Object.entries(filter)) {
if (keyFilter === "") {
delete filter[key]
}
// relationship, switch to table ID
const tableRelated = tables.find(
table =>
table.originalName && key.includes(tableInFilter(table.originalName))
)
if (tableRelated && tableRelated.originalName) {
filter[
key.replace(
tableInFilter(tableRelated.originalName),
tableInFilter(tableRelated._id!)
)
] = filter[key]
delete filter[key]
}
}
}
return filters
}
function buildTableMap(tables: Table[]) {
const tableMap: Record<string, Table> = {}
for (let table of tables) {
// update the table name, should never query by name for SQLite
table.originalName = table.name
table.name = table._id!
tableMap[table._id!] = table
}
return tableMap
}
export async function search(
options: RowSearchParams
): Promise<SearchResponse<Row>> {
const { tableId, paginate, query, ...params } = options
const builder = new SqlQueryBuilder(SqlClient.SQL_LITE)
const allTables = await sdk.tables.getAllInternalTables()
const allTablesMap = buildTableMap(allTables)
const table = allTables.find(table => table._id === tableId)
if (!table) {
throw new Error("Unable to find table")
}
const relationships = buildInternalRelationships(table)
const request: QueryJson = {
endpoint: {
// not important, we query ourselves
datasourceId: "internal",
entityId: table._id!,
operation: Operation.READ,
},
filters: cleanupFilters(query, allTables),
table,
meta: {
table,
tables: allTablesMap,
},
resource: {
fields: buildInternalFieldList(table, allTables),
},
relationships,
}
// make sure only rows returned
request.filters!.equal = {
...request.filters?.equal,
type: "row",
}
if (params.sort && !params.sortType) {
const sortField = table.schema[params.sort]
const sortType =
sortField.type === FieldType.NUMBER ? SortType.NUMBER : SortType.STRING
const sortDirection =
params.sortOrder === SortOrder.ASCENDING
? SortDirection.ASCENDING
: SortDirection.DESCENDING
request.sort = {
[sortField.name]: {
direction: sortDirection,
type: sortType as SortType,
},
}
}
if (paginate && params.limit) {
request.paginate = {
limit: params.limit,
page: params.bookmark,
}
}
try {
const query = builder._query(request, {
disableReturning: true,
disableBindings: true,
})
if (Array.isArray(query)) {
throw new Error("SQS cannot currently handle multiple queries")
}
let sql = query.sql
// quick hack for docIds
sql = sql.replace(/`doc1`.`rowId`/g, "`doc1.rowId`")
sql = sql.replace(/`doc2`.`rowId`/g, "`doc2.rowId`")
const db = context.getAppDB()
const rows = await db.sql<Row>(sql)
return {
rows: await sqlOutputProcessing(
rows,
table!,
allTablesMap,
relationships,
{
sqs: true,
}
),
}
} catch (err: any) {
const msg = typeof err === "string" ? err : err.message
throw new Error(`Unable to search by SQL - ${msg}`)
}
}

8
packages/server/src/sdk/app/rows/search/tests/external.spec.ts
View file

@ -6,7 +6,7 @@ import {
Row,
SourceName,
Table,
SearchParams,
RowSearchParams,
TableSourceType,
} from "@budibase/types"
@ -108,7 +108,7 @@ describe("external search", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
}
@ -125,7 +125,7 @@ describe("external search", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
fields: ["name", "age"],
@ -149,7 +149,7 @@ describe("external search", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {
oneOf: {

6
packages/server/src/sdk/app/rows/search/tests/internal.spec.ts
View file

@ -2,7 +2,7 @@ import {
FieldType,
Row,
Table,
SearchParams,
RowSearchParams,
INTERNAL_TABLE_SOURCE_ID,
TableSourceType,
} from "@budibase/types"
@ -77,7 +77,7 @@ describe("internal", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
}
@ -94,7 +94,7 @@ describe("internal", () => {
await config.doInContext(config.appId, async () => {
const tableId = config.table!._id!
const searchParams: SearchParams = {
const searchParams: RowSearchParams = {
tableId,
query: {},
fields: ["name", "age"],

12
packages/server/src/api/routes/tests/internalSearch.spec.ts → packages/server/src/sdk/app/rows/search/tests/lucene.ts
View file

@ -1,18 +1,18 @@
const nodeFetch = require("node-fetch")
nodeFetch.mockSearch()
import { SearchParams } from "@budibase/backend-core"
import * as search from "../../../sdk/app/rows/search/internalSearch"
import { Row } from "@budibase/types"
import * as search from "../utils"
import { RowSearchParams, SortOrder, SortType } from "@budibase/types"
// this will be mocked out for _search endpoint
const PARAMS: SearchParams<Row> = {
const PARAMS: RowSearchParams = {
query: {},
tableId: "ta_12345679abcdef",
version: "1",
bookmark: undefined,
sort: undefined,
sortOrder: "ascending",
sortType: "string",
sortOrder: SortOrder.ASCENDING,
sortType: SortType.STRING,
}
function checkLucene(resp: any, expected: any, params = PARAMS) {

8
packages/server/src/sdk/app/rows/search/tests/utils.spec.ts
View file

@ -4,7 +4,7 @@ import {
FieldType,
FieldTypeSubtypes,
INTERNAL_TABLE_SOURCE_ID,
SearchParams,
RowSearchParams,
Table,
TableSourceType,
} from "@budibase/types"
@ -47,7 +47,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
const userMedataId = dbCore.generateUserMetadataID(globalUserId)
it("should be able to map ro_ to global user IDs", () => {
const params: SearchParams = {
const params: RowSearchParams = {
tableId,
query: {
equal: {
@ -60,7 +60,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
})
it("should handle array of user IDs", () => {
const params: SearchParams = {
const params: RowSearchParams = {
tableId,
query: {
oneOf: {
@ -77,7 +77,7 @@ describe.each([tableWithUserCol, tableWithUsersCol])(
it("shouldn't change any other input", () => {
const email = "test@example.com"
const params: SearchParams = {
const params: RowSearchParams = {
tableId,
query: {
equal: {

19
packages/server/src/sdk/app/rows/search/types.ts Normal file
View file

@ -0,0 +1,19 @@
import { Format } from "../../../../api/controllers/view/exporters"
import { SearchFilters, SortOrder } from "@budibase/types"
export interface ExportRowsParams {
tableId: string
format: Format
delimiter?: string
rowIds?: string[]
columns?: string[]
query?: SearchFilters
sort?: string
sortOrder?: SortOrder
customHeaders?: { [key: string]: string }
}
export interface ExportRowsResult {
fileName: string
content: string
}

30
packages/server/src/sdk/app/rows/search/utils.ts
View file

@ -1,17 +1,37 @@
import {
FieldType,
SearchParams,
Table,
DocumentType,
SEPARATOR,
FieldSubtype,
SearchFilters,
SearchIndex,
SearchResponse,
Row,
RowSearchParams,
} from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
import { db as dbCore, context } from "@budibase/backend-core"
import { utils } from "@budibase/shared-core"
export async function paginatedSearch(
query: SearchFilters,
params: RowSearchParams
): Promise<SearchResponse<Row>> {
const appId = context.getAppId()
return dbCore.paginatedSearch(appId!, SearchIndex.ROWS, query, params)
}
export async function fullSearch(
query: SearchFilters,
params: RowSearchParams
): Promise<{ rows: Row[] }> {
const appId = context.getAppId()
return dbCore.fullSearch(appId!, SearchIndex.ROWS, query, params)
}
function findColumnInQueries(
column: string,
options: SearchParams,
options: RowSearchParams,
callback: (filter: any) => any
) {
if (!options.query) {
@ -29,7 +49,7 @@ function findColumnInQueries(
}
}
function userColumnMapping(column: string, options: SearchParams) {
function userColumnMapping(column: string, options: RowSearchParams) {
findColumnInQueries(column, options, (filterValue: any): any => {
const isArray = Array.isArray(filterValue),
isString = typeof filterValue === "string"
@ -60,7 +80,7 @@ function userColumnMapping(column: string, options: SearchParams) {
// maps through the search parameters to check if any of the inputs are invalid
// based on the table schema, converts them to something that is valid.
export function searchInputMapping(table: Table, options: SearchParams) {
export function searchInputMapping(table: Table, options: RowSearchParams) {
if (!table?.schema) {
return options
}

2
packages/server/src/sdk/app/rows/utils.ts
View file

@ -49,7 +49,7 @@ export function getSQLClient(datasource: Datasource): SqlClient {
export async function getDatasourceAndQuery(
json: QueryJson
): DatasourcePlusQueryResponse {
): Promise<DatasourcePlusQueryResponse> {
const datasourceId = json.endpoint.datasourceId
const datasource = await sdk.datasources.get(datasourceId)
return makeExternalQuery(datasource, json)

16
packages/server/src/sdk/app/tables/external/index.ts vendored
View file

@ -48,6 +48,18 @@ export async function save(
oldTable = await getTable(tableId)
}
if (
!oldTable &&
(tableToSave.primary == null || tableToSave.primary.length === 0)
) {
tableToSave.primary = ["id"]
tableToSave.schema.id = {
type: FieldType.NUMBER,
autocolumn: true,
name: "id",
}
}
if (hasTypeChanged(tableToSave, oldTable)) {
throw new Error("A column type has changed.")
}
@ -183,6 +195,10 @@ export async function save(
// that the datasource definition changed
const updatedDatasource = await datasourceSdk.get(datasource._id!)
if (updatedDatasource.isSQL) {
tableToSave.sql = true
}
return { datasource: updatedDatasource, table: tableToSave }
}

4
packages/server/src/sdk/app/tables/getters.ts
View file

@ -142,7 +142,9 @@ export function enrichViewSchemas(table: Table): TableResponse {
return {
...table,
views: Object.values(table.views ?? [])
.map(v => sdk.views.enrichSchema(v, table.schema))
.map(v =>
sdk.views.isV2(v) ? sdk.views.enrichSchema(v, table.schema) : v
)
.reduce((p, v) => {
p[v.name!] = v
return p

2
packages/server/src/sdk/app/tables/index.ts
View file

@ -3,6 +3,7 @@ import * as getters from "./getters"
import * as updates from "./update"
import * as utils from "./utils"
import { migrate } from "./migration"
import * as sqs from "./internal/sqs"
export default {
populateExternalTableSchemas,
@ -10,4 +11,5 @@ export default {
...getters,
...utils,
migrate,
sqs,
}

81
packages/server/src/sdk/app/tables/internal/sqs.ts Normal file
View file

@ -0,0 +1,81 @@
import { context, SQLITE_DESIGN_DOC_ID } from "@budibase/backend-core"
import { FieldType, SQLiteDefinition, SQLiteType, Table } from "@budibase/types"
import { cloneDeep } from "lodash"
import tablesSdk from "../"
import { CONSTANT_INTERNAL_ROW_COLS } from "../../../../db/utils"
const BASIC_SQLITE_DOC: SQLiteDefinition = {
_id: SQLITE_DESIGN_DOC_ID,
language: "sqlite",
sql: {
tables: {},
options: {
table_name: "tableId",
},
},
}
const FieldTypeMap: Record<FieldType, SQLiteType> = {
[FieldType.BOOLEAN]: SQLiteType.NUMERIC,
[FieldType.DATETIME]: SQLiteType.TEXT,
[FieldType.FORMULA]: SQLiteType.TEXT,
[FieldType.LONGFORM]: SQLiteType.TEXT,
[FieldType.NUMBER]: SQLiteType.REAL,
[FieldType.STRING]: SQLiteType.TEXT,
[FieldType.AUTO]: SQLiteType.TEXT,
[FieldType.OPTIONS]: SQLiteType.TEXT,
[FieldType.JSON]: SQLiteType.BLOB,
[FieldType.INTERNAL]: SQLiteType.BLOB,
[FieldType.BARCODEQR]: SQLiteType.BLOB,
[FieldType.ATTACHMENT]: SQLiteType.BLOB,
[FieldType.ARRAY]: SQLiteType.BLOB,
[FieldType.LINK]: SQLiteType.BLOB,
[FieldType.BIGINT]: SQLiteType.REAL,
// TODO: consider the difference between multi-user and single user types (subtyping)
[FieldType.BB_REFERENCE]: SQLiteType.TEXT,
}
function mapTable(table: Table): { [key: string]: SQLiteType } {
const fields: Record<string, SQLiteType> = {}
for (let [key, column] of Object.entries(table.schema)) {
if (!FieldTypeMap[column.type]) {
throw new Error(`Unable to map type "${column.type}" to SQLite type`)
}
fields[key] = FieldTypeMap[column.type]
}
// there are some extra columns to map - add these in
const constantMap: Record<string, SQLiteType> = {}
CONSTANT_INTERNAL_ROW_COLS.forEach(col => {
constantMap[col] = SQLiteType.TEXT
})
return {
...constantMap,
...fields,
}
}
// nothing exists, need to iterate though existing tables
async function buildBaseDefinition(): Promise<SQLiteDefinition> {
const tables = await tablesSdk.getAllInternalTables()
const definition = cloneDeep(BASIC_SQLITE_DOC)
for (let table of tables) {
definition.sql.tables[table._id!] = {
fields: mapTable(table),
}
}
return definition
}
export async function addTableToSqlite(table: Table) {
const db = context.getAppDB()
let definition: SQLiteDefinition
try {
definition = await db.get(SQLITE_DESIGN_DOC_ID)
} catch (err) {
definition = await buildBaseDefinition()
}
definition.sql.tables[table._id!] = {
fields: mapTable(table),
}
await db.put(definition)
}

30
packages/server/src/sdk/app/views/external.ts
View file

@ -1,4 +1,4 @@
import { ViewV2 } from "@budibase/types"
import { ViewV2, ViewV2Enriched } from "@budibase/types"
import { context, HTTPError } from "@budibase/backend-core"
import sdk from "../../../sdk"
@ -6,26 +6,34 @@ import * as utils from "../../../db/utils"
import { enrichSchema, isV2 } from "."
import { breakExternalTableId } from "../../../integrations/utils"
export async function get(
viewId: string,
opts?: { enriched: boolean }
): Promise<ViewV2> {
export async function get(viewId: string): Promise<ViewV2> {
const { tableId } = utils.extractViewInfoFromID(viewId)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const ds = await sdk.datasources.get(datasourceId!)
const table = ds.entities![tableName!]
const views = Object.values(table.views!)
const found = views.find(v => isV2(v) && v.id === viewId)
const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId)
if (!found) {
throw new Error("No view found")
}
if (opts?.enriched) {
return enrichSchema(found, table.schema) as ViewV2
} else {
return found as ViewV2
return found
}
export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
const { tableId } = utils.extractViewInfoFromID(viewId)
const { datasourceId, tableName } = breakExternalTableId(tableId)
const ds = await sdk.datasources.get(datasourceId!)
const table = ds.entities![tableName!]
const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId)
if (!found) {
throw new Error("No view found")
}
return enrichSchema(found, table.schema)
}
export async function create(

30
packages/server/src/sdk/app/views/index.ts
View file

@ -1,8 +1,13 @@
import { RenameColumn, TableSchema, View, ViewV2 } from "@budibase/types"
import {
RenameColumn,
TableSchema,
View,
ViewV2,
ViewV2Enriched,
} from "@budibase/types"
import { db as dbCore } from "@budibase/backend-core"
import { cloneDeep } from "lodash"
import sdk from "../../../sdk"
import * as utils from "../../../db/utils"
import { isExternalTableID } from "../../../integrations/utils"
@ -16,12 +21,14 @@ function pickApi(tableId: any) {
return internal
}
export async function get(
viewId: string,
opts?: { enriched: boolean }
): Promise<ViewV2> {
export async function get(viewId: string): Promise<ViewV2> {
const { tableId } = utils.extractViewInfoFromID(viewId)
return pickApi(tableId).get(viewId, opts)
return pickApi(tableId).get(viewId)
}
export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
const { tableId } = utils.extractViewInfoFromID(viewId)
return pickApi(tableId).getEnriched(viewId)
}
export async function create(
@ -52,11 +59,10 @@ export function allowedFields(view: View | ViewV2) {
]
}
export function enrichSchema(view: View | ViewV2, tableSchema: TableSchema) {
if (!sdk.views.isV2(view)) {
return view
}
export function enrichSchema(
view: ViewV2,
tableSchema: TableSchema
): ViewV2Enriched {
let schema = cloneDeep(tableSchema)
const anyViewOrder = Object.values(view.schema || {}).some(
ui => ui.order != null

26
packages/server/src/sdk/app/views/internal.ts
View file

@ -1,26 +1,30 @@
import { ViewV2 } from "@budibase/types"
import { ViewV2, ViewV2Enriched } from "@budibase/types"
import { context, HTTPError } from "@budibase/backend-core"
import sdk from "../../../sdk"
import * as utils from "../../../db/utils"
import { enrichSchema, isV2 } from "."
export async function get(
viewId: string,
opts?: { enriched: boolean }
): Promise<ViewV2> {
export async function get(viewId: string): Promise<ViewV2> {
const { tableId } = utils.extractViewInfoFromID(viewId)
const table = await sdk.tables.getTable(tableId)
const views = Object.values(table.views!)
const found = views.find(v => isV2(v) && v.id === viewId)
const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId)
if (!found) {
throw new Error("No view found")
}
if (opts?.enriched) {
return enrichSchema(found, table.schema) as ViewV2
} else {
return found as ViewV2
return found
}
export async function getEnriched(viewId: string): Promise<ViewV2Enriched> {
const { tableId } = utils.extractViewInfoFromID(viewId)
const table = await sdk.tables.getTable(tableId)
const views = Object.values(table.views!).filter(isV2)
const found = views.find(v => v.id === viewId)
if (!found) {
throw new Error("No view found")
}
return enrichSchema(found, table.schema)
}
export async function create(

44
packages/server/src/startup.ts → packages/server/src/startup/index.ts
View file

@ -1,6 +1,6 @@
import env from "./environment"
import * as redis from "./utilities/redis"
import { generateApiKey, getChecklist } from "./utilities/workerRequests"
import env from "../environment"
import * as redis from "../utilities/redis"
import { generateApiKey, getChecklist } from "../utilities/workerRequests"
import {
events,
installation,
@ -9,25 +9,30 @@ import {
users,
cache,
} from "@budibase/backend-core"
import fs from "fs"
import { watch } from "./watch"
import * as automations from "./automations"
import * as fileSystem from "./utilities/fileSystem"
import { default as eventEmitter, init as eventInit } from "./events"
import * as migrations from "./migrations"
import * as bullboard from "./automations/bullboard"
import { watch } from "../watch"
import * as automations from "../automations"
import * as fileSystem from "../utilities/fileSystem"
import { default as eventEmitter, init as eventInit } from "../events"
import * as migrations from "../migrations"
import * as bullboard from "../automations/bullboard"
import * as pro from "@budibase/pro"
import * as api from "./api"
import sdk from "./sdk"
import { initialise as initialiseWebsockets } from "./websockets"
import { automationsEnabled, printFeatures } from "./features"
import * as api from "../api"
import sdk from "../sdk"
import { initialise as initialiseWebsockets } from "../websockets"
import { automationsEnabled, printFeatures } from "../features"
import * as jsRunner from "../jsRunner"
import Koa from "koa"
import { Server } from "http"
import { AddressInfo } from "net"
import * as jsRunner from "./jsRunner"
import fs from "fs"
let STARTUP_RAN = false
if (env.isProd() && env.SQS_SEARCH_ENABLE) {
console.error("Stopping service - SQS search support is not yet available.")
process.exit(-1)
}
async function initRoutes(app: Koa) {
if (!env.isTest()) {
const plugin = await bullboard.init()
@ -61,8 +66,11 @@ function shutdown(server?: Server) {
}
}
export async function startup(app?: Koa, server?: Server) {
if (STARTUP_RAN) {
export async function startup(
opts: { app?: Koa; server?: Server; rerun?: boolean } = {}
) {
const { app, server, rerun } = opts
if (STARTUP_RAN && !rerun) {
return
}
printFeatures()
@ -139,9 +147,9 @@ export async function startup(app?: Koa, server?: Server) {
try {
const user = await users.UserDB.createAdminUser(
bbAdminEmail,
bbAdminPassword,
tenantId,
{
password: bbAdminPassword,
hashPassword: true,
requirePassword: true,
skipPasswordValidation: true,

Some files were not shown because too many files have changed in this diff Show more