CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-09-10 22:46:09 +12:00
Code Issues Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/develop' into feature/whitelabelling

Browse source
This commit is contained in:
Dean 2023-03-08 13:01:55 +00:00
commit e8c6fc6acf
59 changed files with 465 additions and 462 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/ISSUE_TEMPLATE/bug_report.md vendored
View file

@ -2,10 +2,11 @@
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug
labels: bug, linear
assignees: ''
---
**Checklist**
- [ ] I have searched budibase discussions and github issues to check if my issue already exists

65
.github/workflows/deploy-preprod.yml vendored
View file

@ -1,65 +0,0 @@
name: Budibase Deploy Preprod
on:
workflow_dispatch:
env:
INTERCOM_TOKEN: ${{ secrets.INTERCOM_TOKEN }}
SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-1
- name: Get the latest budibase release version
id: version
run: |
release_version=$(cat lerna.json | jq -r '.version')
echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
- name: Pull values.yaml from budibase-infra
run: |
curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
-H 'Accept: application/vnd.github.v3.raw' \
-o values.preprod.yaml \
-L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-preprod/values.yaml
wc -l values.preprod.yaml
- name: Deploy to Preprod Environment
uses: budibase/helm@v1.8.0
with:
release: budibase-preprod
namespace: budibase
chart: charts/budibase
token: ${{ github.token }}
helm: helm3
values: |
globals:
appVersion: v${{ env.RELEASE_VERSION }}
ingress:
enabled: true
nginx: true
value-files: >-
[
"values.preprod.yaml"
]
env:
KUBECONFIG_FILE: '${{ secrets.PREPROD_KUBECONFIG }}'
- name: Discord Webhook Action
uses: tsickert/discord-webhook@v4.0.0
with:
webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
content: "Preprod Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Pre-prod."
embed-title: ${{ env.RELEASE_VERSION }}

88
.github/workflows/deploy-release.yml vendored
View file

@ -1,88 +0,0 @@
name: Budibase Deploy Release
on:
workflow_dispatch:
jobs:
release:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-1
- name: Fail if branch is not develop
if: github.ref != 'refs/heads/develop'
run: |
echo "Ref is not develop, you must run this job from develop."
exit 1
- name: Get the latest budibase release version
id: version
run: |
release_version=$(cat lerna.json | jq -r '.version')
echo "RELEASE_VERSION=$release_version" >> $GITHUB_ENV
- name: Pull values.yaml from budibase-infra
run: |
curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
-H 'Accept: application/vnd.github.v3.raw' \
-o values.release.yaml \
-L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-release/values.yaml
wc -l values.release.yaml
- name: Deploy to Release Environment
uses: budibase/helm@v1.8.0
with:
release: budibase-release
namespace: budibase
chart: charts/budibase
token: ${{ github.token }}
helm: helm3
values: |
globals:
appVersion: develop
ingress:
enabled: true
nginx: true
value-files: >-
[
"values.release.yaml"
]
env:
KUBECONFIG_FILE: '${{ secrets.RELEASE_KUBECONFIG }}'
- name: Re roll app-service
uses: actions-hub/kubectl@master
env:
KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
with:
args: rollout restart deployment app-service -n budibase
- name: Re roll proxy-service
uses: actions-hub/kubectl@master
env:
KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
with:
args: rollout restart deployment proxy-service -n budibase
- name: Re roll worker-service
uses: actions-hub/kubectl@master
env:
KUBE_CONFIG: ${{ secrets.RELEASE_KUBECONFIG_BASE64 }}
with:
args: rollout restart deployment worker-service -n budibase
- name: Discord Webhook Action
uses: tsickert/discord-webhook@v4.0.0
with:
webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
content: "Release Env Deployment Complete: ${{ env.RELEASE_VERSION }} deployed to Budibase Release Env."
embed-title: ${{ env.RELEASE_VERSION }}

2
.github/workflows/release-develop.yml vendored
View file

@ -117,4 +117,4 @@ jobs:
with:
repository: budibase/budibase-deploys
event: budicloud-qa-deploy
github_pat: ${{ secrets.GH_ACCESS_TOKEN }}
github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

100
.github/workflows/release.yml → .github/workflows/release-master.yml vendored
View file

@ -35,9 +35,8 @@ env:
PERSONAL_ACCESS_TOKEN : ${{ secrets.PERSONAL_ACCESS_TOKEN }}
jobs:
release:
release-images:
runs-on: ubuntu-latest
steps:
- name: Fail if branch is not master
if: github.ref != 'refs/heads/master'
@ -57,14 +56,6 @@ jobs:
- run: yarn lint
- run: yarn build
- run: yarn build:sdk
- run: yarn test
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-1
- name: Publish budibase packages to NPM
env:
@ -90,46 +81,59 @@ jobs:
DOCKER_USER: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_API_KEY }}
BUDIBASE_RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}
- name: Configure AWS Credentials
uses: aws-actions/configure-aws-credentials@v1
with:
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
aws-region: eu-west-1
- name: Pull values.yaml from budibase-infra
run: |
curl -H "Authorization: token ${{ secrets.GH_PERSONAL_TOKEN }}" \
-H 'Accept: application/vnd.github.v3.raw' \
-o values.preprod.yaml \
-L https://api.github.com/repos/budibase/budibase-infra/contents/kubernetes/budibase-preprod/values.yaml
wc -l values.preprod.yaml
release-helm-chart:
needs: [release-images]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Setup Helm
uses: azure/setup-helm@v1
id: helm-install
- name: Deploy to Preprod Environment
uses: budibase/helm@v1.8.0
with:
release: budibase-preprod
namespace: budibase
chart: charts/budibase
token: ${{ github.token }}
helm: helm3
values: |
globals:
appVersion: ${{ steps.previoustag.outputs.tag }}
ingress:
enabled: true
nginx: true
value-files: >-
[
"values.preprod.yaml"
]
- name: 'Get Previous tag'
id: previoustag
uses: "WyriHaximus/github-action-get-previous-tag@v1"
# due to helm repo index issue: https://github.com/helm/helm/issues/7363
# we need to create new package in a different dir, merge the index and move the package back
- name: Build and release helm chart
run: |
git config user.name "Budibase Helm Bot"
git config user.email "<>"
git reset --hard
git pull
mkdir sync
echo "Packaging chart to sync dir"
helm package charts/budibase --version 0.0.0-master --app-version "$RELEASE_VERSION" --destination sync
echo "Packaging successful"
git checkout gh-pages
echo "Indexing helm repo"
helm repo index --merge docs/index.yaml sync
mv -f sync/* docs
rm -rf sync
echo "Pushing new helm release"
git add -A
git commit -m "Helm Release: ${{ env.RELEASE_VERSION }}"
git push
env:
KUBECONFIG_FILE: '${{ secrets.PREPROD_KUBECONFIG }}'
RELEASE_VERSION: ${{ steps.previoustag.outputs.tag }}
- name: Discord Webhook Action
uses: tsickert/discord-webhook@v4.0.0
# Trigger deploy to new EKS preprod environment
trigger-deploy-to-preprod-env:
needs: [release-helm-chart]
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: 'Get Previous tag'
id: previoustag
uses: "WyriHaximus/github-action-get-previous-tag@v1"
- uses: passeidireto/trigger-external-workflow-action@main
env:
PAYLOAD_VERSION: ${{ steps.previoustag.outputs.tag }}
with:
webhook-url: ${{ secrets.PROD_DEPLOY_WEBHOOK_URL }}
content: "Preprod Deployment Complete: ${{ steps.previoustag.outputs.tag }} deployed to Budibase Pre-prod."
embed-title: ${{ steps.previoustag.outputs.tag }}
repository: budibase/budibase-deploys
event: budicloud-preprod-deploy
github_pat: ${{ secrets.GH_ACCESS_TOKEN }}

0
.github/workflows/deploy-single-image.yml → .github/workflows/release-singleimage.yml vendored
View file

31
.github/workflows/smoke_test.yaml vendored
View file

@ -1,31 +0,0 @@
name: Budibase Nightly Tests
on:
workflow_dispatch:
schedule:
- cron: "0 5 * * *" # every day at 5AM
jobs:
nightly:
runs-on: [self-hosted, qa]
steps:
- uses: actions/checkout@v2
- name: Use Node.js 14.x
uses: actions/setup-node@v1
with:
node-version: 14.x
- name: QA Core Integration Tests
run: |
cd qa-core
yarn
yarn api:test:ci
env:
BUDIBASE_HOST: budicloud.qa.budibase.net
BUDIBASE_ACCOUNTS_URL: https://account-portal.budicloud.qa.budibase.net
- name: Cypress Discord Notify
run: yarn test:notify
env:
WEBHOOK_URL: ${{ secrets.BUDI_QA_WEBHOOK }}
GITHUB_RUN_URL: $GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID

12
docs/DEV-SETUP-DEBIAN.md
View file

@ -52,4 +52,14 @@ So this command will actually run the application in dev mode. It creates .env f
The dev version will be available on port 10000 i.e.
http://127.0.0.1:10000/builder/admin
http://127.0.0.1:10000/builder/admin
### File descriptor issues with Vite and Chrome in Linux
If your dev environment stalls forever, with some network requests stuck in flight, it's likely that Chrome is trying to open more file descriptors than your system allows.
To fix this, apply the following tweaks.
Debian based distros:
Add `* - nofile 65536` to `/etc/security/limits.conf`.
Arch:
Add `DefaultLimitNOFILE=65536` to `/etc/systemd/system.conf`.

3
hosting/docker-compose.yaml
View file

@ -59,7 +59,8 @@ services:
minio-service:
restart: unless-stopped
image: minio/minio
# Last version that supports the "fs" backend
image: minio/minio:RELEASE.2022-10-24T18-35-07Z
volumes:
- minio_data:/data
environment:

4
hosting/proxy/nginx.prod.conf
View file

@ -55,12 +55,12 @@ http {
set $csp_style "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net https://fonts.googleapis.com https://rsms.me https://maxcdn.bootstrapcdn.com";
set $csp_object "object-src 'none'";
set $csp_base_uri "base-uri 'self'";
set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.*.amazonaws.com https://s3.*.amazonaws.com https://api.github.com";
set $csp_connect "connect-src 'self' https://*.budibase.net https://api-iam.intercom.io https://api-iam.intercom.io https://api-ping.intercom.io https://app.posthog.com wss://nexus-websocket-a.intercom.io wss://nexus-websocket-b.intercom.io https://nexus-websocket-a.intercom.io https://nexus-websocket-b.intercom.io https://uploads.intercomcdn.com https://uploads.intercomusercontent.com https://*.s3.amazonaws.com https://*.s3.us-east-2.amazonaws.com https://*.s3.us-east-1.amazonaws.com https://*.s3.us-west-1.amazonaws.com https://*.s3.us-west-2.amazonaws.com https://*.s3.af-south-1.amazonaws.com https://*.s3.ap-east-1.amazonaws.com https://*.s3.ap-southeast-3.amazonaws.com https://*.s3.ap-south-1.amazonaws.com https://*.s3.ap-northeast-3.amazonaws.com https://*.s3.ap-northeast-2.amazonaws.com https://*.s3.ap-southeast-1.amazonaws.com https://*.s3.ap-southeast-2.amazonaws.com https://*.s3.ap-northeast-1.amazonaws.com https://*.s3.ca-central-1.amazonaws.com https://*.s3.cn-north-1.amazonaws.com https://*.s3.cn-northwest-1.amazonaws.com https://*.s3.eu-central-1.amazonaws.com https://*.s3.eu-west-1.amazonaws.com https://*.s3.eu-west-2.amazonaws.com https://*.s3.eu-south-1.amazonaws.com https://*.s3.eu-west-3.amazonaws.com https://*.s3.eu-north-1.amazonaws.com https://*.s3.sa-east-1.amazonaws.com https://*.s3.me-south-1.amazonaws.com https://*.s3.us-gov-east-1.amazonaws.com https://*.s3.us-gov-west-1.amazonaws.com https://api.github.com";
set $csp_font "font-src 'self' data: https://cdn.jsdelivr.net https://fonts.gstatic.com https://rsms.me https://maxcdn.bootstrapcdn.com https://js.intercomcdn.com https://fonts.intercomcdn.com";
set $csp_frame "frame-src 'self' https:";
set $csp_img "img-src http: https: data: blob:";
set $csp_manifest "manifest-src 'self'";
set $csp_media "media-src 'self' https://js.intercomcdn.com";
set $csp_media "media-src 'self' https://js.intercomcdn.com https://cdn.budi.live";
set $csp_worker "worker-src 'none'";
error_page 502 503 504 /error.html;

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"npmClient": "yarn",
"packages": [
"packages/*"

4
packages/backend-core/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/backend-core",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase backend core libraries used in server and worker",
"main": "dist/src/index.js",
"types": "dist/src/index.d.ts",
@ -24,7 +24,7 @@
"dependencies": {
"@budibase/nano": "10.1.2",
"@budibase/pouchdb-replication-stream": "1.2.10",
"@budibase/types": "2.3.21-alpha.1",
"@budibase/types": "2.4.6-alpha.0",
"@shopify/jest-koa-mocks": "5.0.1",
"@techpass/passport-openidconnect": "0.3.2",
"aws-cloudfront-sign": "2.2.0",

94
packages/backend-core/src/cache/tests/writethrough.spec.ts vendored
View file

@ -1,10 +1,13 @@
import { structures, DBTestConfiguration } from "../../../tests"
import {
structures,
DBTestConfiguration,
expectFunctionWasCalledTimesWith,
} from "../../../tests"
import { Writethrough } from "../writethrough"
import { getDB } from "../../db"
import tk from "timekeeper"
const START_DATE = Date.now()
tk.freeze(START_DATE)
tk.freeze(Date.now())
const DELAY = 5000
@ -17,34 +20,99 @@ describe("writethrough", () => {
const writethrough = new Writethrough(db, DELAY)
const writethrough2 = new Writethrough(db2, DELAY)
const docId = structures.uuid()
beforeEach(() => {
jest.clearAllMocks()
})
describe("put", () => {
let first: any
let current: any
it("should be able to store, will go to DB", async () => {
await config.doInTenant(async () => {
const response = await writethrough.put({ _id: "test", value: 1 })
const response = await writethrough.put({
_id: docId,
value: 1,
})
const output = await db.get(response.id)
first = output
current = output
expect(output.value).toBe(1)
})
})
it("second put shouldn't update DB", async () => {
await config.doInTenant(async () => {
const response = await writethrough.put({ ...first, value: 2 })
const response = await writethrough.put({ ...current, value: 2 })
const output = await db.get(response.id)
expect(first._rev).toBe(output._rev)
expect(current._rev).toBe(output._rev)
expect(output.value).toBe(1)
})
})
it("should put it again after delay period", async () => {
await config.doInTenant(async () => {
tk.freeze(START_DATE + DELAY + 1)
const response = await writethrough.put({ ...first, value: 3 })
tk.freeze(Date.now() + DELAY + 1)
const response = await writethrough.put({ ...current, value: 3 })
const output = await db.get(response.id)
expect(response.rev).not.toBe(first._rev)
expect(response.rev).not.toBe(current._rev)
expect(output.value).toBe(3)
current = output
})
})
it("should handle parallel DB updates ignoring conflicts", async () => {
await config.doInTenant(async () => {
tk.freeze(Date.now() + DELAY + 1)
const responses = await Promise.all([
writethrough.put({ ...current, value: 4 }),
writethrough.put({ ...current, value: 4 }),
writethrough.put({ ...current, value: 4 }),
])
const newRev = responses.map(x => x.rev).find(x => x !== current._rev)
expect(newRev).toBeDefined()
expect(responses.map(x => x.rev)).toEqual(
expect.arrayContaining([current._rev, current._rev, newRev])
)
expectFunctionWasCalledTimesWith(
console.warn,
2,
"bb-warn: Ignoring redlock conflict in write-through cache"
)
const output = await db.get(current._id)
expect(output.value).toBe(4)
expect(output._rev).toBe(newRev)
current = output
})
})
it("should handle updates with documents falling behind", async () => {
await config.doInTenant(async () => {
tk.freeze(Date.now() + DELAY + 1)
const id = structures.uuid()
await writethrough.put({ _id: id, value: 1 })
const doc = await writethrough.get(id)
// Updating document
tk.freeze(Date.now() + DELAY + 1)
await writethrough.put({ ...doc, value: 2 })
// Update with the old rev value
tk.freeze(Date.now() + DELAY + 1)
const res = await writethrough.put({
...doc,
value: 3,
})
expect(res.ok).toBe(true)
const output = await db.get(id)
expect(output.value).toBe(3)
expect(output._rev).toBe(res.rev)
})
})
})
@ -52,8 +120,8 @@ describe("writethrough", () => {
describe("get", () => {
it("should be able to retrieve", async () => {
await config.doInTenant(async () => {
const response = await writethrough.get("test")
expect(response.value).toBe(3)
const response = await writethrough.get(docId)
expect(response.value).toBe(4)
})
})
})

71
packages/backend-core/src/cache/writethrough.ts vendored
View file

@ -1,7 +1,8 @@
import BaseCache from "./base"
import { getWritethroughClient } from "../redis/init"
import { logWarn } from "../logging"
import { Database } from "@budibase/types"
import { Database, Document, LockName, LockType } from "@budibase/types"
import * as locks from "../redis/redlockImpl"
const DEFAULT_WRITE_RATE_MS = 10000
let CACHE: BaseCache | null = null
@ -27,44 +28,62 @@ function makeCacheItem(doc: any, lastWrite: number | null = null): CacheItem {
return { doc, lastWrite: lastWrite || Date.now() }
}
export async function put(
async function put(
db: Database,
doc: any,
doc: Document,
writeRateMs: number = DEFAULT_WRITE_RATE_MS
) {
const cache = await getCache()
const key = doc._id
let cacheItem: CacheItem | undefined = await cache.get(makeCacheKey(db, key))
let cacheItem: CacheItem | undefined
if (key) {
cacheItem = await cache.get(makeCacheKey(db, key))
}
const updateDb = !cacheItem || cacheItem.lastWrite < Date.now() - writeRateMs
let output = doc
if (updateDb) {
const writeDb = async (toWrite: any) => {
// doc should contain the _id and _rev
const response = await db.put(toWrite)
output = {
...doc,
_id: response.id,
_rev: response.rev,
}
}
try {
await writeDb(doc)
} catch (err: any) {
if (err.status !== 409) {
throw err
} else {
// Swallow 409s but log them
logWarn(`Ignoring conflict in write-through cache`)
const lockResponse = await locks.doWithLock(
{
type: LockType.TRY_ONCE,
name: LockName.PERSIST_WRITETHROUGH,
resource: key,
ttl: 1000,
},
async () => {
const writeDb = async (toWrite: any) => {
// doc should contain the _id and _rev
const response = await db.put(toWrite, { force: true })
output = {
...doc,
_id: response.id,
_rev: response.rev,
}
}
try {
await writeDb(doc)
} catch (err: any) {
if (err.status !== 409) {
throw err
} else {
// Swallow 409s but log them
logWarn(`Ignoring conflict in write-through cache`)
}
}
}
)
if (!lockResponse.executed) {
logWarn(`Ignoring redlock conflict in write-through cache`)
}
}
// if we are updating the DB then need to set the lastWrite to now
cacheItem = makeCacheItem(output, updateDb ? null : cacheItem?.lastWrite)
await cache.store(makeCacheKey(db, key), cacheItem)
if (output._id) {
await cache.store(makeCacheKey(db, output._id), cacheItem)
}
return { ok: true, id: output._id, rev: output._rev }
}
export async function get(db: Database, id: string): Promise<any> {
async function get(db: Database, id: string): Promise<any> {
const cache = await getCache()
const cacheKey = makeCacheKey(db, id)
let cacheItem: CacheItem = await cache.get(cacheKey)
@ -76,11 +95,7 @@ export async function get(db: Database, id: string): Promise<any> {
return cacheItem.doc
}
export async function remove(
db: Database,
docOrId: any,
rev?: any
): Promise<void> {
async function remove(db: Database, docOrId: any, rev?: any): Promise<void> {
const cache = await getCache()
if (!docOrId) {
throw new Error("No ID/Rev provided.")

29
packages/backend-core/src/redis/redlockImpl.ts
View file

@ -24,7 +24,7 @@ const getClient = async (type: LockType): Promise<Redlock> => {
}
}
export const OPTIONS = {
const OPTIONS = {
TRY_ONCE: {
// immediately throws an error if the lock is already held
retryCount: 0,
@ -56,14 +56,29 @@ export const OPTIONS = {
},
}
export const newRedlock = async (opts: Options = {}) => {
const newRedlock = async (opts: Options = {}) => {
let options = { ...OPTIONS.DEFAULT, ...opts }
const redisWrapper = await getLockClient()
const client = redisWrapper.getClient()
return new Redlock([client], options)
}
export const doWithLock = async (opts: LockOptions, task: any) => {
type SuccessfulRedlockExecution<T> = {
executed: true
result: T
}
type UnsuccessfulRedlockExecution = {
executed: false
}
type RedlockExecution<T> =
| SuccessfulRedlockExecution<T>
| UnsuccessfulRedlockExecution
export const doWithLock = async <T>(
opts: LockOptions,
task: () => Promise<T>
): Promise<RedlockExecution<T>> => {
const redlock = await getClient(opts.type)
let lock
try {
@ -73,8 +88,8 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
let name: string = `lock:${prefix}_${opts.name}`
// add additional unique name if required
if (opts.nameSuffix) {
name = name + `_${opts.nameSuffix}`
if (opts.resource) {
name = name + `_${opts.resource}`
}
// create the lock
@ -83,7 +98,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
// perform locked task
// need to await to ensure completion before unlocking
const result = await task()
return result
return { executed: true, result }
} catch (e: any) {
console.warn("lock error")
// lock limit exceeded
@ -92,7 +107,7 @@ export const doWithLock = async (opts: LockOptions, task: any) => {
// don't throw for try-once locks, they will always error
// due to retry count (0) exceeded
console.warn(e)
return
return { executed: false }
} else {
console.error(e)
throw e

2
packages/backend-core/tests/utilities/index.ts
View file

@ -4,4 +4,6 @@ export { generator } from "./structures"
export * as testEnv from "./testEnv"
export * as testContainerUtils from "./testContainerUtils"
export * from "./jestUtils"
export { default as DBTestConfiguration } from "./DBTestConfiguration"

9
packages/backend-core/tests/utilities/jestUtils.ts Normal file
View file

@ -0,0 +1,9 @@
export function expectFunctionWasCalledTimesWith(
jestFunction: any,
times: number,
argument: any
) {
expect(
jestFunction.mock.calls.filter((call: any) => call[0] === argument).length
).toBe(times)
}

7
packages/backend-core/tests/utilities/structures/db.ts
View file

@ -1,5 +1,12 @@
import { structures } from ".."
import { newid } from "../../../src/newid"
export function id() {
return `db_${newid()}`
}
export function rev() {
return `${structures.generator.character({
numeric: true,
})}-${structures.uuid().replace(/-/, "")}`
}

4
packages/bbui/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"license": "MPL-2.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",
@ -38,7 +38,7 @@
],
"dependencies": {
"@adobe/spectrum-css-workflow-icons": "1.2.1",
"@budibase/string-templates": "2.3.21-alpha.1",
"@budibase/string-templates": "2.4.6-alpha.0",
"@spectrum-css/accordion": "3.0.24",
"@spectrum-css/actionbutton": "1.0.1",
"@spectrum-css/actiongroup": "1.0.1",

10
packages/builder/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"license": "GPL-3.0",
"private": true,
"scripts": {
@ -58,10 +58,10 @@
}
},
"dependencies": {
"@budibase/bbui": "2.3.21-alpha.1",
"@budibase/client": "2.3.21-alpha.1",
"@budibase/frontend-core": "2.3.21-alpha.1",
"@budibase/string-templates": "2.3.21-alpha.1",
"@budibase/bbui": "2.4.6-alpha.0",
"@budibase/client": "2.4.6-alpha.0",
"@budibase/frontend-core": "2.4.6-alpha.0",
"@budibase/string-templates": "2.4.6-alpha.0",
"@fortawesome/fontawesome-svg-core": "^6.2.1",
"@fortawesome/free-brands-svg-icons": "^6.2.1",
"@fortawesome/free-solid-svg-icons": "^6.2.1",

8
packages/builder/src/components/design/settings/controls/ColumnEditor/ColumnEditor.svelte
View file

@ -8,6 +8,7 @@
getSchemaForDatasource,
} from "builderStore/dataBinding"
import { currentAsset } from "builderStore"
import { getFields } from "helpers/searchFields"
export let componentInstance
export let value = []
@ -21,9 +22,14 @@
$: datasource = getDatasourceForProvider($currentAsset, componentInstance)
$: schema = getSchema($currentAsset, datasource)
$: options = Object.keys(schema || {})
$: options = allowCellEditing
? Object.keys(schema || {})
: enrichedSchemaFields?.map(field => field.name)
$: sanitisedValue = getValidColumns(value, options)
$: updateBoundValue(sanitisedValue)
$: enrichedSchemaFields = getFields(Object.values(schema) || [], {
allowLinks: true,
})
const getSchema = (asset, datasource) => {
const schema = getSchemaForDatasource(asset, datasource).schema

2
packages/builder/src/components/integration/RestQueryViewer.svelte
View file

@ -120,7 +120,7 @@
const cleanUrl = inputUrl =>
url
?.replace(/(http)|(https)|[{}:]/g, "")
?.replace(/(https)|(http)|[{}:]/g, "")
?.replaceAll(".", "_")
?.replaceAll("/", " ")
?.trim() || inputUrl

9
packages/builder/src/components/portal/onboarding/tours.js
View file

@ -1,8 +1,9 @@
import { get } from "svelte/store"
import { store } from "builderStore"
import { users, auth } from "stores/portal"
import { auth } from "stores/portal"
import analytics from "analytics"
import { OnboardingData, OnboardingDesign, OnboardingPublish } from "./steps"
import { API } from "api"
const ONBOARDING_EVENT_PREFIX = "onboarding"
export const TOUR_STEP_KEYS = {
@ -83,8 +84,7 @@ const getTours = () => {
// Mark the users onboarding as complete
// Clear all tour related state
if (get(auth).user) {
await users.save({
...get(auth).user,
await API.updateSelf({
onboardedAt: new Date().toISOString(),
})
@ -114,8 +114,7 @@ const getTours = () => {
onComplete: async () => {
// Push the onboarding forward
if (get(auth).user) {
await users.save({
...get(auth).user,
await API.updateSelf({
onboardedAt: new Date().toISOString(),
})

1
packages/builder/src/components/settings/ProfileModal.svelte
View file

@ -13,6 +13,7 @@
await auth.updateSelf($values)
notifications.success("Information updated successfully")
} catch (error) {
console.error(error)
notifications.error("Failed to update information")
}
}

3
packages/builder/src/pages/builder/app/[application]/_components/BuilderSidePanel.svelte
View file

@ -671,6 +671,7 @@
align-items: center;
gap: var(--spacing-m);
color: var(--spectrum-global-color-gray-900);
overflow: hidden;
}
.auth-entity .user-email {
@ -751,11 +752,11 @@
}
.builder-side-panel-header {
height: 58px;
display: flex;
flex-direction: row;
justify-content: space-between;
align-items: center;
flex: 0 0 58px;
}
.invite-header {

11
packages/builder/src/stores/portal/auth.js
View file

@ -154,9 +154,14 @@ export function createAuthStore() {
await setInitInfo({})
},
updateSelf: async fields => {
const newUser = { ...get(auth).user, ...fields }
await API.updateSelf(newUser)
setUser(newUser)
await API.updateSelf({ ...fields })
// Refetch to enrich after update.
try {
const user = await API.fetchBuilderSelf()
setUser(user)
} catch (error) {
setUser(null)
}
},
forgotPassword: async email => {
const tenantId = get(store).tenantId

10
packages/cli/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "dist/index.js",
"bin": {
@ -29,14 +29,14 @@
"outputPath": "build"
},
"dependencies": {
"@budibase/backend-core": "2.3.21-alpha.1",
"@budibase/string-templates": "2.3.21-alpha.1",
"@budibase/types": "2.3.21-alpha.1",
"@budibase/backend-core": "2.4.6-alpha.0",
"@budibase/string-templates": "2.4.6-alpha.0",
"@budibase/types": "2.4.6-alpha.0",
"axios": "0.21.2",
"chalk": "4.1.0",
"cli-progress": "3.11.2",
"commander": "7.1.0",
"docker-compose": "0.23.6",
"docker-compose": "0.23.12",
"dotenv": "16.0.1",
"download": "8.0.0",
"find-free-port": "^2.0.0",

15
packages/cli/yarn.lock
View file

@ -1235,10 +1235,12 @@ dir-glob@^3.0.1:
dependencies:
path-type "^4.0.0"
docker-compose@0.23.6:
version "0.23.6"
resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.6.tgz#bd21e17d599f17fcf7a4b5d607cff0358a9c378b"
integrity sha512-y3Q8MkwG862rNqkvEQG59/7Fi2/fzs3NYDCvqUAAD+z0WGs2qcJ9hRcn34hWgWv9ouPkFqe3Vwca0h+4bIIRWw==
docker-compose@0.23.12:
version "0.23.12"
resolved "https://registry.yarnpkg.com/docker-compose/-/docker-compose-0.23.12.tgz#fa883b98be08f6926143d06bf9e522ef7ed3210c"
integrity sha512-KFbSMqQBuHjTGZGmYDOCO0L4SaML3BsWTId5oSUyaBa22vALuFHNv+UdDWs3HcMylHWKsxCbLB7hnM/nCosWZw==
dependencies:
yaml "^1.10.2"
doctrine@^3.0.0:
version "3.0.0"
@ -3689,6 +3691,11 @@ yallist@^4.0.0:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yaml@^1.10.2:
version "1.10.2"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"
integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==
yaml@^2.1.1:
version "2.1.1"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.1.1.tgz#1e06fb4ca46e60d9da07e4f786ea370ed3c3cfec"

8
packages/client/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "2.3.21-alpha.1",
"@budibase/frontend-core": "2.3.21-alpha.1",
"@budibase/string-templates": "2.3.21-alpha.1",
"@budibase/bbui": "2.4.6-alpha.0",
"@budibase/frontend-core": "2.4.6-alpha.0",
"@budibase/string-templates": "2.4.6-alpha.0",
"@spectrum-css/button": "^3.0.3",
"@spectrum-css/card": "^3.0.3",
"@spectrum-css/divider": "^1.0.3",

3
packages/client/src/components/app/dynamic-filter/FilterModal.svelte
View file

@ -57,6 +57,9 @@
const onFieldChange = (expression, field) => {
// Update the field type
expression.type = schemaFields.find(x => x.name === field)?.type
expression.externalType = schemaFields.find(
x => x.name === field
)?.externalType
// Ensure a valid operator is set
const validOperators = LuceneUtils.getValidOperatorsForType(

41
packages/client/src/components/app/embedded-map/EmbeddedMap.svelte
View file

@ -283,27 +283,32 @@
if (mapInstance) {
mapInstance.remove()
}
mapInstance = L.map(embeddedMapId, mapOptions)
mapMarkerGroup.addTo(mapInstance)
candidateMarkerGroup.addTo(mapInstance)
// Add attribution
const cleanAttribution = sanitizeHtml(attribution, {
allowedTags: ["a"],
allowedAttributes: {
a: ["href", "target"],
},
})
L.tileLayer(tileURL, {
attribution: "&copy; " + cleanAttribution,
zoom,
}).addTo(mapInstance)
try {
mapInstance = L.map(embeddedMapId, mapOptions)
mapMarkerGroup.addTo(mapInstance)
candidateMarkerGroup.addTo(mapInstance)
// Add click handler
mapInstance.on("click", handleMapClick)
// Add attribution
const cleanAttribution = sanitizeHtml(attribution, {
allowedTags: ["a"],
allowedAttributes: {
a: ["href", "target"],
},
})
L.tileLayer(tileURL, {
attribution: "&copy; " + cleanAttribution,
zoom,
}).addTo(mapInstance)
// Reset view
resetView()
// Add click handler
mapInstance.on("click", handleMapClick)
// Reset view
resetView()
} catch (e) {
console.log("There was a problem with the map", e)
}
}
const handleMapClick = e => {

1
packages/client/src/utils/schema.js
View file

@ -81,6 +81,7 @@ export const getRelationshipSchemaAdditions = async schema => {
Object.keys(linkSchema || {}).forEach(linkKey => {
relationshipAdditions[`${fieldKey}.${linkKey}`] = {
type: linkSchema[linkKey].type,
externalType: linkSchema[linkKey].externalType,
}
})
}

4
packages/frontend-core/package.json
View file

@ -1,12 +1,12 @@
{
"name": "@budibase/frontend-core",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase frontend core libraries used in builder and client",
"author": "Budibase",
"license": "MPL-2.0",
"svelte": "src/index.js",
"dependencies": {
"@budibase/bbui": "2.3.21-alpha.1",
"@budibase/bbui": "2.4.6-alpha.0",
"lodash": "^4.17.21",
"svelte": "^3.46.2"
}

3
packages/frontend-core/src/api/user.js
View file

@ -194,8 +194,7 @@ export const buildUserEndpoints = API => ({
},
/**
* Retrieves the invitation associated with a provided code.
* @param code The unique code for the target invite
* Retrieves all user invitations for the current tenant.
*/
getUserInvites: async () => {
return await API.get({

2
packages/sdk/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/sdk",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase Public API SDK",
"author": "Budibase",
"license": "MPL-2.0",

12
packages/server/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase Web Server",
"main": "src/index.ts",
"repository": {
@ -43,11 +43,11 @@
"license": "GPL-3.0",
"dependencies": {
"@apidevtools/swagger-parser": "10.0.3",
"@budibase/backend-core": "2.3.21-alpha.1",
"@budibase/client": "2.3.21-alpha.1",
"@budibase/pro": "2.3.21-alpha.1",
"@budibase/string-templates": "2.3.21-alpha.1",
"@budibase/types": "2.3.21-alpha.1",
"@budibase/backend-core": "2.4.6-alpha.0",
"@budibase/client": "2.4.6-alpha.0",
"@budibase/pro": "2.4.6-alpha.0",
"@budibase/string-templates": "2.4.6-alpha.0",
"@budibase/types": "2.4.6-alpha.0",
"@bull-board/api": "3.7.0",
"@bull-board/koa": "3.9.4",
"@elastic/elasticsearch": "7.10.0",

4
packages/server/src/integrations/tests/googlesheets.spec.ts
View file

@ -35,6 +35,10 @@ describe("Google Sheets Integration", () => {
let integration: any,
config = new TestConfiguration()
beforeAll(() => {
config.setGoogleAuth("test")
})
beforeEach(async () => {
integration = new GoogleSheetsIntegration.integration({
spreadsheetId: "randomId",

5
packages/server/src/integrations/utils.ts
View file

@ -223,14 +223,15 @@ function shouldCopySpecialColumn(
column: { type: string },
fetchedColumn: { type: string } | undefined
) {
const isFormula = column.type === FieldTypes.FORMULA
const specialTypes = [
FieldTypes.OPTIONS,
FieldTypes.LONGFORM,
FieldTypes.ARRAY,
FieldTypes.FORMULA,
]
// column has been deleted, remove
if (column && !fetchedColumn) {
// column has been deleted, remove - formulas will never exist, always copy
if (!isFormula && column && !fetchedColumn) {
return false
}
const fetchedIsNumber =

7
packages/server/src/tests/utilities/TestConfiguration.ts
View file

@ -181,6 +181,13 @@ class TestConfiguration {
coreEnv._set("SELF_HOSTED", value)
}
setGoogleAuth = (value: string) => {
env._set("GOOGLE_CLIENT_ID", value)
env._set("GOOGLE_CLIENT_SECRET", value)
coreEnv._set("GOOGLE_CLIENT_ID", value)
coreEnv._set("GOOGLE_CLIENT_SECRET", value)
}
modeCloud = () => {
this.setSelfHosted(false)
}

2
packages/server/src/utilities/global.ts
View file

@ -35,8 +35,6 @@ export function updateAppRole(
user.roleId = roles.BUILTIN_ROLE_IDS.ADMIN
} else if (!user.roleId && !user?.userGroups?.length) {
user.roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
} else if (user?.userGroups?.length) {
user.roleId = undefined
}
delete user.roles

30
packages/server/yarn.lock
View file

@ -1278,14 +1278,14 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.3.21-alpha.1":
version "2.3.21-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.21-alpha.1.tgz#151eaee7f30a3c9bf8e5ab3c30bdbb67a34f9b26"
integrity sha512-YyclBgY7YGjlzjI2fNRSj4vJfudXB+1KjZBvZ5EhGKvPZDqhIE+I+7nXRxxFxTtEe0m+p673KjntgREybXPZaQ==
"@budibase/backend-core@2.4.6-alpha.0":
version "2.4.6-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.4.6-alpha.0.tgz#9c44f638621e87aa4cddb034d9333eab71b8ab3b"
integrity sha512-DJY65pqBVd7ndTJa2mz43E33rBkODzESQbUcHYGZAVKCnYhfo5vuHI2/iI9FpfjOPDyVVXyxaUfe6dQj2WHEkw==
dependencies:
"@budibase/nano" "10.1.2"
"@budibase/pouchdb-replication-stream" "1.2.10"
"@budibase/types" "2.3.21-alpha.1"
"@budibase/types" "2.4.6-alpha.0"
"@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2"
aws-cloudfront-sign "2.2.0"
@ -1417,14 +1417,14 @@
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
"@budibase/pro@2.3.21-alpha.1":
version "2.3.21-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.21-alpha.1.tgz#c7ccff06c36b7dafbdaafdfb525c2ca72ebfd050"
integrity sha512-oxOM+0le4SNk2b9jI9sym8qBwi4Uu1soJG68TSdoLluHXzQsXJNrwueO5FY8PS+50Y2HmeIQPCVw2fGWRC7Cjg==
"@budibase/pro@2.4.6-alpha.0":
version "2.4.6-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.4.6-alpha.0.tgz#190f05762a4bd78b9032ebe0e489f00229cae371"
integrity sha512-ED3RxOt5Z7omM9lH1G5ujDEK+j8RLfrul0byxIMk+Qdvglw4gx4gHcsQ+F+8KNgmrcoJNTycqjHxsjV9KfM0gQ==
dependencies:
"@budibase/backend-core" "2.3.21-alpha.1"
"@budibase/backend-core" "2.4.6-alpha.0"
"@budibase/string-templates" "2.3.20"
"@budibase/types" "2.3.21-alpha.1"
"@budibase/types" "2.4.6-alpha.0"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"
@ -1463,10 +1463,10 @@
lodash "^4.17.20"
vm2 "^3.9.4"
"@budibase/types@2.3.21-alpha.1":
version "2.3.21-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.21-alpha.1.tgz#d606f5f8d47ad5e50f80b09ae0434802b810c041"
integrity sha512-55Hk7/s7IV4+u1IjuKwlHKPKutsKbHbNbKPK0FStiZgCMG7/+GPysRWdSY9VgLL9XPHOlQ5dtI6jzuWU7ihH/g==
"@budibase/types@2.4.6-alpha.0":
version "2.4.6-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.6-alpha.0.tgz#774b0c8cd7ad5e00164345a995296090a193b828"
integrity sha512-Y3/dcp6KozrtxIe4QyxWxwdqeQP1sVfZjPS5oii/UbxtT2FTglxRQOwRZ5dD5ADKketvhyML6weJDMJ8ioFzOg==
"@bull-board/api@3.7.0":
version "3.7.0"

2
packages/string-templates/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

2
packages/types/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/types",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase types",
"main": "dist/index.js",
"types": "dist/index.d.ts",

1
packages/types/src/api/web/auth.ts
View file

@ -17,6 +17,7 @@ export interface UpdateSelfRequest {
lastName?: string
password?: string
forceResetPassword?: boolean
onboardedAt?: string
}
export interface UpdateSelfResponse {

5
packages/types/src/sdk/locks.ts
View file

@ -13,6 +13,7 @@ export enum LockName {
TRIGGER_QUOTA = "trigger_quota",
SYNC_ACCOUNT_LICENSE = "sync_account_license",
UPDATE_TENANTS_DOC = "update_tenants_doc",
PERSIST_WRITETHROUGH = "persist_writethrough",
}
export interface LockOptions {
@ -29,9 +30,9 @@ export interface LockOptions {
*/
ttl: number
/**
* The suffix to add to the lock name for additional uniqueness
* The individual resource to lock. This is useful for locking around very specific identifiers, e.g. a document that is prone to conflicts
*/
nameSuffix?: string
resource?: string
/**
* This is a system-wide lock - don't use tenancy in lock key
*/

7
packages/types/src/sdk/user.ts
View file

@ -1,10 +1,3 @@
export interface UpdateSelf {
firstName?: string
lastName?: string
password?: string
forceResetPassword?: boolean
}
export interface SaveUserOpts {
hashPassword?: boolean
requirePassword?: boolean

10
packages/worker/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "2.3.21-alpha.1",
"version": "2.4.6-alpha.0",
"description": "Budibase background service",
"main": "src/index.ts",
"repository": {
@ -36,10 +36,10 @@
"author": "Budibase",
"license": "GPL-3.0",
"dependencies": {
"@budibase/backend-core": "2.3.21-alpha.1",
"@budibase/pro": "2.3.21-alpha.1",
"@budibase/string-templates": "2.3.21-alpha.1",
"@budibase/types": "2.3.21-alpha.1",
"@budibase/backend-core": "2.4.6-alpha.0",
"@budibase/pro": "2.4.6-alpha.0",
"@budibase/string-templates": "2.4.6-alpha.0",
"@budibase/types": "2.4.6-alpha.0",
"@koa/router": "8.0.8",
"@sentry/node": "6.17.7",
"@techpass/passport-openidconnect": "0.3.2",

22
packages/worker/src/api/controllers/global/self.ts
View file

@ -10,12 +10,7 @@ import {
} from "@budibase/backend-core"
import env from "../../../environment"
import { groups } from "@budibase/pro"
import {
UpdateSelfRequest,
UpdateSelfResponse,
UpdateSelf,
UserCtx,
} from "@budibase/types"
import { UpdateSelfRequest, UpdateSelfResponse, UserCtx } from "@budibase/types"
const { getCookie, clearCookie, newid } = utils
function newTestApiKey() {
@ -122,15 +117,14 @@ export async function getSelf(ctx: any) {
export async function updateSelf(
ctx: UserCtx<UpdateSelfRequest, UpdateSelfResponse>
) {
const body = ctx.request.body
const update: UpdateSelf = {
firstName: body.firstName,
lastName: body.lastName,
password: body.password,
forceResetPassword: body.forceResetPassword,
}
const update = ctx.request.body
const user = await userSdk.updateSelf(ctx.user._id!, update)
let user = await userSdk.getUser(ctx.user._id!)
user = {
...user,
...update,
}
user = await userSdk.save(user, { requirePassword: false })
if (update.password) {
// Log all other sessions out apart from the current one

2
packages/worker/src/api/controllers/global/users.ts
View file

@ -341,7 +341,7 @@ export const getUserInvites = async (ctx: any) => {
let invites
try {
// Restricted to the currently authenticated tenant
invites = await getInviteCodes([ctx.user.tenantId])
invites = await getInviteCodes()
} catch (e) {
ctx.throw(400, "There was a problem fetching invites")
}

2
packages/worker/src/api/routes/global/self.ts
View file

@ -11,7 +11,7 @@ router
.get("/api/global/self", controller.getSelf)
.post(
"/api/global/self",
users.buildUserSaveValidation(true),
users.buildSelfSaveValidation(),
controller.updateSelf
)

38
packages/worker/src/api/routes/global/tests/self.spec.ts
View file

@ -18,30 +18,26 @@ describe("/api/global/self", () => {
})
describe("update", () => {
it("should update self", async () => {
it("should reject updates with forbidden keys", async () => {
const user = await config.createUser()
await config.createSession(user)
delete user.password
const res = await config.api.self.updateSelf(user)
const dbUser = await config.getUser(user.email)
user._rev = dbUser._rev
user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
expect(res.body._id).toBe(user._id)
expect(events.user.updated).toBeCalledTimes(1)
expect(events.user.updated).toBeCalledWith(dbUser)
expect(events.user.passwordUpdated).not.toBeCalled()
await config.api.self.updateSelf(user, user).expect(400)
})
it("should update password", async () => {
const user = await config.createUser()
await config.createSession(user)
user.password = "newPassword"
const res = await config.api.self.updateSelf(user)
const res = await config.api.self
.updateSelf(user, {
password: "newPassword",
})
.expect(200)
const dbUser = await config.getUser(user.email)
user._rev = dbUser._rev
user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
expect(res.body._id).toBe(user._id)
@ -51,4 +47,22 @@ describe("/api/global/self", () => {
expect(events.user.passwordUpdated).toBeCalledWith(dbUser)
})
})
it("should update onboarding", async () => {
const user = await config.createUser()
await config.createSession(user)
const res = await config.api.self
.updateSelf(user, {
onboardedAt: "2023-03-07T14:10:54.869Z",
})
.expect(200)
const dbUser = await config.getUser(user.email)
user._rev = dbUser._rev
user.dayPassRecordedAt = mocks.date.MOCK_DATE.toISOString()
expect(dbUser.onboardedAt).toBe("2023-03-07T14:10:54.869Z")
expect(res.body._id).toBe(user._id)
})
})

15
packages/worker/src/api/routes/global/tests/users.spec.ts
View file

@ -4,6 +4,7 @@ jest.mock("nodemailer")
import { TestConfiguration, mocks, structures } from "../../../../tests"
const sendMailMock = mocks.email.mock()
import { events, tenancy, accounts as _accounts } from "@budibase/backend-core"
import * as userSdk from "../../../../sdk/users"
const accounts = jest.mocked(_accounts)
@ -468,6 +469,20 @@ describe("/api/global/users", () => {
config.authHeaders(nonAdmin)
)
})
describe("sso users", () => {
function createSSOUser() {
return config.doInTenant(() => {
const user = structures.users.ssoUser()
return userSdk.save(user, { requirePassword: false })
})
}
it("should be able to update an sso user that has no password", async () => {
const user = await createSSOUser()
await config.api.users.saveUser(user)
})
})
})
describe("POST /api/global/users/bulk (delete)", () => {

2
packages/worker/src/api/routes/global/users.ts
View file

@ -128,7 +128,7 @@ router
.get("/api/global/users/self", selfController.getSelf)
.post(
"/api/global/users/self",
users.buildUserSaveValidation(true),
users.buildUserSaveValidation(),
selfController.updateSelf
)

23
packages/worker/src/api/routes/validation/users.ts
View file

@ -17,13 +17,22 @@ let schema: any = {
roles: Joi.object().pattern(/.*/, Joi.string()).required().unknown(true),
}
export const buildUserSaveValidation = (isSelf = false) => {
if (!isSelf) {
schema = {
...schema,
_id: Joi.string(),
_rev: Joi.string(),
}
export const buildSelfSaveValidation = () => {
schema = {
password: Joi.string().optional(),
forceResetPassword: Joi.boolean().optional(),
firstName: Joi.string().allow("").optional(),
lastName: Joi.string().allow("").optional(),
onboardedAt: Joi.string().optional(),
}
return auth.joiValidator.body(Joi.object(schema).required().unknown(false))
}
export const buildUserSaveValidation = () => {
schema = {
...schema,
_id: Joi.string(),
_rev: Joi.string(),
}
return auth.joiValidator.body(Joi.object(schema).required().unknown(true))
}

15
packages/worker/src/sdk/users/users.ts
View file

@ -29,7 +29,6 @@ import {
PlatformUserByEmail,
RowResponse,
SearchUsersRequest,
UpdateSelf,
User,
SaveUserOpts,
} from "@budibase/types"
@ -132,6 +131,11 @@ const buildUser = async (
): Promise<User> => {
let { password, _id } = user
// don't require a password if the db user doesn't already have one
if (dbUser && !dbUser.password) {
opts.requirePassword = false
}
let hashedPassword
if (password) {
if (await isPreventPasswordActions(user)) {
@ -227,15 +231,6 @@ export async function isPreventPasswordActions(user: User) {
return !!(account && isSSOAccount(account))
}
export async function updateSelf(id: string, data: UpdateSelf) {
let user = await getUser(id)
user = {
...user,
...data,
}
return save(user)
}
export const save = async (
user: User,
opts: SaveUserOpts = {}

5
packages/worker/src/tests/api/self.ts
View file

@ -7,13 +7,12 @@ export class SelfAPI extends TestAPI {
super(config)
}
updateSelf = (user: User) => {
updateSelf = (user: User, update: any) => {
return this.request
.post(`/api/global/self`)
.send(user)
.send(update)
.set(this.config.authHeaders(user))
.expect("Content-Type", /json/)
.expect(200)
}
getSelf = (user: User) => {

21
packages/worker/src/utilities/redis.ts
View file

@ -1,4 +1,5 @@
import { redis, utils } from "@budibase/backend-core"
import { redis, utils, tenancy } from "@budibase/backend-core"
import env from "../environment"
function getExpirySecondsForDB(db: string) {
switch (db) {
@ -129,10 +130,9 @@ export async function checkInviteCode(
}
/**
Get all currently available user invitations.
@return {Object[]} A list of all objects containing invite metadata
Get all currently available user invitations for the current tenant.
**/
export async function getInviteCodes(tenantIds?: string[]) {
export async function getInviteCodes() {
const client = await getClient(redis.utils.Databases.INVITATIONS)
const invites: any[] = await client.scan()
@ -142,12 +142,9 @@ export async function getInviteCodes(tenantIds?: string[]) {
code: invite.key,
}
})
return results.reduce((acc, invite) => {
if (tenantIds?.length && tenantIds.includes(invite.info.tenantId)) {
acc.push(invite)
} else {
acc.push(invite)
}
return acc
}, [])
if (!env.MULTI_TENANCY) {
return results
}
const tenantId = tenancy.getTenantId()
return results.filter(invite => tenantId === invite.info.tenantId)
}

30
packages/worker/yarn.lock
View file

@ -475,14 +475,14 @@
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
"@budibase/backend-core@2.3.21-alpha.1":
version "2.3.21-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.3.21-alpha.1.tgz#151eaee7f30a3c9bf8e5ab3c30bdbb67a34f9b26"
integrity sha512-YyclBgY7YGjlzjI2fNRSj4vJfudXB+1KjZBvZ5EhGKvPZDqhIE+I+7nXRxxFxTtEe0m+p673KjntgREybXPZaQ==
"@budibase/backend-core@2.4.6-alpha.0":
version "2.4.6-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-2.4.6-alpha.0.tgz#9c44f638621e87aa4cddb034d9333eab71b8ab3b"
integrity sha512-DJY65pqBVd7ndTJa2mz43E33rBkODzESQbUcHYGZAVKCnYhfo5vuHI2/iI9FpfjOPDyVVXyxaUfe6dQj2WHEkw==
dependencies:
"@budibase/nano" "10.1.2"
"@budibase/pouchdb-replication-stream" "1.2.10"
"@budibase/types" "2.3.21-alpha.1"
"@budibase/types" "2.4.6-alpha.0"
"@shopify/jest-koa-mocks" "5.0.1"
"@techpass/passport-openidconnect" "0.3.2"
aws-cloudfront-sign "2.2.0"
@ -564,14 +564,14 @@
pouchdb-promise "^6.0.4"
through2 "^2.0.0"
"@budibase/pro@2.3.21-alpha.1":
version "2.3.21-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.3.21-alpha.1.tgz#c7ccff06c36b7dafbdaafdfb525c2ca72ebfd050"
integrity sha512-oxOM+0le4SNk2b9jI9sym8qBwi4Uu1soJG68TSdoLluHXzQsXJNrwueO5FY8PS+50Y2HmeIQPCVw2fGWRC7Cjg==
"@budibase/pro@2.4.6-alpha.0":
version "2.4.6-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-2.4.6-alpha.0.tgz#190f05762a4bd78b9032ebe0e489f00229cae371"
integrity sha512-ED3RxOt5Z7omM9lH1G5ujDEK+j8RLfrul0byxIMk+Qdvglw4gx4gHcsQ+F+8KNgmrcoJNTycqjHxsjV9KfM0gQ==
dependencies:
"@budibase/backend-core" "2.3.21-alpha.1"
"@budibase/backend-core" "2.4.6-alpha.0"
"@budibase/string-templates" "2.3.20"
"@budibase/types" "2.3.21-alpha.1"
"@budibase/types" "2.4.6-alpha.0"
"@koa/router" "8.0.8"
bull "4.10.1"
joi "17.6.0"
@ -592,10 +592,10 @@
lodash "^4.17.20"
vm2 "^3.9.4"
"@budibase/types@2.3.21-alpha.1":
version "2.3.21-alpha.1"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.3.21-alpha.1.tgz#d606f5f8d47ad5e50f80b09ae0434802b810c041"
integrity sha512-55Hk7/s7IV4+u1IjuKwlHKPKutsKbHbNbKPK0FStiZgCMG7/+GPysRWdSY9VgLL9XPHOlQ5dtI6jzuWU7ihH/g==
"@budibase/types@2.4.6-alpha.0":
version "2.4.6-alpha.0"
resolved "https://registry.yarnpkg.com/@budibase/types/-/types-2.4.6-alpha.0.tgz#774b0c8cd7ad5e00164345a995296090a193b828"
integrity sha512-Y3/dcp6KozrtxIe4QyxWxwdqeQP1sVfZjPS5oii/UbxtT2FTglxRQOwRZ5dD5ADKketvhyML6weJDMJ8ioFzOg==
"@cspotcode/source-map-support@^0.8.0":
version "0.8.1"

4
scripts/install-minio.sh
View file

@ -2,9 +2,9 @@
if [[ $TARGETARCH == arm* ]] ;
then
echo "INSTALLING ARM64 MINIO"
wget https://dl.min.io/server/minio/release/linux-arm64/minio
wget https://dl.min.io/server/minio/release/linux-arm64/archive/minio.RELEASE.2022-10-24T18-35-07Z
else
echo "INSTALLING AMD64 MINIO"
wget https://dl.min.io/server/minio/release/linux-amd64/minio
wget https://dl.min.io/server/minio/release/linux-amd64/minio.RELEASE.2022-10-24T18-35-07Z
fi
chmod +x minio