diff --git a/.github/workflows/release-singleimage.yml b/.github/workflows/release-singleimage.yml index 4d35916f4d..a3444d5e7a 100644 --- a/.github/workflows/release-singleimage.yml +++ b/.github/workflows/release-singleimage.yml @@ -66,14 +66,21 @@ jobs: context: . push: true platforms: linux/amd64,linux/arm64 + build-args: BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase,budibase/budibase:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} - name: Tag and release Budibase Azure App Service docker image uses: docker/build-push-action@v2 with: context: . push: true platforms: linux/amd64 - build-args: TARGETBUILD=aas + build-args: | + TARGETBUILD=aas + BUDIBASE_VERSION=$BUDIBASE_VERSION tags: budibase/budibase-aas,budibase/budibase-aas:${{ env.RELEASE_VERSION }} file: ./hosting/single/Dockerfile.v2 + env: + BUDIBASE_VERSION: ${{ env.RELEASE_VERSION }} diff --git a/hosting/scripts/install-minio.sh b/hosting/scripts/install-minio.sh deleted file mode 100755 index 8297593599..0000000000 --- a/hosting/scripts/install-minio.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -if [[ $TARGETARCH == arm* ]] ; -then - echo "INSTALLING ARM64 MINIO" - wget https://dl.min.io/server/minio/release/linux-arm64/minio -else - echo "INSTALLING AMD64 MINIO" - wget https://dl.min.io/server/minio/release/linux-amd64/minio -fi -chmod +x minio diff --git a/hosting/single/Dockerfile.v2 b/hosting/single/Dockerfile.v2 index 5b07a51b27..ec03a1b5a2 100644 --- a/hosting/single/Dockerfile.v2 +++ b/hosting/single/Dockerfile.v2 @@ -42,6 +42,7 @@ COPY packages/string-templates packages/string-templates FROM budibase/couchdb as runner ARG TARGETARCH ENV TARGETARCH $TARGETARCH +ENV NODE_MAJOR 18 #TARGETBUILD can be set to single (for single docker image) or aas (for azure app service) # e.g. docker build --build-arg TARGETBUILD=aas .... ARG TARGETBUILD=single @@ -49,10 +50,10 @@ ENV TARGETBUILD $TARGETBUILD # install base dependencies RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server + apt-get install -y --no-install-recommends software-properties-common nginx uuid-runtime redis-server libaio1 # Install postgres client for pg_dump utils -RUN apt install software-properties-common apt-transport-https gpg -y \ +RUN apt install -y software-properties-common apt-transport-https ca-certificates gnupg \ && curl -fsSl https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor | tee /usr/share/keyrings/postgresql.gpg > /dev/null \ && echo deb [arch=amd64,arm64,ppc64el signed-by=/usr/share/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main | tee /etc/apt/sources.list.d/postgresql.list \ && apt update -y \ @@ -61,10 +62,8 @@ RUN apt install software-properties-common apt-transport-https gpg -y \ # install other dependencies, nodejs, oracle requirements, jdk8, redis, nginx WORKDIR /nodejs -RUN curl -sL https://deb.nodesource.com/setup_18.x -o /tmp/nodesource_setup.sh && \ - bash /tmp/nodesource_setup.sh && \ - apt-get install -y --no-install-recommends libaio1 nodejs && \ - npm install --global yarn pm2 +COPY scripts/install-node.sh ./install.sh +RUN chmod +x install.sh && ./install.sh # setup nginx COPY hosting/single/nginx/nginx.conf /etc/nginx diff --git a/hosting/single/runner.sh b/hosting/single/runner.sh index 9dc7aa25d8..770b23eec1 100644 --- a/hosting/single/runner.sh +++ b/hosting/single/runner.sh @@ -77,7 +77,7 @@ mkdir -p ${DATA_DIR}/minio chown -R couchdb:couchdb ${DATA_DIR}/couch redis-server --requirepass $REDIS_PASSWORD > /dev/stdout 2>&1 & /bbcouch-runner.sh & -/minio/minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & +minio server --console-address ":9001" ${DATA_DIR}/minio > /dev/stdout 2>&1 & /etc/init.d/nginx restart if [[ ! -z "${CUSTOM_DOMAIN}" ]]; then # Add monthly cron job to renew certbot certificate diff --git a/lerna.json b/lerna.json index 6df4a4c4cd..f0f51242d1 100644 --- a/lerna.json +++ b/lerna.json @@ -1,5 +1,5 @@ { - "version": "2.12.1", + "version": "2.12.4", "npmClient": "yarn", "packages": [ "packages/*" diff --git a/packages/backend-core/__mocks__/aws-sdk.ts b/packages/backend-core/__mocks__/aws-sdk.ts index b8d91dbaa9..e3be511d08 100644 --- a/packages/backend-core/__mocks__/aws-sdk.ts +++ b/packages/backend-core/__mocks__/aws-sdk.ts @@ -3,6 +3,7 @@ const mockS3 = { deleteObject: jest.fn().mockReturnThis(), deleteObjects: jest.fn().mockReturnThis(), createBucket: jest.fn().mockReturnThis(), + getObject: jest.fn().mockReturnThis(), listObject: jest.fn().mockReturnThis(), getSignedUrl: jest.fn((operation: string, params: any) => { return `http://s3.example.com/${params.Bucket}/${params.Key}` diff --git a/packages/backend-core/src/index.ts b/packages/backend-core/src/index.ts index ffffd8240a..c7cf9f56cc 100644 --- a/packages/backend-core/src/index.ts +++ b/packages/backend-core/src/index.ts @@ -30,6 +30,7 @@ export * as timers from "./timers" export { default as env } from "./environment" export * as blacklist from "./blacklist" export * as docUpdates from "./docUpdates" +export * from "./utils/Duration" export { SearchParams } from "./db" // Add context to tenancy for backwards compatibility // only do this for external usages to prevent internal diff --git a/packages/backend-core/src/queue/inMemoryQueue.ts b/packages/backend-core/src/queue/inMemoryQueue.ts index af2ec6dbaa..a8add7ecb6 100644 --- a/packages/backend-core/src/queue/inMemoryQueue.ts +++ b/packages/backend-core/src/queue/inMemoryQueue.ts @@ -36,7 +36,7 @@ class InMemoryQueue { * @param opts This is not used by the in memory queue as there is no real use * case when in memory, but is the same API as Bull */ - constructor(name: string, opts = null) { + constructor(name: string, opts?: any) { this._name = name this._opts = opts this._messages = [] diff --git a/packages/backend-core/src/queue/queue.ts b/packages/backend-core/src/queue/queue.ts index 0658147709..c0d1861de3 100644 --- a/packages/backend-core/src/queue/queue.ts +++ b/packages/backend-core/src/queue/queue.ts @@ -2,11 +2,18 @@ import env from "../environment" import { getRedisOptions } from "../redis/utils" import { JobQueue } from "./constants" import InMemoryQueue from "./inMemoryQueue" -import BullQueue from "bull" +import BullQueue, { QueueOptions } from "bull" import { addListeners, StalledFn } from "./listeners" +import { Duration } from "../utils" import * as timers from "../timers" +import * as Redis from "ioredis" -const CLEANUP_PERIOD_MS = 60 * 1000 +// the queue lock is held for 5 minutes +const QUEUE_LOCK_MS = Duration.fromMinutes(5).toMs() +// queue lock is refreshed every 30 seconds +const QUEUE_LOCK_RENEW_INTERNAL_MS = Duration.fromSeconds(30).toMs() +// cleanup the queue every 60 seconds +const CLEANUP_PERIOD_MS = Duration.fromSeconds(60).toMs() let QUEUES: BullQueue.Queue[] | InMemoryQueue[] = [] let cleanupInterval: NodeJS.Timeout @@ -21,7 +28,14 @@ export function createQueue( opts: { removeStalledCb?: StalledFn } = {} ): BullQueue.Queue { const { opts: redisOpts, redisProtocolUrl } = getRedisOptions() - const queueConfig: any = redisProtocolUrl || { redis: redisOpts } + const queueConfig: QueueOptions = { + redis: redisProtocolUrl! || (redisOpts as Redis.RedisOptions), + settings: { + maxStalledCount: 0, + lockDuration: QUEUE_LOCK_MS, + lockRenewTime: QUEUE_LOCK_RENEW_INTERNAL_MS, + }, + } let queue: any if (!env.isTest()) { queue = new BullQueue(jobQueue, queueConfig) diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index ea4a17214d..6dc8750b62 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -249,7 +249,8 @@ export const paginatedUsers = async ({ limit, }: SearchUsersRequest = {}) => { const db = getGlobalDB() - const pageLimit = limit ? limit + 1 : PAGE_LIMIT + 1 + const pageSize = limit ?? PAGE_LIMIT + const pageLimit = pageSize + 1 // get one extra document, to have the next page const opts: DatabaseQueryOpts = { include_docs: true, @@ -276,7 +277,7 @@ export const paginatedUsers = async ({ const response = await db.allDocs(getGlobalUserParams(null, opts)) userList = response.rows.map((row: any) => row.doc) } - return pagination(userList, pageLimit, { + return pagination(userList, pageSize, { paginate: true, property, getKey, diff --git a/packages/backend-core/src/utils/Duration.ts b/packages/backend-core/src/utils/Duration.ts new file mode 100644 index 0000000000..f376c2f7c7 --- /dev/null +++ b/packages/backend-core/src/utils/Duration.ts @@ -0,0 +1,49 @@ +export enum DurationType { + MILLISECONDS = "milliseconds", + SECONDS = "seconds", + MINUTES = "minutes", + HOURS = "hours", + DAYS = "days", +} + +const conversion: Record = { + milliseconds: 1, + seconds: 1000, + minutes: 60 * 1000, + hours: 60 * 60 * 1000, + days: 24 * 60 * 60 * 1000, +} + +export class Duration { + static convert(from: DurationType, to: DurationType, duration: number) { + const milliseconds = duration * conversion[from] + return milliseconds / conversion[to] + } + + static from(from: DurationType, duration: number) { + return { + to: (to: DurationType) => { + return Duration.convert(from, to, duration) + }, + toMs: () => { + return Duration.convert(from, DurationType.MILLISECONDS, duration) + }, + } + } + + static fromSeconds(duration: number) { + return Duration.from(DurationType.SECONDS, duration) + } + + static fromMinutes(duration: number) { + return Duration.from(DurationType.MINUTES, duration) + } + + static fromHours(duration: number) { + return Duration.from(DurationType.HOURS, duration) + } + + static fromDays(duration: number) { + return Duration.from(DurationType.DAYS, duration) + } +} diff --git a/packages/backend-core/src/utils/index.ts b/packages/backend-core/src/utils/index.ts index 318a7f13ba..ac17227459 100644 --- a/packages/backend-core/src/utils/index.ts +++ b/packages/backend-core/src/utils/index.ts @@ -1,3 +1,4 @@ export * from "./hashing" export * from "./utils" export * from "./stringUtils" +export * from "./Duration" diff --git a/packages/backend-core/src/utils/tests/Duration.spec.ts b/packages/backend-core/src/utils/tests/Duration.spec.ts new file mode 100644 index 0000000000..46b996f788 --- /dev/null +++ b/packages/backend-core/src/utils/tests/Duration.spec.ts @@ -0,0 +1,19 @@ +import { Duration, DurationType } from "../Duration" + +describe("duration", () => { + it("should convert minutes to milliseconds", () => { + expect(Duration.fromMinutes(5).toMs()).toBe(300000) + }) + + it("should convert seconds to milliseconds", () => { + expect(Duration.fromSeconds(30).toMs()).toBe(30000) + }) + + it("should convert days to milliseconds", () => { + expect(Duration.fromDays(1).toMs()).toBe(86400000) + }) + + it("should convert minutes to days", () => { + expect(Duration.fromMinutes(1440).to(DurationType.DAYS)).toBe(1) + }) +}) diff --git a/packages/backend-core/tests/core/utilities/mocks/date.ts b/packages/backend-core/tests/core/utilities/mocks/date.ts index f580b68349..1e6d105d93 100644 --- a/packages/backend-core/tests/core/utilities/mocks/date.ts +++ b/packages/backend-core/tests/core/utilities/mocks/date.ts @@ -1,2 +1,3 @@ export const MOCK_DATE = new Date("2020-01-01T00:00:00.000Z") + export const MOCK_DATE_TIMESTAMP = 1577836800000 diff --git a/packages/builder/src/pages/builder/portal/users/users/index.svelte b/packages/builder/src/pages/builder/portal/users/users/index.svelte index e1fc0ca7eb..ff8b749602 100644 --- a/packages/builder/src/pages/builder/portal/users/users/index.svelte +++ b/packages/builder/src/pages/builder/portal/users/users/index.svelte @@ -3,7 +3,6 @@ Heading, Body, Button, - ButtonGroup, Table, Layout, Modal, @@ -46,6 +45,10 @@ datasource: { type: "user", }, + options: { + paginate: true, + limit: 10, + }, }) let groupsLoaded = !$licensing.groupsEnabled || $groups?.length @@ -65,10 +68,12 @@ { column: "role", component: RoleTableRenderer }, ] let userData = [] + let invitesLoaded = false + let pendingInvites = [] + let parsedInvites = [] $: isOwner = $auth.accountPortalAccess && $admin.cloud $: readonly = !sdk.users.isAdmin($auth.user) || $features.isScimEnabled - $: debouncedUpdateFetch(searchEmail) $: schema = { email: { @@ -88,16 +93,6 @@ width: "1fr", }, } - - const getPendingSchema = tblSchema => { - if (!tblSchema) { - return {} - } - let pendingSchema = JSON.parse(JSON.stringify(tblSchema)) - pendingSchema.email.displayName = "Pending Invites" - return pendingSchema - } - $: pendingSchema = getPendingSchema(schema) $: userData = [] $: inviteUsersResponse = { successful: [], unsuccessful: [] } @@ -121,9 +116,15 @@ } }) } - let invitesLoaded = false - let pendingInvites = [] - let parsedInvites = [] + + const getPendingSchema = tblSchema => { + if (!tblSchema) { + return {} + } + let pendingSchema = JSON.parse(JSON.stringify(tblSchema)) + pendingSchema.email.displayName = "Pending Invites" + return pendingSchema + } const invitesToSchema = invites => { return invites.map(invite => { @@ -143,7 +144,9 @@ const updateFetch = email => { fetch.update({ query: { - email, + string: { + email, + }, }, }) } @@ -296,7 +299,7 @@ {/if}
{#if !readonly} - +
- +
{:else} {/if} @@ -390,12 +393,15 @@