Merge branch 'master' of github.com:Budibase/budibase into develop
This commit is contained in:
commit
1ba804f507
20 changed files with 156 additions and 50 deletions
|
@ -75,7 +75,9 @@ services:
|
|||
ports:
|
||||
- "${MAIN_PORT}:10000"
|
||||
container_name: bbproxy
|
||||
image: budibase/proxy
|
||||
image: proxy-service
|
||||
environment:
|
||||
- PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
|
||||
depends_on:
|
||||
- minio-service
|
||||
- worker-service
|
||||
|
|
|
@ -9,7 +9,11 @@ events {
|
|||
}
|
||||
|
||||
http {
|
||||
# rate limiting
|
||||
limit_req_status 429;
|
||||
limit_req_zone $binary_remote_addr zone=ratelimit:10m rate=20r/s;
|
||||
limit_req_zone $binary_remote_addr zone=webhooks:10m rate=${PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND}r/s;
|
||||
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
proxy_set_header Host $host;
|
||||
|
@ -126,6 +130,25 @@ http {
|
|||
proxy_pass http://$apps:4002;
|
||||
}
|
||||
|
||||
location /api/webhooks/ {
|
||||
# calls to webhooks are rate limited
|
||||
limit_req zone=webhooks nodelay;
|
||||
|
||||
# Rest of configuration copied from /api/ location above
|
||||
# 120s timeout on API requests
|
||||
proxy_read_timeout 120s;
|
||||
proxy_connect_timeout 120s;
|
||||
proxy_send_timeout 120s;
|
||||
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Connection $connection_upgrade;
|
||||
proxy_set_header Upgrade $http_upgrade;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
|
||||
proxy_pass http://$apps:4002;
|
||||
}
|
||||
|
||||
location /db/ {
|
||||
proxy_pass http://$couchdb:5984;
|
||||
rewrite ^/db/(.*)$ /$1 break;
|
||||
|
|
|
@ -1,3 +1,13 @@
|
|||
FROM nginx:latest
|
||||
COPY .generated-nginx.prod.conf /etc/nginx/nginx.conf
|
||||
COPY error.html /usr/share/nginx/html/error.html
|
||||
|
||||
# nginx.conf
|
||||
# use the default nginx behaviour for *.template files which are processed with envsubst
|
||||
# override the output dir to output directly to /etc/nginx instead of /etc/nginx/conf.d
|
||||
ENV NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx
|
||||
COPY .generated-nginx.prod.conf /etc/nginx/templates/nginx.conf.template
|
||||
|
||||
# Error handling
|
||||
COPY error.html /usr/share/nginx/html/error.html
|
||||
|
||||
# Default environment
|
||||
ENV PROXY_RATE_LIMIT_WEBHOOKS_PER_SECOND=10
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"npmClient": "yarn",
|
||||
"packages": [
|
||||
"packages/*"
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/backend-core",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Budibase backend core libraries used in server and worker",
|
||||
"main": "dist/src/index.js",
|
||||
"types": "dist/src/index.d.ts",
|
||||
|
@ -20,7 +20,7 @@
|
|||
"test:watch": "jest --watchAll"
|
||||
},
|
||||
"dependencies": {
|
||||
"@budibase/types": "1.2.26-alpha.0",
|
||||
"@budibase/types": "^1.2.27",
|
||||
"@techpass/passport-openidconnect": "0.3.2",
|
||||
"aws-sdk": "2.1030.0",
|
||||
"bcrypt": "5.0.1",
|
||||
|
|
27
packages/backend-core/src/events/emit/BBEventEmitter.ts
Normal file
27
packages/backend-core/src/events/emit/BBEventEmitter.ts
Normal file
|
@ -0,0 +1,27 @@
|
|||
import { EventEmitter } from "events"
|
||||
import * as context from "../../context"
|
||||
import { Identity, Event } from "@budibase/types"
|
||||
|
||||
export interface EmittedEvent {
|
||||
tenantId: string
|
||||
identity: Identity
|
||||
appId: string | undefined
|
||||
properties: any
|
||||
}
|
||||
|
||||
class BBEventEmitter extends EventEmitter {
|
||||
emitEvent(event: Event, properties: any, identity: Identity) {
|
||||
const tenantId = context.getTenantId()
|
||||
const appId = context.getAppId()
|
||||
|
||||
const emittedEvent: EmittedEvent = {
|
||||
tenantId,
|
||||
identity,
|
||||
appId,
|
||||
properties,
|
||||
}
|
||||
this.emit(event, emittedEvent)
|
||||
}
|
||||
}
|
||||
|
||||
export const emitter = new BBEventEmitter()
|
1
packages/backend-core/src/events/emit/index.ts
Normal file
1
packages/backend-core/src/events/emit/index.ts
Normal file
|
@ -0,0 +1 @@
|
|||
export * from "./BBEventEmitter"
|
|
@ -2,6 +2,41 @@ import { Event } from "@budibase/types"
|
|||
import { processors } from "./processors"
|
||||
import * as identification from "./identification"
|
||||
import * as backfill from "./backfill"
|
||||
import { emitter, EmittedEvent } from "./emit"
|
||||
import * as context from "../context"
|
||||
import * as logging from "../logging"
|
||||
|
||||
const USE_EMITTER: any[] = [
|
||||
Event.SERVED_BUILDER,
|
||||
Event.SERVED_APP,
|
||||
Event.SERVED_APP_PREVIEW,
|
||||
]
|
||||
|
||||
for (let event of USE_EMITTER) {
|
||||
emitter.on(event, async (props: EmittedEvent) => {
|
||||
try {
|
||||
await context.doInTenant(props.tenantId, async () => {
|
||||
if (props.appId) {
|
||||
await context.doInAppContext(props.appId, async () => {
|
||||
await processors.processEvent(
|
||||
event as Event,
|
||||
props.identity,
|
||||
props.properties
|
||||
)
|
||||
})
|
||||
} else {
|
||||
await processors.processEvent(
|
||||
event as Event,
|
||||
props.identity,
|
||||
props.properties
|
||||
)
|
||||
}
|
||||
})
|
||||
} catch (e) {
|
||||
logging.logAlert(`Unable to process async event ${event}`, e)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
export const publishEvent = async (
|
||||
event: Event,
|
||||
|
@ -11,6 +46,11 @@ export const publishEvent = async (
|
|||
// in future this should use async events via a distributed queue.
|
||||
const identity = await identification.getCurrentIdentity()
|
||||
|
||||
if (USE_EMITTER.includes(event)) {
|
||||
emitter.emitEvent(event, properties, identity)
|
||||
return
|
||||
}
|
||||
|
||||
const backfilling = await backfill.isBackfillingEvent(event)
|
||||
// no backfill - send the event and exit
|
||||
if (!backfilling) {
|
||||
|
|
|
@ -32,7 +32,7 @@ export default class AnalyticsProcessor implements EventProcessor {
|
|||
return
|
||||
}
|
||||
if (this.posthog) {
|
||||
this.posthog.processEvent(event, identity, properties, timestamp)
|
||||
await this.posthog.processEvent(event, identity, properties, timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -45,14 +45,14 @@ export default class AnalyticsProcessor implements EventProcessor {
|
|||
return
|
||||
}
|
||||
if (this.posthog) {
|
||||
this.posthog.identify(identity, timestamp)
|
||||
await this.posthog.identify(identity, timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
async identifyGroup(group: Group, timestamp?: string | number) {
|
||||
// Group indentifications (tenant and installation) always on
|
||||
if (this.posthog) {
|
||||
this.posthog.identifyGroup(group, timestamp)
|
||||
await this.posthog.identifyGroup(group, timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -45,7 +45,7 @@ export const limited = async (event: Event): Promise<boolean> => {
|
|||
return false
|
||||
}
|
||||
|
||||
const cachedEvent = (await readEvent(event)) as EventProperties
|
||||
const cachedEvent = await readEvent(event)
|
||||
if (cachedEvent) {
|
||||
const timestamp = new Date(cachedEvent.timestamp)
|
||||
const limit = RATE_LIMITS[event]
|
||||
|
@ -76,14 +76,17 @@ export const limited = async (event: Event): Promise<boolean> => {
|
|||
const eventKey = (event: RateLimitedEvent) => {
|
||||
let key = `${CacheKeys.EVENTS_RATE_LIMIT}:${event}`
|
||||
if (isPerApp(event)) {
|
||||
key = key + context.getAppId()
|
||||
key = key + ":" + context.getAppId()
|
||||
}
|
||||
return key
|
||||
}
|
||||
|
||||
const readEvent = async (event: RateLimitedEvent) => {
|
||||
const readEvent = async (
|
||||
event: RateLimitedEvent
|
||||
): Promise<EventProperties | undefined> => {
|
||||
const key = eventKey(event)
|
||||
return cache.get(key)
|
||||
const result = await cache.get(key)
|
||||
return result as EventProperties
|
||||
}
|
||||
|
||||
const recordEvent = async (
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@budibase/bbui",
|
||||
"description": "A UI solution used in the different Budibase projects.",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"license": "MPL-2.0",
|
||||
"svelte": "src/index.js",
|
||||
"module": "dist/bbui.es.js",
|
||||
|
@ -38,7 +38,7 @@
|
|||
],
|
||||
"dependencies": {
|
||||
"@adobe/spectrum-css-workflow-icons": "^1.2.1",
|
||||
"@budibase/string-templates": "1.2.26-alpha.0",
|
||||
"@budibase/string-templates": "^1.2.27",
|
||||
"@spectrum-css/actionbutton": "^1.0.1",
|
||||
"@spectrum-css/actiongroup": "^1.0.1",
|
||||
"@spectrum-css/avatar": "^3.0.2",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/builder",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"license": "GPL-3.0",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
@ -69,10 +69,10 @@
|
|||
}
|
||||
},
|
||||
"dependencies": {
|
||||
"@budibase/bbui": "1.2.26-alpha.0",
|
||||
"@budibase/client": "1.2.26-alpha.0",
|
||||
"@budibase/frontend-core": "1.2.26-alpha.0",
|
||||
"@budibase/string-templates": "1.2.26-alpha.0",
|
||||
"@budibase/bbui": "^1.2.27",
|
||||
"@budibase/client": "^1.2.27",
|
||||
"@budibase/frontend-core": "^1.2.27",
|
||||
"@budibase/string-templates": "^1.2.27",
|
||||
"@sentry/browser": "5.19.1",
|
||||
"@spectrum-css/page": "^3.0.1",
|
||||
"@spectrum-css/vars": "^3.0.1",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/cli",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Budibase CLI, for developers, self hosting and migrations.",
|
||||
"main": "src/index.js",
|
||||
"bin": {
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/client",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"license": "MPL-2.0",
|
||||
"module": "dist/budibase-client.js",
|
||||
"main": "dist/budibase-client.js",
|
||||
|
@ -19,9 +19,9 @@
|
|||
"dev:builder": "rollup -cw"
|
||||
},
|
||||
"dependencies": {
|
||||
"@budibase/bbui": "1.2.26-alpha.0",
|
||||
"@budibase/frontend-core": "1.2.26-alpha.0",
|
||||
"@budibase/string-templates": "1.2.26-alpha.0",
|
||||
"@budibase/bbui": "^1.2.27",
|
||||
"@budibase/frontend-core": "^1.2.27",
|
||||
"@budibase/string-templates": "^1.2.27",
|
||||
"@spectrum-css/button": "^3.0.3",
|
||||
"@spectrum-css/card": "^3.0.3",
|
||||
"@spectrum-css/divider": "^1.0.3",
|
||||
|
|
|
@ -1,12 +1,12 @@
|
|||
{
|
||||
"name": "@budibase/frontend-core",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Budibase frontend core libraries used in builder and client",
|
||||
"author": "Budibase",
|
||||
"license": "MPL-2.0",
|
||||
"svelte": "src/index.js",
|
||||
"dependencies": {
|
||||
"@budibase/bbui": "1.2.26-alpha.0",
|
||||
"@budibase/bbui": "^1.2.27",
|
||||
"lodash": "^4.17.21",
|
||||
"svelte": "^3.46.2"
|
||||
}
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@budibase/server",
|
||||
"email": "hi@budibase.com",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Budibase Web Server",
|
||||
"main": "src/index.ts",
|
||||
"repository": {
|
||||
|
@ -77,11 +77,11 @@
|
|||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"@apidevtools/swagger-parser": "10.0.3",
|
||||
"@budibase/backend-core": "1.2.26-alpha.0",
|
||||
"@budibase/client": "1.2.26-alpha.0",
|
||||
"@budibase/pro": "1.2.26-alpha.0",
|
||||
"@budibase/string-templates": "1.2.26-alpha.0",
|
||||
"@budibase/types": "1.2.26-alpha.0",
|
||||
"@budibase/backend-core": "^1.2.27",
|
||||
"@budibase/client": "^1.2.27",
|
||||
"@budibase/pro": "1.2.27",
|
||||
"@budibase/string-templates": "^1.2.27",
|
||||
"@budibase/types": "^1.2.27",
|
||||
"@bull-board/api": "3.7.0",
|
||||
"@bull-board/koa": "3.9.4",
|
||||
"@elastic/elasticsearch": "7.10.0",
|
||||
|
|
|
@ -18,14 +18,14 @@ const { DocumentTypes, isDevAppID } = require("../../../db/utils")
|
|||
const { getAppDB, getAppId } = require("@budibase/backend-core/context")
|
||||
const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
|
||||
const AWS = require("aws-sdk")
|
||||
const { events } = require("@budibase/backend-core")
|
||||
import { events } from "@budibase/backend-core"
|
||||
|
||||
const fs = require("fs")
|
||||
const {
|
||||
downloadTarballDirect,
|
||||
} = require("../../../utilities/fileSystem/utilities")
|
||||
|
||||
async function prepareUpload({ s3Key, bucket, metadata, file }) {
|
||||
async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
|
||||
const response = await upload({
|
||||
bucket,
|
||||
metadata,
|
||||
|
@ -44,7 +44,7 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
|
|||
}
|
||||
}
|
||||
|
||||
exports.toggleBetaUiFeature = async function (ctx) {
|
||||
export const toggleBetaUiFeature = async function (ctx: any) {
|
||||
const cookieName = `beta:${ctx.params.feature}`
|
||||
|
||||
if (ctx.cookies.get(cookieName)) {
|
||||
|
@ -72,21 +72,21 @@ exports.toggleBetaUiFeature = async function (ctx) {
|
|||
}
|
||||
}
|
||||
|
||||
exports.serveBuilder = async function (ctx) {
|
||||
export const serveBuilder = async function (ctx: any) {
|
||||
const builderPath = resolve(TOP_LEVEL_PATH, "builder")
|
||||
await send(ctx, ctx.file, { root: builderPath })
|
||||
if (!ctx.file.includes("assets/")) {
|
||||
if (ctx.file === "index.html") {
|
||||
await events.serve.servedBuilder()
|
||||
}
|
||||
}
|
||||
|
||||
exports.uploadFile = async function (ctx) {
|
||||
export const uploadFile = async function (ctx: any) {
|
||||
let files =
|
||||
ctx.request.files.file.length > 1
|
||||
? Array.from(ctx.request.files.file)
|
||||
: [ctx.request.files.file]
|
||||
|
||||
const uploads = files.map(async file => {
|
||||
const uploads = files.map(async (file: any) => {
|
||||
const fileExtension = [...file.name.split(".")].pop()
|
||||
// filenames converted to UUIDs so they are unique
|
||||
const processedFileName = `${uuid.v4()}.${fileExtension}`
|
||||
|
@ -101,7 +101,7 @@ exports.uploadFile = async function (ctx) {
|
|||
ctx.body = await Promise.all(uploads)
|
||||
}
|
||||
|
||||
exports.serveApp = async function (ctx) {
|
||||
export const serveApp = async function (ctx: any) {
|
||||
const db = getAppDB({ skip_setup: true })
|
||||
const appInfo = await db.get(DocumentTypes.APP_METADATA)
|
||||
let appId = getAppId()
|
||||
|
@ -134,13 +134,13 @@ exports.serveApp = async function (ctx) {
|
|||
}
|
||||
}
|
||||
|
||||
exports.serveClientLibrary = async function (ctx) {
|
||||
export const serveClientLibrary = async function (ctx: any) {
|
||||
return send(ctx, "budibase-client.js", {
|
||||
root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
|
||||
})
|
||||
}
|
||||
|
||||
exports.getSignedUploadURL = async function (ctx) {
|
||||
export const getSignedUploadURL = async function (ctx: any) {
|
||||
const database = getAppDB()
|
||||
|
||||
// Ensure datasource is valid
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/string-templates",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Handlebars wrapper for Budibase templating.",
|
||||
"main": "src/index.cjs",
|
||||
"module": "dist/bundle.mjs",
|
||||
|
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "@budibase/types",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Budibase types",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"name": "@budibase/worker",
|
||||
"email": "hi@budibase.com",
|
||||
"version": "1.2.26-alpha.0",
|
||||
"version": "1.2.27",
|
||||
"description": "Budibase background service",
|
||||
"main": "src/index.ts",
|
||||
"repository": {
|
||||
|
@ -35,10 +35,10 @@
|
|||
"author": "Budibase",
|
||||
"license": "GPL-3.0",
|
||||
"dependencies": {
|
||||
"@budibase/backend-core": "1.2.26-alpha.0",
|
||||
"@budibase/pro": "1.2.26-alpha.0",
|
||||
"@budibase/string-templates": "1.2.26-alpha.0",
|
||||
"@budibase/types": "1.2.26-alpha.0",
|
||||
"@budibase/backend-core": "^1.2.27",
|
||||
"@budibase/pro": "1.2.27",
|
||||
"@budibase/string-templates": "^1.2.27",
|
||||
"@budibase/types": "^1.2.27",
|
||||
"@koa/router": "8.0.8",
|
||||
"@sentry/node": "6.17.7",
|
||||
"@techpass/passport-openidconnect": "0.3.2",
|
||||
|
|
Loading…
Reference in a new issue