1
0
Fork 0
mirror of synced 2024-06-13 16:05:06 +12:00

Lots of failure handling and caching

This commit is contained in:
Rory Powell 2022-05-30 21:46:08 +01:00
parent f2f6bf779d
commit fd845284d3
39 changed files with 669 additions and 441 deletions

View file

@ -24,7 +24,7 @@
"setup": "node ./hosting/scripts/setup.js && yarn && yarn bootstrap && yarn build && yarn dev",
"bootstrap": "lerna link && lerna bootstrap && ./scripts/link-dependencies.sh",
"build": "lerna run build",
"build:watch": "lerna run build:watch --stream --parallel",
"build:watch": "lerna run build:watch --ignore @budibase/backend-core --stream --parallel",
"release": "lerna publish patch --yes --force-publish && yarn release:pro",
"release:develop": "lerna publish prerelease --yes --force-publish --dist-tag develop && yarn release:pro:develop",
"release:pro": "bash scripts/pro/release.sh",

View file

@ -8,6 +8,7 @@
"license": "GPL-3.0",
"scripts": {
"build": "rimraf dist/ && tsc -p tsconfig.build.json",
"build:watch": "rimraf dist/ && tsc --build tsconfig.build.json --watch --preserveWatchOutput",
"test": "jest",
"test:watch": "jest --watchAll"
},

View file

@ -4,6 +4,9 @@ const { getTenantId } = require("../context")
exports.CacheKeys = {
CHECKLIST: "checklist",
INSTALLATION: "installation",
ANALYTICS_ENABLED: "analyticsEnabled",
UNIQUE_TENANT_ID: "uniqueTenantId",
}
exports.TTL = {
@ -17,8 +20,8 @@ function generateTenantKey(key) {
return `${key}:${tenantId}`
}
exports.withCache = async (key, ttl, fetchFn) => {
key = generateTenantKey(key)
exports.withCache = async (key, ttl, fetchFn, opts = { useTenancy: true }) => {
key = opts.useTenancy ? generateTenantKey(key) : key
const client = await redis.getCacheClient()
const cachedValue = await client.get(key)
if (cachedValue) {

View file

@ -391,6 +391,7 @@ export const getScopedFullConfig = async function (
// defaults
scopedConfig = {
doc: {
_id: generateConfigID({ type, user, workspace }),
config: {
platformUrl: await getPlatformUrl(),
analyticsEnabled: await events.analytics.enabled(),

View file

@ -2,8 +2,8 @@ import env from "../environment"
import * as tenancy from "../tenancy"
import * as dbUtils from "../db/utils"
import { Configs } from "../constants"
import { withCache, TTL, CacheKeys } from "../cache/generic"
// TODO: cache in redis
export const enabled = async () => {
// cloud - always use the environment variable
if (!env.SELF_HOSTED) {
@ -11,13 +11,24 @@ export const enabled = async () => {
}
// self host - prefer the settings doc
// check for explicit true/false values to support
// backwards compatibility where setting may not exist
const settings = await getSettingsDoc()
if (settings?.config?.analyticsEnabled === false) {
return false
} else if (settings?.config?.analyticsEnabled === true) {
return true
// use cache as events have high throughput
const enabledInDB = await withCache(
CacheKeys.ANALYTICS_ENABLED,
TTL.ONE_DAY,
async () => {
const settings = await getSettingsDoc()
// need to do explicit checks in case the field is not set
if (settings?.config?.analyticsEnabled === false) {
return false
} else if (settings?.config?.analyticsEnabled === true) {
return true
}
}
)
if (enabledInDB !== undefined) {
return enabledInDB
}
// fallback to the environment variable

View file

@ -23,6 +23,7 @@ import * as dbUtils from "../db/utils"
import { Configs } from "../constants"
import * as hashing from "../hashing"
import * as installation from "../installation"
import { withCache, TTL, CacheKeys } from "../cache/generic"
const pkg = require("../../package.json")
@ -53,7 +54,7 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
}
} else if (identityType === IdentityType.TENANT) {
const installationId = await getInstallationId()
const tenantId = await getCurrentTenantId()
const tenantId = await getEventTenantId(context.getTenantId())
return {
id: formatDistinctId(tenantId, identityType),
@ -63,7 +64,7 @@ export const getCurrentIdentity = async (): Promise<Identity> => {
}
} else if (identityType === IdentityType.USER) {
const userContext = identityContext as UserContext
const tenantId = await getCurrentTenantId()
const tenantId = await getEventTenantId(context.getTenantId())
let installationId: string | undefined
// self host account users won't have installation
@ -109,7 +110,7 @@ export const identifyTenantGroup = async (
account: Account | undefined,
timestamp?: string | number
): Promise<void> => {
const id = await getGlobalTenantId(tenantId)
const id = await getEventTenantId(tenantId)
const type = IdentityType.TENANT
let hosting: Hosting
@ -144,7 +145,7 @@ export const identifyUser = async (
timestamp?: string | number
) => {
const id = user._id as string
const tenantId = await getGlobalTenantId(user.tenantId)
const tenantId = await getEventTenantId(user.tenantId)
const type = IdentityType.USER
let builder = user.builder?.global || false
let admin = user.admin?.global || false
@ -214,8 +215,6 @@ const getHostingFromEnv = () => {
return env.SELF_HOSTED ? Hosting.SELF : Hosting.CLOUD
}
export const getCurrentTenantId = () => getGlobalTenantId(context.getTenantId())
export const getInstallationId = async () => {
if (isAccountPortal()) {
return "account-portal"
@ -224,31 +223,35 @@ export const getInstallationId = async () => {
return install.installId
}
const getGlobalTenantId = async (tenantId: string): Promise<string> => {
const getEventTenantId = async (tenantId: string): Promise<string> => {
if (env.SELF_HOSTED) {
return getGlobalId(tenantId)
return getUniqueTenantId(tenantId)
} else {
// tenant id's in the cloud are already unique
return tenantId
}
}
// TODO: cache in redis
const getGlobalId = async (tenantId: string): Promise<string> => {
const db = context.getGlobalDB()
const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
type: Configs.SETTINGS,
})
const getUniqueTenantId = async (tenantId: string): Promise<string> => {
// make sure this tenantId always matches the tenantId in context
return context.doInTenant(tenantId, () => {
return withCache(CacheKeys.UNIQUE_TENANT_ID, TTL.ONE_DAY, async () => {
const db = context.getGlobalDB()
const config: SettingsConfig = await dbUtils.getScopedFullConfig(db, {
type: Configs.SETTINGS,
})
let globalId: string
if (config.config.globalId) {
return config.config.globalId
} else {
globalId = `${hashing.newid()}_${tenantId}`
config.config.globalId = globalId
await db.put(config)
return globalId
}
let uniqueTenantId: string
if (config.config.uniqueTenantId) {
return config.config.uniqueTenantId
} else {
uniqueTenantId = `${hashing.newid()}_${tenantId}`
config.config.uniqueTenantId = uniqueTenantId
await db.put(config)
return uniqueTenantId
}
})
})
}
const isAccountPortal = () => {

View file

@ -9,6 +9,7 @@ import {
AutomationStepCreatedEvent,
AutomationStepDeletedEvent,
AutomationTriggerUpdatedEvent,
AutomationsRunEvent,
} from "@budibase/types"
export async function created(automation: Automation, timestamp?: string) {
@ -51,6 +52,13 @@ export async function tested(automation: Automation) {
await publishEvent(Event.AUTOMATION_TESTED, properties)
}
export const run = async (count: number, timestamp?: string | number) => {
const properties: AutomationsRunEvent = {
count,
}
await publishEvent(Event.AUTOMATIONS_RUN, properties, timestamp)
}
export async function stepCreated(
automation: Automation,
step: AutomationStep,

View file

@ -0,0 +1,67 @@
import { publishEvent } from "../events"
import {
Event,
AppBackfillSucceededEvent,
AppBackfillFailedEvent,
TenantBackfillSucceededEvent,
TenantBackfillFailedEvent,
InstallationBackfillSucceededEvent,
InstallationBackfillFailedEvent,
} from "@budibase/types"
const env = require("../../environment")
const shouldSkip = !env.SELF_HOSTED && !env.isDev()
export async function appSucceeded(properties: AppBackfillSucceededEvent) {
if (shouldSkip) {
return
}
await publishEvent(Event.APP_BACKFILL_SUCCEEDED, properties)
}
export async function appFailed(error: any) {
if (shouldSkip) {
return
}
const properties: AppBackfillFailedEvent = {
error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
}
await publishEvent(Event.APP_BACKFILL_FAILED, properties)
}
export async function tenantSucceeded(
properties: TenantBackfillSucceededEvent
) {
if (shouldSkip) {
return
}
await publishEvent(Event.TENANT_BACKFILL_SUCCEEDED, properties)
}
export async function tenantFailed(error: any) {
if (shouldSkip) {
return
}
const properties: TenantBackfillFailedEvent = {
error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
}
await publishEvent(Event.TENANT_BACKFILL_FAILED, properties)
}
export async function installationSucceeded() {
if (shouldSkip) {
return
}
const properties: InstallationBackfillSucceededEvent = {}
await publishEvent(Event.INSTALLATION_BACKFILL_SUCCEEDED, properties)
}
export async function installationFailed(error: any) {
if (shouldSkip) {
return
}
const properties: InstallationBackfillFailedEvent = {
error: JSON.stringify(error, Object.getOwnPropertyNames(error)),
}
await publishEvent(Event.INSTALLATION_BACKFILL_FAILED, properties)
}

View file

@ -16,3 +16,4 @@ export * as serve from "./serve"
export * as user from "./user"
export * as view from "./view"
export * as version from "./version"
export * as backfill from "./backfill"

View file

@ -8,6 +8,7 @@ import {
QueryDeletedEvent,
QueryImportedEvent,
QueryPreviewedEvent,
QueriesRunEvent,
} from "@budibase/types"
/* eslint-disable */
@ -40,6 +41,13 @@ export const imported = async (
await publishEvent(Event.QUERY_IMPORT, properties)
}
export const run = async (count: number, timestamp?: string | number) => {
const properties: QueriesRunEvent = {
count,
}
await publishEvent(Event.QUERIES_RUN, properties, timestamp)
}
export const previewed = async (datasource: Datasource) => {
const properties: QueryPreviewedEvent = {}
await publishEvent(Event.QUERY_PREVIEWED, properties)

View file

@ -5,10 +5,17 @@ import { doWithDB } from "./db"
import { Installation, IdentityType } from "@budibase/types"
import * as context from "./context"
import semver from "semver"
import { bustCache, withCache, TTL, CacheKeys } from "./cache/generic"
const pkg = require("../package.json")
export const getInstall = async (): Promise<Installation> => {
return withCache(CacheKeys.INSTALLATION, TTL.ONE_DAY, getInstallFromDB, {
useTenancy: false,
})
}
const getInstallFromDB = async (): Promise<Installation> => {
return doWithDB(
StaticDatabases.PLATFORM_INFO.name,
async (platformDb: any) => {
@ -44,6 +51,7 @@ const updateVersion = async (version: string): Promise<boolean> => {
const install = await getInstall()
install.version = version
await platformDb.put(install)
await bustCache(CacheKeys.INSTALLATION)
},
{}
)

View file

@ -1,135 +1,106 @@
jest.mock("../../../events", () => {
return {
identification: {
identifyTenantGroup: jest.fn(),
identifyUser: jest.fn(),
},
analytics: {
enabled: () => false,
},
shutdown: () => {},
account: {
created: jest.fn(),
deleted: jest.fn(),
verified: jest.fn(),
},
app: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
published: jest.fn(),
unpublished: jest.fn(),
templateImported: jest.fn(),
fileImported: jest.fn(),
versionUpdated: jest.fn(),
versionReverted: jest.fn(),
reverted: jest.fn(),
exported: jest.fn(),
},
auth: {
login: jest.fn(),
logout: jest.fn(),
SSOCreated: jest.fn(),
SSOUpdated: jest.fn(),
SSOActivated: jest.fn(),
SSODeactivated: jest.fn(),
},
automation: {
created: jest.fn(),
deleted: jest.fn(),
tested: jest.fn(),
// run: jest.fn(),
stepCreated: jest.fn(),
stepDeleted: jest.fn(),
triggerUpdated: jest.fn(),
},
datasource: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
},
email: {
SMTPCreated: jest.fn(),
SMTPUpdated: jest.fn(),
},
layout: {
created: jest.fn(),
deleted: jest.fn(),
},
org: {
nameUpdated: jest.fn(),
logoUpdated: jest.fn(),
platformURLUpdated: jest.fn(),
analyticsOptOut: jest.fn(),
},
version: {
checked: jest.fn(),
},
query: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
imported: jest.fn(),
previewed: jest.fn(),
},
role: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
assigned: jest.fn(),
unassigned: jest.fn(),
},
rows: {
imported: jest.fn(),
created: jest.fn(),
},
screen: {
created: jest.fn(),
deleted: jest.fn(),
},
user: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
permissionAdminAssigned: jest.fn(),
permissionAdminRemoved: jest.fn(),
permissionBuilderAssigned: jest.fn(),
permissionBuilderRemoved: jest.fn(),
invited: jest.fn(),
inviteAccepted: jest.fn(),
passwordForceReset: jest.fn(),
passwordUpdated: jest.fn(),
passwordResetRequested: jest.fn(),
passwordReset: jest.fn(),
},
serve: {
servedBuilder: jest.fn(),
servedApp: jest.fn(),
servedAppPreview: jest.fn(),
},
table: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
exported: jest.fn(),
imported: jest.fn(),
permissionUpdated: jest.fn(),
},
view: {
created: jest.fn(),
updated: jest.fn(),
deleted: jest.fn(),
exported: jest.fn(),
filterCreated: jest.fn(),
filterUpdated: jest.fn(),
filterDeleted: jest.fn(),
calculationCreated: jest.fn(),
calculationUpdated: jest.fn(),
calculationDeleted: jest.fn(),
},
}
})
const events = require("../../../events")
jest.spyOn(events.identification, "identifyTenantGroup")
jest.spyOn(events.identification, "identifyUser")
jest.spyOn(events.account, "created")
jest.spyOn(events.account, "deleted")
jest.spyOn(events.account, "verified")
jest.spyOn(events.app, "created")
jest.spyOn(events.app, "updated")
jest.spyOn(events.app, "deleted")
jest.spyOn(events.app, "published")
jest.spyOn(events.app, "unpublished")
jest.spyOn(events.app, "templateImported")
jest.spyOn(events.app, "fileImported")
jest.spyOn(events.app, "versionUpdated")
jest.spyOn(events.app, "versionReverted")
jest.spyOn(events.app, "reverted")
jest.spyOn(events.app, "exported")
jest.spyOn(events.auth, "login")
jest.spyOn(events.auth, "logout")
jest.spyOn(events.auth, "SSOCreated")
jest.spyOn(events.auth, "SSOUpdated")
jest.spyOn(events.auth, "SSOActivated")
jest.spyOn(events.auth, "SSODeactivated")
jest.spyOn(events.automation, "created")
jest.spyOn(events.automation, "deleted")
jest.spyOn(events.automation, "tested")
jest.spyOn(events.automation, "stepCreated")
jest.spyOn(events.automation, "stepDeleted")
jest.spyOn(events.automation, "triggerUpdated")
jest.spyOn(events.datasource, "created")
jest.spyOn(events.datasource, "updated")
jest.spyOn(events.datasource, "deleted")
jest.spyOn(events.email, "SMTPCreated")
jest.spyOn(events.email, "SMTPUpdated")
jest.spyOn(events.layout, "created")
jest.spyOn(events.layout, "deleted")
jest.spyOn(events.org, "nameUpdated")
jest.spyOn(events.org, "logoUpdated")
jest.spyOn(events.org, "platformURLUpdated")
jest.spyOn(events.org, "analyticsOptOut")
jest.spyOn(events.version, "checked")
jest.spyOn(events.query, "created")
jest.spyOn(events.query, "updated")
jest.spyOn(events.query, "deleted")
jest.spyOn(events.query, "imported")
jest.spyOn(events.query, "previewed")
jest.spyOn(events.role, "created")
jest.spyOn(events.role, "updated")
jest.spyOn(events.role, "deleted")
jest.spyOn(events.role, "assigned")
jest.spyOn(events.role, "unassigned")
jest.spyOn(events.rows, "imported")
jest.spyOn(events.rows, "created")
jest.spyOn(events.screen, "created")
jest.spyOn(events.screen, "deleted")
jest.spyOn(events.user, "created")
jest.spyOn(events.user, "updated")
jest.spyOn(events.user, "deleted")
jest.spyOn(events.user, "permissionAdminAssigned")
jest.spyOn(events.user, "permissionAdminRemoved")
jest.spyOn(events.user, "permissionBuilderAssigned")
jest.spyOn(events.user, "permissionBuilderRemoved")
jest.spyOn(events.user, "invited")
jest.spyOn(events.user, "inviteAccepted")
jest.spyOn(events.user, "passwordForceReset")
jest.spyOn(events.user, "passwordUpdated")
jest.spyOn(events.user, "passwordResetRequested")
jest.spyOn(events.user, "passwordReset")
jest.spyOn(events.serve, "servedBuilder")
jest.spyOn(events.serve, "servedApp")
jest.spyOn(events.serve, "servedAppPreview")
jest.spyOn(events.table, "created")
jest.spyOn(events.table, "updated")
jest.spyOn(events.table, "deleted")
jest.spyOn(events.table, "exported")
jest.spyOn(events.table, "imported")
jest.spyOn(events.view, "created")
jest.spyOn(events.view, "updated")
jest.spyOn(events.view, "deleted")
jest.spyOn(events.view, "exported")
jest.spyOn(events.view, "filterCreated")
jest.spyOn(events.view, "filterUpdated")
jest.spyOn(events.view, "filterDeleted")
jest.spyOn(events.view, "calculationCreated")
jest.spyOn(events.view, "calculationUpdated")
jest.spyOn(events.view, "calculationDeleted")
module.exports = events

View file

@ -1,6 +1,9 @@
{
// Used for building with tsc
"extends": "./tsconfig.json",
"references": [
{ "path": "../types" }
],
"exclude": [
"node_modules",
"dist/**/*",

View file

@ -895,6 +895,13 @@
resolved "https://registry.yarnpkg.com/@types/http-errors/-/http-errors-1.8.2.tgz#7315b4c4c54f82d13fa61c228ec5c2ea5cc9e0e1"
integrity sha512-EqX+YQxINb+MeXaIqYDASb6U6FCHbWjkj4a1CKDBks3d/QiB2+PqBLyO72vLDgAO1wUI4O+9gweRcQK11bTL/w==
"@types/ioredis@^4.27.1":
version "4.28.10"
resolved "https://registry.yarnpkg.com/@types/ioredis/-/ioredis-4.28.10.tgz#40ceb157a4141088d1394bb87c98ed09a75a06ff"
integrity sha512-69LyhUgrXdgcNDv7ogs1qXZomnfOEnSmrmMFqKgt1XMJxmoOSG/u3wYy13yACIfKuMJ8IhKgHafDO3sx19zVQQ==
dependencies:
"@types/node" "*"
"@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1":
version "2.0.3"
resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.3.tgz#4ba8ddb720221f432e443bd5f9117fd22cfd4762"
@ -1092,11 +1099,6 @@ acorn-walk@^7.1.1:
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc"
integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==
acorn@^5.2.1:
version "5.7.4"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.7.4.tgz#3e8d8a9947d0599a1796d10225d7432f4a4acf5e"
integrity sha512-1D++VG7BhrtvQpNbBzovKNc1FLGGEE/oGe7b9xJm/RFHMBeUaUGpluV9RLjZa47YFdPcDAenEYuq9pQPcMdLJg==
acorn@^7.1.1:
version "7.4.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
@ -1132,11 +1134,6 @@ ajv@^6.12.3:
json-schema-traverse "^0.4.1"
uri-js "^4.2.2"
amdefine@>=0.0.4:
version "1.0.1"
resolved "https://registry.yarnpkg.com/amdefine/-/amdefine-1.0.1.tgz#4a5282ac164729e93619bcfd3ad151f817ce91f5"
integrity sha1-SlKCrBZHKek2Gbz9OtFR+BfOkfU=
ansi-align@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/ansi-align/-/ansi-align-3.0.1.tgz#0cdf12e111ace773a86e9a1fad1225c43cb19a59"
@ -1228,11 +1225,6 @@ assert-plus@^0.2.0:
resolved "https://registry.yarnpkg.com/assert-plus/-/assert-plus-0.2.0.tgz#d74e1b87e7affc0db8aadb7021f3fe48101ab234"
integrity sha1-104bh+ev/A24qttwIfP+SBAasjQ=
ast-types@0.9.6:
version "0.9.6"
resolved "https://registry.yarnpkg.com/ast-types/-/ast-types-0.9.6.tgz#102c9e9e9005d3e7e3829bf0c4fa24ee862ee9b9"
integrity sha1-ECyenpAF0+fjgpvwxPok7oYu6bk=
async@~2.1.4:
version "2.1.5"
resolved "https://registry.yarnpkg.com/async/-/async-2.1.5.tgz#e587c68580994ac67fc56ff86d3ac56bdbe810bc"
@ -1361,11 +1353,6 @@ balanced-match@^1.0.0:
resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee"
integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==
base62@^1.1.0:
version "1.2.8"
resolved "https://registry.yarnpkg.com/base62/-/base62-1.2.8.tgz#1264cb0fb848d875792877479dbe8bae6bae3428"
integrity sha512-V6YHUbjLxN1ymqNLb1DPHoU1CpfdL7d2YTIp5W3U4hhoG4hhxNmsFDs66M9EXxBiSEke5Bt5dwdfMwwZF70iLA==
base64-js@^1.0.2, base64-js@^1.3.1:
version "1.5.1"
resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a"
@ -1721,26 +1708,6 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6:
dependencies:
delayed-stream "~1.0.0"
commander@^2.5.0:
version "2.20.3"
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
commoner@^0.10.1:
version "0.10.8"
resolved "https://registry.yarnpkg.com/commoner/-/commoner-0.10.8.tgz#34fc3672cd24393e8bb47e70caa0293811f4f2c5"
integrity sha1-NPw2cs0kOT6LtH5wyqApOBH08sU=
dependencies:
commander "^2.5.0"
detective "^4.3.1"
glob "^5.0.15"
graceful-fs "^4.1.2"
iconv-lite "^0.4.5"
mkdirp "^0.5.0"
private "^0.1.6"
q "^1.1.2"
recast "^0.11.17"
component-type@^1.2.1:
version "1.2.1"
resolved "https://registry.yarnpkg.com/component-type/-/component-type-1.2.1.tgz#8a47901700238e4fc32269771230226f24b415a9"
@ -1945,11 +1912,6 @@ deferred-leveldown@~5.3.0:
abstract-leveldown "~6.2.1"
inherits "^2.0.3"
defined@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.0.tgz#c98d9bcef75674188e110969151199e39b1fa693"
integrity sha1-yY2bzvdWdBiOEQlpFRGZ45sfppM=
delayed-stream@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619"
@ -1990,14 +1952,6 @@ detect-newline@^3.0.0:
resolved "https://registry.yarnpkg.com/detect-newline/-/detect-newline-3.1.0.tgz#576f5dfc63ae1a192ff192d8ad3af6308991b651"
integrity sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==
detective@^4.3.1:
version "4.7.1"
resolved "https://registry.yarnpkg.com/detective/-/detective-4.7.1.tgz#0eca7314338442febb6d65da54c10bb1c82b246e"
integrity sha512-H6PmeeUcZloWtdt4DAkFyzFL94arpHr3NOwwmVILFiy+9Qd4JTxxXrzfyGk/lmct2qVGBwTSwSXagqu2BxmWig==
dependencies:
acorn "^5.2.1"
defined "^1.0.0"
diff-sequences@^27.5.1:
version "27.5.1"
resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327"
@ -2134,15 +2088,6 @@ error-inject@^1.0.0:
resolved "https://registry.yarnpkg.com/error-inject/-/error-inject-1.0.0.tgz#e2b3d91b54aed672f309d950d154850fa11d4f37"
integrity sha1-4rPZG1Su1nLzCdlQ0VSFD6EdTzc=
es3ify@^0.2.2:
version "0.2.2"
resolved "https://registry.yarnpkg.com/es3ify/-/es3ify-0.2.2.tgz#5dae3e650e5be3684b88066513d528d092629862"
integrity sha1-Xa4+ZQ5b42hLiAZlE9Uo0JJimGI=
dependencies:
esprima "^2.7.1"
jstransform "~11.0.0"
through "~2.3.4"
escalade@^3.1.1:
version "3.1.1"
resolved "https://registry.yarnpkg.com/escalade/-/escalade-3.1.1.tgz#d8cfdc7000965c5a0174b4a82eaa5c0552742e40"
@ -2180,26 +2125,11 @@ escodegen@^2.0.0:
optionalDependencies:
source-map "~0.6.1"
esprima-fb@^15001.1.0-dev-harmony-fb:
version "15001.1.0-dev-harmony-fb"
resolved "https://registry.yarnpkg.com/esprima-fb/-/esprima-fb-15001.1.0-dev-harmony-fb.tgz#30a947303c6b8d5e955bee2b99b1d233206a6901"
integrity sha512-59dDGQo2b3M/JfKIws0/z8dcXH2mnVHkfSPRhCYS91JNGfGNwr7GsSF6qzWZuOGvw5Ii0w9TtylrX07MGmlOoQ==
esprima@^2.7.1:
version "2.7.3"
resolved "https://registry.yarnpkg.com/esprima/-/esprima-2.7.3.tgz#96e3b70d5779f6ad49cd032673d1c312767ba581"
integrity sha512-OarPfz0lFCiW4/AV2Oy1Rp9qu0iusTKqykwTspGCZtPxmF81JR4MmIebvF1F9+UOKth2ZubLQ4XGGaU+hSn99A==
esprima@^4.0.0, esprima@^4.0.1:
version "4.0.1"
resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71"
integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==
esprima@~3.1.0:
version "3.1.3"
resolved "https://registry.yarnpkg.com/esprima/-/esprima-3.1.3.tgz#fdca51cee6133895e3c88d535ce49dbff62a4633"
integrity sha512-AWwVMNxwhN8+NIPQzAQZCm7RkLC4RbM3B1OobMuyp3i+w73X57KCKaVIxaRZb+DYCojq7rspo+fmuQfAboyhFg==
estraverse@^5.2.0:
version "5.3.0"
resolved "https://registry.yarnpkg.com/estraverse/-/estraverse-5.3.0.tgz#2eea5290702f26ab8fe5370370ff86c965d21123"
@ -2467,17 +2397,6 @@ glob-parent@~5.1.2:
dependencies:
is-glob "^4.0.1"
glob@^5.0.15:
version "5.0.15"
resolved "https://registry.yarnpkg.com/glob/-/glob-5.0.15.tgz#1bc936b9e02f4a603fcc222ecf7633d30b8b93b1"
integrity sha1-G8k2ueAvSmA/zCIuz3Yz0wuLk7E=
dependencies:
inflight "^1.0.4"
inherits "2"
minimatch "2 || 3"
once "^1.3.0"
path-is-absolute "^1.0.0"
glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.1.4:
version "7.2.0"
resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.0.tgz#d15535af7732e02e948f4c41628bd910293f6023"
@ -2716,7 +2635,7 @@ human-signals@^2.1.0:
resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0"
integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw==
iconv-lite@0.4.24, iconv-lite@^0.4.5:
iconv-lite@0.4.24:
version "0.4.24"
resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
integrity sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==
@ -3550,17 +3469,6 @@ jsprim@^1.2.2:
json-schema "0.2.3"
verror "1.3.6"
jstransform@~11.0.0:
version "11.0.3"
resolved "https://registry.yarnpkg.com/jstransform/-/jstransform-11.0.3.tgz#09a78993e0ae4d4ef4487f6155a91f6190cb4223"
integrity sha1-CaeJk+CuTU70SH9hVakfYZDLQiM=
dependencies:
base62 "^1.1.0"
commoner "^0.10.1"
esprima-fb "^15001.1.0-dev-harmony-fb"
object-assign "^2.0.0"
source-map "^0.4.2"
jwa@^1.1.4:
version "1.1.5"
resolved "https://registry.yarnpkg.com/jwa/-/jwa-1.1.5.tgz#a0552ce0220742cd52e153774a32905c30e756e5"
@ -4069,13 +3977,6 @@ mimic-response@^1.0.0, mimic-response@^1.0.1:
resolved "https://registry.yarnpkg.com/mimic-response/-/mimic-response-1.0.1.tgz#4923538878eef42063cb8a3e3b0798781487ab1b"
integrity sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==
"minimatch@2 || 3":
version "3.1.2"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.1.2.tgz#19cd194bfd3e428f049a70817c038d89ab4be35b"
integrity sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==
dependencies:
brace-expansion "^1.1.7"
minimatch@^3.0.4:
version "3.0.4"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
@ -4083,7 +3984,7 @@ minimatch@^3.0.4:
dependencies:
brace-expansion "^1.1.7"
minimist@^1.2.0, minimist@^1.2.5, minimist@^1.2.6:
minimist@^1.2.0, minimist@^1.2.5:
version "1.2.6"
resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.6.tgz#8637a5b759ea0d6e98702cfb3a9283323c93af44"
integrity sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==
@ -4108,13 +4009,6 @@ mkdirp-classic@^0.5.2:
resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113"
integrity sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==
mkdirp@^0.5.0:
version "0.5.6"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.6.tgz#7def03d2432dcae4ba1d611445c48396062255f6"
integrity sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==
dependencies:
minimist "^1.2.6"
mkdirp@^1.0.3:
version "1.0.4"
resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-1.0.4.tgz#3eb5ed62622756d79a5f0e2a221dfebad75c2f7e"
@ -4295,11 +4189,6 @@ oauth@0.9.x, oauth@^0.9.15:
resolved "https://registry.yarnpkg.com/oauth/-/oauth-0.9.15.tgz#bd1fefaf686c96b75475aed5196412ff60cfb9c1"
integrity sha1-vR/vr2hslrdUda7VGWQS/2DPucE=
object-assign@^2.0.0:
version "2.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-2.1.1.tgz#43c36e5d569ff8e4816c4efa8be02d26967c18aa"
integrity sha1-Q8NuXVaf+OSBbE76i+AtJpZ8GKo=
object-assign@^4.1.1:
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
@ -4624,17 +4513,6 @@ pouchdb-adapter-utils@7.2.2:
pouchdb-merge "7.2.2"
pouchdb-utils "7.2.2"
pouchdb-all-dbs@^1.0.2:
version "1.1.1"
resolved "https://registry.yarnpkg.com/pouchdb-all-dbs/-/pouchdb-all-dbs-1.1.1.tgz#85f04a39cafda52497ec49abf1c93bb5e72813f6"
integrity sha512-UUnsdmcnRSQ8MAOYSJjfTwKkQNb/6fvOfd/f7dNNivWZ2YDYVuMfgw1WQdL634yEtcXTxAENZ/EyLRdzPCB41A==
dependencies:
argsarray "0.0.1"
es3ify "^0.2.2"
inherits "~2.0.1"
pouchdb-promise "6.4.3"
tiny-queue "^0.2.0"
pouchdb-binary-utils@7.2.2:
version "7.2.2"
resolved "https://registry.yarnpkg.com/pouchdb-binary-utils/-/pouchdb-binary-utils-7.2.2.tgz#0690b348052c543b1e67f032f47092ca82bcb10e"
@ -4711,7 +4589,7 @@ pouchdb-merge@7.2.2:
resolved "https://registry.yarnpkg.com/pouchdb-merge/-/pouchdb-merge-7.2.2.tgz#940d85a2b532d6a93a6cab4b250f5648511bcc16"
integrity sha512-6yzKJfjIchBaS7Tusuk8280WJdESzFfQ0sb4jeMUNnrqs4Cx3b0DIEOYTRRD9EJDM+je7D3AZZ4AT0tFw8gb4A==
pouchdb-promise@6.4.3, pouchdb-promise@^6.0.4:
pouchdb-promise@^6.0.4:
version "6.4.3"
resolved "https://registry.yarnpkg.com/pouchdb-promise/-/pouchdb-promise-6.4.3.tgz#74516f4acf74957b54debd0fb2c0e5b5a68ca7b3"
integrity sha512-ruJaSFXwzsxRHQfwNHjQfsj58LBOY1RzGzde4PM5CWINZwFjCQAhZwfMrch2o/0oZT6d+Xtt0HTWhq35p3b0qw==
@ -4798,11 +4676,6 @@ pretty-format@^27.0.0, pretty-format@^27.5.1:
ansi-styles "^5.0.0"
react-is "^17.0.1"
private@^0.1.6, private@~0.1.5:
version "0.1.8"
resolved "https://registry.yarnpkg.com/private/-/private-0.1.8.tgz#2381edb3689f7a53d653190060fcf822d2f368ff"
integrity sha512-VvivMrbvd2nKkiG38qjULzlc+4Vx4wm/whI9pQD35YrARNnhxeiRktSOhSukRLFNlzg6Br/cJPet5J/u19r/mg==
process-nextick-args@~2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
@ -4861,11 +4734,6 @@ pupa@^2.1.1:
dependencies:
escape-goat "^2.0.0"
q@^1.1.2:
version "1.5.1"
resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=
qs@~6.4.0:
version "6.4.0"
resolved "https://registry.yarnpkg.com/qs/-/qs-6.4.0.tgz#13e26d28ad6b0ffaa91312cd3bf708ed351e7233"
@ -4950,16 +4818,6 @@ readline-sync@^1.4.9:
resolved "https://registry.yarnpkg.com/readline-sync/-/readline-sync-1.4.10.tgz#41df7fbb4b6312d673011594145705bf56d8873b"
integrity sha512-gNva8/6UAe8QYepIQH/jQ2qn91Qj0B9sYjMBBs3QOB8F2CXcKgLxQaJRP76sWVRQt+QU+8fAkCbCvjjMFu7Ycw==
recast@^0.11.17:
version "0.11.23"
resolved "https://registry.yarnpkg.com/recast/-/recast-0.11.23.tgz#451fd3004ab1e4df9b4e4b66376b2a21912462d3"
integrity sha1-RR/TAEqx5N+bTktmN2sqIZEkYtM=
dependencies:
ast-types "0.9.6"
esprima "~3.1.0"
private "~0.1.5"
source-map "~0.5.0"
redis-commands@1.7.0:
version "1.7.0"
resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89"
@ -5230,14 +5088,7 @@ source-map-support@^0.5.6:
buffer-from "^1.0.0"
source-map "^0.6.0"
source-map@^0.4.2:
version "0.4.4"
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.4.4.tgz#eba4f5da9c0dc999de68032d8b4f76173652036b"
integrity sha1-66T12pwNyZneaAMti092FzZSA2s=
dependencies:
amdefine ">=0.0.4"
source-map@^0.5.0, source-map@~0.5.0:
source-map@^0.5.0:
version "0.5.7"
resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc"
integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=
@ -5510,21 +5361,11 @@ through2@^2.0.0, through2@^2.0.2, through2@^2.0.3:
readable-stream "~2.3.6"
xtend "~4.0.1"
through@~2.3.4:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"
integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=
timekeeper@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/timekeeper/-/timekeeper-2.2.0.tgz#9645731fce9e3280a18614a57a9d1b72af3ca368"
integrity sha512-W3AmPTJWZkRwu+iSNxPIsLZ2ByADsOLbbLxe46UJyWj3mlYLlwucKiq+/dPm0l9wTzqoF3/2PH0AGFCebjq23A==
tiny-queue@^0.2.0:
version "0.2.1"
resolved "https://registry.yarnpkg.com/tiny-queue/-/tiny-queue-0.2.1.tgz#25a67f2c6e253b2ca941977b5ef7442ef97a6046"
integrity sha1-JaZ/LG4lOyypQZd7XvdELvl6YEY=
tmp@^0.0.33:
version "0.0.33"
resolved "https://registry.yarnpkg.com/tmp/-/tmp-0.0.33.tgz#6d34335889768d21b2bcda0aa277ced3b1bfadf9"

View file

@ -1,11 +1,5 @@
const { tmpdir } = require("os")
const env = require("../src/environment")
const { mocks } = require("@budibase/backend-core/testUtils")
// mock all dates to 2020-01-01T00:00:00.000Z
// use tk.reset() to use real dates in individual tests
const tk = require("timekeeper")
tk.freeze(mocks.date.MOCK_DATE)
env._set("SELF_HOSTED", "1")
env._set("NODE_ENV", "jest")
@ -15,4 +9,11 @@ env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
env._set("LOG_LEVEL", "silent")
env._set("PORT", 0)
const { mocks } = require("@budibase/backend-core/testUtils")
// mock all dates to 2020-01-01T00:00:00.000Z
// use tk.reset() to use real dates in individual tests
const tk = require("timekeeper")
tk.freeze(mocks.date.MOCK_DATE)
global.console.log = jest.fn() // console.log are ignored in tests

View file

@ -71,9 +71,9 @@ exports.create = async function (ctx) {
newAuto: automation,
})
const response = await db.put(automation)
await events.automation.created()
await events.automation.created(automation)
for (let step of automation.definition.steps) {
await events.automation.stepCreated(step)
await events.automation.stepCreated(automation, step)
}
automation._rev = response.rev

View file

@ -144,7 +144,7 @@ exports.save = async function (ctx) {
}
const dbResp = await db.put(datasource)
await events.datasource.created()
await events.datasource.created(datasource)
datasource._rev = dbResp.rev
// Drain connection pools when configuration is changed

View file

@ -53,9 +53,11 @@ describe("Rest Importer", () => {
}
const runTest = async (test, assertions) => {
for (let [key, data] of Object.entries(datasets)) {
await test(key, data, assertions)
}
config.doInContext(config.appId, async () => {
for (let [key, data] of Object.entries(datasets)) {
await test(key, data, assertions)
}
})
}
const testGetInfo = async (key, data, assertions) => {

View file

@ -83,6 +83,22 @@ module.exports = server.listen(env.PORT || 0, async () => {
eventEmitter.emitPort(env.PORT)
fileSystem.init()
await redis.init()
// run migrations on startup if not done via http
// not recommended in a clustered environment
if (!env.HTTP_MIGRATIONS && !env.isTest()) {
try {
await migrations.migrate()
} catch (e) {
console.error("Error performing migrations. Exiting.\n", e)
shutdown()
}
}
// check for version updates
await installation.checkInstallVersion()
// done last - this will never complete
await automations.init()
})
@ -99,15 +115,3 @@ process.on("uncaughtException", err => {
process.on("SIGTERM", () => {
shutdown()
})
// run migrations on startup if not done via http
// not recommended in a clustered environment
if (!env.HTTP_MIGRATIONS && !env.isTest()) {
migrations.migrate().catch(err => {
console.error("Error performing migrations. Exiting.\n", err)
shutdown()
})
}
// check for version updates
installation.checkInstallVersion()

View file

@ -6,8 +6,21 @@ import * as roles from "./app/roles"
import * as tables from "./app/tables"
import * as screens from "./app/screens"
import * as global from "./global"
import { App } from "@budibase/types"
import { App, AppBackfillSucceededEvent } from "@budibase/types"
import { db as dbUtils, events } from "@budibase/backend-core"
import env from "../../../environment"
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
/**
* Date:
@ -18,28 +31,92 @@ import { db as dbUtils, events } from "@budibase/backend-core"
*/
export const run = async (appDb: any) => {
if (await global.isComplete()) {
// make sure new apps aren't backfilled
// return if the global migration for this tenant is complete
// which runs after the app migrations
return
}
try {
if (await global.isComplete()) {
// make sure new apps aren't backfilled
// return if the global migration for this tenant is complete
// which runs after the app migrations
return
}
const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA)
const timestamp = app.createdAt as string
const app: App = await appDb.get(dbUtils.DocumentTypes.APP_METADATA)
const timestamp = app.createdAt as string
if (dbUtils.isDevAppID(app.appId)) {
await events.app.created(app, timestamp)
await automations.backfill(appDb, timestamp)
await datasources.backfill(appDb, timestamp)
await layouts.backfill(appDb, timestamp)
await queries.backfill(appDb, timestamp)
await roles.backfill(appDb, timestamp)
await tables.backfill(appDb, timestamp)
await screens.backfill(appDb, timestamp)
}
if (dbUtils.isProdAppID(app.appId)) {
await events.app.published(app, timestamp)
}
if (dbUtils.isProdAppID(app.appId)) {
await events.app.published(app, timestamp)
const totals: any = {}
const errors: any = []
if (dbUtils.isDevAppID(app.appId)) {
await events.app.created(app, timestamp)
try {
totals.automations = await automations.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.datasources = await datasources.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.layouts = await layouts.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.queries = await queries.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.roles = await roles.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.screens = await screens.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.tables = await tables.backfill(appDb, timestamp)
} catch (e) {
handleError(e, errors)
}
}
const properties: AppBackfillSucceededEvent = {
appId: app.appId,
automations: totals.automations,
datasources: totals.datasources,
layouts: totals.layouts,
queries: totals.queries,
roles: totals.roles,
tables: totals.tables,
screens: totals.screens,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.appSucceeded(properties)
} catch (e) {
handleError(e)
await events.backfill.appFailed(e)
}
}

View file

@ -21,4 +21,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
await events.automation.stepCreated(automation, step, timestamp)
}
}
return automations.length
}

View file

@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
for (const datasource of datasources) {
await events.datasource.created(datasource, timestamp)
}
return datasources.length
}

View file

@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
for (const layout of layouts) {
await events.layout.created(layout, timestamp)
}
return layouts.length
}

View file

@ -28,4 +28,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
)
await events.query.created(datasource, query, timestamp)
}
return queries.length
}

View file

@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
for (const role of roles) {
await events.role.created(role, timestamp)
}
return roles.length
}

View file

@ -17,4 +17,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
for (const screen of screens) {
await events.screen.created(screen, timestamp)
}
return screens.length
}

View file

@ -31,4 +31,6 @@ export const backfill = async (appDb: any, timestamp: string) => {
}
}
}
return tables.length
}

View file

@ -1,10 +1,62 @@
import { TenantBackfillSucceededEvent } from "./../../../../../types/src/events/backfill"
import * as users from "./global/users"
import * as rows from "./global/rows"
import * as configs from "./global/configs"
import { tenancy, events, migrations, accounts } from "@budibase/backend-core"
import { CloudAccount } from "@budibase/types"
import * as quotas from "./global/quotas"
import {
tenancy,
events,
migrations,
accounts,
db as dbUtils,
} from "@budibase/backend-core"
import { QuotaUsage } from "@budibase/pro"
import { CloudAccount, App } from "@budibase/types"
import env from "../../../environment"
const failGraceful = env.SELF_HOSTED && !env.isDev()
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
const formatUsage = (usage: QuotaUsage) => {
let maxAutomations = 0
let maxQueries = 0
let rows = 0
let developers = 0
if (usage) {
if (usage.usageQuota) {
rows = usage.usageQuota.rows
developers = usage.usageQuota.developers
}
if (usage.monthly) {
for (const value of Object.values(usage.monthly)) {
if (value.automations > maxAutomations) {
maxAutomations = value.automations
}
if (value.queries > maxQueries) {
maxQueries = value.queries
}
}
}
}
return {
maxAutomations,
maxQueries,
rows,
developers,
}
}
/**
* Date:
* May 2022
@ -14,23 +66,75 @@ import env from "../../../environment"
*/
export const run = async (db: any) => {
const tenantId = tenancy.getTenantId()
const installTimestamp = (await getInstallTimestamp(db)) as number
let account: CloudAccount | undefined
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
account = await accounts.getAccountByTenantId(tenantId)
try {
const tenantId = tenancy.getTenantId()
let installTimestamp
const totals: any = {}
const errors: any = []
try {
installTimestamp = await getInstallTimestamp(db)
} catch (e) {
handleError(e, errors)
}
let account: CloudAccount | undefined
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
account = await accounts.getAccountByTenantId(tenantId)
}
try {
await events.identification.identifyTenantGroup(
tenantId,
account,
installTimestamp
)
} catch (e) {
handleError(e, errors)
}
try {
await configs.backfill(db, installTimestamp)
} catch (e) {
handleError(e, errors)
}
try {
totals.users = await users.backfill(db, account)
} catch (e) {
handleError(e, errors)
}
try {
const allApps: App[] = await dbUtils.getAllApps({ dev: true })
totals.apps = allApps.length
totals.usage = await quotas.backfill(allApps)
} catch (e) {
handleError(e, errors)
}
const properties: TenantBackfillSucceededEvent = {
apps: totals.apps,
users: totals.users,
...formatUsage(totals.usage),
usage: totals.usage,
}
if (errors.length) {
properties.errors = errors.map((e: any) =>
JSON.stringify(e, Object.getOwnPropertyNames(e))
)
properties.errorCount = errors.length
} else {
properties.errorCount = 0
}
await events.backfill.tenantSucceeded(properties)
} catch (e) {
handleError(e)
await events.backfill.tenantFailed(e)
}
await events.identification.identifyTenantGroup(
tenantId,
account,
installTimestamp
)
await configs.backfill(db, installTimestamp)
// users and rows provide their own timestamp
await users.backfill(db, account)
await rows.backfill()
}
export const isComplete = async (): Promise<boolean> => {

View file

@ -0,0 +1,53 @@
import { events } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
import { App } from "@budibase/types"
const getOldestCreatedAt = (allApps: App[]): string | undefined => {
const timestamps = allApps
.filter(app => !!app.createdAt)
.map(app => app.createdAt as string)
.sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
if (timestamps.length) {
return timestamps[0]
}
}
const getMonthTimestamp = (monthString: string): number => {
const parts = monthString.split("-")
const month = parseInt(parts[0]) - 1 // we already do +1 in month string calculation
const year = parseInt(parts[1])
// using 0 as the day in next month gives us last day in previous month
const date = new Date(year, month + 1, 0).getTime()
const now = new Date().getTime()
if (date > now) {
return now
} else {
return date
}
}
export const backfill = async (allApps: App[]) => {
const usage = await quotas.getQuotaUsage()
const rows = usage.usageQuota.rows
const rowsTimestamp = getOldestCreatedAt(allApps)
await events.rows.created(rows, rowsTimestamp)
for (const [monthString, quotas] of Object.entries(usage.monthly)) {
if (monthString === "current") {
continue
}
const monthTimestamp = getMonthTimestamp(monthString)
const queries = quotas.queries
await events.query.run(queries, monthTimestamp)
const automations = quotas.automations
await events.automation.run(automations, monthTimestamp)
}
return usage
}

View file

@ -1,28 +0,0 @@
import { events, db as dbUtils } from "@budibase/backend-core"
import { Row, App } from "@budibase/types"
import { getUniqueRows } from "../../../../utilities/usageQuota/rows"
// Rows is a special circumstance where we get rows across all apps
// therefore migration is performed at the global level
const getOldestCreatedAt = (allApps: App[]): string | undefined => {
const timestamps = allApps
.filter(app => !!app.createdAt)
.map(app => app.createdAt as string)
.sort((a, b) => new Date(a).getTime() - new Date(b).getTime())
if (timestamps.length) {
return timestamps[0]
}
}
export const backfill = async () => {
const allApps: App[] = await dbUtils.getAllApps({ dev: true })
const timestamp = getOldestCreatedAt(allApps)
const appIds = allApps ? allApps.map((app: { appId: any }) => app.appId) : []
const rows: Row[] = await getUniqueRows(appIds)
const rowCount = rows ? rows.length : 0
if (rowCount) {
await events.rows.created(rowCount, timestamp)
}
}

View file

@ -40,4 +40,6 @@ export const backfill = async (
}
}
}
return users.length
}

View file

@ -1,6 +1,19 @@
import { events, tenancy, installation } from "@budibase/backend-core"
import { Installation } from "@budibase/types"
import * as global from "./global"
import env from "../../../environment"
const failGraceful = env.SELF_HOSTED
const handleError = (e: any, errors?: any) => {
if (failGraceful) {
if (errors) {
errors.push(e)
}
return
}
throw e
}
/**
* Date:
@ -11,14 +24,21 @@ import * as global from "./global"
*/
export const run = async () => {
// need to use the default tenant to try to get the installation time
await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
const installTimestamp = (await global.getInstallTimestamp(db)) as number
const install: Installation = await installation.getInstall()
await events.identification.identifyInstallationGroup(
install.installId,
installTimestamp
)
})
try {
// need to use the default tenant to try to get the installation time
await tenancy.doInTenant(tenancy.DEFAULT_TENANT_ID, async () => {
const db = tenancy.getGlobalDB()
const installTimestamp = (await global.getInstallTimestamp(db)) as number
const install: Installation = await installation.getInstall()
await events.identification.identifyInstallationGroup(
install.installId,
installTimestamp
)
})
await events.backfill.installationSucceeded()
throw new Error("fail")
} catch (e) {
handleError(e)
await events.backfill.installationFailed(e)
}
}

View file

@ -19,7 +19,7 @@ export interface SettingsConfig extends Config {
company: string
logoUrl: string
platformUrl: string
globalId?: string
uniqueTenantId?: string
}
}

View file

@ -43,3 +43,7 @@ export interface AutomationStepDeletedEvent {
stepId: string
stepType: string
}
export interface AutomationsRunEvent {
count: number
}

View file

@ -0,0 +1,35 @@
export interface AppBackfillSucceededEvent {
appId: string
automations: number
datasources: number
layouts: number
queries: number
roles: number
tables: number
screens: number
errors?: string[]
errorCount?: number
}
export interface AppBackfillFailedEvent {
error: string
}
export interface TenantBackfillSucceededEvent {
apps: number
users: number
usage: any
errors?: [string]
errorCount?: number
}
export interface TenantBackfillFailedEvent {
error: string
}
export interface InstallationBackfillSucceededEvent {}
export interface InstallationBackfillFailedEvent {
error: string
}

View file

@ -81,7 +81,7 @@ export enum Event {
QUERY_UPDATED = "query:updated",
QUERY_DELETED = "query:deleted",
QUERY_IMPORT = "query:import",
// QUERY_RUN = "query:run",
QUERIES_RUN = "queries:run",
QUERY_PREVIEWED = "query:previewed",
// TABLE
@ -124,7 +124,7 @@ export enum Event {
AUTOMATION_CREATED = "automation:created",
AUTOMATION_DELETED = "automation:deleted",
AUTOMATION_TESTED = "automation:tested",
// AUTOMATION_RUN = "automation:run",
AUTOMATIONS_RUN = "automations:run",
AUTOMATION_STEP_CREATED = "automation:step:created",
AUTOMATION_STEP_DELETED = "automation:step:deleted",
AUTOMATION_TRIGGER_UPDATED = "automation:trigger:updated",
@ -140,6 +140,14 @@ export enum Event {
ACCOUNT_CREATED = "account:created",
ACCOUNT_DELETED = "account:deleted",
ACCOUNT_VERIFIED = "account:verified",
// BACKFILL
APP_BACKFILL_SUCCEEDED = "app:backfill:succeeded",
APP_BACKFILL_FAILED = "app:backfill:failed",
TENANT_BACKFILL_SUCCEEDED = "tenant:backfill:succeeded",
TENANT_BACKFILL_FAILED = "tenant:backfill:failed",
INSTALLATION_BACKFILL_SUCCEEDED = "installation:backfill:succeeded",
INSTALLATION_BACKFILL_FAILED = "installation:backfill:failed",
}
export type RowImportFormat = "csv"

View file

@ -16,3 +16,4 @@ export * from "./table"
export * from "./user"
export * from "./view"
export * from "./account"
export * from "./backfill"

View file

@ -7,3 +7,7 @@ export interface QueryDeletedEvent {}
export interface QueryImportedEvent {}
export interface QueryPreviewedEvent {}
export interface QueriesRunEvent {
count: number
}

View file

@ -167,6 +167,7 @@ exports.save = async function (ctx) {
try {
const response = await db.put(ctx.request.body)
await bustCache(CacheKeys.CHECKLIST)
await bustCache(CacheKeys.ANALYTICS_ENABLED)
for (const fn of eventFns) {
await fn()