CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-06-13 16:05:06 +12:00
Code Issues Projects Releases Wiki Activity

Published apps, automations and query count quotas

Browse source
This commit is contained in:
Rory Powell 2022-03-20 01:13:54 +00:00
parent 62b360b9ea
commit 336f0ff4a0
22 changed files with 344 additions and 439 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
packages/backend-core/src/errors/index.js
View file

@ -21,7 +21,7 @@ const getPublicError = err => {
type: err.type,
}
if (err.code) {
if (err.code && context[err.code]) {
error = {
...error,
// get any additional context from this error

2
packages/bbui/src/ProgressBar/ProgressBar.svelte
View file

@ -28,7 +28,7 @@
aria-valuenow={$progress}
aria-valuemin="0"
aria-valuemax="100"
style={width ? `width: ${width}px;` : ""}
style={width ? `width: ${width};` : ""}
>
{#if $$slots}
<div

147
packages/server/src/api/controllers/application.js → packages/server/src/api/controllers/application.ts
View file

@ -1,29 +1,29 @@
const env = require("../../environment")
const packageJson = require("../../../package.json")
const {
import env from "../../environment"
import packageJson from "../../../package.json"
import {
createLinkView,
createRoutingView,
createAllSearchIndex,
} = require("../../db/views/staticViews")
const {
} from "../../db/views/staticViews"
import {
getTemplateStream,
createApp,
deleteApp,
} = require("../../utilities/fileSystem")
const {
} from "../../utilities/fileSystem"
import {
generateAppID,
getLayoutParams,
getScreenParams,
generateDevAppID,
DocumentTypes,
AppStatus,
} = require("../../db/utils")
} from "../../db/utils"
const {
BUILTIN_ROLE_IDS,
AccessController,
} = require("@budibase/backend-core/roles")
const { BASE_LAYOUTS } = require("../../constants/layouts")
const { cloneDeep } = require("lodash/fp")
import { BASE_LAYOUTS } from "../../constants/layouts"
import { cloneDeep } from "lodash/fp"
const { processObject } = require("@budibase/string-templates")
const {
getAllApps,
@ -31,24 +31,27 @@ const {
getProdAppID,
Replication,
} = require("@budibase/backend-core/db")
const { USERS_TABLE_SCHEMA } = require("../../constants")
const { removeAppFromUserRoles } = require("../../utilities/workerRequests")
const { clientLibraryPath, stringToReadStream } = require("../../utilities")
const { getAllLocks } = require("../../utilities/redis")
const {
import { USERS_TABLE_SCHEMA } from "../../constants"
import { removeAppFromUserRoles } from "../../utilities/workerRequests"
import { clientLibraryPath, stringToReadStream } from "../../utilities"
import { getAllLocks } from "../../utilities/redis"
import {
updateClientLibrary,
backupClientLibrary,
revertClientLibrary,
} = require("../../utilities/fileSystem/clientLibrary")
} from "../../utilities/fileSystem/clientLibrary"
const { getTenantId, isMultiTenant } = require("@budibase/backend-core/tenancy")
const { syncGlobalUsers } = require("./user")
import { syncGlobalUsers } from "./user"
const { app: appCache } = require("@budibase/backend-core/cache")
const { cleanupAutomations } = require("../../automations/utils")
import { cleanupAutomations } from "../../automations/utils"
const {
getAppDB,
getProdAppDB,
updateAppId,
} = require("@budibase/backend-core/context")
import { getUniqueRows } from "../../utilities/usageQuota/rows"
import { quotas } from "@budibase/pro"
import { errors } from "@budibase/backend-core"
const URL_REGEX_SLASH = /\/|\\/g
@ -61,7 +64,7 @@ async function getLayouts() {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map((row: any) => row.doc)
}
async function getScreens() {
@ -72,16 +75,16 @@ async function getScreens() {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map((row: any) => row.doc)
}
function getUserRoleId(ctx) {
function getUserRoleId(ctx: any) {
return !ctx.user.role || !ctx.user.role._id
? BUILTIN_ROLE_IDS.PUBLIC
: ctx.user.role._id
}
exports.getAppUrl = ctx => {
export const getAppUrl = (ctx: any) => {
// construct the url
let url
if (ctx.request.body.url) {
@ -97,29 +100,34 @@ exports.getAppUrl = ctx => {
return url
}
const checkAppUrl = (ctx, apps, url, currentAppId) => {
const checkAppUrl = (ctx: any, apps: any, url: any, currentAppId?: string) => {
if (currentAppId) {
apps = apps.filter(app => app.appId !== currentAppId)
apps = apps.filter((app: any) => app.appId !== currentAppId)
}
if (apps.some(app => app.url === url)) {
if (apps.some((app: any) => app.url === url)) {
ctx.throw(400, "App URL is already in use.")
}
}
const checkAppName = (ctx, apps, name, currentAppId) => {
const checkAppName = (
ctx: any,
apps: any,
name: any,
currentAppId?: string
) => {
// TODO: Replace with Joi
if (!name) {
ctx.throw(400, "Name is required")
}
if (currentAppId) {
apps = apps.filter(app => app.appId !== currentAppId)
apps = apps.filter((app: any) => app.appId !== currentAppId)
}
if (apps.some(app => app.name === name)) {
if (apps.some((app: any) => app.name === name)) {
ctx.throw(400, "App name is already in use.")
}
}
async function createInstance(template) {
async function createInstance(template: any) {
const tenantId = isMultiTenant() ? getTenantId() : null
const baseAppId = generateAppID(tenantId)
const appId = generateDevAppID(baseAppId)
@ -160,7 +168,7 @@ async function createInstance(template) {
return { _id: appId }
}
exports.fetch = async ctx => {
export const fetch = async (ctx: any) => {
const dev = ctx.query && ctx.query.status === AppStatus.DEV
const all = ctx.query && ctx.query.status === AppStatus.ALL
const apps = await getAllApps({ dev, all })
@ -172,7 +180,7 @@ exports.fetch = async ctx => {
if (app.status !== "development") {
continue
}
const lock = locks.find(lock => lock.appId === app.appId)
const lock = locks.find((lock: any) => lock.appId === app.appId)
if (lock) {
app.lockedBy = lock.user
} else {
@ -185,7 +193,7 @@ exports.fetch = async ctx => {
ctx.body = apps
}
exports.fetchAppDefinition = async ctx => {
export const fetchAppDefinition = async (ctx: any) => {
const layouts = await getLayouts()
const userRoleId = getUserRoleId(ctx)
const accessController = new AccessController()
@ -200,7 +208,7 @@ exports.fetchAppDefinition = async ctx => {
}
}
exports.fetchAppPackage = async ctx => {
export const fetchAppPackage = async (ctx: any) => {
const db = getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA)
const layouts = await getLayouts()
@ -221,7 +229,7 @@ exports.fetchAppPackage = async ctx => {
}
}
exports.create = async ctx => {
const performAppCreate = async (ctx: any) => {
const apps = await getAllApps({ dev: true })
const name = ctx.request.body.name
checkAppName(ctx, apps, name)
@ -229,7 +237,7 @@ exports.create = async ctx => {
checkAppUrl(ctx, apps, url)
const { useTemplate, templateKey, templateString } = ctx.request.body
const instanceConfig = {
const instanceConfig: any = {
useTemplate,
key: templateKey,
templateString,
@ -279,13 +287,41 @@ exports.create = async ctx => {
}
await appCache.invalidateAppMetadata(appId, newApplication)
ctx.status = 200
return newApplication
}
const appPostCreate = async (ctx: any, appId: string) => {
// app import & template creation
if (ctx.request.body.useTemplate === "true") {
const rows = await getUniqueRows([appId])
const rowCount = rows ? rows.length : 0
if (rowCount) {
try {
await quotas.addRows(rowCount)
} catch (err: any) {
if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
// this import resulted in row usage exceeding the quota
// delete the app
// skip pre and post steps as no rows have been added to quotas yet
ctx.params.appId = appId
await destroyApp(ctx)
}
throw err
}
}
}
}
export const create = async (ctx: any) => {
const newApplication = await quotas.addApp(() => performAppCreate(ctx))
await appPostCreate(ctx, newApplication.appId)
ctx.body = newApplication
ctx.status = 200
}
// This endpoint currently operates as a PATCH rather than a PUT
// Thus name and url fields are handled only if present
exports.update = async ctx => {
export const update = async (ctx: any) => {
const apps = await getAllApps({ dev: true })
// validation
const name = ctx.request.body.name
@ -303,7 +339,7 @@ exports.update = async ctx => {
ctx.body = data
}
exports.updateClient = async ctx => {
export const updateClient = async (ctx: any) => {
// Get current app version
const db = getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA)
@ -325,7 +361,7 @@ exports.updateClient = async ctx => {
ctx.body = data
}
exports.revertClient = async ctx => {
export const revertClient = async (ctx: any) => {
// Check app can be reverted
const db = getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA)
@ -348,10 +384,15 @@ exports.revertClient = async ctx => {
ctx.body = data
}
exports.delete = async ctx => {
const destroyApp = async (ctx: any) => {
const db = getAppDB()
const result = await db.destroy()
if (ctx.query.unpublish) {
await quotas.removePublishedApp()
} else {
await quotas.removeApp()
}
/* istanbul ignore next */
if (!env.isTest() && !ctx.query.unpublish) {
await deleteApp(ctx.params.appId)
@ -362,12 +403,30 @@ exports.delete = async ctx => {
// make sure the app/role doesn't stick around after the app has been deleted
await removeAppFromUserRoles(ctx, ctx.params.appId)
await appCache.invalidateAppMetadata(ctx.params.appId)
return result
}
const preDestroyApp = async (ctx: any) => {
const rows = await getUniqueRows([ctx.appId])
ctx.rowCount = rows.length
}
const postDestroyApp = async (ctx: any) => {
const rowCount = ctx.rowCount
if (rowCount) {
await quotas.removeRows(rowCount)
}
}
export const destroy = async (ctx: any) => {
await preDestroyApp(ctx)
const result = await destroyApp(ctx)
await postDestroyApp(ctx)
ctx.status = 200
ctx.body = result
}
exports.sync = async (ctx, next) => {
export const sync = async (ctx: any, next: any) => {
const appId = ctx.params.appId
if (!isDevAppID(appId)) {
ctx.throw(400, "This action cannot be performed for production apps")
@ -397,7 +456,7 @@ exports.sync = async (ctx, next) => {
let error
try {
await replication.replicate({
filter: function (doc) {
filter: function (doc: any) {
return doc._id !== DocumentTypes.APP_METADATA
},
})
@ -417,7 +476,7 @@ exports.sync = async (ctx, next) => {
}
}
const updateAppPackage = async (appPackage, appId) => {
const updateAppPackage = async (appPackage: any, appId: any) => {
const db = getAppDB()
const application = await db.get(DocumentTypes.APP_METADATA)
@ -436,7 +495,7 @@ const updateAppPackage = async (appPackage, appId) => {
return response
}
const createEmptyAppPackage = async (ctx, app) => {
const createEmptyAppPackage = async (ctx: any, app: any) => {
const db = getAppDB()
let screensAndLayouts = []

73
packages/server/src/api/controllers/deploy/index.js → packages/server/src/api/controllers/deploy/index.ts
View file

@ -1,20 +1,18 @@
const Deployment = require("./Deployment")
const {
import Deployment from "./Deployment"
import {
Replication,
getProdAppID,
getDevelopmentAppID,
} = require("@budibase/backend-core/db")
const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
const {
disableAllCrons,
enableCronTrigger,
} = require("../../../automations/utils")
const { app: appCache } = require("@budibase/backend-core/cache")
const {
} from "@budibase/backend-core/db"
import { DocumentTypes, getAutomationParams } from "../../../db/utils"
import { disableAllCrons, enableCronTrigger } from "../../../automations/utils"
import { app as appCache } from "@budibase/backend-core/cache"
import {
getAppId,
getAppDB,
getProdAppDB,
} = require("@budibase/backend-core/context")
} from "@budibase/backend-core/context"
import { quotas } from "@budibase/pro"
// the max time we can wait for an invalidation to complete before considering it failed
const MAX_PENDING_TIME_MS = 30 * 60000
@ -25,9 +23,10 @@ const DeploymentStatus = {
}
// checks that deployments are in a good state, any pending will be updated
async function checkAllDeployments(deployments) {
async function checkAllDeployments(deployments: any) {
let updated = false
for (let deployment of Object.values(deployments.history)) {
let deployment: any
for (deployment of Object.values(deployments.history)) {
// check that no deployments have crashed etc and are now stuck
if (
deployment.status === DeploymentStatus.PENDING &&
@ -41,7 +40,7 @@ async function checkAllDeployments(deployments) {
return { updated, deployments }
}
async function storeDeploymentHistory(deployment) {
async function storeDeploymentHistory(deployment: any) {
const deploymentJSON = deployment.getJSON()
const db = getAppDB()
@ -70,7 +69,7 @@ async function storeDeploymentHistory(deployment) {
return deployment
}
async function initDeployedApp(prodAppId) {
async function initDeployedApp(prodAppId: any) {
const db = getProdAppDB()
console.log("Reading automation docs")
const automations = (
@ -79,7 +78,7 @@ async function initDeployedApp(prodAppId) {
include_docs: true,
})
)
).rows.map(row => row.doc)
).rows.map((row: any) => row.doc)
console.log("You have " + automations.length + " automations")
const promises = []
console.log("Disabling prod crons..")
@ -93,16 +92,17 @@ async function initDeployedApp(prodAppId) {
console.log("Enabled cron triggers for deployed app..")
}
async function deployApp(deployment) {
async function deployApp(deployment: any) {
try {
const appId = getAppId()
const devAppId = getDevelopmentAppID(appId)
const productionAppId = getProdAppID(appId)
const replication = new Replication({
const config: any = {
source: devAppId,
target: productionAppId,
})
}
const replication = new Replication(config)
console.log("Replication object created")
@ -119,7 +119,7 @@ async function deployApp(deployment) {
console.log("Deployed app initialised, setting deployment to successful")
deployment.setStatus(DeploymentStatus.SUCCESS)
await storeDeploymentHistory(deployment)
} catch (err) {
} catch (err: any) {
deployment.setStatus(DeploymentStatus.FAILURE, err.message)
await storeDeploymentHistory(deployment)
throw {
@ -129,14 +129,11 @@ async function deployApp(deployment) {
}
}
exports.fetchDeployments = async function (ctx) {
export async function fetchDeployments(ctx: any) {
try {
const db = getAppDB()
const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
const { updated, deployments } = await checkAllDeployments(
deploymentDoc,
ctx.user
)
const { updated, deployments } = await checkAllDeployments(deploymentDoc)
if (updated) {
await db.put(deployments)
}
@ -146,7 +143,7 @@ exports.fetchDeployments = async function (ctx) {
}
}
exports.deploymentProgress = async function (ctx) {
export async function deploymentProgress(ctx: any) {
try {
const db = getAppDB()
const deploymentDoc = await db.get(DocumentTypes.DEPLOYMENTS)
@ -159,7 +156,20 @@ exports.deploymentProgress = async function (ctx) {
}
}
exports.deployApp = async function (ctx) {
const isFirstDeploy = async () => {
try {
const db = getProdAppDB()
await db.get(DocumentTypes.APP_METADATA)
} catch (e: any) {
if (e.status === 404) {
return true
}
throw e
}
return false
}
const _deployApp = async function (ctx: any) {
let deployment = new Deployment()
console.log("Deployment object created")
deployment.setStatus(DeploymentStatus.PENDING)
@ -168,7 +178,14 @@ exports.deployApp = async function (ctx) {
console.log("Stored deployment history")
console.log("Deploying app...")
await deployApp(deployment)
if (await isFirstDeploy()) {
await quotas.addPublishedApp(() => deployApp(deployment))
} else {
await deployApp(deployment)
}
ctx.body = deployment
}
export { _deployApp as deployApp }

97
packages/server/src/api/controllers/query/index.js → packages/server/src/api/controllers/query/index.ts
View file

@ -1,22 +1,19 @@
const {
generateQueryID,
getQueryParams,
isProdAppID,
} = require("../../../db/utils")
const { BaseQueryVerbs } = require("../../../constants")
const { Thread, ThreadType } = require("../../../threads")
const { save: saveDatasource } = require("../datasource")
const { RestImporter } = require("./import")
const { invalidateDynamicVariables } = require("../../../threads/utils")
const environment = require("../../../environment")
const { getAppDB } = require("@budibase/backend-core/context")
import { generateQueryID, getQueryParams, isProdAppID } from "../../../db/utils"
import { BaseQueryVerbs } from "../../../constants"
import { Thread, ThreadType } from "../../../threads"
import { save as saveDatasource } from "../datasource"
import { RestImporter } from "./import"
import { invalidateDynamicVariables } from "../../../threads/utils"
import { QUERY_THREAD_TIMEOUT } from "../../../environment"
import { getAppDB } from "@budibase/backend-core/context"
import { quotas } from "@budibase/pro"
const Runner = new Thread(ThreadType.QUERY, {
timeoutMs: environment.QUERY_THREAD_TIMEOUT || 10000,
timeoutMs: QUERY_THREAD_TIMEOUT || 10000,
})
// simple function to append "readable" to all read queries
function enrichQueries(input) {
function enrichQueries(input: any) {
const wasArray = Array.isArray(input)
const queries = wasArray ? input : [input]
for (let query of queries) {
@ -27,7 +24,7 @@ function enrichQueries(input) {
return wasArray ? queries : queries[0]
}
exports.fetch = async function (ctx) {
export async function fetch(ctx: any) {
const db = getAppDB()
const body = await db.allDocs(
@ -36,10 +33,10 @@ exports.fetch = async function (ctx) {
})
)
ctx.body = enrichQueries(body.rows.map(row => row.doc))
ctx.body = enrichQueries(body.rows.map((row: any) => row.doc))
}
exports.import = async ctx => {
const _import = async (ctx: any) => {
const body = ctx.request.body
const data = body.data
@ -49,7 +46,7 @@ exports.import = async ctx => {
let datasourceId
if (!body.datasourceId) {
// construct new datasource
const info = await importer.getInfo()
const info: any = await importer.getInfo()
let datasource = {
type: "datasource",
source: "REST",
@ -77,8 +74,9 @@ exports.import = async ctx => {
}
ctx.status = 200
}
export { _import as import }
exports.save = async function (ctx) {
export async function save(ctx: any) {
const db = getAppDB()
const query = ctx.request.body
@ -93,7 +91,7 @@ exports.save = async function (ctx) {
ctx.message = `Query ${query.name} saved successfully.`
}
exports.find = async function (ctx) {
export async function find(ctx: any) {
const db = getAppDB()
const query = enrichQueries(await db.get(ctx.params.queryId))
// remove properties that could be dangerous in real app
@ -104,7 +102,7 @@ exports.find = async function (ctx) {
ctx.body = query
}
exports.preview = async function (ctx) {
export async function preview(ctx: any) {
const db = getAppDB()
const datasource = await db.get(ctx.request.body.datasourceId)
@ -114,16 +112,18 @@ exports.preview = async function (ctx) {
ctx.request.body
try {
const { rows, keys, info, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb,
fields,
parameters,
transformer,
queryId,
})
const runFn = () =>
Runner.run({
appId: ctx.appId,
datasource,
queryVerb,
fields,
parameters,
transformer,
queryId,
})
const { rows, keys, info, extra } = await quotas.addQuery(runFn)
ctx.body = {
rows,
schemaFields: [...new Set(keys)],
@ -135,7 +135,7 @@ exports.preview = async function (ctx) {
}
}
async function execute(ctx, opts = { rowsOnly: false }) {
async function execute(ctx: any, opts = { rowsOnly: false }) {
const db = getAppDB()
const query = await db.get(ctx.params.queryId)
@ -153,16 +153,19 @@ async function execute(ctx, opts = { rowsOnly: false }) {
// call the relevant CRUD method on the integration class
try {
const { rows, pagination, extra } = await Runner.run({
appId: ctx.appId,
datasource,
queryVerb: query.queryVerb,
fields: query.fields,
pagination: ctx.request.body.pagination,
parameters: enrichedParameters,
transformer: query.transformer,
queryId: ctx.params.queryId,
})
const runFn = () =>
Runner.run({
appId: ctx.appId,
datasource,
queryVerb: query.queryVerb,
fields: query.fields,
pagination: ctx.request.body.pagination,
parameters: enrichedParameters,
transformer: query.transformer,
queryId: ctx.params.queryId,
})
const { rows, pagination, extra } = await quotas.addQuery(runFn)
if (opts && opts.rowsOnly) {
ctx.body = rows
} else {
@ -173,15 +176,15 @@ async function execute(ctx, opts = { rowsOnly: false }) {
}
}
exports.executeV1 = async function (ctx) {
export async function executeV1(ctx: any) {
return execute(ctx, { rowsOnly: true })
}
exports.executeV2 = async function (ctx) {
export async function executeV2(ctx: any) {
return execute(ctx, { rowsOnly: false })
}
const removeDynamicVariables = async queryId => {
const removeDynamicVariables = async (queryId: any) => {
const db = getAppDB()
const query = await db.get(queryId)
const datasource = await db.get(query.datasourceId)
@ -190,19 +193,19 @@ const removeDynamicVariables = async queryId => {
if (dynamicVariables) {
// delete dynamic variables from the datasource
datasource.config.dynamicVariables = dynamicVariables.filter(
dv => dv.queryId !== queryId
(dv: any) => dv.queryId !== queryId
)
await db.put(datasource)
// invalidate the deleted variables
const variablesToDelete = dynamicVariables.filter(
dv => dv.queryId === queryId
(dv: any) => dv.queryId === queryId
)
await invalidateDynamicVariables(variablesToDelete)
}
}
exports.destroy = async function (ctx) {
export async function destroy(ctx: any) {
const db = getAppDB()
await removeDynamicVariables(ctx.params.queryId)
await db.remove(ctx.params.queryId, ctx.params.revId)

39
packages/server/src/api/controllers/row/index.js → packages/server/src/api/controllers/row/index.ts
View file

@ -1,15 +1,16 @@
const internal = require("./internal")
const external = require("./external")
const { isExternalTable } = require("../../../integrations/utils")
import { quotas } from "@budibase/pro"
import internal from "./internal"
import external from "./external"
import { isExternalTable } from "../../../integrations/utils"
function pickApi(tableId) {
function pickApi(tableId: any) {
if (isExternalTable(tableId)) {
return external
}
return internal
}
function getTableId(ctx) {
function getTableId(ctx: any) {
if (ctx.request.body && ctx.request.body.tableId) {
return ctx.request.body.tableId
}
@ -21,13 +22,13 @@ function getTableId(ctx) {
}
}
exports.patch = async ctx => {
export async function patch(ctx: any): Promise<any> {
const appId = ctx.appId
const tableId = getTableId(ctx)
const body = ctx.request.body
// if it doesn't have an _id then its save
if (body && !body._id) {
return exports.save(ctx)
return save(ctx)
}
try {
const { row, table } = await pickApi(tableId).patch(ctx)
@ -41,13 +42,13 @@ exports.patch = async ctx => {
}
}
exports.save = async function (ctx) {
const saveRow = async (ctx: any) => {
const appId = ctx.appId
const tableId = getTableId(ctx)
const body = ctx.request.body
// if it has an ID already then its a patch
if (body && body._id) {
return exports.patch(ctx)
return patch(ctx)
}
try {
const { row, table } = await pickApi(tableId).save(ctx)
@ -60,7 +61,11 @@ exports.save = async function (ctx) {
}
}
exports.fetchView = async function (ctx) {
export async function save(ctx: any) {
await quotas.addRow(() => saveRow(ctx))
}
export async function fetchView(ctx: any) {
const tableId = getTableId(ctx)
try {
ctx.body = await pickApi(tableId).fetchView(ctx)
@ -69,7 +74,7 @@ exports.fetchView = async function (ctx) {
}
}
exports.fetch = async function (ctx) {
export async function fetch(ctx: any) {
const tableId = getTableId(ctx)
try {
ctx.body = await pickApi(tableId).fetch(ctx)
@ -78,7 +83,7 @@ exports.fetch = async function (ctx) {
}
}
exports.find = async function (ctx) {
export async function find(ctx: any) {
const tableId = getTableId(ctx)
try {
ctx.body = await pickApi(tableId).find(ctx)
@ -87,19 +92,21 @@ exports.find = async function (ctx) {
}
}
exports.destroy = async function (ctx) {
export async function destroy(ctx: any) {
const appId = ctx.appId
const inputs = ctx.request.body
const tableId = getTableId(ctx)
let response, row
if (inputs.rows) {
let { rows } = await pickApi(tableId).bulkDestroy(ctx)
await quotas.removeRows(rows.length)
response = rows
for (let row of rows) {
ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
}
} else {
let resp = await pickApi(tableId).destroy(ctx)
await quotas.removeRow()
response = resp.response
row = resp.row
ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
@ -110,7 +117,7 @@ exports.destroy = async function (ctx) {
ctx.body = response
}
exports.search = async ctx => {
export async function search(ctx: any) {
const tableId = getTableId(ctx)
try {
ctx.status = 200
@ -120,7 +127,7 @@ exports.search = async ctx => {
}
}
exports.validate = async function (ctx) {
export async function validate(ctx: any) {
const tableId = getTableId(ctx)
try {
ctx.body = await pickApi(tableId).validate(ctx)
@ -129,7 +136,7 @@ exports.validate = async function (ctx) {
}
}
exports.fetchEnrichedRow = async function (ctx) {
export async function fetchEnrichedRow(ctx: any) {
const tableId = getTableId(ctx)
try {
ctx.body = await pickApi(tableId).fetchEnrichedRow(ctx)

6
packages/server/src/api/controllers/table/internal.ts
View file

@ -120,11 +120,7 @@ export async function destroy(ctx: any) {
await db.bulkDocs(
rows.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
)
await quotas.updateUsage(
-rows.rows.length,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
await quotas.removeRows(rows.rows.length)
// update linked rows
await updateLinks({

7
packages/server/src/api/controllers/table/utils.ts
View file

@ -147,12 +147,7 @@ export async function handleDataImport(user: any, table: any, dataImport: any) {
finalData.push(row)
}
await quotas.tryUpdateUsage(
() => db.bulkDocs(finalData),
finalData.length,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
await quotas.addRows(finalData.length, () => db.bulkDocs(finalData))
let response = await db.put(table)
table._rev = response._rev
return table

16
packages/server/src/api/routes/application.js → packages/server/src/api/routes/application.ts
View file

@ -1,14 +1,13 @@
const Router = require("@koa/router")
const controller = require("../controllers/application")
const authorized = require("../../middleware/authorized")
import * as controller from "../controllers/application"
import authorized from "../../middleware/authorized"
const { BUILDER } = require("@budibase/backend-core/permissions")
const usage = require("../../middleware/usageQuota")
const router = Router()
router
.post("/api/applications/:appId/sync", authorized(BUILDER), controller.sync)
.post("/api/applications", authorized(BUILDER), usage, controller.create)
.post("/api/applications", authorized(BUILDER), controller.create)
.get("/api/applications/:appId/definition", controller.fetchAppDefinition)
.get("/api/applications", controller.fetch)
.get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
@ -23,11 +22,6 @@ router
authorized(BUILDER),
controller.revertClient
)
.delete(
"/api/applications/:appId",
authorized(BUILDER),
usage,
controller.delete
)
.delete("/api/applications/:appId", authorized(BUILDER), controller.destroy)
module.exports = router
export default router

59
packages/server/src/api/routes/index.js
View file

@ -1,32 +1,33 @@
const authRoutes = require("./auth")
const layoutRoutes = require("./layout")
const screenRoutes = require("./screen")
const userRoutes = require("./user")
const applicationRoutes = require("./application")
const tableRoutes = require("./table")
const rowRoutes = require("./row")
const viewRoutes = require("./view")
const staticRoutes = require("./static")
const componentRoutes = require("./component")
const automationRoutes = require("./automation")
const webhookRoutes = require("./webhook")
const roleRoutes = require("./role")
const deployRoutes = require("./deploy")
const apiKeysRoutes = require("./apikeys")
const templatesRoutes = require("./templates")
const analyticsRoutes = require("./analytics")
const routingRoutes = require("./routing")
const integrationRoutes = require("./integration")
const permissionRoutes = require("./permission")
const datasourceRoutes = require("./datasource")
const queryRoutes = require("./query")
const backupRoutes = require("./backup")
const metadataRoutes = require("./metadata")
const devRoutes = require("./dev")
const cloudRoutes = require("./cloud")
const migrationRoutes = require("./migrations")
import authRoutes from "./auth"
import layoutRoutes from "./layout"
import screenRoutes from "./screen"
import userRoutes from "./user"
import applicationRoutes from "./application"
import tableRoutes from "./table"
import rowRoutes from "./row"
import viewRoutes from "./view"
import componentRoutes from "./component"
import automationRoutes from "./automation"
import webhookRoutes from "./webhook"
import roleRoutes from "./role"
import deployRoutes from "./deploy"
import apiKeysRoutes from "./apikeys"
import templatesRoutes from "./templates"
import analyticsRoutes from "./analytics"
import routingRoutes from "./routing"
import integrationRoutes from "./integration"
import permissionRoutes from "./permission"
import datasourceRoutes from "./datasource"
import queryRoutes from "./query"
import backupRoutes from "./backup"
import metadataRoutes from "./metadata"
import devRoutes from "./dev"
import cloudRoutes from "./cloud"
import migrationRoutes from "./migrations"
exports.mainRoutes = [
export { default as staticRoutes } from "./static"
export const mainRoutes = [
authRoutes,
deployRoutes,
layoutRoutes,
@ -56,5 +57,3 @@ exports.mainRoutes = [
rowRoutes,
migrationRoutes,
]
exports.staticRoutes = staticRoutes

14
packages/server/src/api/routes/row.js → packages/server/src/api/routes/row.ts
View file

@ -1,11 +1,7 @@
const Router = require("@koa/router")
const rowController = require("../controllers/row")
const authorized = require("../../middleware/authorized")
const usage = require("../../middleware/usageQuota")
const {
paramResource,
paramSubResource,
} = require("../../middleware/resourceId")
import * as rowController from "../controllers/row"
import authorized from "../../middleware/authorized"
import { paramResource, paramSubResource } from "../../middleware/resourceId"
const {
PermissionLevels,
PermissionTypes,
@ -180,7 +176,6 @@ router
"/api/:tableId/rows",
paramResource("tableId"),
authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
usage,
rowController.save
)
/**
@ -247,8 +242,7 @@ router
"/api/:tableId/rows",
paramResource("tableId"),
authorized(PermissionTypes.TABLE, PermissionLevels.WRITE),
usage,
rowController.destroy
)
module.exports = router
export default router

16
packages/server/src/api/routes/static.js
View file

@ -1,14 +1,14 @@
const Router = require("@koa/router")
const controller = require("../controllers/static")
const { budibaseTempDir } = require("../../utilities/budibaseDir")
const authorized = require("../../middleware/authorized")
const {
import * as controller from "../controllers/static"
import { budibaseTempDir } from "../../utilities/budibaseDir"
import authorized from "../../middleware/authorized"
import {
BUILDER,
PermissionTypes,
PermissionLevels,
} = require("@budibase/backend-core/permissions")
const env = require("../../environment")
const { paramResource } = require("../../middleware/resourceId")
} from "@budibase/backend-core/permissions"
import * as env from "../../environment"
import { paramResource } from "../../middleware/resourceId"
const router = Router()
@ -52,4 +52,4 @@ router
controller.getSignedUploadURL
)
module.exports = router
export default router

9
packages/server/src/automations/steps/createRow.ts
View file

@ -1,4 +1,4 @@
import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
import { quotas } from "@budibase/pro"
import { save } from "../../api/controllers/row"
import { cleanUpRow, getError } from "../automationUtils"
import { buildCtx } from "./utils"
@ -78,12 +78,7 @@ export async function run({ inputs, appId, emitter }: any) {
try {
inputs.row = await cleanUpRow(inputs.row.tableId, inputs.row)
await quotas.tryUpdateUsage(
() => save(ctx),
1,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
await quotas.addRow(() => save(ctx))
return {
row: inputs.row,
response: ctx.body,

4
packages/server/src/automations/steps/deleteRow.ts
View file

@ -1,7 +1,7 @@
import { destroy } from "../../api/controllers/row"
import { buildCtx } from "./utils"
import { getError } from "../automationUtils"
import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
import { quotas } from "@budibase/pro"
export const definition = {
description: "Delete a row from your database",
@ -73,8 +73,8 @@ export async function run({ inputs, appId, emitter }: any) {
})
try {
await quotas.updateUsage(-1, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
await destroy(ctx)
await quotas.removeRow()
return {
response: ctx.body,
row: ctx.row,

74
packages/server/src/automations/utils.js → packages/server/src/automations/utils.ts
View file

@ -1,23 +1,29 @@
const { Thread, ThreadType } = require("../threads")
const { definitions } = require("./triggerInfo")
const webhooks = require("../api/controllers/webhook")
const CouchDB = require("../db")
const { queue } = require("./bullboard")
const newid = require("../db/newid")
const { updateEntityMetadata } = require("../utilities")
const { MetadataTypes } = require("../constants")
const { getProdAppID } = require("@budibase/backend-core/db")
const { cloneDeep } = require("lodash/fp")
const { getAppDB, getAppId } = require("@budibase/backend-core/context")
import { Thread, ThreadType } from "../threads"
import { definitions } from "./triggerInfo"
import { destroy, Webhook, WebhookType, save } from "../api/controllers/webhook"
import CouchDB from "../db"
import { queue } from "./bullboard"
import newid from "../db/newid"
import { updateEntityMetadata } from "../utilities"
import { MetadataTypes } from "../constants"
import { getProdAppID } from "@budibase/backend-core/db"
import { cloneDeep } from "lodash/fp"
import { getAppDB, getAppId } from "@budibase/backend-core/context"
import { tenancy } from "@budibase/backend-core"
import { quotas } from "@budibase/pro"
const WH_STEP_ID = definitions.WEBHOOK.stepId
const CRON_STEP_ID = definitions.CRON.stepId
const Runner = new Thread(ThreadType.AUTOMATION)
exports.processEvent = async job => {
export async function processEvent(job: any) {
try {
// need to actually await these so that an error can be captured properly
return await Runner.run(job)
const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
return await tenancy.doInTenant(tenantId, async () => {
// need to actually await these so that an error can be captured properly
const runFn = () => Runner.run(job)
return quotas.addAutomation(runFn)
})
} catch (err) {
console.error(
`${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
@ -26,11 +32,15 @@ exports.processEvent = async job => {
}
}
exports.updateTestHistory = async (appId, automation, history) => {
export async function updateTestHistory(
appId: any,
automation: any,
history: any
) {
return updateEntityMetadata(
MetadataTypes.AUTOMATION_TEST_HISTORY,
automation._id,
metadata => {
(metadata: any) => {
if (metadata && Array.isArray(metadata.history)) {
metadata.history.push(history)
} else {
@ -43,7 +53,7 @@ exports.updateTestHistory = async (appId, automation, history) => {
)
}
exports.removeDeprecated = definitions => {
export function removeDeprecated(definitions: any) {
const base = cloneDeep(definitions)
for (let key of Object.keys(base)) {
if (base[key].deprecated) {
@ -54,13 +64,15 @@ exports.removeDeprecated = definitions => {
}
// end the repetition and the job itself
exports.disableAllCrons = async appId => {
export async function disableAllCrons(appId: any) {
const promises = []
const jobs = await queue.getRepeatableJobs()
for (let job of jobs) {
if (job.key.includes(`${appId}_cron`)) {
promises.push(queue.removeRepeatableByKey(job.key))
promises.push(queue.removeJobs(job.id))
if (job.id) {
promises.push(queue.removeJobs(job.id))
}
}
}
return Promise.all(promises)
@ -71,9 +83,9 @@ exports.disableAllCrons = async appId => {
* @param {string} appId The ID of the app in which we are checking for webhooks
* @param {object|undefined} automation The automation object to be updated.
*/
exports.enableCronTrigger = async (appId, automation) => {
export async function enableCronTrigger(appId: any, automation: any) {
const trigger = automation ? automation.definition.trigger : null
function isCronTrigger(auto) {
function isCronTrigger(auto: any) {
return (
auto &&
auto.definition.trigger &&
@ -84,7 +96,7 @@ exports.enableCronTrigger = async (appId, automation) => {
if (isCronTrigger(automation)) {
// make a job id rather than letting Bull decide, makes it easier to handle on way out
const jobId = `${appId}_cron_${newid()}`
const job = await queue.add(
const job: any = await queue.add(
{
automation,
event: { appId, timestamp: Date.now() },
@ -112,13 +124,13 @@ exports.enableCronTrigger = async (appId, automation) => {
* @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
* written to DB (this does not write to DB as it would be wasteful to repeat).
*/
exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
export async function checkForWebhooks({ oldAuto, newAuto }: any) {
const appId = getAppId()
const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
const newTrigger = newAuto ? newAuto.definition.trigger : null
const triggerChanged =
oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id
function isWebhookTrigger(auto) {
function isWebhookTrigger(auto: any) {
return (
auto &&
auto.definition.trigger &&
@ -144,7 +156,7 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
delete newTrigger.webhookId
newTrigger.inputs = {}
}
await webhooks.destroy(ctx)
await destroy(ctx)
} catch (err) {
// don't worry about not being able to delete, if it doesn't exist all good
}
@ -154,17 +166,17 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
(!isWebhookTrigger(oldAuto) || triggerChanged) &&
isWebhookTrigger(newAuto)
) {
const ctx = {
const ctx: any = {
appId,
request: {
body: new webhooks.Webhook(
body: new Webhook(
"Automation webhook",
webhooks.WebhookType.AUTOMATION,
WebhookType.AUTOMATION,
newAuto._id
),
},
}
await webhooks.save(ctx)
await save(ctx)
const id = ctx.body.webhook._id
newTrigger.webhookId = id
// the app ID has to be development for this endpoint
@ -184,6 +196,6 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
* @param appId {string} the app that is being removed.
* @return {Promise<void>} clean is complete if this succeeds.
*/
exports.cleanupAutomations = async appId => {
await exports.disableAllCrons(appId)
export async function cleanupAutomations(appId: any) {
await disableAllCrons(appId)
}

172
packages/server/src/middleware/usageQuota.ts
View file

@ -1,172 +0,0 @@
import { quotas, StaticQuotaName, QuotaUsageType } from "@budibase/pro"
const { getUniqueRows } = require("../utilities/usageQuota/rows")
const {
isExternalTable,
isRowId: isExternalRowId,
} = require("../integrations/utils")
const { getAppDB } = require("@budibase/backend-core/context")
// currently only counting new writes and deletes
const METHOD_MAP: any = {
POST: 1,
DELETE: -1,
}
const DOMAIN_MAP: any = {
rows: {
name: StaticQuotaName.ROWS,
type: QuotaUsageType.STATIC,
},
applications: {
name: StaticQuotaName.APPS,
type: QuotaUsageType.STATIC,
},
}
function getQuotaInfo(url: string) {
for (let domain of Object.keys(DOMAIN_MAP)) {
if (url.indexOf(domain) !== -1) {
return DOMAIN_MAP[domain]
}
}
}
module.exports = async (ctx: any, next: any) => {
if (!quotas.useQuotas()) {
return next()
}
let usage = METHOD_MAP[ctx.req.method]
const quotaInfo = getQuotaInfo(ctx.req.url)
if (usage == null || quotaInfo == null) {
return next()
}
// post request could be a save of a pre-existing entry
if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) {
const usageId = ctx.request.body._id
try {
if (ctx.appId) {
const db = getAppDB()
await db.get(usageId)
}
return next()
} catch (err) {
if (
isExternalTable(usageId) ||
(ctx.request.body.tableId &&
isExternalTable(ctx.request.body.tableId)) ||
isExternalRowId(usageId)
) {
return next()
} else {
ctx.throw(404, `${usageId} does not exist`)
}
}
}
try {
await performRequest(ctx, next, quotaInfo, usage)
} catch (err) {
ctx.throw(400, err)
}
}
const performRequest = async (
ctx: any,
next: any,
quotaInfo: any,
usage: number
) => {
const usageContext = {
skipNext: false,
skipUsage: false,
[StaticQuotaName.APPS]: {},
}
const quotaName = quotaInfo.name
if (usage === -1) {
if (PRE_DELETE[quotaName]) {
await PRE_DELETE[quotaName](ctx, usageContext)
}
} else {
if (PRE_CREATE[quotaName]) {
await PRE_CREATE[quotaName](ctx, usageContext)
}
}
// run the request
if (!usageContext.skipNext) {
await quotas.updateUsage(usage, quotaName, quotaInfo.type, {
dryRun: true,
})
await next()
}
if (usage === -1) {
if (POST_DELETE[quotaName]) {
await POST_DELETE[quotaName](ctx, usageContext)
}
} else {
if (POST_CREATE[quotaName]) {
await POST_CREATE[quotaName](ctx)
}
}
// update the usage
if (!usageContext.skipUsage) {
await quotas.updateUsage(usage, quotaName, quotaInfo.type)
}
}
const appPreDelete = async (ctx: any, usageContext: any) => {
if (ctx.query.unpublish) {
// don't run usage decrement for unpublish
usageContext.skipUsage = true
return
}
// store the row count to delete
const rows = await getUniqueRows([ctx.appId])
if (rows.length) {
usageContext[StaticQuotaName.APPS] = { rowCount: rows.length }
}
}
const appPostDelete = async (ctx: any, usageContext: any) => {
// delete the app rows from usage
const rowCount = usageContext[StaticQuotaName.APPS].rowCount
if (rowCount) {
await quotas.updateUsage(
-rowCount,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}
}
const appPostCreate = async (ctx: any) => {
// app import & template creation
if (ctx.request.body.useTemplate === "true") {
const rows = await getUniqueRows([ctx.response.body.appId])
const rowCount = rows ? rows.length : 0
await quotas.updateUsage(
rowCount,
StaticQuotaName.ROWS,
QuotaUsageType.STATIC
)
}
}
const PRE_DELETE: any = {
[StaticQuotaName.APPS]: appPreDelete,
}
const POST_DELETE: any = {
[StaticQuotaName.APPS]: appPostDelete,
}
const PRE_CREATE: any = {}
const POST_CREATE: any = {
[StaticQuotaName.APPS]: appPostCreate,
}

5
packages/server/src/migrations/functions/usageQuotas/index.ts
View file

@ -1,8 +1,3 @@
import { quotas } from "@budibase/pro"
export const runQuotaMigration = async (migration: Function) => {
if (!quotas.useQuotas()) {
return
}
await migration()
}

2
packages/server/src/module.d.ts vendored
View file

@ -1,3 +1,5 @@
declare module "@budibase/backend-core"
declare module "@budibase/backend-core/tenancy"
declare module "@budibase/backend-core/db"
declare module "@budibase/backend-core/context"
declare module "@budibase/backend-core/cache"

26
packages/server/src/threads/index.js → packages/server/src/threads/index.ts
View file

@ -1,12 +1,12 @@
const workerFarm = require("worker-farm")
const env = require("../environment")
import workerFarm from "worker-farm"
import * as env from "../environment"
const ThreadType = {
export const ThreadType = {
QUERY: "query",
AUTOMATION: "automation",
}
function typeToFile(type) {
function typeToFile(type: any) {
let filename = null
switch (type) {
case ThreadType.QUERY:
@ -21,8 +21,13 @@ function typeToFile(type) {
return require.resolve(filename)
}
class Thread {
constructor(type, opts = { timeoutMs: null, count: 1 }) {
export class Thread {
type: any
count: any
disableThreading: any
workers: any
constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
this.type = type
this.count = opts.count ? opts.count : 1
this.disableThreading =
@ -31,7 +36,7 @@ class Thread {
this.count === 0 ||
env.isInThread()
if (!this.disableThreading) {
const workerOpts = {
const workerOpts: any = {
autoStart: true,
maxConcurrentWorkers: this.count,
}
@ -42,7 +47,7 @@ class Thread {
}
}
run(data) {
run(data: any) {
return new Promise((resolve, reject) => {
let fncToCall
// if in test then don't use threading
@ -51,7 +56,7 @@ class Thread {
} else {
fncToCall = this.workers
}
fncToCall(data, (err, response) => {
fncToCall(data, (err: any, response: any) => {
if (err) {
reject(err)
} else {
@ -61,6 +66,3 @@ class Thread {
})
}
}
module.exports.Thread = Thread
module.exports.ThreadType = ThreadType

8
packages/server/src/utilities/queue/inMemoryQueue.js
View file

@ -66,7 +66,8 @@ class InMemoryQueue {
* @param {object} msg A message to be transported over the queue, this should be
* a JSON message as this is required by Bull.
*/
add(msg) {
// eslint-disable-next-line no-unused-vars
add(msg, repeat) {
if (typeof msg !== "object") {
throw "Queue only supports carrying JSON."
}
@ -90,6 +91,11 @@ class InMemoryQueue {
return []
}
// eslint-disable-next-line no-unused-vars
removeJobs(pattern) {
// no-op
}
/**
* Implemented for tests
*/

2
packages/worker/src/api/controllers/global/auth.ts
View file

@ -224,7 +224,7 @@ async function oidcStrategyFactory(ctx: any, configId: any) {
const chosenConfig = config.configs.filter((c: any) => c.uuid === configId)[0]
let callbackUrl = await exports.oidcCallbackUrl(chosenConfig)
return oidc.strategyFactory(chosenConfig, callbackUrl)
return oidc.strategyFactory(chosenConfig, callbackUrl, users.save)
}
/**

3
packages/worker/src/api/controllers/global/users.ts
View file

@ -37,7 +37,7 @@ const allUsers = async () => {
export const save = async (ctx: any) => {
try {
const user = await users.save(ctx.request.body, getTenantId())
const user: any = await users.save(ctx.request.body, getTenantId())
// let server know to sync user
await syncUserInApps(user._id)
ctx.body = user
@ -129,6 +129,7 @@ export const destroy = async (ctx: any) => {
await removeUserFromInfoDB(dbUser)
await db.remove(dbUser._id, dbUser._rev)
await quotas.removeUser(dbUser)
await userCache.invalidateUser(dbUser._id)
await invalidateSessions(dbUser._id)
// let server know to sync user