CodeMirror/budibase
1
0
Fork 0
mirror of synced 2024-07-03 13:30:46 +12:00
Code Issues Projects Releases Wiki Activity

Merge branch 'develop' into feature/search-bar

Browse source
This commit is contained in:
Martin McKeaveney 2021-09-30 15:12:59 +01:00 committed by GitHub
parent 9f6961a90c d3a752a45a
commit 76c83ed51e
41 changed files with 2215 additions and 124 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
lerna.json
View file

@ -1,5 +1,5 @@
{
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"npmClient": "yarn",
"packages": [
"packages/*"

2
packages/auth/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/auth",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"description": "Authentication middlewares for budibase builder and apps",
"main": "src/index.js",
"author": "Budibase",

2
packages/auth/src/cache/user.js vendored
View file

@ -12,7 +12,7 @@ const populateFromDB = async (userId, tenantId) => {
const user = await getGlobalDB(tenantId).get(userId)
user.budibaseAccess = true
if (!env.SELF_HOSTED) {
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
const account = await accounts.getAccount(user.email)
if (account) {
user.account = account

1
packages/auth/src/environment.js
View file

@ -21,6 +21,7 @@ module.exports = {
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
MULTI_TENANCY: process.env.MULTI_TENANCY,
ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
SELF_HOSTED: !!parseInt(process.env.SELF_HOSTED),
COOKIE_DOMAIN: process.env.COOKIE_DOMAIN,
isTest,

16
packages/auth/src/tenancy/deprovision.js
View file

@ -19,6 +19,22 @@ const removeTenantFromInfoDB = async tenantId => {
}
}
exports.removeUserFromInfoDB = async dbUser => {
const infoDb = getDB(PLATFORM_INFO_DB)
const keys = [dbUser._id, dbUser.email]
const userDocs = await infoDb.allDocs({
keys,
include_docs: true,
})
const toDelete = userDocs.rows.map(row => {
return {
...row.doc,
_deleted: true,
}
})
await infoDb.bulkDocs(toDelete)
}
const removeUsersFromInfoDB = async tenantId => {
try {
const globalDb = getGlobalDB(tenantId)

7
packages/auth/src/tenancy/tenancy.js
View file

@ -73,7 +73,7 @@ exports.tryAddTenant = async (tenantId, userId, email) => {
await Promise.all(promises)
}
exports.getGlobalDB = (tenantId = null) => {
exports.getGlobalDBName = (tenantId = null) => {
// tenant ID can be set externally, for example user API where
// new tenants are being created, this may be the case
if (!tenantId) {
@ -81,13 +81,16 @@ exports.getGlobalDB = (tenantId = null) => {
}
let dbName
if (tenantId === DEFAULT_TENANT_ID) {
dbName = StaticDatabases.GLOBAL.name
} else {
dbName = `${tenantId}${SEPARATOR}${StaticDatabases.GLOBAL.name}`
}
return dbName
}
exports.getGlobalDB = (tenantId = null) => {
const dbName = exports.getGlobalDBName(tenantId)
return getDB(dbName)
}

2
packages/bbui/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/bbui",
"description": "A UI solution used in the different Budibase projects.",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"license": "AGPL-3.0",
"svelte": "src/index.js",
"module": "dist/bbui.es.js",

8
packages/builder/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/builder",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"license": "AGPL-3.0",
"private": true,
"scripts": {
@ -65,10 +65,10 @@
}
},
"dependencies": {
"@budibase/bbui": "^0.9.146-alpha.3",
"@budibase/client": "^0.9.146-alpha.3",
"@budibase/bbui": "^0.9.146",
"@budibase/client": "^0.9.146",
"@budibase/colorpicker": "1.1.2",
"@budibase/string-templates": "^0.9.146-alpha.3",
"@budibase/string-templates": "^0.9.146",
"@sentry/browser": "5.19.1",
"@spectrum-css/page": "^3.0.1",
"@spectrum-css/vars": "^3.0.1",

10
packages/builder/src/components/backend/DataTable/DataTable.svelte
View file

@ -56,6 +56,12 @@
schema,
})
}
// Fetch data whenever rows are modified. Unfortunately we have to lose
// our pagination place, as our bookmarks will have shifted.
const onUpdateRows = () => {
search.update()
}
</script>
<div>
@ -71,7 +77,7 @@
allowEditing
disableSorting
on:updatecolumns={onUpdateColumns}
on:updaterows={search.refresh}
on:updaterows={onUpdateRows}
>
{#if isInternal}
<CreateColumnButton on:updatecolumns={onUpdateColumns} />
@ -79,7 +85,7 @@
{#if schema && Object.keys(schema).length > 0}
{#if !isUsersTable}
<CreateRowButton
on:updaterows={search.refresh}
on:updaterows={onUpdateRows}
title={"Create row"}
modalContentComponent={CreateEditRow}
/>

40
packages/builder/src/pages/builder/_layout.svelte
View file

@ -4,6 +4,9 @@
import { onMount } from "svelte"
let loaded = false
// don't react to these
let cloud = $admin.cloud
let shouldRedirect = !cloud || $admin.disableAccountPortal
$: multiTenancyEnabled = $admin.multiTenancy
$: hasAdminUser = $admin?.checklist?.adminUser?.checked
@ -39,30 +42,35 @@
$: {
// We should never see the org or admin user creation screens in the cloud
if (!cloud) {
const apiReady = $admin.loaded && $auth.loaded
// if tenant is not set go to it
if (loaded && apiReady && multiTenancyEnabled && !tenantSet) {
$redirect("./auth/org")
}
// Force creation of an admin user if one doesn't exist
else if (loaded && apiReady && !hasAdminUser) {
$redirect("./admin")
}
}
}
// Redirect to log in at any time if the user isn't authenticated
$: {
const apiReady = $admin.loaded && $auth.loaded
// if tenant is not set go to it
if (
loaded &&
shouldRedirect &&
apiReady &&
multiTenancyEnabled &&
!tenantSet
) {
$redirect("./auth/org")
}
// Force creation of an admin user if one doesn't exist
else if (loaded && shouldRedirect && apiReady && !hasAdminUser) {
$redirect("./admin")
}
// Redirect to log in at any time if the user isn't authenticated
else if (
loaded &&
(hasAdminUser || cloud) &&
!$auth.user &&
!$isActive("./auth") &&
!$isActive("./invite")
!$isActive("./invite") &&
!$isActive("./admin")
) {
const returnUrl = encodeURIComponent(window.location.pathname)
$redirect("./auth?", { returnUrl })
} else if ($auth?.user?.forceResetPassword) {
}
// check if password reset required for user
else if ($auth.user?.forceResetPassword) {
$redirect("./auth/reset")
}
}

50
packages/builder/src/pages/builder/admin/_components/ImportAppsModal.svelte Normal file
View file

@ -0,0 +1,50 @@
<script>
import { notifications, ModalContent, Dropzone, Body } from "@budibase/bbui"
import { post } from "builderStore/api"
let submitting = false
$: value = { file: null }
async function importApps() {
submitting = true
try {
// Create form data to create app
let data = new FormData()
data.append("importFile", value.file)
// Create App
const importResp = await post("/api/cloud/import", data, {})
const importJson = await importResp.json()
if (!importResp.ok) {
throw new Error(importJson.message)
}
// now reload to get to login
window.location.reload()
} catch (error) {
notifications.error(error)
submitting = false
}
}
</script>
<ModalContent
title="Import apps"
confirmText="Import apps"
onConfirm={importApps}
disabled={!value.file}
>
<Body
>Please upload the file that was exported from your Cloud environment to get
started</Body
>
<Dropzone
gallery={false}
label="File to import"
value={[value.file]}
on:change={e => {
value.file = e.detail?.[0]
}}
/>
</ModalContent>

16
packages/builder/src/pages/builder/admin/index.svelte
View file

@ -7,18 +7,22 @@
Input,
Body,
ActionButton,
Modal,
} from "@budibase/bbui"
import { goto } from "@roxi/routify"
import api from "builderStore/api"
import { admin, auth } from "stores/portal"
import PasswordRepeatInput from "components/common/users/PasswordRepeatInput.svelte"
import ImportAppsModal from "./_components/ImportAppsModal.svelte"
import Logo from "assets/bb-emblem.svg"
let adminUser = {}
let error
let modal
$: tenantId = $auth.tenantId
$: multiTenancyEnabled = $admin.multiTenancy
$: cloud = $admin.cloud
async function save() {
try {
@ -38,6 +42,9 @@
}
</script>
<Modal bind:this={modal} padding={false} width="600px">
<ImportAppsModal />
</Modal>
<section>
<div class="container">
<Layout>
@ -66,6 +73,15 @@
>
Change organisation
</ActionButton>
{:else if !cloud}
<ActionButton
quiet
on:click={() => {
modal.show()
}}
>
Import from cloud
</ActionButton>
{/if}
</Layout>
</Layout>

2
packages/builder/src/pages/builder/auth/_layout.svelte
View file

@ -9,10 +9,10 @@
$redirect("../")
}
// redirect to account portal for authentication in the cloud
if (
!$auth.user &&
$admin.cloud &&
!$admin.disableAccountPortal &&
$admin.accountPortalUrl &&
!$admin?.checklist?.sso?.checked
) {

3
packages/builder/src/pages/builder/auth/org.svelte
View file

@ -9,6 +9,7 @@
let tenantId = get(auth).tenantSet ? get(auth).tenantId : ""
$: multiTenancyEnabled = $admin.multiTenancy
$: cloud = $admin.cloud
$: disableAccountPortal = $admin.disableAccountPortal
async function setOrg() {
if (tenantId == null || tenantId === "") {
@ -26,7 +27,7 @@
onMount(async () => {
await auth.checkQueryString()
if (!multiTenancyEnabled || cloud) {
if (!multiTenancyEnabled || (cloud && !disableAccountPortal)) {
$goto("../")
} else {
admin.unload()

6
packages/builder/src/pages/builder/index.svelte
View file

@ -5,11 +5,9 @@
auth.checkQueryString()
$: {
if (!$auth.user) {
$redirect(`./auth`)
} else if ($auth.user.builder?.global) {
if ($auth.user?.builder?.global) {
$redirect(`./portal`)
} else {
} else if ($auth.user) {
$redirect(`./apps`)
}
}

13
packages/builder/src/pages/builder/portal/apps/index.svelte
View file

@ -37,6 +37,7 @@
let creatingApp = false
let loaded = false
let searchTerm = ""
let cloud = $admin.cloud
$: enrichedApps = enrichApps($apps, $auth.user, sortBy)
$: filteredApps = enrichedApps.filter(app =>
@ -76,6 +77,15 @@
creatingApp = true
}
const initiateAppsExport = () => {
try {
download(`/api/cloud/export`)
notifications.success("Apps exported successfully")
} catch (err) {
notifications.error(`Error exporting apps: ${err}`)
}
}
const initiateAppImport = () => {
template = { fromFile: true }
creationModal.show()
@ -196,6 +206,9 @@
<div class="title">
<Heading>Apps</Heading>
<ButtonGroup>
{#if cloud}
<Button secondary on:click={initiateAppsExport}>Export apps</Button>
{/if}
<Button secondary on:click={initiateAppImport}>Import app</Button>
<Button cta on:click={initiateAppCreation}>Create app</Button>
</ButtonGroup>

4
packages/builder/src/stores/portal/admin.js
View file

@ -7,6 +7,7 @@ export function createAdminStore() {
loaded: false,
multiTenancy: false,
cloud: false,
disableAccountPortal: false,
accountPortalUrl: "",
onboardingProgress: 0,
checklist: {
@ -47,12 +48,14 @@ export function createAdminStore() {
async function getEnvironment() {
let multiTenancyEnabled = false
let cloud = false
let disableAccountPortal = false
let accountPortalUrl = ""
try {
const response = await api.get(`/api/system/environment`)
const json = await response.json()
multiTenancyEnabled = json.multiTenancy
cloud = json.cloud
disableAccountPortal = json.disableAccountPortal
accountPortalUrl = json.accountPortalUrl
} catch (err) {
// just let it stay disabled
@ -60,6 +63,7 @@ export function createAdminStore() {
admin.update(store => {
store.multiTenancy = multiTenancyEnabled
store.cloud = cloud
store.disableAccountPortal = disableAccountPortal
store.accountPortalUrl = accountPortalUrl
return store
})

2
packages/cli/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/cli",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"description": "Budibase CLI, for developers, self hosting and migrations.",
"main": "src/index.js",
"bin": {

6
packages/client/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/client",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"license": "MPL-2.0",
"module": "dist/budibase-client.js",
"main": "dist/budibase-client.js",
@ -19,9 +19,9 @@
"dev:builder": "rollup -cw"
},
"dependencies": {
"@budibase/bbui": "^0.9.146-alpha.3",
"@budibase/bbui": "^0.9.146",
"@budibase/standard-components": "^0.9.139",
"@budibase/string-templates": "^0.9.146-alpha.3",
"@budibase/string-templates": "^0.9.146",
"regexparam": "^1.3.0",
"shortid": "^2.2.15",
"svelte-spa-router": "^3.0.5"

1008
packages/client/yarn.lock
View file

File diff suppressed because it is too large Load diff

9
packages/server/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/server",
"email": "hi@budibase.com",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"description": "Budibase Web Server",
"main": "src/index.js",
"repository": {
@ -64,9 +64,9 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/auth": "^0.9.146-alpha.3",
"@budibase/client": "^0.9.146-alpha.3",
"@budibase/string-templates": "^0.9.146-alpha.3",
"@budibase/auth": "^0.9.146",
"@budibase/client": "^0.9.146",
"@budibase/string-templates": "^0.9.146",
"@elastic/elasticsearch": "7.10.0",
"@koa/router": "8.0.0",
"@sendgrid/mail": "7.1.1",
@ -96,6 +96,7 @@
"koa-session": "5.12.0",
"koa-static": "5.0.0",
"lodash": "4.17.21",
"memorystream": "^0.3.1",
"mongodb": "3.6.3",
"mssql": "6.2.3",
"mysql": "2.18.1",

3
packages/server/scripts/dev/manage.js
View file

@ -37,7 +37,7 @@ async function init() {
const envFileJson = {
PORT: 4001,
MINIO_URL: "http://localhost:10000/",
COUCH_DB_URL: "http://@localhost:10000/db/",
COUCH_DB_URL: "http://budibase:budibase@localhost:10000/db/",
REDIS_URL: "localhost:6379",
WORKER_URL: "http://localhost:4002",
INTERNAL_API_KEY: "budibase",
@ -48,6 +48,7 @@ async function init() {
COUCH_DB_PASSWORD: "budibase",
COUCH_DB_USER: "budibase",
SELF_HOSTED: 1,
DISABLE_ACCOUNT_PORTAL: "",
MULTI_TENANCY: "",
}
let envFile = ""

14
packages/server/src/api/controllers/application.js
View file

@ -31,7 +31,7 @@ const {
getDeployedApps,
removeAppFromUserRoles,
} = require("../../utilities/workerRequests")
const { clientLibraryPath } = require("../../utilities")
const { clientLibraryPath, stringToReadStream } = require("../../utilities")
const { getAllLocks } = require("../../utilities/redis")
const {
updateClientLibrary,
@ -114,8 +114,13 @@ async function createInstance(template) {
// replicate the template data to the instance DB
// this is currently very hard to test, downloading and importing template files
/* istanbul ignore next */
if (template && template.useTemplate === "true") {
if (template && template.templateString) {
const { ok } = await db.load(stringToReadStream(template.templateString))
if (!ok) {
throw "Error loading database dump from memory."
}
} else if (template && template.useTemplate === "true") {
/* istanbul ignore next */
const { ok } = await db.load(await getTemplateStream(template))
if (!ok) {
throw "Error loading database dump from template."
@ -191,10 +196,11 @@ exports.fetchAppPackage = async function (ctx) {
}
exports.create = async function (ctx) {
const { useTemplate, templateKey } = ctx.request.body
const { useTemplate, templateKey, templateString } = ctx.request.body
const instanceConfig = {
useTemplate,
key: templateKey,
templateString,
}
if (ctx.request.files && ctx.request.files.templateFile) {
instanceConfig.file = ctx.request.files.templateFile

92
packages/server/src/api/controllers/cloud.js Normal file
View file

@ -0,0 +1,92 @@
const env = require("../../environment")
const { getAllApps } = require("@budibase/auth/db")
const CouchDB = require("../../db")
const {
exportDB,
sendTempFile,
readFileSync,
} = require("../../utilities/fileSystem")
const { stringToReadStream } = require("../../utilities")
const { getGlobalDBName, getGlobalDB } = require("@budibase/auth/tenancy")
const { create } = require("./application")
const { getDocParams, DocumentTypes, isDevAppID } = require("../../db/utils")
async function createApp(appName, appImport) {
const ctx = {
request: {
body: {
templateString: appImport,
name: appName,
},
},
}
return create(ctx)
}
exports.exportApps = async ctx => {
if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
}
const apps = await getAllApps(CouchDB, { all: true })
const globalDBString = await exportDB(getGlobalDBName())
let allDBs = {
global: globalDBString,
}
for (let app of apps) {
// only export the dev apps as they will be the latest, the user can republish the apps
// in their self hosted environment
if (isDevAppID(app._id)) {
allDBs[app.name] = await exportDB(app._id)
}
}
const filename = `cloud-export-${new Date().getTime()}.txt`
ctx.attachment(filename)
ctx.body = sendTempFile(JSON.stringify(allDBs))
}
async function getAllDocType(db, docType) {
const response = await db.allDocs(
getDocParams(docType, null, {
include_docs: true,
})
)
return response.rows.map(row => row.doc)
}
exports.importApps = async ctx => {
if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
ctx.throw(400, "Importing only allowed in self hosted environments.")
}
const apps = await getAllApps(CouchDB, { all: true })
if (
apps.length !== 0 ||
!ctx.request.files ||
!ctx.request.files.importFile
) {
ctx.throw(
400,
"Import file is required and environment must be fresh to import apps."
)
}
const importFile = ctx.request.files.importFile
const importString = readFileSync(importFile.path)
const dbs = JSON.parse(importString)
const globalDbImport = dbs.global
// remove from the list of apps
delete dbs.global
const globalDb = getGlobalDB()
// load the global db first
await globalDb.load(stringToReadStream(globalDbImport))
for (let [appName, appImport] of Object.entries(dbs)) {
await createApp(appName, appImport)
}
// once apps are created clean up the global db
let users = await getAllDocType(globalDb, DocumentTypes.USER)
for (let user of users) {
delete user.tenantId
}
await globalDb.bulkDocs(users)
ctx.body = {
message: "Apps successfully imported.",
}
}

47
packages/server/src/api/controllers/row/internal.js
View file

@ -5,7 +5,6 @@ const {
generateRowID,
DocumentTypes,
InternalTables,
generateMemoryViewID,
} = require("../../../db/utils")
const userController = require("../user")
const {
@ -20,7 +19,12 @@ const { fullSearch, paginatedSearch } = require("./internalSearch")
const { getGlobalUsersFromMetadata } = require("../../../utilities/global")
const inMemoryViews = require("../../../db/inMemoryView")
const env = require("../../../environment")
const { migrateToInMemoryView } = require("../view/utils")
const {
migrateToInMemoryView,
migrateToDesignView,
getFromDesignDoc,
getFromMemoryDoc,
} = require("../view/utils")
const CALCULATION_TYPES = {
SUM: "sum",
@ -74,33 +78,24 @@ async function getRawTableData(ctx, db, tableId) {
}
async function getView(db, viewName) {
let viewInfo
async function getFromDesignDoc() {
const designDoc = await db.get("_design/database")
viewInfo = designDoc.views[viewName]
return viewInfo
}
let migrate = false
if (env.SELF_HOSTED) {
viewInfo = await getFromDesignDoc()
} else {
try {
viewInfo = await db.get(generateMemoryViewID(viewName))
if (viewInfo) {
viewInfo = viewInfo.view
}
} catch (err) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await getFromDesignDoc()
migrate = !!viewInfo
}
let mainGetter = env.SELF_HOSTED ? getFromDesignDoc : getFromMemoryDoc
let secondaryGetter = env.SELF_HOSTED ? getFromMemoryDoc : getFromDesignDoc
let migration = env.SELF_HOSTED ? migrateToDesignView : migrateToInMemoryView
let viewInfo,
migrate = false
try {
viewInfo = await mainGetter(db, viewName)
} catch (err) {
// check if it can be retrieved from design doc (needs migrated)
if (err.status !== 404) {
viewInfo = null
} else {
viewInfo = await secondaryGetter(db, viewName)
migrate = !!viewInfo
}
}
if (migrate) {
await migrateToInMemoryView(db, viewName)
await migration(db, viewName)
}
if (!viewInfo) {
throw "View does not exist."

27
packages/server/src/api/controllers/view/utils.js
View file

@ -107,3 +107,30 @@ exports.migrateToInMemoryView = async (db, viewName) => {
await db.put(designDoc)
await exports.saveView(db, null, viewName, view)
}
exports.migrateToDesignView = async (db, viewName) => {
let view = await db.get(generateMemoryViewID(viewName))
const designDoc = await db.get("_design/database")
designDoc.views[viewName] = view.view
await db.put(designDoc)
await db.remove(view._id, view._rev)
}
exports.getFromDesignDoc = async (db, viewName) => {
const designDoc = await db.get("_design/database")
let view = designDoc.views[viewName]
if (view == null) {
throw { status: 404, message: "Unable to get view" }
}
return view
}
exports.getFromMemoryDoc = async (db, viewName) => {
let view = await db.get(generateMemoryViewID(viewName))
if (view) {
view = view.view
} else {
throw { status: 404, message: "Unable to get view" }
}
return view
}

13
packages/server/src/api/routes/cloud.js Normal file
View file

@ -0,0 +1,13 @@
const Router = require("@koa/router")
const controller = require("../controllers/cloud")
const authorized = require("../../middleware/authorized")
const { BUILDER } = require("@budibase/auth/permissions")
const router = Router()
router
.get("/api/cloud/export", authorized(BUILDER), controller.exportApps)
// has to be public, only run if apps don't exist
.post("/api/cloud/import", controller.importApps)
module.exports = router

2
packages/server/src/api/routes/index.js
View file

@ -24,6 +24,7 @@ const hostingRoutes = require("./hosting")
const backupRoutes = require("./backup")
const metadataRoutes = require("./metadata")
const devRoutes = require("./dev")
const cloudRoutes = require("./cloud")
exports.mainRoutes = [
authRoutes,
@ -49,6 +50,7 @@ exports.mainRoutes = [
backupRoutes,
metadataRoutes,
devRoutes,
cloudRoutes,
// these need to be handled last as they still use /api/:tableId
// this could be breaking as koa may recognise other routes as this
tableRoutes,

4
packages/server/src/api/routes/tests/row.spec.js
View file

@ -317,7 +317,7 @@ describe("/rows", () => {
await request
.get(`/api/views/derp`)
.set(config.defaultHeaders())
.expect(400)
.expect(404)
})
it("should be able to run on a view", async () => {
@ -394,4 +394,4 @@ describe("/rows", () => {
})
})
})
})
})

2
packages/server/src/db/utils.js
View file

@ -110,6 +110,8 @@ function getDocParams(docType, docId = null, otherProps = {}) {
}
}
exports.getDocParams = getDocParams
/**
* Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
*/

1
packages/server/src/environment.js
View file

@ -44,6 +44,7 @@ module.exports = {
NODE_ENV: process.env.NODE_ENV,
JEST_WORKER_ID: process.env.JEST_WORKER_ID,
BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
// minor
SALT_ROUNDS: process.env.SALT_ROUNDS,
LOGGER: process.env.LOGGER,

22
packages/server/src/integrations/mongodb.ts
View file

@ -85,10 +85,10 @@ module MongoDBModule {
// which method we want to call on the collection
switch (query.extra.actionTypes) {
case "insertOne": {
return collection.insertOne(query.json)
return await collection.insertOne(query.json)
}
case "insertMany": {
return collection.insertOne(query.json).toArray()
return await collection.insertOne(query.json).toArray()
}
default: {
throw new Error(
@ -112,19 +112,19 @@ module MongoDBModule {
switch (query.extra.actionTypes) {
case "find": {
return collection.find(query.json).toArray()
return await collection.find(query.json).toArray()
}
case "findOne": {
return collection.findOne(query.json)
return await collection.findOne(query.json)
}
case "findOneAndUpdate": {
return collection.findOneAndUpdate(query.json)
return await collection.findOneAndUpdate(query.json)
}
case "count": {
return collection.countDocuments(query.json)
return await collection.countDocuments(query.json)
}
case "distinct": {
return collection.distinct(query.json)
return await collection.distinct(query.json)
}
default: {
throw new Error(
@ -148,10 +148,10 @@ module MongoDBModule {
switch (query.extra.actionTypes) {
case "updateOne": {
return collection.updateOne(query.json)
return await collection.updateOne(query.json)
}
case "updateMany": {
return collection.updateMany(query.json).toArray()
return await collection.updateMany(query.json).toArray()
}
default: {
throw new Error(
@ -175,10 +175,10 @@ module MongoDBModule {
switch (query.extra.actionTypes) {
case "deleteOne": {
return collection.deleteOne(query.json)
return await collection.deleteOne(query.json)
}
case "deleteMany": {
return collection.deleteMany(query.json).toArray()
return await collection.deleteMany(query.json).toArray()
}
default: {
throw new Error(

74
packages/server/src/utilities/fileSystem/index.js
View file

@ -19,6 +19,7 @@ const {
USER_METDATA_PREFIX,
LINK_USER_METADATA_PREFIX,
} = require("../../db/utils")
const MemoryStream = require("memorystream")
const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
@ -111,29 +112,88 @@ exports.apiFileReturn = contents => {
* to the temporary backup file (to return via API if required).
*/
exports.performBackup = async (appId, backupName) => {
const path = join(budibaseTempDir(), backupName)
const writeStream = fs.createWriteStream(path)
// perform couch dump
const instanceDb = new CouchDB(appId)
await instanceDb.dump(writeStream, {
// filter out anything that has a user metadata structure in its ID
return exports.exportDB(appId, {
exportName: backupName,
filter: doc =>
!(
doc._id.includes(USER_METDATA_PREFIX) ||
doc.includes(LINK_USER_METADATA_PREFIX)
),
})
}
/**
* exports a DB to either file or a variable (memory).
* @param {string} dbName the DB which is to be exported.
* @param {string} exportName optional - the file name to export to, if not in memory.
* @param {function} filter optional - a filter function to clear out any un-wanted docs.
* @return Either the file stream or the variable (if no export name provided).
*/
exports.exportDB = async (
dbName,
{ exportName, filter } = { exportName: undefined, filter: undefined }
) => {
let stream,
appString = "",
path = null
if (exportName) {
path = join(budibaseTempDir(), exportName)
stream = fs.createWriteStream(path)
} else {
stream = new MemoryStream()
stream.on("data", chunk => {
appString += chunk.toString()
})
}
// perform couch dump
const instanceDb = new CouchDB(dbName)
await instanceDb.dump(stream, {
filter,
})
// just in memory, return the final string
if (!exportName) {
return appString
}
// write the file to the object store
if (env.SELF_HOSTED) {
await streamUpload(
ObjectStoreBuckets.BACKUPS,
join(appId, backupName),
join(dbName, exportName),
fs.createReadStream(path)
)
}
return fs.createReadStream(path)
}
/**
* Writes the provided contents to a temporary file, which can be used briefly.
* @param {string} fileContents contents which will be written to a temp file.
* @return {string} the path to the temp file.
*/
exports.storeTempFile = fileContents => {
const path = join(budibaseTempDir(), uuid())
fs.writeFileSync(path, fileContents)
return path
}
/**
* Utility function for getting a file read stream - a simple in memory buffered read
* stream doesn't work for pouchdb.
*/
exports.stringToFileStream = contents => {
const path = exports.storeTempFile(contents)
return fs.createReadStream(path)
}
/**
* Creates a temp file and returns it from the API.
* @param {string} fileContents the contents to be returned in file.
*/
exports.sendTempFile = fileContents => {
const path = exports.storeTempFile(fileContents)
return fs.createReadStream(path)
}
/**
* Uploads the latest client library to the object store.
* @param {string} appId The ID of the app which is being created.

10
packages/server/src/utilities/index.js
View file

@ -3,6 +3,7 @@ const { OBJ_STORE_DIRECTORY } = require("../constants")
const { sanitizeKey } = require("@budibase/auth/src/objectStore")
const CouchDB = require("../db")
const { generateMetadataID } = require("../db/utils")
const Readable = require("stream").Readable
const BB_CDN = "https://cdn.budi.live"
@ -124,3 +125,12 @@ exports.escapeDangerousCharacters = string => {
.replace(/[\r]/g, "\\r")
.replace(/[\t]/g, "\\t")
}
exports.stringToReadStream = string => {
return new Readable({
read() {
this.push(string)
this.push(null)
},
})
}

791
packages/server/yarn.lock
View file

File diff suppressed because it is too large Load diff

2
packages/string-templates/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@budibase/string-templates",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"description": "Handlebars wrapper for Budibase templating.",
"main": "src/index.cjs",
"module": "dist/bundle.mjs",

6
packages/worker/package.json
View file

@ -1,7 +1,7 @@
{
"name": "@budibase/worker",
"email": "hi@budibase.com",
"version": "0.9.146-alpha.3",
"version": "0.9.146",
"description": "Budibase background service",
"main": "src/index.js",
"repository": {
@ -25,8 +25,8 @@
"author": "Budibase",
"license": "AGPL-3.0-or-later",
"dependencies": {
"@budibase/auth": "^0.9.146-alpha.3",
"@budibase/string-templates": "^0.9.146-alpha.3",
"@budibase/auth": "^0.9.146",
"@budibase/string-templates": "^0.9.146",
"@koa/router": "^8.0.0",
"@techpass/passport-openidconnect": "^0.3.0",
"aws-sdk": "^2.811.0",

1
packages/worker/scripts/dev/manage.js
View file

@ -21,6 +21,7 @@ async function init() {
COUCH_DB_PASSWORD: "budibase",
// empty string is false
MULTI_TENANCY: "",
DISABLE_ACCOUNT_PORTAL: "",
ACCOUNT_PORTAL_URL: "http://localhost:10001",
}
let envFile = ""

17
packages/worker/src/api/controllers/global/users.js
View file

@ -19,6 +19,7 @@ const {
tryAddTenant,
updateTenantId,
} = require("@budibase/auth/tenancy")
const { removeUserFromInfoDB } = require("@budibase/auth/deprovision")
const env = require("../../../environment")
const PLATFORM_INFO_DB = StaticDatabases.PLATFORM_INFO.name
@ -65,7 +66,7 @@ async function saveUser(
}
// check root account users in account portal
if (!env.SELF_HOSTED) {
if (!env.SELF_HOSTED && !env.DISABLE_ACCOUNT_PORTAL) {
const account = await accounts.getAccount(email)
if (account && account.verified && account.tenantId !== tenantId) {
throw `Email address ${email} already in use.`
@ -132,7 +133,7 @@ exports.save = async ctx => {
}
const parseBooleanParam = param => {
if (param && param == "false") {
if (param && param === "false") {
return false
} else {
return true
@ -160,6 +161,17 @@ exports.adminUser = async ctx => {
// write usage quotas for cloud
if (!env.SELF_HOSTED) {
// could be a scenario where it exists, make sure its clean
try {
const usageQuota = await db.get(
StaticDatabases.PLATFORM_INFO.docs.usageQuota
)
if (usageQuota) {
await db.remove(usageQuota._id, usageQuota._rev)
}
} catch (err) {
// don't worry about errors
}
await db.post(generateNewUsageQuotaDoc())
}
@ -193,6 +205,7 @@ exports.adminUser = async ctx => {
exports.destroy = async ctx => {
const db = getGlobalDB()
const dbUser = await db.get(ctx.params.id)
await removeUserFromInfoDB(dbUser)
await db.remove(dbUser._id, dbUser._rev)
await userCache.invalidateUser(dbUser._id)
await invalidateSessions(dbUser._id)

1
packages/worker/src/api/controllers/system/environment.js
View file

@ -5,5 +5,6 @@ exports.fetch = async ctx => {
multiTenancy: !!env.MULTI_TENANCY,
cloud: !env.SELF_HOSTED,
accountPortalUrl: env.ACCOUNT_PORTAL_URL,
disableAccountPortal: env.DISABLE_ACCOUNT_PORTAL,
}
}

1
packages/worker/src/environment.js
View file

@ -32,6 +32,7 @@ module.exports = {
REDIS_PASSWORD: process.env.REDIS_PASSWORD,
INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
MULTI_TENANCY: process.env.MULTI_TENANCY,
DISABLE_ACCOUNT_PORTAL: process.env.DISABLE_ACCOUNT_PORTAL,
ACCOUNT_PORTAL_URL: process.env.ACCOUNT_PORTAL_URL,
SMTP_FALLBACK_ENABLED: process.env.SMTP_FALLBACK_ENABLED,
SMTP_USER: process.env.SMTP_USER,