diff --git a/hosting/docker-compose.yaml b/hosting/docker-compose.yaml index 66b24f4e49..c94d1520a1 100644 --- a/hosting/docker-compose.yaml +++ b/hosting/docker-compose.yaml @@ -45,6 +45,7 @@ services: MINIO_ACCESS_KEY: ${MINIO_ACCESS_KEY} MINIO_SECRET_KEY: ${MINIO_SECRET_KEY} MINIO_URL: http://minio-service:9000 + APPS_URL: http://app-service:4002 COUCH_DB_USERNAME: ${COUCH_DB_USER} COUCH_DB_PASSWORD: ${COUCH_DB_PASSWORD} COUCH_DB_URL: http://${COUCH_DB_USER}:${COUCH_DB_PASSWORD}@couchdb-service:5984 diff --git a/hosting/kubernetes/budibase/templates/worker-service-deployment.yaml b/hosting/kubernetes/budibase/templates/worker-service-deployment.yaml index d7d8702567..6cded8545f 100644 --- a/hosting/kubernetes/budibase/templates/worker-service-deployment.yaml +++ b/hosting/kubernetes/budibase/templates/worker-service-deployment.yaml @@ -113,6 +113,8 @@ spec: value: {{ .Values.globals.smtp.port | quote }} - name: SMTP_FROM_ADDRESS value: {{ .Values.globals.smtp.from | quote }} + - name: APPS_URL + value: http://app-service:{{ .Values.services.apps.port }} image: budibase/worker imagePullPolicy: Always name: bbworker diff --git a/packages/auth/src/db/utils.js b/packages/auth/src/db/utils.js index fa162603e6..b956089660 100644 --- a/packages/auth/src/db/utils.js +++ b/packages/auth/src/db/utils.js @@ -152,6 +152,17 @@ exports.getDeployedAppID = appId => { return appId } +/** + * Convert a deployed app ID to a development app ID. + */ +exports.getDevelopmentAppID = appId => { + if (!appId.startsWith(exports.APP_DEV_PREFIX)) { + const id = appId.split(exports.APP_PREFIX)[1] + return `${exports.APP_DEV_PREFIX}${id}` + } + return appId +} + exports.getCouchUrl = () => { if (!env.COUCH_DB_URL) return @@ -248,6 +259,24 @@ exports.getAllApps = async (CouchDB, { dev, all, idsOnly } = {}) => { } } +/** + * Utility function for getAllApps but filters to production apps only. + */ +exports.getDeployedAppIDs = async CouchDB => { + return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter( + id => !exports.isDevAppID(id) + ) +} + +/** + * Utility function for the inverse of above. + */ +exports.getDevAppIDs = async CouchDB => { + return (await exports.getAllApps(CouchDB, { idsOnly: true })).filter(id => + exports.isDevAppID(id) + ) +} + exports.dbExists = async (CouchDB, dbName) => { let exists = false try { diff --git a/packages/server/src/api/controllers/application.js b/packages/server/src/api/controllers/application.js index e2e42c20f9..c38f098db4 100644 --- a/packages/server/src/api/controllers/application.js +++ b/packages/server/src/api/controllers/application.js @@ -44,6 +44,7 @@ const { revertClientLibrary, } = require("../../utilities/fileSystem/clientLibrary") const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy") +const { syncGlobalUsers } = require("./user") const URL_REGEX_SLASH = /\/|\\/g @@ -328,6 +329,8 @@ exports.sync = async ctx => { if (!isDevAppID(appId)) { ctx.throw(400, "This action cannot be performed for production apps") } + + // replicate prod to dev const prodAppId = getDeployedAppID(appId) const replication = new Replication({ source: prodAppId, @@ -343,6 +346,10 @@ exports.sync = async ctx => { } catch (err) { error = err } + + // sync the users + await syncGlobalUsers(appId) + if (error) { ctx.throw(400, error) } else { diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js index 2bcdae4c05..53d8ecbacb 100644 --- a/packages/server/src/api/controllers/auth.js +++ b/packages/server/src/api/controllers/auth.js @@ -14,6 +14,8 @@ exports.fetchSelf = async ctx => { } const user = await getFullUser(ctx, userId) + // this shouldn't be returned by the app self + delete user.roles if (appId) { const db = new CouchDB(appId) @@ -36,6 +38,7 @@ exports.fetchSelf = async ctx => { // user has a role of some sort, return them else if (err.status === 404) { const metadata = { + ...user, _id: userId, } const dbResp = await db.put(metadata) diff --git a/packages/server/src/api/controllers/user.js b/packages/server/src/api/controllers/user.js index 3ff67278b4..908018fe51 100644 --- a/packages/server/src/api/controllers/user.js +++ b/packages/server/src/api/controllers/user.js @@ -4,27 +4,139 @@ const { getUserMetadataParams, } = require("../../db/utils") const { InternalTables } = require("../../db/utils") -const { getGlobalUsers } = require("../../utilities/global") +const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global") const { getFullUser } = require("../../utilities/users") +const { isEqual } = require("lodash") +const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles") +const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db") +const { doesDatabaseExist } = require("../../utilities") +const { UserStatus } = require("@budibase/auth/constants") -function removeGlobalProps(user) { - // make sure to always remove some of the global user props - delete user.password - delete user.roles - delete user.builder - return user -} - -exports.fetchMetadata = async function (ctx) { - const database = new CouchDB(ctx.appId) - const global = await getGlobalUsers(ctx.appId) - const metadata = ( - await database.allDocs( +async function rawMetadata(db) { + return ( + await db.allDocs( getUserMetadataParams(null, { include_docs: true, }) ) ).rows.map(row => row.doc) +} + +function combineMetadataAndUser(user, metadata) { + // skip users with no access + if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) { + return null + } + delete user._rev + const metadataId = generateUserMetadataID(user._id) + const newDoc = { + ...user, + _id: metadataId, + tableId: InternalTables.USER_METADATA, + } + const found = Array.isArray(metadata) + ? metadata.find(doc => doc._id === metadataId) + : metadata + // copy rev over for the purposes of equality check + if (found) { + newDoc._rev = found._rev + } + if (found == null || !isEqual(newDoc, found)) { + return { + ...found, + ...newDoc, + } + } + return null +} + +exports.syncGlobalUsers = async appId => { + // sync user metadata + const db = new CouchDB(appId) + const [users, metadata] = await Promise.all([ + getGlobalUsers(appId), + rawMetadata(db), + ]) + const toWrite = [] + for (let user of users) { + const combined = await combineMetadataAndUser(user, metadata) + if (combined) { + toWrite.push(combined) + } + } + await db.bulkDocs(toWrite) +} + +exports.syncUser = async function (ctx) { + let deleting = false, + user + const userId = ctx.params.id + try { + user = await getRawGlobalUser(userId) + } catch (err) { + if (err && err.status === 404) { + user = {} + deleting = true + } else { + throw err + } + } + const roles = user.roles + // remove props which aren't useful to metadata + delete user.password + delete user.forceResetPassword + delete user.roles + // run through all production appIDs in the users roles + let prodAppIds + // if they are a builder then get all production app IDs + if ((user.builder && user.builder.global) || deleting) { + prodAppIds = await getDeployedAppIDs(CouchDB) + } else { + prodAppIds = Object.entries(roles) + .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC) + .map(([appId]) => appId) + } + for (let prodAppId of prodAppIds) { + const devAppId = getDevelopmentAppID(prodAppId) + for (let appId of [prodAppId, devAppId]) { + if (!(await doesDatabaseExist(appId))) { + continue + } + const db = new CouchDB(appId) + const metadataId = generateUserMetadataID(userId) + let metadata + try { + metadata = await db.get(metadataId) + } catch (err) { + if (deleting) { + continue + } + metadata = { + tableId: InternalTables.USER_METADATA, + } + } + let combined + if (deleting) { + combined = { + ...metadata, + status: UserStatus.INACTIVE, + metadata: BUILTIN_ROLE_IDS.PUBLIC, + } + } else { + combined = combineMetadataAndUser(user, metadata) + } + await db.put(combined) + } + } + ctx.body = { + message: "User synced.", + } +} + +exports.fetchMetadata = async function (ctx) { + const database = new CouchDB(ctx.appId) + const global = await getGlobalUsers(ctx.appId) + const metadata = await rawMetadata(database) const users = [] for (let user of global) { // find the metadata that matches up to the global ID @@ -52,7 +164,9 @@ exports.updateSelfMetadata = async function (ctx) { exports.updateMetadata = async function (ctx) { const appId = ctx.appId const db = new CouchDB(appId) - const user = removeGlobalProps(ctx.request.body) + const user = ctx.request.body + // this isn't applicable to the user + delete user.roles const metadata = { tableId: InternalTables.USER_METADATA, ...user, diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js index d171870215..a3043b5af1 100644 --- a/packages/server/src/api/routes/user.js +++ b/packages/server/src/api/routes/user.js @@ -34,5 +34,10 @@ router authorized(PermissionTypes.USER, PermissionLevels.WRITE), controller.destroyMetadata ) + .post( + "/api/users/metadata/sync/:id", + authorized(PermissionTypes.USER, PermissionLevels.WRITE), + controller.syncUser + ) module.exports = router diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js index d91311e165..37b0c7ac42 100644 --- a/packages/server/src/middleware/authorized.js +++ b/packages/server/src/middleware/authorized.js @@ -15,7 +15,8 @@ module.exports = (permType, permLevel = null) => async (ctx, next) => { // webhooks don't need authentication, each webhook unique - if (isWebhookEndpoint(ctx)) { + // also internal requests (between services) don't need authorized + if (isWebhookEndpoint(ctx) || ctx.internal) { return next() } diff --git a/packages/server/src/utilities/global.js b/packages/server/src/utilities/global.js index 4b9bbcba8c..6527aa0601 100644 --- a/packages/server/src/utilities/global.js +++ b/packages/server/src/utilities/global.js @@ -46,9 +46,13 @@ exports.getCachedSelf = async (ctx, appId) => { return processUser(appId, user) } -exports.getGlobalUser = async (appId, userId) => { +exports.getRawGlobalUser = async userId => { const db = getGlobalDB() - let user = await db.get(getGlobalIDFromUserMetadataID(userId)) + return db.get(getGlobalIDFromUserMetadataID(userId)) +} + +exports.getGlobalUser = async (appId, userId) => { + let user = await exports.getRawGlobalUser(userId) return processUser(appId, user) } @@ -73,6 +77,7 @@ exports.getGlobalUsers = async (appId = null, users = null) => { .filter(user => user != null) .map(user => { delete user.password + delete user.forceResetPassword return user }) if (!appId) { diff --git a/packages/server/src/utilities/index.js b/packages/server/src/utilities/index.js index e568ba063c..266ee09f9f 100644 --- a/packages/server/src/utilities/index.js +++ b/packages/server/src/utilities/index.js @@ -134,3 +134,13 @@ exports.stringToReadStream = string => { }, }) } + +exports.doesDatabaseExist = async dbName => { + try { + const db = new CouchDB(dbName, { skip_setup: true }) + const info = await db.info() + return info && !info.error + } catch (err) { + return false + } +} diff --git a/packages/worker/scripts/dev/manage.js b/packages/worker/scripts/dev/manage.js index e0b8c3586a..179167883f 100644 --- a/packages/worker/scripts/dev/manage.js +++ b/packages/worker/scripts/dev/manage.js @@ -25,6 +25,7 @@ async function init() { ACCOUNT_PORTAL_URL: "http://localhost:10001", ACCOUNT_PORTAL_API_KEY: "budibase", PLATFORM_URL: "http://localhost:10000", + APPS_URL: "http://localhost:4001", } let envFile = "" Object.keys(envFileJson).forEach(key => { diff --git a/packages/worker/src/api/controllers/global/users.js b/packages/worker/src/api/controllers/global/users.js index ed70d6122e..42166faad7 100644 --- a/packages/worker/src/api/controllers/global/users.js +++ b/packages/worker/src/api/controllers/global/users.js @@ -19,6 +19,7 @@ const { } = require("@budibase/auth/tenancy") const { removeUserFromInfoDB } = require("@budibase/auth/deprovision") const env = require("../../../environment") +const { syncUserInApps } = require("../../../utilities/appService") async function allUsers() { const db = getGlobalDB() @@ -32,7 +33,10 @@ async function allUsers() { exports.save = async ctx => { try { - ctx.body = await saveUser(ctx.request.body, getTenantId()) + const user = await saveUser(ctx.request.body, getTenantId()) + // let server know to sync user + await syncUserInApps(user._id) + ctx.body = user } catch (err) { ctx.throw(err.status || 400, err) } @@ -129,6 +133,8 @@ exports.destroy = async ctx => { await db.remove(dbUser._id, dbUser._rev) await userCache.invalidateUser(dbUser._id) await invalidateSessions(dbUser._id) + // let server know to sync user + await syncUserInApps(dbUser._id) ctx.body = { message: `User ${ctx.params.id} deleted.`, } diff --git a/packages/worker/src/environment.js b/packages/worker/src/environment.js index a1fab84112..91f06ea46d 100644 --- a/packages/worker/src/environment.js +++ b/packages/worker/src/environment.js @@ -42,6 +42,7 @@ module.exports = { SMTP_FROM_ADDRESS: process.env.SMTP_FROM_ADDRESS, PLATFORM_URL: process.env.PLATFORM_URL, COOKIE_DOMAIN: process.env.COOKIE_DOMAIN, + APPS_URL: process.env.APPS_URL, _set(key, value) { process.env[key] = value module.exports[key] = value @@ -53,6 +54,13 @@ module.exports = { }, } +// if some var haven't been set, define them +if (!module.exports.APPS_URL) { + module.exports.APPS_URL = isDev() + ? "http://localhost:4001" + : "http://app-service:4002" +} + // clean up any environment variable edge cases for (let [key, value] of Object.entries(module.exports)) { // handle the edge case of "0" to disable an environment variable diff --git a/packages/worker/src/utilities/appService.js b/packages/worker/src/utilities/appService.js new file mode 100644 index 0000000000..23c5581510 --- /dev/null +++ b/packages/worker/src/utilities/appService.js @@ -0,0 +1,33 @@ +const fetch = require("node-fetch") +const { Headers } = require("@budibase/auth/constants") +const { getTenantId, isTenantIdSet } = require("@budibase/auth/tenancy") +const { checkSlashesInUrl } = require("../utilities") +const env = require("../environment") + +async function makeAppRequest(url, method, body) { + if (env.isTest()) { + return + } + const request = { headers: {} } + request.headers[Headers.API_KEY] = env.INTERNAL_API_KEY + if (isTenantIdSet()) { + request.headers[Headers.TENANT_ID] = getTenantId() + } + if (body) { + request.headers["Content-Type"] = "application/json" + request.body = JSON.stringify(body) + } + request.method = method + return fetch(checkSlashesInUrl(env.APPS_URL + url), request) +} + +exports.syncUserInApps = async userId => { + const response = await makeAppRequest( + `/api/users/metadata/sync/${userId}`, + "POST", + {} + ) + if (response && response.status !== 200) { + throw "Unable to sync user." + } +}