diff --git a/packages/backend-core/src/constants/db.ts b/packages/backend-core/src/constants/db.ts index b33b4835a9..bb944556af 100644 --- a/packages/backend-core/src/constants/db.ts +++ b/packages/backend-core/src/constants/db.ts @@ -28,7 +28,7 @@ export enum ViewName { APP_BACKUP_BY_TRIGGER = "by_trigger", } -export const DeprecatedViews = { +export const DeprecatedViews: Record = { [ViewName.USER_BY_EMAIL]: [ // removed due to inaccuracy in view doc filter logic "by_email", diff --git a/packages/backend-core/src/db/couch/DatabaseImpl.ts b/packages/backend-core/src/db/couch/DatabaseImpl.ts index 29ca4123f5..330b15e680 100644 --- a/packages/backend-core/src/db/couch/DatabaseImpl.ts +++ b/packages/backend-core/src/db/couch/DatabaseImpl.ts @@ -175,12 +175,14 @@ export class DatabaseImpl implements Database { return this.updateOutput(() => db.bulk({ docs: documents })) } - async allDocs(params: DatabaseQueryOpts): Promise> { + async allDocs( + params: DatabaseQueryOpts + ): Promise> { const db = await this.checkSetup() return this.updateOutput(() => db.list(params)) } - async query( + async query( viewName: string, params: DatabaseQueryOpts ): Promise> { diff --git a/packages/backend-core/src/db/views.ts b/packages/backend-core/src/db/views.ts index f0980ad217..5d9c5b74d3 100644 --- a/packages/backend-core/src/db/views.ts +++ b/packages/backend-core/src/db/views.ts @@ -7,12 +7,19 @@ import { } from "../constants" import { getGlobalDB } from "../context" import { doWithDB } from "./" -import { AllDocsResponse, Database, DatabaseQueryOpts } from "@budibase/types" +import { + AllDocsResponse, + Database, + DatabaseQueryOpts, + Document, + DesignDocument, + DBView, +} from "@budibase/types" import env from "../environment" const DESIGN_DB = "_design/database" -function DesignDoc() { +function DesignDoc(): DesignDocument { return { _id: DESIGN_DB, // view collation information, read before writing any complex views: @@ -21,20 +28,14 @@ function DesignDoc() { } } -interface DesignDocument { - views: any -} - async function removeDeprecated(db: Database, viewName: ViewName) { - // @ts-ignore if (!DeprecatedViews[viewName]) { return } try { const designDoc = await db.get(DESIGN_DB) - // @ts-ignore for (let deprecatedNames of DeprecatedViews[viewName]) { - delete designDoc.views[deprecatedNames] + delete designDoc.views?.[deprecatedNames] } await db.put(designDoc) } catch (err) { @@ -43,18 +44,18 @@ async function removeDeprecated(db: Database, viewName: ViewName) { } export async function createView( - db: any, + db: Database, viewJs: string, viewName: string ): Promise { let designDoc try { - designDoc = (await db.get(DESIGN_DB)) as DesignDocument + designDoc = await db.get(DESIGN_DB) } catch (err) { // no design doc, make one designDoc = DesignDoc() } - const view = { + const view: DBView = { map: viewJs, } designDoc.views = { @@ -109,7 +110,7 @@ export interface QueryViewOptions { arrayResponse?: boolean } -export async function queryViewRaw( +export async function queryViewRaw( viewName: ViewName, params: DatabaseQueryOpts, db: Database, @@ -137,18 +138,16 @@ export async function queryViewRaw( } } -export const queryView = async ( +export const queryView = async ( viewName: ViewName, params: DatabaseQueryOpts, db: Database, createFunc: any, opts?: QueryViewOptions -): Promise => { +): Promise => { const response = await queryViewRaw(viewName, params, db, createFunc, opts) const rows = response.rows - const docs = rows.map((row: any) => - params.include_docs ? row.doc : row.value - ) + const docs = rows.map(row => (params.include_docs ? row.doc! : row.value)) // if arrayResponse has been requested, always return array regardless of length if (opts?.arrayResponse) { @@ -198,11 +197,11 @@ export const createPlatformUserView = async () => { await createPlatformView(viewJs, ViewName.PLATFORM_USERS_LOWERCASE) } -export const queryPlatformView = async ( +export const queryPlatformView = async ( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions -): Promise => { +): Promise => { const CreateFuncByName: any = { [ViewName.ACCOUNT_BY_EMAIL]: createPlatformAccountEmailView, [ViewName.PLATFORM_USERS_LOWERCASE]: createPlatformUserView, @@ -220,7 +219,7 @@ const CreateFuncByName: any = { [ViewName.USER_BY_APP]: createUserAppView, } -export const queryGlobalView = async ( +export const queryGlobalView = async ( viewName: ViewName, params: DatabaseQueryOpts, db?: Database, @@ -231,10 +230,10 @@ export const queryGlobalView = async ( db = getGlobalDB() } const createFn = CreateFuncByName[viewName] - return queryView(viewName, params, db!, createFn, opts) + return queryView(viewName, params, db!, createFn, opts) } -export async function queryGlobalViewRaw( +export async function queryGlobalViewRaw( viewName: ViewName, params: DatabaseQueryOpts, opts?: QueryViewOptions diff --git a/packages/backend-core/src/users/db.ts b/packages/backend-core/src/users/db.ts index c071064713..59f698d99c 100644 --- a/packages/backend-core/src/users/db.ts +++ b/packages/backend-core/src/users/db.ts @@ -413,15 +413,13 @@ export class UserDB { } // Get users and delete - const allDocsResponse: AllDocsResponse = await db.allDocs({ + const allDocsResponse = await db.allDocs({ include_docs: true, keys: userIds, }) - const usersToDelete: User[] = allDocsResponse.rows.map( - (user: RowResponse) => { - return user.doc - } - ) + const usersToDelete = allDocsResponse.rows.map(user => { + return user.doc! + }) // Delete from DB const toDelete = usersToDelete.map(user => ({ diff --git a/packages/backend-core/src/users/users.ts b/packages/backend-core/src/users/users.ts index 6dc8750b62..9f4a41f6df 100644 --- a/packages/backend-core/src/users/users.ts +++ b/packages/backend-core/src/users/users.ts @@ -151,7 +151,7 @@ export const searchGlobalUsersByApp = async ( include_docs: true, }) params.startkey = opts && opts.startkey ? opts.startkey : params.startkey - let response = await queryGlobalView(ViewName.USER_BY_APP, params) + let response = await queryGlobalView(ViewName.USER_BY_APP, params) if (!response) { response = [] diff --git a/packages/server/src/api/controllers/layout.ts b/packages/server/src/api/controllers/layout.ts index c00252d643..69e4ad91ed 100644 --- a/packages/server/src/api/controllers/layout.ts +++ b/packages/server/src/api/controllers/layout.ts @@ -1,7 +1,7 @@ import { EMPTY_LAYOUT } from "../../constants/layouts" import { generateLayoutID, getScreenParams } from "../../db/utils" import { events, context } from "@budibase/backend-core" -import { BBContext } from "@budibase/types" +import { BBContext, Layout } from "@budibase/types" export async function save(ctx: BBContext) { const db = context.getAppDB() @@ -30,12 +30,12 @@ export async function destroy(ctx: BBContext) { layoutRev = ctx.params.layoutRev const layoutsUsedByScreens = ( - await db.allDocs( + await db.allDocs( getScreenParams(null, { include_docs: true, }) ) - ).rows.map(element => element.doc.layoutId) + ).rows.map(element => element.doc!.layoutId) if (layoutsUsedByScreens.includes(layoutId)) { ctx.throw(400, "Cannot delete a layout that's being used by a screen") } diff --git a/packages/server/src/api/controllers/permission.ts b/packages/server/src/api/controllers/permission.ts index a9cd686674..e2bd6c40e5 100644 --- a/packages/server/src/api/controllers/permission.ts +++ b/packages/server/src/api/controllers/permission.ts @@ -25,12 +25,12 @@ const SUPPORTED_LEVELS = CURRENTLY_SUPPORTED_LEVELS // utility function to stop this repetition - permissions always stored under roles async function getAllDBRoles(db: Database) { - const body = await db.allDocs( + const body = await db.allDocs( getRoleParams(null, { include_docs: true, }) ) - return body.rows.map(row => row.doc) + return body.rows.map(row => row.doc!) } async function updatePermissionOnRole( @@ -79,7 +79,7 @@ async function updatePermissionOnRole( ) { rolePermissions[resourceId] = typeof rolePermissions[resourceId] === "string" - ? [rolePermissions[resourceId]] + ? [rolePermissions[resourceId] as unknown as string] : [] } // handle the removal/updating the role which has this permission first diff --git a/packages/server/src/api/controllers/role.ts b/packages/server/src/api/controllers/role.ts index 3697bbe925..ae6b89e6d4 100644 --- a/packages/server/src/api/controllers/role.ts +++ b/packages/server/src/api/controllers/role.ts @@ -6,7 +6,13 @@ import { Header, } from "@budibase/backend-core" import { getUserMetadataParams, InternalTables } from "../../db/utils" -import { Database, Role, UserCtx, UserRoles } from "@budibase/types" +import { + Database, + Role, + UserCtx, + UserMetadata, + UserRoles, +} from "@budibase/types" import { sdk as sharedSdk } from "@budibase/shared-core" import sdk from "../../sdk" @@ -115,12 +121,12 @@ export async function destroy(ctx: UserCtx) { const role = await db.get(roleId) // first check no users actively attached to role const users = ( - await db.allDocs( + await db.allDocs( getUserMetadataParams(undefined, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) const usersWithRole = users.filter(user => user.roleId === roleId) if (usersWithRole.length !== 0) { ctx.throw(400, "Cannot delete role when it is in use.") diff --git a/packages/server/src/api/controllers/row/internal.ts b/packages/server/src/api/controllers/row/internal.ts index 80a69cf92b..d53345a239 100644 --- a/packages/server/src/api/controllers/row/internal.ts +++ b/packages/server/src/api/controllers/row/internal.ts @@ -233,17 +233,16 @@ export async function fetchEnrichedRow(ctx: UserCtx) { const tableId = utils.getTableId(ctx) const rowId = ctx.params.rowId as string // need table to work out where links go in row, as well as the link docs - let response = await Promise.all([ + const [table, row, links] = await Promise.all([ sdk.tables.getTable(tableId), utils.findRow(ctx, tableId, rowId), linkRows.getLinkDocuments({ tableId, rowId, fieldName }), ]) - const table = response[0] as Table - const row = response[1] as Row - const linkVals = response[2] as LinkDocumentValue[] + const linkVals = links as LinkDocumentValue[] + // look up the actual rows based on the ids const params = getMultiIDParams(linkVals.map(linkVal => linkVal.id)) - let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc) + let linkedRows = (await db.allDocs(params)).rows.map(row => row.doc!) // get the linked tables const linkTableIds = getLinkedTableIDs(table as Table) diff --git a/packages/server/src/api/controllers/row/staticFormula.ts b/packages/server/src/api/controllers/row/staticFormula.ts index 6f426c6fa0..87d551bc4b 100644 --- a/packages/server/src/api/controllers/row/staticFormula.ts +++ b/packages/server/src/api/controllers/row/staticFormula.ts @@ -86,12 +86,12 @@ export async function updateAllFormulasInTable(table: Table) { const db = context.getAppDB() // start by getting the raw rows (which will be written back to DB after update) let rows = ( - await db.allDocs( + await db.allDocs( getRowParams(table._id, null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) // now enrich the rows, note the clone so that we have the base state of the // rows so that we don't write any of the enriched information back let enrichedRows = await outputProcessing(table, cloneDeep(rows), { diff --git a/packages/server/src/api/controllers/table/utils.ts b/packages/server/src/api/controllers/table/utils.ts index da1573715c..ee295ff7af 100644 --- a/packages/server/src/api/controllers/table/utils.ts +++ b/packages/server/src/api/controllers/table/utils.ts @@ -333,29 +333,33 @@ export async function checkForViewUpdates( columnRename?: RenameColumn ) { const views = await getViews() - const tableViews = views.filter(view => view.meta.tableId === table._id) + const tableViews = views.filter(view => view.meta?.tableId === table._id) // Check each table view to see if impacted by this table action for (let view of tableViews) { let needsUpdated = false + const viewMetadata = view.meta as any + if (!viewMetadata) { + continue + } // First check for renames, otherwise check for deletions if (columnRename) { // Update calculation field if required - if (view.meta.field === columnRename.old) { - view.meta.field = columnRename.updated + if (viewMetadata.field === columnRename.old) { + viewMetadata.field = columnRename.updated needsUpdated = true } // Update group by field if required - if (view.meta.groupBy === columnRename.old) { - view.meta.groupBy = columnRename.updated + if (viewMetadata.groupBy === columnRename.old) { + viewMetadata.groupBy = columnRename.updated needsUpdated = true } // Update filters if required - if (view.meta.filters) { - view.meta.filters.forEach((filter: any) => { + if (viewMetadata.filters) { + viewMetadata.filters.forEach((filter: any) => { if (filter.key === columnRename.old) { filter.key = columnRename.updated needsUpdated = true @@ -365,26 +369,26 @@ export async function checkForViewUpdates( } else if (deletedColumns) { deletedColumns.forEach((column: string) => { // Remove calculation statement if required - if (view.meta.field === column) { - delete view.meta.field - delete view.meta.calculation - delete view.meta.groupBy + if (viewMetadata.field === column) { + delete viewMetadata.field + delete viewMetadata.calculation + delete viewMetadata.groupBy needsUpdated = true } // Remove group by field if required - if (view.meta.groupBy === column) { - delete view.meta.groupBy + if (viewMetadata.groupBy === column) { + delete viewMetadata.groupBy needsUpdated = true } // Remove filters referencing deleted field if required - if (view.meta.filters && view.meta.filters.length) { - const initialLength = view.meta.filters.length - view.meta.filters = view.meta.filters.filter((filter: any) => { + if (viewMetadata.filters && viewMetadata.filters.length) { + const initialLength = viewMetadata.filters.length + viewMetadata.filters = viewMetadata.filters.filter((filter: any) => { return filter.key !== column }) - if (initialLength !== view.meta.filters.length) { + if (initialLength !== viewMetadata.filters.length) { needsUpdated = true } } @@ -397,15 +401,16 @@ export async function checkForViewUpdates( (field: any) => field.name == view.groupBy ) const newViewTemplate = viewTemplate( - view.meta, + viewMetadata, groupByField?.type === FieldTypes.ARRAY ) - await saveView(null, view.name, newViewTemplate) - if (!newViewTemplate.meta.schema) { - newViewTemplate.meta.schema = table.schema + const viewName = view.name! + await saveView(null, viewName, newViewTemplate) + if (!newViewTemplate.meta?.schema) { + newViewTemplate.meta!.schema = table.schema } - if (table.views?.[view.name]) { - table.views[view.name] = newViewTemplate.meta as View + if (table.views?.[viewName]) { + table.views[viewName] = newViewTemplate.meta as View } } } diff --git a/packages/server/src/api/controllers/view/utils.ts b/packages/server/src/api/controllers/view/utils.ts index 189c4ede51..7f9ae1a9bc 100644 --- a/packages/server/src/api/controllers/view/utils.ts +++ b/packages/server/src/api/controllers/view/utils.ts @@ -8,13 +8,13 @@ import { import env from "../../../environment" import { context } from "@budibase/backend-core" import viewBuilder from "./viewBuilder" -import { Database } from "@budibase/types" +import { Database, DBView, DesignDocument, InMemoryView } from "@budibase/types" export async function getView(viewName: string) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - return designDoc.views[viewName] + const designDoc = await db.get("_design/database") + return designDoc.views?.[viewName] } else { // This is a table view, don't read the view from the DB if (viewName.startsWith(DocumentType.TABLE + SEPARATOR)) { @@ -22,7 +22,7 @@ export async function getView(viewName: string) { } try { - const viewDoc = await db.get(generateMemoryViewID(viewName)) + const viewDoc = await db.get(generateMemoryViewID(viewName)) return viewDoc.view } catch (err: any) { // Return null when PouchDB doesn't found the view @@ -35,30 +35,33 @@ export async function getView(viewName: string) { } } -export async function getViews() { +export async function getViews(): Promise { const db = context.getAppDB() - const response = [] + const response: DBView[] = [] if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - for (let name of Object.keys(designDoc.views)) { + const designDoc = await db.get("_design/database") + for (let name of Object.keys(designDoc.views || {})) { // Only return custom views, not built ins const viewNames = Object.values(ViewName) as string[] if (viewNames.indexOf(name) !== -1) { continue } - response.push({ - name, - ...designDoc.views[name], - }) + const view = designDoc.views?.[name] + if (view) { + response.push({ + name, + ...view, + }) + } } } else { const views = ( - await db.allDocs( + await db.allDocs( getMemoryViewParams({ include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) for (let viewDoc of views) { response.push({ name: viewDoc.name, @@ -72,11 +75,11 @@ export async function getViews() { export async function saveView( originalName: string | null, viewName: string, - viewTemplate: any + viewTemplate: DBView ) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") designDoc.views = { ...designDoc.views, [viewName]: viewTemplate, @@ -89,17 +92,17 @@ export async function saveView( } else { const id = generateMemoryViewID(viewName) const originalId = originalName ? generateMemoryViewID(originalName) : null - const viewDoc: any = { + const viewDoc: InMemoryView = { _id: id, view: viewTemplate, name: viewName, - tableId: viewTemplate.meta.tableId, + tableId: viewTemplate.meta!.tableId, } try { - const old = await db.get(id) + const old = await db.get(id) if (originalId) { - const originalDoc = await db.get(originalId) - await db.remove(originalDoc._id, originalDoc._rev) + const originalDoc = await db.get(originalId) + await db.remove(originalDoc._id!, originalDoc._rev) } if (old && old._rev) { viewDoc._rev = old._rev @@ -114,52 +117,65 @@ export async function saveView( export async function deleteView(viewName: string) { const db = context.getAppDB() if (env.SELF_HOSTED) { - const designDoc = await db.get("_design/database") - const view = designDoc.views[viewName] - delete designDoc.views[viewName] + const designDoc = await db.get("_design/database") + const view = designDoc.views?.[viewName] + delete designDoc.views?.[viewName] await db.put(designDoc) return view } else { const id = generateMemoryViewID(viewName) - const viewDoc = await db.get(id) - await db.remove(viewDoc._id, viewDoc._rev) + const viewDoc = await db.get(id) + await db.remove(viewDoc._id!, viewDoc._rev) return viewDoc.view } } export async function migrateToInMemoryView(db: Database, viewName: string) { // delete the view initially - const designDoc = await db.get("_design/database") + const designDoc = await db.get("_design/database") + const meta = designDoc.views?.[viewName].meta + if (!meta) { + throw new Error("Unable to migrate view - no metadata") + } // run the view back through the view builder to update it - const view = viewBuilder(designDoc.views[viewName].meta) - delete designDoc.views[viewName] + const view = viewBuilder(meta) + delete designDoc.views?.[viewName] await db.put(designDoc) - await exports.saveView(db, null, viewName, view) + await saveView(null, viewName, view) } export async function migrateToDesignView(db: Database, viewName: string) { - let view = await db.get(generateMemoryViewID(viewName)) - const designDoc = await db.get("_design/database") - designDoc.views[viewName] = viewBuilder(view.view.meta) + let view = await db.get(generateMemoryViewID(viewName)) + const designDoc = await db.get("_design/database") + const meta = view.view.meta + if (!meta) { + throw new Error("Unable to migrate view - no metadata") + } + if (!designDoc.views) { + designDoc.views = {} + } + designDoc.views[viewName] = viewBuilder(meta) await db.put(designDoc) - await db.remove(view._id, view._rev) + await db.remove(view._id!, view._rev) } export async function getFromDesignDoc(db: Database, viewName: string) { - const designDoc = await db.get("_design/database") - let view = designDoc.views[viewName] + const designDoc = await db.get("_design/database") + let view = designDoc.views?.[viewName] if (view == null) { throw { status: 404, message: "Unable to get view" } } return view } -export async function getFromMemoryDoc(db: Database, viewName: string) { - let view = await db.get(generateMemoryViewID(viewName)) +export async function getFromMemoryDoc( + db: Database, + viewName: string +): Promise { + let view = await db.get(generateMemoryViewID(viewName)) if (view) { - view = view.view + return view.view } else { throw { status: 404, message: "Unable to get view" } } - return view } diff --git a/packages/server/src/api/controllers/view/viewBuilder.ts b/packages/server/src/api/controllers/view/viewBuilder.ts index cbe7e72d04..3df9df6657 100644 --- a/packages/server/src/api/controllers/view/viewBuilder.ts +++ b/packages/server/src/api/controllers/view/viewBuilder.ts @@ -1,13 +1,4 @@ -import { ViewFilter } from "@budibase/types" - -type ViewTemplateOpts = { - field: string - tableId: string - groupBy: string - filters: ViewFilter[] - calculation: string - groupByMulti: boolean -} +import { ViewFilter, ViewTemplateOpts, DBView } from "@budibase/types" const TOKEN_MAP: Record = { EQUALS: "===", @@ -146,7 +137,7 @@ function parseEmitExpression(field: string, groupBy: string) { export default function ( { field, tableId, groupBy, filters = [], calculation }: ViewTemplateOpts, groupByMulti?: boolean -) { +): DBView { // first filter can't have a conjunction if (filters && filters.length > 0 && filters[0].conjunction) { delete filters[0].conjunction diff --git a/packages/server/src/api/controllers/view/views.ts b/packages/server/src/api/controllers/view/views.ts index 86deff09ef..d311565524 100644 --- a/packages/server/src/api/controllers/view/views.ts +++ b/packages/server/src/api/controllers/view/views.ts @@ -47,8 +47,11 @@ export async function save(ctx: Ctx) { // add views to table document if (!table.views) table.views = {} - if (!view.meta.schema) { - view.meta.schema = table.schema + if (!view.meta?.schema) { + view.meta = { + ...view.meta!, + schema: table.schema, + } } table.views[viewName] = { ...view.meta, name: viewName } if (originalName) { @@ -125,10 +128,13 @@ export async function destroy(ctx: Ctx) { const db = context.getAppDB() const viewName = decodeURIComponent(ctx.params.viewName) const view = await deleteView(viewName) + if (!view || !view.meta) { + ctx.throw(400, "Unable to delete view - no metadata/view not found.") + } const table = await sdk.tables.getTable(view.meta.tableId) delete table.views![viewName] await db.put(table) - await events.view.deleted(view) + await events.view.deleted(view as View) ctx.body = view builderSocket?.emitTableUpdate(ctx, table) @@ -147,7 +153,7 @@ export async function exportView(ctx: Ctx) { ) } - if (view) { + if (view && view.meta) { ctx.params.viewName = viewName // Fetch view rows ctx.query = { diff --git a/packages/server/src/automations/triggers.ts b/packages/server/src/automations/triggers.ts index 922bc10343..f0eca759f5 100644 --- a/packages/server/src/automations/triggers.ts +++ b/packages/server/src/automations/triggers.ts @@ -20,10 +20,10 @@ const JOB_OPTS = { async function getAllAutomations() { const db = context.getAppDB() - let automations = await db.allDocs( + let automations = await db.allDocs( getAutomationParams(null, { include_docs: true }) ) - return automations.rows.map(row => row.doc) + return automations.rows.map(row => row.doc!) } async function queueRelevantRowAutomations( @@ -45,19 +45,19 @@ async function queueRelevantRowAutomations( for (let automation of automations) { let automationDef = automation.definition - let automationTrigger = automationDef ? automationDef.trigger : {} + let automationTrigger = automationDef?.trigger // don't queue events which are for dev apps, only way to test automations is // running tests on them, in production the test flag will never // be checked due to lazy evaluation (first always false) if ( !env.ALLOW_DEV_AUTOMATIONS && isDevAppID(event.appId) && - !(await checkTestFlag(automation._id)) + !(await checkTestFlag(automation._id!)) ) { continue } if ( - automationTrigger.inputs && + automationTrigger?.inputs && automationTrigger.inputs.tableId === event.row.tableId ) { await automationQueue.add({ automation, event }, JOB_OPTS) diff --git a/packages/server/src/db/inMemoryView.ts b/packages/server/src/db/inMemoryView.ts index 4e9301f4ee..724bc725ce 100644 --- a/packages/server/src/db/inMemoryView.ts +++ b/packages/server/src/db/inMemoryView.ts @@ -1,5 +1,5 @@ import newid from "./newid" -import { Row, View, Document } from "@budibase/types" +import { Row, Document, DBView } from "@budibase/types" // bypass the main application db config // use in memory pouchdb directly @@ -7,7 +7,7 @@ import { db as dbCore } from "@budibase/backend-core" const Pouch = dbCore.getPouch({ inMemory: true }) export async function runView( - view: View, + view: DBView, calculation: string, group: boolean, data: Row[] diff --git a/packages/server/src/db/linkedRows/index.ts b/packages/server/src/db/linkedRows/index.ts index 7a7a06551e..8cccf1b96a 100644 --- a/packages/server/src/db/linkedRows/index.ts +++ b/packages/server/src/db/linkedRows/index.ts @@ -14,7 +14,14 @@ import partition from "lodash/partition" import { getGlobalUsersFromMetadata } from "../../utilities/global" import { processFormulas } from "../../utilities/rowProcessor" import { context } from "@budibase/backend-core" -import { Table, Row, LinkDocumentValue, FieldType } from "@budibase/types" +import { + Table, + Row, + LinkDocumentValue, + FieldType, + LinkDocument, + ContextUser, +} from "@budibase/types" import sdk from "../../sdk" export { IncludeDocs, getLinkDocuments, createLinkView } from "./linkUtils" @@ -73,18 +80,18 @@ async function getFullLinkedDocs(links: LinkDocumentValue[]) { const db = context.getAppDB() const linkedRowIds = links.map(link => link.id) const uniqueRowIds = [...new Set(linkedRowIds)] - let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( - row => row.doc + let dbRows = (await db.allDocs(getMultiIDParams(uniqueRowIds))).rows.map( + row => row.doc! ) // convert the unique db rows back to a full list of linked rows const linked = linkedRowIds .map(id => dbRows.find(row => row && row._id === id)) - .filter(row => row != null) + .filter(row => row != null) as Row[] // need to handle users as specific cases let [users, other] = partition(linked, linkRow => - linkRow._id.startsWith(USER_METDATA_PREFIX) + linkRow._id!.startsWith(USER_METDATA_PREFIX) ) - users = await getGlobalUsersFromMetadata(users) + users = await getGlobalUsersFromMetadata(users as ContextUser[]) return [...other, ...users] } @@ -176,7 +183,7 @@ export async function attachFullLinkedDocs( // clear any existing links that could be dupe'd rows = clearRelationshipFields(table, rows) // now get the docs and combine into the rows - let linked = [] + let linked: Row[] = [] if (linksWithoutFromRow.length > 0) { linked = await getFullLinkedDocs(linksWithoutFromRow) } @@ -189,7 +196,7 @@ export async function attachFullLinkedDocs( if (opts?.fromRow && opts?.fromRow?._id === link.id) { linkedRow = opts.fromRow! } else { - linkedRow = linked.find(row => row._id === link.id) + linkedRow = linked.find(row => row._id === link.id)! } if (linkedRow) { const linkedTableId = diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts index 158e4772b2..c3415bdb36 100644 --- a/packages/server/src/sdk/app/applications/import.ts +++ b/packages/server/src/sdk/app/applications/import.ts @@ -91,7 +91,7 @@ async function getImportableDocuments(db: Database) { // map the responses to the document itself let documents: Document[] = [] for (let response of await Promise.all(docPromises)) { - documents = documents.concat(response.rows.map(row => row.doc)) + documents = documents.concat(response.rows.map(row => row.doc!)) } // remove the _rev, stops it being written documents.forEach(doc => { diff --git a/packages/server/src/sdk/app/applications/sync.ts b/packages/server/src/sdk/app/applications/sync.ts index 6e1e6747e1..e73b3396d9 100644 --- a/packages/server/src/sdk/app/applications/sync.ts +++ b/packages/server/src/sdk/app/applications/sync.ts @@ -3,7 +3,11 @@ import { db as dbCore, context, logging, roles } from "@budibase/backend-core" import { User, ContextUser, UserGroup } from "@budibase/types" import { sdk as proSdk } from "@budibase/pro" import sdk from "../../" -import { getGlobalUsers, processUser } from "../../../utilities/global" +import { + getGlobalUsers, + getRawGlobalUsers, + processUser, +} from "../../../utilities/global" import { generateUserMetadataID, InternalTables } from "../../../db/utils" type DeletedUser = { _id: string; deleted: boolean } @@ -77,9 +81,7 @@ async function syncUsersToApp( export async function syncUsersToAllApps(userIds: string[]) { // list of users, if one has been deleted it will be undefined in array - const users = (await getGlobalUsers(userIds, { - noProcessing: true, - })) as User[] + const users = await getRawGlobalUsers(userIds) const groups = await proSdk.groups.fetch() const finalUsers: (User | DeletedUser)[] = [] for (let userId of userIds) { diff --git a/packages/server/src/sdk/app/datasources/datasources.ts b/packages/server/src/sdk/app/datasources/datasources.ts index fb5d04b03e..51cceeab94 100644 --- a/packages/server/src/sdk/app/datasources/datasources.ts +++ b/packages/server/src/sdk/app/datasources/datasources.ts @@ -51,12 +51,12 @@ export async function fetch(opts?: { // Get external datasources const datasources = ( - await db.allDocs( + await db.allDocs( getDatasourceParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([ bbInternalDb, @@ -271,5 +271,5 @@ export async function getExternalDatasources(): Promise { }) ) - return externalDatasources.rows.map(r => r.doc) + return externalDatasources.rows.map(r => r.doc!) } diff --git a/packages/server/src/sdk/app/rows/search/internal.ts b/packages/server/src/sdk/app/rows/search/internal.ts index e31bda1a15..87a33c0ba0 100644 --- a/packages/server/src/sdk/app/rows/search/internal.ts +++ b/packages/server/src/sdk/app/rows/search/internal.ts @@ -185,8 +185,8 @@ export async function fetchView( group: !!group, }) } else { - const tableId = viewInfo.meta.tableId - const data = await fetchRaw(tableId) + const tableId = viewInfo.meta!.tableId + const data = await fetchRaw(tableId!) response = await inMemoryViews.runView( viewInfo, calculation as string, @@ -200,7 +200,7 @@ export async function fetchView( response.rows = response.rows.map(row => row.doc) let table: Table try { - table = await sdk.tables.getTable(viewInfo.meta.tableId) + table = await sdk.tables.getTable(viewInfo.meta!.tableId) } catch (err) { throw new Error("Unable to retrieve view table.") } diff --git a/packages/server/src/sdk/app/tables/getters.ts b/packages/server/src/sdk/app/tables/getters.ts index c0d5fe8da8..a7074f95b2 100644 --- a/packages/server/src/sdk/app/tables/getters.ts +++ b/packages/server/src/sdk/app/tables/getters.ts @@ -48,7 +48,7 @@ export async function getAllInternalTables(db?: Database): Promise { if (!db) { db = context.getAppDB() } - const internalTables = await db.allDocs( + const internalTables = await db.allDocs( getTableParams(null, { include_docs: true, }) @@ -124,7 +124,7 @@ export async function getTables(tableIds: string[]): Promise { } if (internalTableIds.length) { const db = context.getAppDB() - const internalTableDocs = await db.allDocs( + const internalTableDocs = await db.allDocs
( getMultiIDParams(internalTableIds) ) tables = tables.concat(internalTableDocs.rows.map(row => row.doc!)) diff --git a/packages/server/src/sdk/users/utils.ts b/packages/server/src/sdk/users/utils.ts index 03ddc954e9..9632ac29d8 100644 --- a/packages/server/src/sdk/users/utils.ts +++ b/packages/server/src/sdk/users/utils.ts @@ -7,12 +7,17 @@ import { InternalTables, } from "../../db/utils" import isEqual from "lodash/isEqual" -import { ContextUser, UserMetadata, User, Database } from "@budibase/types" +import { + ContextUser, + UserMetadata, + Database, + ContextUserMetadata, +} from "@budibase/types" export function combineMetadataAndUser( user: ContextUser, metadata: UserMetadata | UserMetadata[] -) { +): ContextUserMetadata | null { const metadataId = generateUserMetadataID(user._id!) const found = Array.isArray(metadata) ? metadata.find(doc => doc._id === metadataId) @@ -51,33 +56,33 @@ export function combineMetadataAndUser( return null } -export async function rawUserMetadata(db?: Database) { +export async function rawUserMetadata(db?: Database): Promise { if (!db) { db = context.getAppDB() } return ( - await db.allDocs( + await db.allDocs( getUserMetadataParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) } -export async function fetchMetadata() { +export async function fetchMetadata(): Promise { const global = await getGlobalUsers() const metadata = await rawUserMetadata() - const users = [] + const users: ContextUserMetadata[] = [] for (let user of global) { // find the metadata that matches up to the global ID - const info = metadata.find(meta => meta._id.includes(user._id)) + const info = metadata.find(meta => meta._id!.includes(user._id!)) // remove these props, not for the correct DB users.push({ ...user, ...info, tableId: InternalTables.USER_METADATA, // make sure the ID is always a local ID, not a global one - _id: generateUserMetadataID(user._id), + _id: generateUserMetadataID(user._id!), }) } return users @@ -90,9 +95,10 @@ export async function syncGlobalUsers() { if (!(await db.exists())) { continue } - const resp = await Promise.all([getGlobalUsers(), rawUserMetadata(db)]) - const users = resp[0] as User[] - const metadata = resp[1] as UserMetadata[] + const [users, metadata] = await Promise.all([ + getGlobalUsers(), + rawUserMetadata(db), + ]) const toWrite = [] for (let user of users) { const combined = combineMetadataAndUser(user, metadata) diff --git a/packages/server/src/utilities/global.ts b/packages/server/src/utilities/global.ts index 5aa201990c..cdc2d84513 100644 --- a/packages/server/src/utilities/global.ts +++ b/packages/server/src/utilities/global.ts @@ -71,69 +71,67 @@ export async function processUser( return user } -export async function getCachedSelf(ctx: UserCtx, appId: string) { +export async function getCachedSelf( + ctx: UserCtx, + appId: string +): Promise { // this has to be tenant aware, can't depend on the context to find it out // running some middlewares before the tenancy causes context to break const user = await cache.user.getUser(ctx.user?._id!) return processUser(user, { appId }) } -export async function getRawGlobalUser(userId: string) { +export async function getRawGlobalUser(userId: string): Promise { const db = tenancy.getGlobalDB() return db.get(getGlobalIDFromUserMetadataID(userId)) } -export async function getGlobalUser(userId: string) { +export async function getGlobalUser(userId: string): Promise { const appId = context.getAppId() let user = await getRawGlobalUser(userId) return processUser(user, { appId }) } -export async function getGlobalUsers( - userIds?: string[], - opts?: { noProcessing?: boolean } -) { - const appId = context.getAppId() +export async function getRawGlobalUsers(userIds?: string[]): Promise { const db = tenancy.getGlobalDB() - let globalUsers + let globalUsers: User[] if (userIds) { - globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( - row => row.doc + globalUsers = (await db.allDocs(getMultiIDParams(userIds))).rows.map( + row => row.doc! ) } else { globalUsers = ( - await db.allDocs( + await db.allDocs( dbCore.getGlobalUserParams(null, { include_docs: true, }) ) - ).rows.map(row => row.doc) + ).rows.map(row => row.doc!) } - globalUsers = globalUsers + return globalUsers .filter(user => user != null) .map(user => { delete user.password delete user.forceResetPassword return user }) +} - if (opts?.noProcessing || !appId) { - return globalUsers - } else { - // pass in the groups, meaning we don't actually need to retrieve them for - // each user individually - const allGroups = await groups.fetch() - return Promise.all( - globalUsers.map(user => processUser(user, { groups: allGroups })) - ) - } +export async function getGlobalUsers( + userIds?: string[] +): Promise { + const users = await getRawGlobalUsers(userIds) + const allGroups = await groups.fetch() + return Promise.all( + users.map(user => processUser(user, { groups: allGroups })) + ) } export async function getGlobalUsersFromMetadata(users: ContextUser[]) { const globalUsers = await getGlobalUsers(users.map(user => user._id!)) return users.map(user => { const globalUser = globalUsers.find( - globalUser => globalUser && user._id?.includes(globalUser._id) + globalUser => globalUser && user._id?.includes(globalUser._id!) ) return { ...globalUser, diff --git a/packages/server/src/utilities/routing/index.ts b/packages/server/src/utilities/routing/index.ts index de966a946b..82d45743ce 100644 --- a/packages/server/src/utilities/routing/index.ts +++ b/packages/server/src/utilities/routing/index.ts @@ -1,9 +1,9 @@ import { createRoutingView } from "../../db/views/staticViews" import { ViewName, getQueryIndex, UNICODE_MAX } from "../../db/utils" import { context } from "@budibase/backend-core" -import { ScreenRouting } from "@budibase/types" +import { ScreenRouting, Document } from "@budibase/types" -type ScreenRoutesView = { +interface ScreenRoutesView extends Document { id: string routing: ScreenRouting } diff --git a/packages/types/src/documents/app/layout.ts b/packages/types/src/documents/app/layout.ts index db046e3d92..06542f680d 100644 --- a/packages/types/src/documents/app/layout.ts +++ b/packages/types/src/documents/app/layout.ts @@ -2,4 +2,5 @@ import { Document } from "../document" export interface Layout extends Document { props: any + layoutId?: string } diff --git a/packages/types/src/documents/app/user.ts b/packages/types/src/documents/app/user.ts index 4defd4a414..207997245e 100644 --- a/packages/types/src/documents/app/user.ts +++ b/packages/types/src/documents/app/user.ts @@ -1,6 +1,6 @@ -import { Document } from "../document" +import { User } from "../global" +import { Row } from "./row" +import { ContextUser } from "../../sdk" -export interface UserMetadata extends Document { - roleId: string - email?: string -} +export type UserMetadata = User & Row +export type ContextUserMetadata = ContextUser & Row diff --git a/packages/types/src/documents/app/view.ts b/packages/types/src/documents/app/view.ts index 0d79e2c505..b5a22ec592 100644 --- a/packages/types/src/documents/app/view.ts +++ b/packages/types/src/documents/app/view.ts @@ -1,5 +1,24 @@ import { SearchFilter, SortOrder, SortType } from "../../api" import { UIFieldMetadata } from "./table" +import { Document } from "../document" +import { DBView } from "../../sdk" + +export type ViewTemplateOpts = { + field: string + tableId: string + groupBy: string + filters: ViewFilter[] + schema: any + calculation: string + groupByMulti?: boolean +} + +export interface InMemoryView extends Document { + view: DBView + name: string + tableId: string + groupBy?: string +} export interface View { name?: string @@ -10,7 +29,7 @@ export interface View { calculation?: ViewCalculation map?: string reduce?: any - meta?: Record + meta?: ViewTemplateOpts } export interface ViewV2 { diff --git a/packages/types/src/documents/pouch.ts b/packages/types/src/documents/pouch.ts index d484f4700d..11efc502be 100644 --- a/packages/types/src/documents/pouch.ts +++ b/packages/types/src/documents/pouch.ts @@ -1,17 +1,19 @@ +import { Document } from "../" + export interface RowValue { rev: string deleted: boolean } -export interface RowResponse { +export interface RowResponse { id: string key: string error: string value: T | RowValue - doc?: T | any + doc?: T } -export interface AllDocsResponse { +export interface AllDocsResponse { offset: number total_rows: number rows: RowResponse[] diff --git a/packages/types/src/sdk/db.ts b/packages/types/src/sdk/db.ts index 05f72f5524..26807d99ce 100644 --- a/packages/types/src/sdk/db.ts +++ b/packages/types/src/sdk/db.ts @@ -1,5 +1,5 @@ import Nano from "@budibase/nano" -import { AllDocsResponse, AnyDocument, Document } from "../" +import { AllDocsResponse, AnyDocument, Document, ViewTemplateOpts } from "../" import { Writable } from "stream" export enum SearchIndex { @@ -20,6 +20,37 @@ export enum SortOption { DESCENDING = "desc", } +export type IndexAnalyzer = { + name: string + default?: string + fields?: Record +} + +export type DBView = { + name?: string + map: string + reduce?: string + meta?: ViewTemplateOpts + groupBy?: string +} + +export interface DesignDocument extends Document { + // we use this static reference for all design documents + _id: "_design/database" + language?: string + // CouchDB views + views?: { + [viewName: string]: DBView + } + // Lucene indexes + indexes?: { + [indexName: string]: { + index: string + analyzer?: string | IndexAnalyzer + } + } +} + export type CouchFindOptions = { selector: PouchDB.Find.Selector fields?: string[] @@ -101,8 +132,10 @@ export interface Database { opts?: DatabasePutOpts ): Promise bulkDocs(documents: AnyDocument[]): Promise - allDocs(params: DatabaseQueryOpts): Promise> - query( + allDocs( + params: DatabaseQueryOpts + ): Promise> + query( viewName: string, params: DatabaseQueryOpts ): Promise> diff --git a/packages/worker/src/constants/templates/index.ts b/packages/worker/src/constants/templates/index.ts index 1feac62040..6dd3f556a6 100644 --- a/packages/worker/src/constants/templates/index.ts +++ b/packages/worker/src/constants/templates/index.ts @@ -56,12 +56,12 @@ export async function getTemplates({ id, }: { ownerId?: string; type?: string; id?: string } = {}) { const db = tenancy.getGlobalDB() - const response = await db.allDocs( + const response = await db.allDocs