Merge pull request #5881 from Budibase/fix/5850
Fixing issues with calculation views when self hosting and using real CouchDB views
This commit is contained in:
commit
931c14198f
7 changed files with 138 additions and 38 deletions
|
@ -71,7 +71,7 @@ describe("oidc", () => {
|
||||||
|
|
||||||
describe("authenticate", () => {
|
describe("authenticate", () => {
|
||||||
afterEach(() => {
|
afterEach(() => {
|
||||||
jest.clearAllMocks();
|
jest.clearAllMocks()
|
||||||
});
|
});
|
||||||
|
|
||||||
// mock third party common authentication
|
// mock third party common authentication
|
||||||
|
@ -80,10 +80,10 @@ describe("oidc", () => {
|
||||||
|
|
||||||
// mock the passport callback
|
// mock the passport callback
|
||||||
const mockDone = jest.fn()
|
const mockDone = jest.fn()
|
||||||
|
const mockSaveUserFn = jest.fn()
|
||||||
|
|
||||||
async function doAuthenticate() {
|
async function doAuthenticate() {
|
||||||
const oidc = require("../oidc")
|
const oidc = require("../oidc")
|
||||||
const mockSaveUserFn = jest.fn()
|
|
||||||
const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
|
const authenticate = await oidc.buildVerifyFn(mockSaveUserFn)
|
||||||
|
|
||||||
await authenticate(
|
await authenticate(
|
||||||
|
@ -105,11 +105,13 @@ describe("oidc", () => {
|
||||||
expect(authenticateThirdParty).toHaveBeenCalledWith(
|
expect(authenticateThirdParty).toHaveBeenCalledWith(
|
||||||
user,
|
user,
|
||||||
false,
|
false,
|
||||||
mockDone)
|
mockDone,
|
||||||
|
mockSaveUserFn,
|
||||||
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
it("delegates authentication to third party common", async () => {
|
it("delegates authentication to third party common", async () => {
|
||||||
doTest()
|
await doTest()
|
||||||
})
|
})
|
||||||
|
|
||||||
it("uses JWT email to get email", async () => {
|
it("uses JWT email to get email", async () => {
|
||||||
|
@ -118,7 +120,7 @@ describe("oidc", () => {
|
||||||
email : "mock@budibase.com"
|
email : "mock@budibase.com"
|
||||||
}
|
}
|
||||||
|
|
||||||
doTest()
|
await doTest()
|
||||||
})
|
})
|
||||||
|
|
||||||
it("uses JWT username to get email", async () => {
|
it("uses JWT username to get email", async () => {
|
||||||
|
@ -127,7 +129,7 @@ describe("oidc", () => {
|
||||||
preferred_username : "mock@budibase.com"
|
preferred_username : "mock@budibase.com"
|
||||||
}
|
}
|
||||||
|
|
||||||
doTest()
|
await doTest()
|
||||||
})
|
})
|
||||||
|
|
||||||
it("uses JWT invalid username to get email", async () => {
|
it("uses JWT invalid username to get email", async () => {
|
||||||
|
|
|
@ -1,9 +1,66 @@
|
||||||
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
// Jest Snapshot v1, https://goo.gl/fbAQLP
|
||||||
|
|
||||||
|
exports[`viewBuilder Calculate and filter creates a view with the calculation statistics and filter schema 1`] = `
|
||||||
|
Object {
|
||||||
|
"map": "function (doc) {
|
||||||
|
if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
|
||||||
|
doc[\\"myField\\"] === undefined ||
|
||||||
|
doc[\\"myField\\"] === null ||
|
||||||
|
doc[\\"myField\\"] === \\"\\" ||
|
||||||
|
(Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
|
||||||
|
)) && (doc[\\"age\\"] > 17)) {
|
||||||
|
emit(doc[\\"_id\\"], doc[\\"myField\\"]);
|
||||||
|
}
|
||||||
|
}",
|
||||||
|
"meta": Object {
|
||||||
|
"calculation": "stats",
|
||||||
|
"field": "myField",
|
||||||
|
"filters": Array [
|
||||||
|
Object {
|
||||||
|
"condition": "MT",
|
||||||
|
"key": "age",
|
||||||
|
"value": 17,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
"groupBy": undefined,
|
||||||
|
"schema": Object {
|
||||||
|
"avg": Object {
|
||||||
|
"type": "number",
|
||||||
|
},
|
||||||
|
"count": Object {
|
||||||
|
"type": "number",
|
||||||
|
},
|
||||||
|
"field": Object {
|
||||||
|
"type": "string",
|
||||||
|
},
|
||||||
|
"max": Object {
|
||||||
|
"type": "number",
|
||||||
|
},
|
||||||
|
"min": Object {
|
||||||
|
"type": "number",
|
||||||
|
},
|
||||||
|
"sum": Object {
|
||||||
|
"type": "number",
|
||||||
|
},
|
||||||
|
"sumsqr": Object {
|
||||||
|
"type": "number",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
|
||||||
|
},
|
||||||
|
"reduce": "_stats",
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
|
||||||
exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
|
exports[`viewBuilder Calculate creates a view with the calculation statistics schema 1`] = `
|
||||||
Object {
|
Object {
|
||||||
"map": "function (doc) {
|
"map": "function (doc) {
|
||||||
if (doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" ) {
|
if ((doc.tableId === \\"14f1c4e94d6a47b682ce89d35d4c78b0\\" && !(
|
||||||
|
doc[\\"myField\\"] === undefined ||
|
||||||
|
doc[\\"myField\\"] === null ||
|
||||||
|
doc[\\"myField\\"] === \\"\\" ||
|
||||||
|
(Array.isArray(doc[\\"myField\\"]) && doc[\\"myField\\"].length === 0)
|
||||||
|
)) ) {
|
||||||
emit(doc[\\"_id\\"], doc[\\"myField\\"]);
|
emit(doc[\\"_id\\"], doc[\\"myField\\"]);
|
||||||
}
|
}
|
||||||
}",
|
}",
|
||||||
|
|
|
@ -44,4 +44,22 @@ describe("viewBuilder", () => {
|
||||||
})).toMatchSnapshot()
|
})).toMatchSnapshot()
|
||||||
})
|
})
|
||||||
})
|
})
|
||||||
|
|
||||||
|
describe("Calculate and filter", () => {
|
||||||
|
it("creates a view with the calculation statistics and filter schema", () => {
|
||||||
|
expect(viewTemplate({
|
||||||
|
"name": "Calculate View",
|
||||||
|
"field": "myField",
|
||||||
|
"calculation": "stats",
|
||||||
|
"tableId": "14f1c4e94d6a47b682ce89d35d4c78b0",
|
||||||
|
"filters": [
|
||||||
|
{
|
||||||
|
"value": 17,
|
||||||
|
"condition": "MT",
|
||||||
|
"key": "age",
|
||||||
|
}
|
||||||
|
]
|
||||||
|
})).toMatchSnapshot()
|
||||||
|
})
|
||||||
|
})
|
||||||
});
|
});
|
|
@ -7,6 +7,7 @@ const {
|
||||||
} = require("../../../db/utils")
|
} = require("../../../db/utils")
|
||||||
const env = require("../../../environment")
|
const env = require("../../../environment")
|
||||||
const { getAppDB } = require("@budibase/backend-core/context")
|
const { getAppDB } = require("@budibase/backend-core/context")
|
||||||
|
const viewBuilder = require("./viewBuilder")
|
||||||
|
|
||||||
exports.getView = async viewName => {
|
exports.getView = async viewName => {
|
||||||
const db = getAppDB()
|
const db = getAppDB()
|
||||||
|
@ -114,7 +115,8 @@ exports.deleteView = async viewName => {
|
||||||
exports.migrateToInMemoryView = async (db, viewName) => {
|
exports.migrateToInMemoryView = async (db, viewName) => {
|
||||||
// delete the view initially
|
// delete the view initially
|
||||||
const designDoc = await db.get("_design/database")
|
const designDoc = await db.get("_design/database")
|
||||||
const view = designDoc.views[viewName]
|
// run the view back through the view builder to update it
|
||||||
|
const view = viewBuilder(designDoc.views[viewName].meta)
|
||||||
delete designDoc.views[viewName]
|
delete designDoc.views[viewName]
|
||||||
await db.put(designDoc)
|
await db.put(designDoc)
|
||||||
await exports.saveView(db, null, viewName, view)
|
await exports.saveView(db, null, viewName, view)
|
||||||
|
@ -123,7 +125,7 @@ exports.migrateToInMemoryView = async (db, viewName) => {
|
||||||
exports.migrateToDesignView = async (db, viewName) => {
|
exports.migrateToDesignView = async (db, viewName) => {
|
||||||
let view = await db.get(generateMemoryViewID(viewName))
|
let view = await db.get(generateMemoryViewID(viewName))
|
||||||
const designDoc = await db.get("_design/database")
|
const designDoc = await db.get("_design/database")
|
||||||
designDoc.views[viewName] = view.view
|
designDoc.views[viewName] = viewBuilder(view.view.meta)
|
||||||
await db.put(designDoc)
|
await db.put(designDoc)
|
||||||
await db.remove(view._id, view._rev)
|
await db.remove(view._id, view._rev)
|
||||||
}
|
}
|
||||||
|
|
|
@ -10,6 +10,12 @@ const TOKEN_MAP = {
|
||||||
OR: "||",
|
OR: "||",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const CONDITIONS = {
|
||||||
|
EMPTY: "EMPTY",
|
||||||
|
NOT_EMPTY: "NOT_EMPTY",
|
||||||
|
CONTAINS: "CONTAINS",
|
||||||
|
}
|
||||||
|
|
||||||
const isEmptyExpression = key => {
|
const isEmptyExpression = key => {
|
||||||
return `(
|
return `(
|
||||||
doc["${key}"] === undefined ||
|
doc["${key}"] === undefined ||
|
||||||
|
@ -77,13 +83,13 @@ function parseFilterExpression(filters) {
|
||||||
expression.push(TOKEN_MAP[filter.conjunction])
|
expression.push(TOKEN_MAP[filter.conjunction])
|
||||||
}
|
}
|
||||||
|
|
||||||
if (filter.condition === "CONTAINS") {
|
if (filter.condition === CONDITIONS.CONTAINS) {
|
||||||
expression.push(
|
expression.push(
|
||||||
`doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
|
`doc["${filter.key}"].${TOKEN_MAP[filter.condition]}("${filter.value}")`
|
||||||
)
|
)
|
||||||
} else if (filter.condition === "EMPTY") {
|
} else if (filter.condition === CONDITIONS.EMPTY) {
|
||||||
expression.push(isEmptyExpression(filter.key))
|
expression.push(isEmptyExpression(filter.key))
|
||||||
} else if (filter.condition === "NOT_EMPTY") {
|
} else if (filter.condition === CONDITIONS.NOT_EMPTY) {
|
||||||
expression.push(`!${isEmptyExpression(filter.key)}`)
|
expression.push(`!${isEmptyExpression(filter.key)}`)
|
||||||
} else {
|
} else {
|
||||||
const value =
|
const value =
|
||||||
|
@ -125,21 +131,36 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
|
||||||
if (filters && filters.length > 0 && filters[0].conjunction) {
|
if (filters && filters.length > 0 && filters[0].conjunction) {
|
||||||
delete filters[0].conjunction
|
delete filters[0].conjunction
|
||||||
}
|
}
|
||||||
const parsedFilters = parseFilterExpression(filters)
|
|
||||||
const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
|
|
||||||
|
|
||||||
const emitExpression = parseEmitExpression(field, groupBy)
|
let schema = null,
|
||||||
|
statFilter = null
|
||||||
const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
|
|
||||||
|
|
||||||
let schema = null
|
|
||||||
|
|
||||||
if (calculation) {
|
if (calculation) {
|
||||||
schema = {
|
schema = {
|
||||||
...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
|
...(groupBy ? GROUP_PROPERTY : FIELD_PROPERTY),
|
||||||
...SCHEMA_MAP[calculation],
|
...SCHEMA_MAP[calculation],
|
||||||
}
|
}
|
||||||
|
if (
|
||||||
|
!filters.find(
|
||||||
|
filter =>
|
||||||
|
filter.key === field && filter.condition === CONDITIONS.NOT_EMPTY
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
statFilter = parseFilterExpression([
|
||||||
|
{ key: field, condition: CONDITIONS.NOT_EMPTY },
|
||||||
|
])
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const parsedFilters = parseFilterExpression(filters)
|
||||||
|
const filterExpression = parsedFilters ? `&& (${parsedFilters})` : ""
|
||||||
|
|
||||||
|
const emitExpression = parseEmitExpression(field, groupBy)
|
||||||
|
const tableExpression = `doc.tableId === "${tableId}"`
|
||||||
|
const coreExpression = statFilter
|
||||||
|
? `(${tableExpression} && ${statFilter})`
|
||||||
|
: tableExpression
|
||||||
|
const reduction = field && calculation ? { reduce: `_${calculation}` } : {}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
meta: {
|
meta: {
|
||||||
|
@ -151,7 +172,7 @@ function viewTemplate({ field, tableId, groupBy, filters = [], calculation }) {
|
||||||
calculation,
|
calculation,
|
||||||
},
|
},
|
||||||
map: `function (doc) {
|
map: `function (doc) {
|
||||||
if (doc.tableId === "${tableId}" ${filterExpression}) {
|
if (${coreExpression} ${filterExpression}) {
|
||||||
${emitExpression}
|
${emitExpression}
|
||||||
}
|
}
|
||||||
}`,
|
}`,
|
||||||
|
|
|
@ -1014,10 +1014,10 @@
|
||||||
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
||||||
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
|
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
|
||||||
|
|
||||||
"@budibase/backend-core@1.0.154":
|
"@budibase/backend-core@1.0.160":
|
||||||
version "1.0.154"
|
version "1.0.160"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
|
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
|
||||||
integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
|
integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@techpass/passport-openidconnect" "^0.3.0"
|
"@techpass/passport-openidconnect" "^0.3.0"
|
||||||
aws-sdk "^2.901.0"
|
aws-sdk "^2.901.0"
|
||||||
|
@ -1091,12 +1091,12 @@
|
||||||
svelte-flatpickr "^3.2.3"
|
svelte-flatpickr "^3.2.3"
|
||||||
svelte-portal "^1.0.0"
|
svelte-portal "^1.0.0"
|
||||||
|
|
||||||
"@budibase/pro@1.0.154":
|
"@budibase/pro@1.0.160":
|
||||||
version "1.0.154"
|
version "1.0.160"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
|
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
|
||||||
integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
|
integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@budibase/backend-core" "1.0.154"
|
"@budibase/backend-core" "1.0.160"
|
||||||
node-fetch "^2.6.1"
|
node-fetch "^2.6.1"
|
||||||
|
|
||||||
"@budibase/standard-components@^0.9.139":
|
"@budibase/standard-components@^0.9.139":
|
||||||
|
|
|
@ -293,10 +293,10 @@
|
||||||
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
|
||||||
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
|
integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
|
||||||
|
|
||||||
"@budibase/backend-core@1.0.154":
|
"@budibase/backend-core@1.0.160":
|
||||||
version "1.0.154"
|
version "1.0.160"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.154.tgz#c310834892e7621778b07579464955487c5c9830"
|
resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.160.tgz#000e3b5a3ed91e73a542b4caa202a6f147d91294"
|
||||||
integrity sha512-mcZxt8XhGgOB4XRHKkWTvBEI4HGp2bo8qyzOJRCvDqlg56S9zqGJDl75Z0N/Wc8N3I53QRcxISerj48odX172A==
|
integrity sha512-XfAFU6sRPrCSEKlm58WeuPw8lUoJK+KwO0tcbT+bB2Nb7XCHplskryEgk/PM9ujRU6SMPDx11zKeqRebHlycbA==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@techpass/passport-openidconnect" "^0.3.0"
|
"@techpass/passport-openidconnect" "^0.3.0"
|
||||||
aws-sdk "^2.901.0"
|
aws-sdk "^2.901.0"
|
||||||
|
@ -321,12 +321,12 @@
|
||||||
uuid "^8.3.2"
|
uuid "^8.3.2"
|
||||||
zlib "^1.0.5"
|
zlib "^1.0.5"
|
||||||
|
|
||||||
"@budibase/pro@1.0.154":
|
"@budibase/pro@1.0.160":
|
||||||
version "1.0.154"
|
version "1.0.160"
|
||||||
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.154.tgz#f4e31e30376b206159b711224038141d73a1118e"
|
resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.160.tgz#921c4e3f65b866d84292644dfd8793c4d0b667c7"
|
||||||
integrity sha512-+O6bemrcgyWG4a+D5dIOoZ+LGjW4aN7tRdFeZqoaIPCc1pA6zNtLUkM1nb+Laafuwq2Aht37vEuaRy7jfzVprA==
|
integrity sha512-p+Jhnk1n98CWCJXydSQSO7a+HDpqGAHekGQbOR7aayuwuoYzyOXxTcHNLdBp+3lkXhLSZq9oIwfEGpgdrrhXPA==
|
||||||
dependencies:
|
dependencies:
|
||||||
"@budibase/backend-core" "1.0.154"
|
"@budibase/backend-core" "1.0.160"
|
||||||
node-fetch "^2.6.1"
|
node-fetch "^2.6.1"
|
||||||
|
|
||||||
"@cspotcode/source-map-consumer@0.8.0":
|
"@cspotcode/source-map-consumer@0.8.0":
|
||||||
|
|
Loading…
Reference in a new issue