Progress towards using couchdb for automation logs, after testing redis and realising it won't work.
This commit is contained in:
parent
b0ff30ed0a
commit
702f4057bd
9 changed files with 149 additions and 13 deletions
|
@ -19,6 +19,7 @@ exports.Databases = {
|
|||
QUERY_VARS: "queryVars",
|
||||
LICENSES: "license",
|
||||
GENERIC_CACHE: "data_cache",
|
||||
AUTOMATION_LOGS: "autoLogs",
|
||||
}
|
||||
|
||||
exports.SEPARATOR = SEPARATOR
|
||||
|
|
|
@ -13,6 +13,13 @@
|
|||
return results.steps.filter(x => x.stepId !== "LOOP" || [])
|
||||
}
|
||||
|
||||
function textArea(results, message) {
|
||||
if (!results) {
|
||||
return message
|
||||
}
|
||||
return JSON.stringify(results, null, 2)
|
||||
}
|
||||
|
||||
$: filteredResults = prepTestResults(testResults)
|
||||
|
||||
$: {
|
||||
|
@ -66,11 +73,7 @@
|
|||
<TextArea
|
||||
minHeight="80px"
|
||||
disabled
|
||||
value={JSON.stringify(
|
||||
filteredResults?.[idx]?.inputs,
|
||||
null,
|
||||
2
|
||||
)}
|
||||
value={textArea(filteredResults?.[idx]?.inputs, "No input")}
|
||||
/>
|
||||
</div></Tab
|
||||
>
|
||||
|
@ -79,10 +82,9 @@
|
|||
<TextArea
|
||||
minHeight="100px"
|
||||
disabled
|
||||
value={JSON.stringify(
|
||||
value={textArea(
|
||||
filteredResults?.[idx]?.outputs,
|
||||
null,
|
||||
2
|
||||
"No output"
|
||||
)}
|
||||
/>
|
||||
</div>
|
||||
|
|
|
@ -56,7 +56,7 @@
|
|||
let definitions = await automationStore.actions.definitions()
|
||||
runHistory = enrichHistory(definitions, [
|
||||
{
|
||||
status: "Error",
|
||||
status: "Success",
|
||||
timestamp: "2022-05-11T16:06:14.438Z",
|
||||
name: "automation name",
|
||||
steps: [
|
||||
|
@ -88,7 +88,7 @@
|
|||
],
|
||||
},
|
||||
{
|
||||
status: "Success",
|
||||
status: "Error",
|
||||
timestamp: "2022-05-11T16:03:14.438Z",
|
||||
name: "automation name",
|
||||
steps: [
|
||||
|
@ -151,7 +151,7 @@
|
|||
}
|
||||
|
||||
.panelOpen {
|
||||
grid-template-columns: auto 360px;
|
||||
grid-template-columns: auto 420px;
|
||||
}
|
||||
|
||||
.search {
|
||||
|
@ -178,7 +178,7 @@
|
|||
position: absolute;
|
||||
right: 0;
|
||||
height: 100%;
|
||||
width: 360px;
|
||||
width: 420px;
|
||||
overflow: hidden;
|
||||
background-color: var(--background);
|
||||
}
|
||||
|
|
67
packages/server/src/automations/history/index.ts
Normal file
67
packages/server/src/automations/history/index.ts
Normal file
|
@ -0,0 +1,67 @@
|
|||
import { AutomationResults } from "../../definitions/automation"
|
||||
import { getAppDB } from "@budibase/backend-core/context"
|
||||
import {
|
||||
generateAutomationLogID,
|
||||
getAutomationLogParams,
|
||||
// getQueryIndex,
|
||||
// ViewNames,
|
||||
} from "../../db/utils"
|
||||
// import { createLogByAutomationView } from "../../db/views/staticViews"
|
||||
|
||||
const FREE_EXPIRY_SEC = 86400
|
||||
// const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30
|
||||
|
||||
async function clearOldHistory() {
|
||||
const db = getAppDB()
|
||||
// TODO: handle license lookup for deletion
|
||||
const time = new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000)
|
||||
const queryParams: any = {
|
||||
endIso: time,
|
||||
}
|
||||
const results = await db.allDocs(getAutomationLogParams(queryParams))
|
||||
const toDelete = results.map((doc: any) => ({
|
||||
_id: doc.id,
|
||||
_rev: doc.rev,
|
||||
_deleted: true,
|
||||
}))
|
||||
await db.bulkDocs(toDelete)
|
||||
}
|
||||
|
||||
// async function getByAutomationID(automationId: string) {
|
||||
// const db = getAppDB()
|
||||
// try {
|
||||
// const queryParams: any = {
|
||||
// automationId,
|
||||
// }
|
||||
// return (
|
||||
// await db.query(
|
||||
// getQueryIndex(ViewNames.LOGS_BY_AUTOMATION),
|
||||
// getAutomationLogParams(queryParams, { include_docs: true })
|
||||
// )
|
||||
// ).rows.map((row: any) => row.doc)
|
||||
// } catch (err: any) {
|
||||
// if (err != null && err.name === "not_found") {
|
||||
// await createLogByAutomationView()
|
||||
// return getByAutomationID(automationId)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
|
||||
export async function storeHistory(
|
||||
automationId: string,
|
||||
results: AutomationResults
|
||||
) {
|
||||
const db = getAppDB()
|
||||
const isoDate = new Date().toISOString()
|
||||
const id = generateAutomationLogID(isoDate, automationId)
|
||||
await db.put({
|
||||
// results contain automationId and status for view
|
||||
...results,
|
||||
createdAt: isoDate,
|
||||
_id: id,
|
||||
})
|
||||
// clear up old history for app
|
||||
await clearOldHistory()
|
||||
}
|
||||
|
||||
export async function retrieve() {}
|
|
@ -31,6 +31,7 @@ const DocumentTypes = {
|
|||
ROW: "ro",
|
||||
USER: "us",
|
||||
AUTOMATION: "au",
|
||||
AUTOMATION_LOG: "log_au",
|
||||
LINK: "li",
|
||||
WEBHOOK: "wh",
|
||||
INSTANCE: "inst",
|
||||
|
@ -48,6 +49,7 @@ const DocumentTypes = {
|
|||
const ViewNames = {
|
||||
LINK: "by_link",
|
||||
ROUTING: "screen_routes",
|
||||
LOGS_BY_AUTOMATION: "log_by_auto",
|
||||
}
|
||||
|
||||
const InternalTables = {
|
||||
|
@ -362,6 +364,29 @@ exports.getMemoryViewParams = (otherProps = {}) => {
|
|||
return getDocParams(DocumentTypes.MEM_VIEW, null, otherProps)
|
||||
}
|
||||
|
||||
exports.generateAutomationLogID = (automationId, isoDate) => {
|
||||
return `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}${isoDate}${SEPARATOR}${automationId}`
|
||||
}
|
||||
|
||||
exports.getAutomationLogParams = (
|
||||
{ startIso, endIso, automationId } = {},
|
||||
otherProps = {}
|
||||
) => {
|
||||
const base = `${DocumentTypes.AUTOMATION_LOG}${SEPARATOR}`
|
||||
let start = startIso || "",
|
||||
end = endIso || ""
|
||||
// reverse for view
|
||||
if (automationId) {
|
||||
start = `${automationId}${SEPARATOR}${start}`
|
||||
end = `${automationId}${SEPARATOR}${end}`
|
||||
}
|
||||
return {
|
||||
...otherProps,
|
||||
startkey: `${base}${start}`,
|
||||
endkey: `${base}${end}${UNICODE_MAX}`,
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* This can be used with the db.allDocs to get a list of IDs
|
||||
*/
|
||||
|
|
|
@ -6,6 +6,7 @@ const {
|
|||
SearchIndexes,
|
||||
} = require("../utils")
|
||||
const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
|
||||
const LOG_PREFIX = DocumentTypes.AUTOMATION_LOG + SEPARATOR
|
||||
|
||||
/**************************************************
|
||||
* INFORMATION *
|
||||
|
@ -58,6 +59,28 @@ exports.createLinkView = async () => {
|
|||
await db.put(designDoc)
|
||||
}
|
||||
|
||||
/**
|
||||
* A separate view that allows us to perform queries by the automation ID and time series, while the
|
||||
* main all_docs allows access to time series only
|
||||
*/
|
||||
exports.createLogByAutomationView = async () => {
|
||||
const db = getAppDB()
|
||||
const designDoc = await db.get("_design/database")
|
||||
const view = {
|
||||
map: `function(doc) {
|
||||
if (doc._id.startsWith("${LOG_PREFIX}")) {
|
||||
let key = doc.automationId + ${SEPARATOR} + doc.createdAt
|
||||
emit(key, doc._id)
|
||||
}
|
||||
}`,
|
||||
}
|
||||
designDoc.views = {
|
||||
...designDoc.views,
|
||||
[ViewNames.LOGS_BY_AUTOMATION]: view,
|
||||
}
|
||||
await db.put(designDoc)
|
||||
}
|
||||
|
||||
exports.createRoutingView = async () => {
|
||||
const db = getAppDB()
|
||||
const designDoc = await db.get("_design/database")
|
||||
|
|
13
packages/server/src/definitions/automation.ts
Normal file
13
packages/server/src/definitions/automation.ts
Normal file
|
@ -0,0 +1,13 @@
|
|||
export interface AutomationResults {
|
||||
automationId: string
|
||||
status: string
|
||||
steps: {
|
||||
stepId: string
|
||||
inputs: {
|
||||
[key: string]: any
|
||||
}
|
||||
outputs: {
|
||||
[key: string]: any
|
||||
}
|
||||
}[]
|
||||
}
|
1
packages/server/src/module.d.ts
vendored
1
packages/server/src/module.d.ts
vendored
|
@ -9,3 +9,4 @@ declare module "@budibase/backend-core/constants"
|
|||
declare module "@budibase/backend-core/auth"
|
||||
declare module "@budibase/backend-core/sessions"
|
||||
declare module "@budibase/backend-core/encryption"
|
||||
declare module "@budibase/backend-core/redis"
|
||||
|
|
|
@ -77,7 +77,7 @@ export class Thread {
|
|||
})
|
||||
}
|
||||
|
||||
static shutdown() {
|
||||
static stopThreads() {
|
||||
return new Promise<void>(resolve => {
|
||||
if (Thread.workerRefs.length === 0) {
|
||||
resolve()
|
||||
|
@ -95,4 +95,8 @@ export class Thread {
|
|||
Thread.workerRefs = []
|
||||
})
|
||||
}
|
||||
|
||||
static async shutdown() {
|
||||
await Thread.stopThreads()
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue