Moving get of logs to pro.

packages/backend-core/src/db/constants.js

@@ -1,8 +1,27 @@
 exports.SEPARATOR = "_"
+exports.UNICODE_MAX = "\ufff0"
 
 const PRE_APP = "app"
 const PRE_DEV = "dev"
 
+/**
+ * Can be used to create a few different forms of querying a view.
+ */
+exports.ViewModes = {
+  ALL: "all",
+  AUTOMATION: "auto",
+  STATUS: "status",
+}
+
+exports.ViewNames = {
+  USER_BY_EMAIL: "by_email",
+  BY_API_KEY: "by_api_key",
+  USER_BY_BUILDERS: "by_builders",
+  LINK: "by_link",
+  ROUTING: "screen_routes",
+  AUTO_LOGS: "auto_log",
+}
+
 exports.DocumentTypes = {
   USER: "us",
   WORKSPACE: "workspace",
@@ -15,6 +34,7 @@ exports.DocumentTypes = {
   ROLE: "role",
   MIGRATIONS: "migrations",
   DEV_INFO: "devinfo",
+  AUTOMATION_LOG: "log_au",
 }
 
 exports.StaticDatabases = {

packages/backend-core/src/db/utils.ts

@@ -1,7 +1,7 @@
 import { newid } from "../hashing"
 import { DEFAULT_TENANT_ID, Configs } from "../constants"
 import env from "../environment"
-import { SEPARATOR, DocumentTypes } from "./constants"
+import { SEPARATOR, DocumentTypes, UNICODE_MAX } from "./constants"
 import { getTenantId, getGlobalDBName, getGlobalDB } from "../tenancy"
 import fetch from "node-fetch"
 import { doWithDB, allDbs } from "./index"
@@ -12,14 +12,6 @@ import { isDevApp, isDevAppID } from "./conversions"
 import { APP_PREFIX } from "./constants"
 import * as events from "../events"
 
-const UNICODE_MAX = "\ufff0"
-
-export const ViewNames = {
-  USER_BY_EMAIL: "by_email",
-  BY_API_KEY: "by_api_key",
-  USER_BY_BUILDERS: "by_builders",
-}
-
 export * from "./constants"
 export * from "./conversions"
 export { default as Replication } from "./Replication"
@@ -63,6 +55,13 @@ export function getDocParams(
   }
 }
 
+/**
+ * Retrieve the correct index for a view based on default design DB.
+ */
+export function getQueryIndex(viewName: string) {
+  return `database/${viewName}`
+}
+
 /**
  * Generates a new workspace ID.
  * @returns {string} The new workspace ID which the workspace doc can be stored under.

packages/server/src/api/controllers/automation.js

@@ -1,6 +1,5 @@
 const actions = require("../../automations/actions")
 const triggers = require("../../automations/triggers")
-const { getLogs, oneMonthAgo } = require("../../automations/logging")
 const {
   getAutomationParams,
   generateAutomationID,
@@ -21,6 +20,7 @@ const {
 } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
 const { app } = require("@budibase/backend-core/cache")
+const { logs } = require("@budibase/pro")
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -195,11 +195,12 @@ exports.destroy = async function (ctx) {
 
 exports.logSearch = async function (ctx) {
   let { automationId, status, page, startDate } = ctx.request.body
-  // TODO: need to check maximum allowed in license
+  ctx.body = await logs.automations.getLogs(
-  if (!startDate) {
+    startDate,
-    startDate = oneMonthAgo()
+    status,
-  }
+    automationId,
-  ctx.body = await getLogs(startDate, status, automationId, page)
+    page
+  )
 }
 
 exports.clearLogError = async function (ctx) {

packages/server/src/automations/logging/index.ts

@@ -1,28 +1,20 @@
-import {
-  AutomationLogPage,
-  AutomationResults,
-  AutomationStatus,
-} from "../../definitions/automation"
 import { getAppId, getProdAppDB } from "@budibase/backend-core/context"
 import {
   DocumentTypes,
   generateAutomationLogID,
-  getAutomationLogParams,
-  getQueryIndex,
-  ViewNames,
   SEPARATOR,
   isProdAppID,
 } from "../../db/utils"
-import { createLogByAutomationView } from "../../db/views/staticViews"
 import { Automation, MetadataErrors } from "../../definitions/common"
 import { app } from "@budibase/backend-core/cache"
 import { backOff } from "../../utilities"
 import * as env from "../../environment"
-
+import { logs } from "@budibase/pro"
-const PAGE_SIZE = 9
+import {
-const EARLIEST_DATE = new Date(0).toISOString()
+  AutomationResults,
-const FREE_EXPIRY_SEC = 86400
+  AutomationStatus,
-const PRO_EXPIRY_SEC = FREE_EXPIRY_SEC * 30
+} from "../../definitions/automation"
+const { logAlert } = require("@budibase/backend-core/logging")
 
 function getStatus(results: AutomationResults) {
   let status = AutomationStatus.SUCCESS
@@ -40,108 +32,29 @@ function getStatus(results: AutomationResults) {
   return status
 }
 
-export function oneMonthAgo() {
-  return new Date(new Date().getTime() - PRO_EXPIRY_SEC * 1000).toISOString()
-}
-
-export function oneDayAgo() {
-  return new Date(new Date().getTime() - FREE_EXPIRY_SEC * 1000).toISOString()
-}
-
 async function clearOldHistory() {
   const db = getProdAppDB()
-  // TODO: handle license lookup for deletion
-  const expiredEnd = oneDayAgo()
-  const results = await getAllLogs(EARLIEST_DATE, expiredEnd, {
-    docs: false,
-    paginate: false,
-  })
-  const toDelete = results.data.map((doc: any) => ({
-    _id: doc.id,
-    _rev: doc.value.rev,
-    _deleted: true,
-  }))
-  const errorLogIds = results.data
-    .filter((doc: any) => {
-      const parts = doc.id.split(SEPARATOR)
-      const status = parts[parts.length - 1]
-      return status === AutomationStatus.ERROR
-    })
-    .map((doc: any) => doc.id)
-  await db.bulkDocs(toDelete)
-  if (errorLogIds.length) {
-    await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
-  }
-}
-
-function pagination(
-  response: any,
-  paginate: boolean = true
-): AutomationLogPage {
-  const data = response.rows.map((row: any) => {
-    return row.doc ? row.doc : row
-  })
-  if (!paginate) {
-    return { data, hasNextPage: false }
-  }
-  const hasNextPage = data.length > PAGE_SIZE
-  return {
-    data: data.slice(0, PAGE_SIZE),
-    hasNextPage,
-    nextPage: hasNextPage ? data[PAGE_SIZE]?._id : undefined,
-  }
-}
-
-async function getAllLogs(
-  startDate: string,
-  endDate: string,
-  opts: {
-    docs: boolean
-    status?: string
-    paginate?: boolean
-    page?: string
-  } = { docs: true }
-): Promise<AutomationLogPage> {
-  const db = getProdAppDB()
-  let optional: any = { status: opts.status }
-  const params = getAutomationLogParams(startDate, endDate, optional, {
-    include_docs: opts.docs,
-    limit: opts?.paginate ? PAGE_SIZE + 1 : undefined,
-  })
-  if (opts?.page) {
-    params.startkey = opts.page
-  }
-  let response = await db.allDocs(params)
-  return pagination(response, opts?.paginate)
-}
-
-async function getLogsByView(
-  startDate: string,
-  endDate: string,
-  viewParams: { automationId?: string; status?: string; page?: string } = {}
-): Promise<AutomationLogPage> {
-  const db = getProdAppDB()
-  let response
   try {
-    let optional = {
+    const expired = await logs.automations.getExpiredLogs()
-      automationId: viewParams?.automationId,
+    const toDelete = expired.data.map((doc: any) => ({
-      status: viewParams?.status,
+      _id: doc.id,
-    }
+      _rev: doc.value.rev,
-    const params = getAutomationLogParams(startDate, endDate, optional, {
+      _deleted: true,
-      include_docs: true,
+    }))
-      limit: PAGE_SIZE,
+    const errorLogIds = expired.data
-    })
+      .filter((doc: any) => {
-    if (viewParams?.page) {
+        const parts = doc.id.split(SEPARATOR)
-      params.startkey = viewParams.page
+        const status = parts[parts.length - 1]
-    }
+        return status === AutomationStatus.ERROR
-    response = await db.query(getQueryIndex(ViewNames.AUTO_LOGS), params)
+      })
-  } catch (err: any) {
+      .map((doc: any) => doc.id)
-    if (err != null && err.name === "not_found") {
+    await db.bulkDocs(toDelete)
-      await createLogByAutomationView()
+    if (errorLogIds.length) {
-      return getLogsByView(startDate, endDate, viewParams)
+      await updateAppMetadataWithErrors(errorLogIds, { clearing: true })
     }
+  } catch (err) {
+    logAlert(`Failed to cleanup automation log history - Database "${db.name}"`)
   }
-  return pagination(response)
 }
 
 async function updateAppMetadataWithErrors(
@@ -214,28 +127,3 @@ export async function storeLog(
   // clear up old logging for app
   await clearOldHistory()
 }
-
-export async function getLogs(
-  startDate: string,
-  status?: string,
-  automationId?: string,
-  page?: string
-): Promise<AutomationLogPage> {
-  let response: AutomationLogPage
-  let endDate = new Date().toISOString()
-  if (automationId || status) {
-    response = await getLogsByView(startDate, endDate, {
-      automationId,
-      status,
-      page,
-    })
-  } else {
-    response = await getAllLogs(startDate, endDate, {
-      status,
-      page,
-      docs: true,
-      paginate: true,
-    })
-  }
-  return response
-}

packages/server/src/db/utils.js

@@ -11,6 +11,8 @@ const {
   isProdAppID,
   getDevelopmentAppID,
   generateAppID,
+  getQueryIndex,
+  ViewNames,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -22,16 +24,11 @@ const AppStatus = {
 }
 
 const DocumentTypes = {
-  APP: CoreDocTypes.APP,
+  ...CoreDocTypes,
-  DEV: CoreDocTypes.DEV,
-  APP_DEV: CoreDocTypes.APP_DEV,
-  APP_METADATA: CoreDocTypes.APP_METADATA,
-  ROLE: CoreDocTypes.ROLE,
   TABLE: "ta",
   ROW: "ro",
   USER: "us",
   AUTOMATION: "au",
-  AUTOMATION_LOG: "log_au",
   LINK: "li",
   WEBHOOK: "wh",
   INSTANCE: "inst",
@@ -46,12 +43,6 @@ const DocumentTypes = {
   USER_FLAG: "flag",
 }
 
-const ViewNames = {
-  LINK: "by_link",
-  ROUTING: "screen_routes",
-  AUTO_LOGS: "auto_log",
-}
-
 const ViewModes = {
   ALL: "all",
   AUTOMATION: "auto",
@@ -98,9 +89,7 @@ exports.generateDevAppID = getDevelopmentAppID
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
 
-exports.getQueryIndex = viewName => {
+exports.getQueryIndex = getQueryIndex
-  return `database/${viewName}`
-}
 
 /**
  * If creating DB allDocs/query params with only a single top level ID this can be used, this

packages/server/src/db/views/staticViews.js

@@ -3,11 +3,9 @@ const {
   DocumentTypes,
   SEPARATOR,
   ViewNames,
-  ViewModes,
   SearchIndexes,
 } = require("../utils")
 const SCREEN_PREFIX = DocumentTypes.SCREEN + SEPARATOR
-const LOG_PREFIX = DocumentTypes.AUTOMATION_LOG + SEPARATOR
 
 /**************************************************
  *                  INFORMATION                   *
@@ -60,34 +58,6 @@ exports.createLinkView = async () => {
   await db.put(designDoc)
 }
 
-/**
- * A separate view that allows us to perform queries by the automation ID and time series, while the
- * main all_docs allows access to time series only
- */
-exports.createLogByAutomationView = async () => {
-  const db = getAppDB()
-  const designDoc = await db.get("_design/database")
-  const view = {
-    map: `function(doc) {
-      if (doc._id.startsWith("${LOG_PREFIX}")) {
-        let autoId = doc.automationId + "${SEPARATOR}"
-        let status = doc.status + "${SEPARATOR}"
-        let autoKey = "${ViewModes.AUTOMATION}${SEPARATOR}" + autoId + doc.createdAt
-        let statusKey = "${ViewModes.STATUS}${SEPARATOR}" + status + doc.createdAt
-        let allKey = "${ViewModes.ALL}${SEPARATOR}" + status + autoId + doc.createdAt
-        emit(statusKey)
-        emit(autoKey)
-        emit(allKey)
-      }
-    }`,
-  }
-  designDoc.views = {
-    ...designDoc.views,
-    [ViewNames.AUTO_LOGS]: view,
-  }
-  await db.put(designDoc)
-}
-
 exports.createRoutingView = async () => {
   const db = getAppDB()
   const designDoc = await db.get("_design/database")

packages/server/yarn.lock

@@ -1080,11 +1080,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.211":
+"@budibase/backend-core@1.0.212-alpha.0":
-  version "1.0.211"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.211.tgz#579bcc97acf1df2510e302bb70b43245e5ab0e37"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
-  integrity sha512-ham+Qk6WXQi37Lgnz1Gh/+ItcvCo+dnPZDcVvPESrVxqQkVVIejlxQinZSueIrsgOYHE1j3TUH5zER5M2t6RWw==
+  integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
   dependencies:
+    "@budibase/types" "^1.0.212-alpha.0"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
     bcrypt "5.0.1"
@@ -1160,12 +1161,12 @@
     svelte-flatpickr "^3.2.3"
     svelte-portal "^1.0.0"
 
-"@budibase/pro@1.0.211":
+"@budibase/pro@1.0.212-alpha.0":
-  version "1.0.211"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.211.tgz#d362e9af8c15f6ed386f27b7cca95cc096a91344"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
-  integrity sha512-dfFByJhlTIURT3sXei5mVXq5rczFMM/ij2Scze0uqPZNpmIlWVqiYdGX7/HEcmIFSS0+UfcdBxjCJJlTGiK4/w==
+  integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
   dependencies:
-    "@budibase/backend-core" "1.0.211"
+    "@budibase/backend-core" "1.0.212-alpha.0"
     node-fetch "^2.6.1"
 
 "@budibase/standard-components@^0.9.139":
@@ -1186,6 +1187,11 @@
     svelte-apexcharts "^1.0.2"
     svelte-flatpickr "^3.1.0"
 
+"@budibase/types@^1.0.212-alpha.0":
+  version "1.0.212"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
+  integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
+
 "@bull-board/api@3.7.0":
   version "3.7.0"
   resolved "https://registry.yarnpkg.com/@bull-board/api/-/api-3.7.0.tgz#231f687187c0cb34e0b97f463917b6aaeb4ef6af"
@@ -8832,7 +8838,7 @@ keyv@^3.0.0:
 kind-of@^3.0.2, kind-of@^3.0.3, kind-of@^3.2.0:
   version "3.2.2"
   resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-3.2.2.tgz#31ea21a734bab9bbb0f32466d893aea51e4a3c64"
-  integrity sha1-MeohpzS6ubuw8yRm2JOupR5KPGQ=
+  integrity sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==
   dependencies:
     is-buffer "^1.1.5"
 

packages/worker/yarn.lock

@@ -293,11 +293,12 @@
   resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39"
   integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw==
 
-"@budibase/backend-core@1.0.207-alpha.3":
+"@budibase/backend-core@1.0.212-alpha.0":
-  version "1.0.207-alpha.3"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.207-alpha.3.tgz#98bced0575ec4e2b158239a8c73b39ca2d816719"
+  resolved "https://registry.yarnpkg.com/@budibase/backend-core/-/backend-core-1.0.212-alpha.0.tgz#6ac3448c8272e918f1af1fff0cf8c5773ae61219"
-  integrity sha512-DU4X6jJ+DfhzOv4TTa1w4Dk5ZEdlK/z1joCTruT+SGM5qI75bXrGeol5OX2OaEbNKtXFKJ1zeVTmBCYcu7OFUg==
+  integrity sha512-hFvbQQEbF3w2u9fe/S+RhNw5HUETS6rhu9q5KDTDQ57k05D4YMPcpMBGSh7SPMqmVyEwUDgcL36mkFOc3AgjYQ==
   dependencies:
+    "@budibase/types" "^1.0.212-alpha.0"
     "@techpass/passport-openidconnect" "0.3.2"
     aws-sdk "2.1030.0"
     bcrypt "5.0.1"
@@ -324,14 +325,19 @@
     uuid "8.3.2"
     zlib "1.0.5"
 
-"@budibase/pro@1.0.207-alpha.3":
+"@budibase/pro@1.0.212-alpha.0":
-  version "1.0.207-alpha.3"
+  version "1.0.212-alpha.0"
-  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.207-alpha.3.tgz#9bde845ceb685f1b43286a124620c21fdf891a01"
+  resolved "https://registry.yarnpkg.com/@budibase/pro/-/pro-1.0.212-alpha.0.tgz#843f8291fcc0a2fbcb4d857a750bc116cdaee293"
-  integrity sha512-WFEMujpKTVAMvAgLBnMdw8ou9PxsbM4Oa9Dq+DAUsWpPACsMWOProyHLsdRxJyvHlgGfwVjo5MEusvStjI4j6g==
+  integrity sha512-4nhWxjMcxSQBPXRy/U+37IaVLYOr4/RVe79/fUvnXrr5qAeecbEk/QbkJJd3dU1WaNxB2eGhNtH3uBUPQvcT9A==
   dependencies:
-    "@budibase/backend-core" "1.0.207-alpha.3"
+    "@budibase/backend-core" "1.0.212-alpha.0"
     node-fetch "^2.6.1"
 
+"@budibase/types@^1.0.212-alpha.0":
+  version "1.0.212"
+  resolved "https://registry.yarnpkg.com/@budibase/types/-/types-1.0.212.tgz#e66a15b711544b4fab7767261fd5f2f1dd7f40d7"
+  integrity sha512-DhGyw6snwJQZQlx7havVYnqPZfZERueKZfmVCBySzwInZZt0+sXZaBl1BVjGjYuwpaUQBMDBf7geBgHXp6DIKg==
+
 "@cspotcode/source-map-consumer@0.8.0":
   version "0.8.0"
   resolved "https://registry.yarnpkg.com/@cspotcode/source-map-consumer/-/source-map-consumer-0.8.0.tgz#33bf4b7b39c178821606f669bbc447a6a629786b"