From 61c12d88cfde7ba575464682204d32cc0543909d Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 20 Sep 2023 16:43:50 +0100
Subject: [PATCH] Final work to support app update process.

---
 .../components/start/ImportAppModal.svelte    | 15 +++-
 .../server/src/sdk/app/applications/import.ts | 80 ++++++++++++++++---
 .../server/src/sdk/app/backups/imports.ts     |  5 +-
 packages/types/src/documents/document.ts      | 19 +++++
 4 files changed, 104 insertions(+), 15 deletions(-)
diff --git a/packages/builder/src/components/start/ImportAppModal.svelte b/packages/builder/src/components/start/ImportAppModal.svelte
index 7496b5824d..7d30ded896 100644
--- a/packages/builder/src/components/start/ImportAppModal.svelte
+++ b/packages/builder/src/components/start/ImportAppModal.svelte
@@ -1,5 +1,13 @@
 <script>
-  import { ModalContent, Toggle, Input, Layout, Dropzone, notifications } from "@budibase/bbui"
+  import {
+    ModalContent,
+    Toggle,
+    Input,
+    Layout,
+    Dropzone,
+    notifications,
+    Body,
+  } from "@budibase/bbui"
   import { API } from "api"
   import { automationStore, store } from "../../builderStore"
 
@@ -35,6 +43,11 @@
   onConfirm={updateApp}
   bind:disabled
 >
+  <Body size="S"
+    >Updating an app using an app export will replace all tables, datasources,
+    queries, screens and automations. It is recommended to perform a backup
+    before running this operation.</Body
+  >
   <Layout noPadding gap="XS">
     <Dropzone
       gallery={false}
diff --git a/packages/server/src/sdk/app/applications/import.ts b/packages/server/src/sdk/app/applications/import.ts
index e72300fa4e..22482514a7 100644
--- a/packages/server/src/sdk/app/applications/import.ts
+++ b/packages/server/src/sdk/app/applications/import.ts
@@ -1,4 +1,10 @@
 import { db as dbCore } from "@budibase/backend-core"
+import {
+  DocumentsToImport,
+  Document,
+  Database,
+  RowValue,
+} from "@budibase/types"
 import backups from "../backups"
 
 export type FileAttributes = {
@@ -6,24 +12,74 @@ export type FileAttributes = {
   path: string
 }
 
+async function removeImportableDocuments(db: Database) {
+  // get the references to the documents, not the whole document
+  const docPromises = []
+  for (let docType of DocumentsToImport) {
+    docPromises.push(db.allDocs(dbCore.getDocParams(docType)))
+  }
+  let documentRefs: { _id: string; _rev: string }[] = []
+  for (let response of await Promise.all(docPromises)) {
+    documentRefs = documentRefs.concat(
+      response.rows.map(row => ({
+        _id: row.id,
+        _rev: (row.value as RowValue).rev,
+      }))
+    )
+  }
+  // add deletion key
+  documentRefs = documentRefs.map(ref => ({ _deleted: true, ...ref }))
+  // perform deletion
+  await db.bulkDocs(documentRefs)
+}
+
+async function getImportableDocuments(db: Database) {
+  // get the whole document
+  const docPromises = []
+  for (let docType of DocumentsToImport) {
+    docPromises.push(
+      db.allDocs(dbCore.getDocParams(docType, null, { include_docs: true }))
+    )
+  }
+  // map the responses to the document itself
+  let documents: Document[] = []
+  for (let response of await Promise.all(docPromises)) {
+    documents = documents.concat(response.rows.map(row => row.doc))
+  }
+  // remove the _rev, stops it being written
+  documents.forEach(doc => {
+    delete doc._rev
+  })
+  return documents
+}
+
 export async function updateWithExport(
   appId: string,
   file: FileAttributes,
   password?: string
 ) {
   const devId = dbCore.getDevAppID(appId)
-  // TEMPORARY BEGIN
+  const tempAppName = `temp_${devId}`
+  const tempDb = dbCore.getDB(tempAppName)
   const appDb = dbCore.getDB(devId)
-  await appDb.destroy()
-  // TEMPORARY END
-  // const tempAppName = `temp_${devId}`
-  // const tempDb = dbCore.getDB(tempAppName)
-  const template = {
-    file: {
-      type: file.type!,
-      path: file.path!,
-      password,
-    },
+  try {
+    const template = {
+      file: {
+        type: file.type!,
+        path: file.path!,
+        password,
+      },
+    }
+    // get a temporary version of the import
+    // don't need obj store, the existing app already has everything we need
+    await backups.importApp(devId, tempDb, template, { objStore: false })
+    // get the documents to copy
+    const documents = await getImportableDocuments(tempDb)
+    // clear out the old documents
+    await removeImportableDocuments(appDb)
+    // now write the import documents
+    await appDb.bulkDocs(documents)
+  } finally {
+    await tempDb.destroy()
   }
-  await backups.importApp(devId, appDb, template)
 }
diff --git a/packages/server/src/sdk/app/backups/imports.ts b/packages/server/src/sdk/app/backups/imports.ts
index 619f888329..c61313b441 100644
--- a/packages/server/src/sdk/app/backups/imports.ts
+++ b/packages/server/src/sdk/app/backups/imports.ts
@@ -151,7 +151,8 @@ export function getListOfAppsInMulti(tmpPath: string) {
 export async function importApp(
   appId: string,
   db: Database,
-  template: TemplateType
+  template: TemplateType,
+  opts: { objStore: boolean } = { objStore: true }
 ) {
   let prodAppId = dbCore.getProdAppID(appId)
   let dbStream: any
@@ -165,7 +166,7 @@ export async function importApp(
     }
     const contents = fs.readdirSync(tmpPath)
     // have to handle object import
-    if (contents.length) {
+    if (contents.length && opts.objStore) {
       let promises = []
       let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE]
       for (let filename of contents) {
diff --git a/packages/types/src/documents/document.ts b/packages/types/src/documents/document.ts
index 03e01907b8..65168810e7 100644
--- a/packages/types/src/documents/document.ts
+++ b/packages/types/src/documents/document.ts
@@ -39,6 +39,25 @@ export enum DocumentType {
   AUDIT_LOG = "al",
 }
 
+// these are the core documents that make up the data, design
+// and automation sections of an app. This excludes any internal
+// rows as we shouldn't import data.
+export const DocumentsToImport: DocumentType[] = [
+  DocumentType.ROLE,
+  DocumentType.DATASOURCE,
+  DocumentType.DATASOURCE_PLUS,
+  DocumentType.TABLE,
+  DocumentType.AUTOMATION,
+  DocumentType.WEBHOOK,
+  DocumentType.SCREEN,
+  DocumentType.QUERY,
+  DocumentType.METADATA,
+  DocumentType.MEM_VIEW,
+  // Deprecated but still copied
+  DocumentType.INSTANCE,
+  DocumentType.LAYOUT,
+]
+
 // these documents don't really exist, they are part of other
 // documents or enriched into existence as part of get requests
 export enum VirtualDocumentType {
