From e7ff299c63e5dedb5f594b0cf0ad400d5ac86379 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 29 Jun 2022 19:11:00 +0100
Subject: [PATCH] Adding user pagination, removing usages of the global user
 list from builder and replacing with direct user lookups where possible,
 still need to apply filtering to username/email serverside.

---
 packages/backend-core/src/db/utils.ts         | 27 +++++++++++-
 packages/builder/src/helpers/pagination.js    | 31 ++++++++++++++
 .../builder/portal/manage/users/index.svelte  | 42 +++++++++++++++----
 .../overview/_components/OverviewTab.svelte   | 30 ++++---------
 packages/builder/src/stores/portal/users.js   | 26 +++++++++---
 packages/frontend-core/src/api/user.js        | 15 ++++++-
 .../src/api/controllers/global/users.ts       |  7 ++--
 packages/worker/src/sdk/users/users.ts        | 15 +++++++
 8 files changed, 150 insertions(+), 43 deletions(-)
 create mode 100644 packages/builder/src/helpers/pagination.js

diff --git a/packages/backend-core/src/db/utils.ts b/packages/backend-core/src/db/utils.ts
index dc7a0454c3..fc4094ad9f 100644
--- a/packages/backend-core/src/db/utils.ts
+++ b/packages/backend-core/src/db/utils.ts
@@ -93,13 +93,17 @@ export function generateGlobalUserID(id?: any) {
 /**
  * Gets parameters for retrieving users.
  */
-export function getGlobalUserParams(globalId: any, otherProps = {}) {
+export function getGlobalUserParams(globalId: any, otherProps: any = {}) {
   if (!globalId) {
     globalId = ""
   }
+  const startkey = otherProps?.startkey
   return {
     ...otherProps,
-    startkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}`,
+    // need to include this incase pagination
+    startkey: startkey
+      ? startkey
+      : `${DocumentTypes.USER}${SEPARATOR}${globalId}`,
     endkey: `${DocumentTypes.USER}${SEPARATOR}${globalId}${UNICODE_MAX}`,
   }
 }
@@ -434,6 +438,25 @@ export const getPlatformUrl = async (opts = { tenantAware: true }) => {
   return platformUrl
 }
 
+export function pagination(
+  response: any,
+  pageSize: number,
+  paginate: boolean = true
+) {
+  const data = response.rows.map((row: any) => {
+    return row.doc ? row.doc : row
+  })
+  if (!paginate) {
+    return { data, hasNextPage: false }
+  }
+  const hasNextPage = data.length > pageSize
+  return {
+    data: data.slice(0, pageSize),
+    hasNextPage,
+    nextPage: hasNextPage ? data[pageSize]?._id : undefined,
+  }
+}
+
 export async function getScopedConfig(db: any, params: any) {
   const configDoc = await getScopedFullConfig(db, params)
   return configDoc && configDoc.config ? configDoc.config : configDoc
diff --git a/packages/builder/src/helpers/pagination.js b/packages/builder/src/helpers/pagination.js
new file mode 100644
index 0000000000..8f524d8d9f
--- /dev/null
+++ b/packages/builder/src/helpers/pagination.js
@@ -0,0 +1,31 @@
+export class PageInfo {
+  constructor(fetch) {
+    this.reset()
+    this.fetch = fetch
+  }
+
+  async goToNextPage() {
+    this.pageNumber++
+    this.prevPage = this.page
+    this.page = this.nextPage
+    this.hasPrevPage = this.pageNumber > 1
+    await this.fetch(this.page)
+  }
+
+  async goToPrevPage() {
+    this.pageNumber--
+    this.nextPage = this.page
+    this.page = this.prevPage
+    this.hasPrevPage = this.pageNumber > 1
+    await this.fetch(this.page)
+  }
+
+  reset() {
+    this.prevPage = null
+    this.nextPage = null
+    this.page = undefined
+    this.hasPrevPage = false
+    this.hasNextPage = false
+    this.pageNumber = 1
+  }
+}
diff --git a/packages/builder/src/pages/builder/portal/manage/users/index.svelte b/packages/builder/src/pages/builder/portal/manage/users/index.svelte
index 1d9c245480..ef1ba2a24a 100644
--- a/packages/builder/src/pages/builder/portal/manage/users/index.svelte
+++ b/packages/builder/src/pages/builder/portal/manage/users/index.svelte
@@ -12,25 +12,26 @@
     Layout,
     Modal,
     notifications,
+    Pagination,
   } from "@budibase/bbui"
   import TagsRenderer from "./_components/TagsTableRenderer.svelte"
   import AddUserModal from "./_components/AddUserModal.svelte"
   import { users } from "stores/portal"
+  import { PageInfo } from "helpers/pagination"
   import { onMount } from "svelte"
 
   const schema = {
     email: {},
     developmentAccess: { displayName: "Development Access", type: "boolean" },
     adminAccess: { displayName: "Admin Access", type: "boolean" },
-    // role: { type: "options" },
     group: {},
-    // access: {},
-    // group: {}
   }
 
+  let pageInfo = new PageInfo(fetchUsers)
   let search
-  $: filteredUsers = $users
-    .filter(user => user.email.includes(search || ""))
+  $: checkRefreshed($users.page)
+  $: filteredUsers = $users.data
+    ?.filter(user => user?.email?.includes(search || ""))
     .map(user => ({
       ...user,
       group: ["All users"],
@@ -40,12 +41,28 @@
 
   let createUserModal
 
-  onMount(async () => {
+  async function checkRefreshed(page) {
+    // the users have been reset, go back to first page
+    if (!page && pageInfo.page) {
+      pageInfo.reset()
+      pageInfo.pageNumber = pageInfo.pageNumber
+      pageInfo.hasNextPage = $users.hasNextPage
+      pageInfo.nextPage = $users.nextPage
+    }
+  }
+
+  async function fetchUsers(page) {
     try {
-      await users.init()
+      await users.fetch(page)
+      pageInfo.hasNextPage = $users.hasNextPage
+      pageInfo.nextPage = $users.nextPage
     } catch (error) {
       notifications.error("Error getting user list")
     }
+  }
+
+  onMount(async () => {
+    await fetchUsers()
   })
 </script>
 
@@ -75,12 +92,21 @@
     <Table
       on:click={({ detail }) => $goto(`./${detail._id}`)}
       {schema}
-      data={filteredUsers || $users}
+      data={filteredUsers || $users.data}
       allowEditColumns={false}
       allowEditRows={false}
       allowSelectRows={false}
       customRenderers={[{ column: "group", component: TagsRenderer }]}
     />
+    <div class="pagination">
+      <Pagination
+        page={pageInfo.pageNumber}
+        hasPrevPage={pageInfo.hasPrevPage}
+        hasNextPage={pageInfo.hasNextPage}
+        goToPrevPage={() => pageInfo.goToPrevPage()}
+        goToNextPage={() => pageInfo.goToNextPage()}
+      />
+    </div>
   </Layout>
 </Layout>
 
diff --git a/packages/builder/src/pages/builder/portal/overview/_components/OverviewTab.svelte b/packages/builder/src/pages/builder/portal/overview/_components/OverviewTab.svelte
index 7b7258cada..6957f2ea9a 100644
--- a/packages/builder/src/pages/builder/portal/overview/_components/OverviewTab.svelte
+++ b/packages/builder/src/pages/builder/portal/overview/_components/OverviewTab.svelte
@@ -1,14 +1,7 @@
 <script>
   import DashCard from "components/common/DashCard.svelte"
   import { AppStatus } from "constants"
-  import {
-    Icon,
-    Heading,
-    Link,
-    Avatar,
-    notifications,
-    Layout,
-  } from "@budibase/bbui"
+  import { Icon, Heading, Link, Avatar, Layout } from "@budibase/bbui"
   import { store } from "builderStore"
   import clientPackage from "@budibase/client/package.json"
   import { processStringSync } from "@budibase/string-templates"
@@ -20,29 +13,22 @@
   export let navigateTab
   const dispatch = createEventDispatcher()
 
-  const userInit = async () => {
-    try {
-      await users.init()
-    } catch (error) {
-      notifications.error("Error getting user list")
-    }
-  }
-
   const unpublishApp = () => {
     dispatch("unpublish", app)
   }
 
-  let userPromise = userInit()
+  let appEditor, appEditorPromise
 
   $: updateAvailable = clientPackage.version !== $store.version
   $: isPublished = app && app?.status === AppStatus.DEPLOYED
   $: appEditorId = !app?.updatedBy ? $auth.user._id : app?.updatedBy
   $: appEditorText = appEditor?.firstName || appEditor?.email
-  $: filteredUsers = !appEditorId
-    ? []
-    : $users.filter(user => user._id === appEditorId)
+  $: fetchAppEditor(appEditorId)
 
-  $: appEditor = filteredUsers.length ? filteredUsers[0] : null
+  async function fetchAppEditor(editorId) {
+    appEditorPromise = users.get(editorId)
+    appEditor = await appEditorPromise
+  }
 
   const getInitials = user => {
     let initials = ""
@@ -90,7 +76,7 @@
       </DashCard>
       <DashCard title={"Last Edited"} dataCy={"edited-by"}>
         <div class="last-edited-content">
-          {#await userPromise}
+          {#await appEditorPromise}
             <Avatar size="M" initials={"-"} />
           {:then _}
             <div class="updated-by">
diff --git a/packages/builder/src/stores/portal/users.js b/packages/builder/src/stores/portal/users.js
index 0b6b8f563a..5e29d6e80f 100644
--- a/packages/builder/src/stores/portal/users.js
+++ b/packages/builder/src/stores/portal/users.js
@@ -3,11 +3,23 @@ import { API } from "api"
 import { update } from "lodash"
 
 export function createUsersStore() {
-  const { subscribe, set } = writable([])
+  const { subscribe, set } = writable({})
 
-  async function init() {
-    const users = await API.getUsers()
-    set(users)
+  async function fetch(page) {
+    const paged = await API.getUsers(page)
+    set({
+      ...paged,
+      page,
+    })
+    return paged
+  }
+
+  async function get(userId) {
+    try {
+      return await API.getUser(userId)
+    } catch (err) {
+      return null
+    }
   }
 
   async function invite({ email, builder, admin }) {
@@ -47,7 +59,8 @@ export function createUsersStore() {
       body.admin = { global: true }
     }
     await API.saveUser(body)
-    await init()
+    // re-fetch from first page
+    await fetch()
   }
 
   async function del(id) {
@@ -61,7 +74,8 @@ export function createUsersStore() {
 
   return {
     subscribe,
-    init,
+    fetch,
+    get,
     invite,
     acceptInvite,
     create,
diff --git a/packages/frontend-core/src/api/user.js b/packages/frontend-core/src/api/user.js
index 9eade435da..a82f7f84dc 100644
--- a/packages/frontend-core/src/api/user.js
+++ b/packages/frontend-core/src/api/user.js
@@ -2,9 +2,20 @@ export const buildUserEndpoints = API => ({
   /**
    * Gets a list of users in the current tenant.
    */
-  getUsers: async () => {
+  getUsers: async page => {
+    const input = page ? { page } : {}
+    const params = new URLSearchParams(input)
     return await API.get({
-      url: "/api/global/users",
+      url: `/api/global/users?${params.toString()}`,
+    })
+  },
+
+  /**
+   * Get a single user by ID.
+   */
+  getUser: async userId => {
+    return await API.get({
+      url: `/api/global/users/${userId}`,
     })
   },
 
diff --git a/packages/worker/src/api/controllers/global/users.ts b/packages/worker/src/api/controllers/global/users.ts
index b78c1c5762..27ca780e9e 100644
--- a/packages/worker/src/api/controllers/global/users.ts
+++ b/packages/worker/src/api/controllers/global/users.ts
@@ -98,14 +98,15 @@ export const destroy = async (ctx: any) => {
 
 // called internally by app server user fetch
 export const fetch = async (ctx: any) => {
-  const all = await users.allUsers()
+  const { page } = ctx.request.query
+  const paginated = await users.paginatedUsers(page)
   // user hashed password shouldn't ever be returned
-  for (let user of all) {
+  for (let user of paginated.data) {
     if (user) {
       delete user.password
     }
   }
-  ctx.body = all
+  ctx.body = paginated
 }
 
 // called internally by app server user find
diff --git a/packages/worker/src/sdk/users/users.ts b/packages/worker/src/sdk/users/users.ts
index c2e29c7dae..3391e5f869 100644
--- a/packages/worker/src/sdk/users/users.ts
+++ b/packages/worker/src/sdk/users/users.ts
@@ -17,6 +17,8 @@ import {
 } from "@budibase/backend-core"
 import { MigrationType } from "@budibase/types"
 
+const PAGE_LIMIT = 10
+
 /**
  * Retrieves all users from the current tenancy.
  */
@@ -30,6 +32,19 @@ export const allUsers = async () => {
   return response.rows.map((row: any) => row.doc)
 }
 
+export const paginatedUsers = async (page?: string) => {
+  const db = tenancy.getGlobalDB()
+  // get one extra document, to have the next page
+  const response = await db.allDocs(
+    dbUtils.getGlobalUserParams(null, {
+      include_docs: true,
+      limit: PAGE_LIMIT + 1,
+      startkey: page,
+    })
+  )
+  return dbUtils.pagination(response, PAGE_LIMIT)
+}
+
 /**
  * Gets a user by ID from the global database, based on the current tenancy.
  */