From fc2e2a88040e8cfb0c091631f1498d3e7a9e4e9a Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Tue, 6 Oct 2020 19:13:41 +0100
Subject: [PATCH 1/7] First work towards implementing Dynamo usage in the
 server when running in the cloud; this is for tracking usage against API
 keys.

---
 packages/server/package.json                  |   2 +-
 packages/server/src/api/controllers/auth.js   |   6 +
 packages/server/src/api/controllers/model.js  |  14 +-
 packages/server/src/api/controllers/record.js |   5 +-
 packages/server/src/api/routes/model.js       |   4 +-
 packages/server/src/api/routes/record.js      |   3 +
 .../src/api/routes/tests/couchTestUtils.js    |   3 +
 packages/server/src/db/dynamoClient.js        | 108 ++++++++
 packages/server/src/environment.js            |   3 +
 .../server/src/middleware/authenticated.js    |   1 +
 packages/server/src/middleware/usageQuota.js  |  77 ++++++
 packages/server/yarn.lock                     | 256 +++++++++++++++++-
 12 files changed, 461 insertions(+), 21 deletions(-)
 create mode 100644 packages/server/src/db/dynamoClient.js
 create mode 100644 packages/server/src/middleware/usageQuota.js

diff --git a/packages/server/package.json b/packages/server/package.json
index 1875e44285..6905685324 100644
--- a/packages/server/package.json
+++ b/packages/server/package.json
@@ -46,7 +46,7 @@
     "@koa/router": "^8.0.0",
     "@sendgrid/mail": "^7.1.1",
     "@sentry/node": "^5.19.2",
-    "aws-sdk": "^2.706.0",
+    "aws-sdk": "^2.767.0",
     "bcryptjs": "^2.4.3",
     "chmodr": "^1.2.0",
     "dotenv": "^8.2.0",
diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js
index 828a88bb9b..b7a7c116b9 100644
--- a/packages/server/src/api/controllers/auth.js
+++ b/packages/server/src/api/controllers/auth.js
@@ -2,6 +2,8 @@ const jwt = require("jsonwebtoken")
 const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
 const bcrypt = require("../../utilities/bcrypt")
+const environment = require("../../environment")
+const { apiKeyTable } = require("../../db/dynamoClient")
 const { generateUserID } = require("../../db/utils")
 
 exports.authenticate = async ctx => {
@@ -51,6 +53,10 @@ exports.authenticate = async ctx => {
       appId: ctx.user.appId,
       instanceId,
     }
+    // if in cloud add the user api key
+    if (environment.CLOUD) {
+      payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId })
+    }
 
     const token = jwt.sign(payload, ctx.config.jwtSecret, {
       expiresIn: "1 day",
diff --git a/packages/server/src/api/controllers/model.js b/packages/server/src/api/controllers/model.js
index 2593002d72..29bf2dd90f 100644
--- a/packages/server/src/api/controllers/model.js
+++ b/packages/server/src/api/controllers/model.js
@@ -30,17 +30,9 @@ exports.save = async function(ctx) {
     views: {},
     ...ctx.request.body,
   }
-  // get the model in its previous state for differencing
-  let oldModel
-  let oldModelId = ctx.request.body._id
-  if (oldModelId) {
-    // if it errors then the oldModelId is invalid - can't diff it
-    try {
-      oldModel = await db.get(oldModelId)
-    } catch (err) {
-      oldModel = null
-    }
-  }
+
+  // if the model obj had an _id then it will have been retrieved
+  const oldModel = ctx.preExisting
 
   // rename record fields when table column is renamed
   const { _rename } = modelToSave
diff --git a/packages/server/src/api/controllers/record.js b/packages/server/src/api/controllers/record.js
index 45d983eb2b..7422cc9fc8 100644
--- a/packages/server/src/api/controllers/record.js
+++ b/packages/server/src/api/controllers/record.js
@@ -70,6 +70,9 @@ exports.save = async function(ctx) {
     record._id = generateRecordID(record.modelId)
   }
 
+  // if the record obj had an _id then it will have been retrieved
+  const existingRecord = ctx.preExisting
+
   const model = await db.get(record.modelId)
 
   const validateResult = await validate({
@@ -86,8 +89,6 @@ exports.save = async function(ctx) {
     return
   }
 
-  const existingRecord = record._rev && (await db.get(record._id))
-
   // make sure link records are up to date
   record = await linkRecords.updateLinks({
     instanceId,
diff --git a/packages/server/src/api/routes/model.js b/packages/server/src/api/routes/model.js
index 00eb46d515..118870cc61 100644
--- a/packages/server/src/api/routes/model.js
+++ b/packages/server/src/api/routes/model.js
@@ -1,6 +1,7 @@
 const Router = require("@koa/router")
 const modelController = require("../controllers/model")
 const authorized = require("../../middleware/authorized")
+const usage = require("../../middleware/usageQuota")
 const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels")
 
 const router = Router()
@@ -12,10 +13,11 @@ router
     authorized(READ_MODEL, ctx => ctx.params.id),
     modelController.find
   )
-  .post("/api/models", authorized(BUILDER), modelController.save)
+  .post("/api/models", authorized(BUILDER), usage, modelController.save)
   .delete(
     "/api/models/:modelId/:revId",
     authorized(BUILDER),
+    usage,
     modelController.destroy
   )
 
diff --git a/packages/server/src/api/routes/record.js b/packages/server/src/api/routes/record.js
index ddc26a55af..b3b5e9ed56 100644
--- a/packages/server/src/api/routes/record.js
+++ b/packages/server/src/api/routes/record.js
@@ -1,6 +1,7 @@
 const Router = require("@koa/router")
 const recordController = require("../controllers/record")
 const authorized = require("../../middleware/authorized")
+const usage = require("../../middleware/usageQuota")
 const { READ_MODEL, WRITE_MODEL } = require("../../utilities/accessLevels")
 
 const router = Router()
@@ -25,6 +26,7 @@ router
   .post(
     "/api/:modelId/records",
     authorized(WRITE_MODEL, ctx => ctx.params.modelId),
+    usage,
     recordController.save
   )
   .patch(
@@ -40,6 +42,7 @@ router
   .delete(
     "/api/:modelId/records/:recordId/:revId",
     authorized(WRITE_MODEL, ctx => ctx.params.modelId),
+    usage,
     recordController.destroy
   )
 
diff --git a/packages/server/src/api/routes/tests/couchTestUtils.js b/packages/server/src/api/routes/tests/couchTestUtils.js
index a22a2a427f..c269c14e4d 100644
--- a/packages/server/src/api/routes/tests/couchTestUtils.js
+++ b/packages/server/src/api/routes/tests/couchTestUtils.js
@@ -40,6 +40,9 @@ exports.defaultHeaders = (appId, instanceId) => {
 }
 
 exports.createModel = async (request, appId, instanceId, model) => {
+  if (model != null && model._id) {
+    delete model._id
+  }
   model = model || {
     name: "TestModel",
     type: "model",
diff --git a/packages/server/src/db/dynamoClient.js b/packages/server/src/db/dynamoClient.js
new file mode 100644
index 0000000000..9f30b151da
--- /dev/null
+++ b/packages/server/src/db/dynamoClient.js
@@ -0,0 +1,108 @@
+let _ = require("lodash")
+let environment = require("../environment")
+
+const TableInfo = {
+  API_KEYS: {
+    name: "beta-api-key-table",
+    primary: "pk",
+  },
+  USERS: {
+    name: "prod-budi-table",
+    primary: "pk",
+    sort: "sk",
+  },
+}
+
+let docClient = null
+
+class Table {
+  constructor(tableInfo) {
+    if (!tableInfo.name || !tableInfo.primary) {
+      throw "Table info must specify a name and a primary key"
+    }
+    this._name = tableInfo.name
+    this._primary = tableInfo.primary
+    this._sort = tableInfo.sort
+  }
+
+  async get({ primary, sort, otherProps }) {
+    let params = {
+      TableName: this._name,
+      Key: {
+        [this._primary]: primary,
+      },
+    }
+    if (this._sort && sort) {
+      params.Key[this._sort] = sort
+    }
+    if (otherProps) {
+      params = _.merge(params, otherProps)
+    }
+    let response = await docClient.get(params).promise()
+    return response.Item
+  }
+
+  async update({
+    primary,
+    sort,
+    expression,
+    condition,
+    names,
+    values,
+    otherProps,
+  }) {
+    let params = {
+      TableName: this._name,
+      Key: {
+        [this._primary]: primary,
+      },
+      ExpressionAttributeNames: names,
+      ExpressionAttributeValues: values,
+      UpdateExpression: expression,
+    }
+    if (condition) {
+      params.ConditionExpression = condition
+    }
+    if (this._sort && sort) {
+      params.Key[this._sort] = sort
+    }
+    if (otherProps) {
+      params = _.merge(params, otherProps)
+    }
+    return docClient.update(params).promise()
+  }
+
+  async put({ item, otherProps }) {
+    if (
+      item[this._primary] == null ||
+      (this._sort && item[this._sort] == null)
+    ) {
+      throw "Cannot put item without primary and sort key (if required)"
+    }
+    let params = {
+      TableName: this._name,
+      Item: item,
+    }
+    if (otherProps) {
+      params = _.merge(params, otherProps)
+    }
+    return docClient.put(params).promise()
+  }
+}
+
+exports.init = () => {
+  if (!environment.CLOUD) {
+    return
+  }
+  let AWS = require("aws-sdk")
+  let docClientParams = {
+    correctClockSkew: true,
+  }
+  if (environment.DYNAMO_ENDPOINT) {
+    docClientParams.endpoint = environment.DYNAMO_ENDPOINT
+  }
+  docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
+}
+
+exports.apiKeyTable = new Table(TableInfo.API_KEYS)
+exports.userTable = new Table(TableInfo.USERS)
diff --git a/packages/server/src/environment.js b/packages/server/src/environment.js
index bd08f191aa..f24d495397 100644
--- a/packages/server/src/environment.js
+++ b/packages/server/src/environment.js
@@ -11,4 +11,7 @@ module.exports = {
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
   BUDIBASE_ENVIRONMENT: process.env.BUDIBASE_ENVIRONMENT,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
+  CLOUD: process.env.CLOUD,
+  DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
+  AWS_REGION: process.env.AWS_REGION,
 }
diff --git a/packages/server/src/middleware/authenticated.js b/packages/server/src/middleware/authenticated.js
index 53cb0b2c13..126d616e3b 100644
--- a/packages/server/src/middleware/authenticated.js
+++ b/packages/server/src/middleware/authenticated.js
@@ -20,6 +20,7 @@ module.exports = async (ctx, next) => {
   if (builderToken) {
     try {
       const jwtPayload = jwt.verify(builderToken, ctx.config.jwtSecret)
+      ctx.apiKey = jwtPayload.apiKey
       ctx.isAuthenticated = jwtPayload.accessLevelId === BUILDER_LEVEL_ID
       ctx.user = {
         ...jwtPayload,
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
new file mode 100644
index 0000000000..1d119f25c8
--- /dev/null
+++ b/packages/server/src/middleware/usageQuota.js
@@ -0,0 +1,77 @@
+const CouchDB = require("../db")
+const environment = require("../environment")
+const { apiKeyTable } = require("../db/dynamoClient")
+
+// a normalised month in milliseconds
+const QUOTA_RESET = 2592000000
+
+// currently only counting new writes and deletes
+const METHOD_MAP = {
+  POST: 1,
+  DELETE: -1,
+}
+
+const DOMAIN_MAP = {
+  models: "model",
+  records: "record",
+}
+
+function buildUpdateParams(key, property, usage) {
+  return {
+    primary: key,
+    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
+    expression: "ADD #quota.#prop :usage",
+    names: {
+      "#quota": "usageQuota",
+      "#prop": property,
+      "#limits": "limits",
+      "#quotaReset": "quotaReset",
+    },
+    values: {
+      ":usage": usage,
+      ":now": Date.now(),
+    },
+  }
+}
+
+module.exports = async (ctx, next) => {
+  const db = new CouchDB(ctx.user.instanceId)
+  const usage = METHOD_MAP[ctx.req.method]
+  const domainParts = ctx.req.url.split("/")
+  const property = DOMAIN_MAP[domainParts[domainParts.length - 1]]
+  if (usage == null || property == null) {
+    return next()
+  }
+  // post request could be a save of a pre-existing entry
+  if (ctx.request.body && ctx.request.body._id) {
+    try {
+      ctx.preExisting = await db.get(ctx.request.body._id)
+      return next()
+    } catch (err) {
+      ctx.throw(404, `${ctx.request.body._id} does not exist`)
+      return
+    }
+  }
+  // don't try validate in builder
+  if (!environment.CLOUD) {
+    return next()
+  }
+  try {
+    await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage))
+  } catch (err) {
+    if (err.code !== "ConditionalCheckFailedException") {
+      // get the API key so we can check it
+      let apiKey = await apiKeyTable.get({ primary: ctx.apiKey })
+      // we have infact breached the reset period
+      if (apiKey && apiKey.quotaReset >= Date.now()) {
+        // update the quota reset period and reset the values for all properties
+        apiKey.quotaReset = Date.now() + QUOTA_RESET
+        for (let prop of Object.keys(apiKey.usageQuota)) {
+          apiKey.usageQuota[prop] = 0
+        }
+        await apiKeyTable.put({ item: apiKey })
+      }
+      ctx.throw(403, `Resource limits have been reached`)
+    }
+  }
+}
\ No newline at end of file
diff --git a/packages/server/yarn.lock b/packages/server/yarn.lock
index de283385a8..cb22d75c7b 100644
--- a/packages/server/yarn.lock
+++ b/packages/server/yarn.lock
@@ -172,6 +172,15 @@
     lodash "^4.17.13"
     to-fast-properties "^2.0.0"
 
+"@budibase/client@^0.1.23":
+  version "0.1.23"
+  resolved "https://registry.yarnpkg.com/@budibase/client/-/client-0.1.23.tgz#d72d2b26ff3a2d99f2b6c1b71020b1136880937d"
+  integrity sha512-pZdwdCq5kKLZfZYxasIHBNnqu3BFFrqJLxXMFs0K9ddCVZ0UNons59nn73nFGbeRgNVdWp6yyW71XyMQr8NOEw==
+  dependencies:
+    deep-equal "^2.0.1"
+    mustache "^4.0.1"
+    regexparam "^1.3.0"
+
 "@cnakazawa/watch@^1.0.3":
   version "1.0.4"
   resolved "https://registry.yarnpkg.com/@cnakazawa/watch/-/watch-1.0.4.tgz#f864ae85004d0fcab6f50be9141c4da368d1656a"
@@ -867,6 +876,11 @@ array-equal@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/array-equal/-/array-equal-1.0.0.tgz#8c2a5ef2472fd9ea742b04c77a75093ba2757c93"
 
+array-filter@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/array-filter/-/array-filter-1.0.0.tgz#baf79e62e6ef4c2a4c0b831232daffec251f9d83"
+  integrity sha1-uveeYubvTCpMC4MSMtr/7CUfnYM=
+
 array-unique@^0.3.2:
   version "0.3.2"
   resolved "https://registry.yarnpkg.com/array-unique/-/array-unique-0.3.2.tgz#a894b75d4bc4f6cd679ef3244a9fd8f46ae2d428"
@@ -925,9 +939,17 @@ atomic-sleep@^1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/atomic-sleep/-/atomic-sleep-1.0.0.tgz#eb85b77a601fc932cfe432c5acd364a9e2c9075b"
 
-aws-sdk@^2.706.0:
-  version "2.706.0"
-  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.706.0.tgz#09f65e9a91ecac5a635daf934082abae30eca953"
+available-typed-arrays@^1.0.0, available-typed-arrays@^1.0.2:
+  version "1.0.2"
+  resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.2.tgz#6b098ca9d8039079ee3f77f7b783c4480ba513f5"
+  integrity sha512-XWX3OX8Onv97LMk/ftVyBibpGwY5a8SmuxZPzeOxqmuEqUCOM9ZE+uIaD1VNJ5QnvU2UQusvmKbuM1FR8QWGfQ==
+  dependencies:
+    array-filter "^1.0.0"
+
+aws-sdk@^2.767.0:
+  version "2.767.0"
+  resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.767.0.tgz#9863c8bfd5990106b95f38e9345a547fee782470"
+  integrity sha512-soPZxjNpat0CtuIqm54GO/FDT4SZTlQG0icSptWYfMFYdkXe8b0tJqaPssNn9TzlgoWDCNTdaoepM6TN0rNHkQ==
   dependencies:
     buffer "4.9.2"
     events "1.1.1"
@@ -1725,6 +1747,26 @@ decompress@^4.2.1:
     pify "^2.3.0"
     strip-dirs "^2.0.0"
 
+deep-equal@^2.0.1:
+  version "2.0.4"
+  resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-2.0.4.tgz#6b0b407a074666033169df3acaf128e1c6f3eab6"
+  integrity sha512-BUfaXrVoCfgkOQY/b09QdO9L3XNoF2XH0A3aY9IQwQL/ZjLOe8FQgCNVl1wiolhsFo8kFdO9zdPViCPbmaJA5w==
+  dependencies:
+    es-abstract "^1.18.0-next.1"
+    es-get-iterator "^1.1.0"
+    is-arguments "^1.0.4"
+    is-date-object "^1.0.2"
+    is-regex "^1.1.1"
+    isarray "^2.0.5"
+    object-is "^1.1.3"
+    object-keys "^1.1.1"
+    object.assign "^4.1.1"
+    regexp.prototype.flags "^1.3.0"
+    side-channel "^1.0.3"
+    which-boxed-primitive "^1.0.1"
+    which-collection "^1.0.1"
+    which-typed-array "^1.1.2"
+
 deep-equal@~1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/deep-equal/-/deep-equal-1.0.1.tgz#f5d260292b660e084eff4cdbc9f08ad3247448b5"
@@ -2073,6 +2115,54 @@ es-abstract@^1.17.0-next.1, es-abstract@^1.17.2, es-abstract@^1.17.5:
     string.prototype.trimleft "^2.1.1"
     string.prototype.trimright "^2.1.1"
 
+es-abstract@^1.17.4:
+  version "1.17.7"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.17.7.tgz#a4de61b2f66989fc7421676c1cb9787573ace54c"
+  integrity sha512-VBl/gnfcJ7OercKA9MVaegWsBHFjV492syMudcnQZvt/Dw8ezpcOHYZXa/J96O8vx+g4x65YKhxOwDUh63aS5g==
+  dependencies:
+    es-to-primitive "^1.2.1"
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    has-symbols "^1.0.1"
+    is-callable "^1.2.2"
+    is-regex "^1.1.1"
+    object-inspect "^1.8.0"
+    object-keys "^1.1.1"
+    object.assign "^4.1.1"
+    string.prototype.trimend "^1.0.1"
+    string.prototype.trimstart "^1.0.1"
+
+es-abstract@^1.18.0-next.0, es-abstract@^1.18.0-next.1:
+  version "1.18.0-next.1"
+  resolved "https://registry.yarnpkg.com/es-abstract/-/es-abstract-1.18.0-next.1.tgz#6e3a0a4bda717e5023ab3b8e90bec36108d22c68"
+  integrity sha512-I4UGspA0wpZXWENrdA0uHbnhte683t3qT/1VFH9aX2dA5PPSf6QW5HHXf5HImaqPmjXaVeVk4RGWnaylmV7uAA==
+  dependencies:
+    es-to-primitive "^1.2.1"
+    function-bind "^1.1.1"
+    has "^1.0.3"
+    has-symbols "^1.0.1"
+    is-callable "^1.2.2"
+    is-negative-zero "^2.0.0"
+    is-regex "^1.1.1"
+    object-inspect "^1.8.0"
+    object-keys "^1.1.1"
+    object.assign "^4.1.1"
+    string.prototype.trimend "^1.0.1"
+    string.prototype.trimstart "^1.0.1"
+
+es-get-iterator@^1.1.0:
+  version "1.1.0"
+  resolved "https://registry.yarnpkg.com/es-get-iterator/-/es-get-iterator-1.1.0.tgz#bb98ad9d6d63b31aacdc8f89d5d0ee57bcb5b4c8"
+  integrity sha512-UfrmHuWQlNMTs35e1ypnvikg6jCz3SK8v8ImvmDsh36fCVUR1MqoFDiyn0/k52C8NqO3YsO8Oe0azeesNuqSsQ==
+  dependencies:
+    es-abstract "^1.17.4"
+    has-symbols "^1.0.1"
+    is-arguments "^1.0.4"
+    is-map "^2.0.1"
+    is-set "^2.0.1"
+    is-string "^1.0.5"
+    isarray "^2.0.5"
+
 es-to-primitive@^1.2.1:
   version "1.2.1"
   resolved "https://registry.yarnpkg.com/es-to-primitive/-/es-to-primitive-1.2.1.tgz#e55cd4c9cdc188bcefb03b366c736323fc5c898a"
@@ -3112,6 +3202,11 @@ is-accessor-descriptor@^1.0.0:
   dependencies:
     kind-of "^6.0.0"
 
+is-arguments@^1.0.4:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/is-arguments/-/is-arguments-1.0.4.tgz#3faf966c7cba0ff437fb31f6250082fcf0448cf3"
+  integrity sha512-xPh0Rmt8NE65sNzvyUmWgI1tz3mKq74lGA0mL8LYZcoIzKOzDh6HmrYm3d18k60nHerC8A9Km8kYu87zfSFnLA==
+
 is-arrayish@^0.2.1:
   version "0.2.1"
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d"
@@ -3121,12 +3216,22 @@ is-arrayish@^0.3.1:
   resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"
   integrity sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==
 
+is-bigint@^1.0.0:
+  version "1.0.0"
+  resolved "https://registry.yarnpkg.com/is-bigint/-/is-bigint-1.0.0.tgz#73da8c33208d00f130e9b5e15d23eac9215601c4"
+  integrity sha512-t5mGUXC/xRheCK431ylNiSkGGpBp8bHENBcENTkDT6ppwPzEVxNGZRvgvmOEfbWkFhA7D2GEuE2mmQTr78sl2g==
+
 is-binary-path@~2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/is-binary-path/-/is-binary-path-2.1.0.tgz#ea1f7f3b80f064236e83470f86c09c254fb45b09"
   dependencies:
     binary-extensions "^2.0.0"
 
+is-boolean-object@^1.0.0:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/is-boolean-object/-/is-boolean-object-1.0.1.tgz#10edc0900dd127697a92f6f9807c7617d68ac48e"
+  integrity sha512-TqZuVwa/sppcrhUCAYkGBk7w0yxfQQnxq28fjkO53tnK9FQXmdwz2JS5+GjsWQ6RByES1K40nI+yDic5c9/aAQ==
+
 is-buffer@^1.1.5:
   version "1.1.6"
   resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be"
@@ -3135,6 +3240,11 @@ is-callable@^1.1.4, is-callable@^1.1.5:
   version "1.1.5"
   resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.1.5.tgz#f7e46b596890456db74e7f6e976cb3273d06faab"
 
+is-callable@^1.2.2:
+  version "1.2.2"
+  resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.2.tgz#c7c6715cd22d4ddb48d3e19970223aceabb080d9"
+  integrity sha512-dnMqspv5nU3LoewK2N/y7KLtxtakvTuaCsU9FU50/QDmdbHNy/4/JuRtMHqRU22o3q+W89YQndQEeCVwK+3qrA==
+
 is-ci@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/is-ci/-/is-ci-2.0.0.tgz#6bc6334181810e04b5c22b3d589fdca55026404c"
@@ -3157,7 +3267,7 @@ is-data-descriptor@^1.0.0:
   dependencies:
     kind-of "^6.0.0"
 
-is-date-object@^1.0.1:
+is-date-object@^1.0.1, is-date-object@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-date-object/-/is-date-object-1.0.2.tgz#bda736f2cd8fd06d32844e7743bfa7494c3bfd7e"
 
@@ -3227,15 +3337,30 @@ is-installed-globally@^0.3.1:
     global-dirs "^2.0.1"
     is-path-inside "^3.0.1"
 
+is-map@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-map/-/is-map-2.0.1.tgz#520dafc4307bb8ebc33b813de5ce7c9400d644a1"
+  integrity sha512-T/S49scO8plUiAOA2DBTBG3JHpn1yiw0kRp6dgiZ0v2/6twi5eiB0rHtHFH9ZIrvlWc6+4O+m4zg5+Z833aXgw==
+
 is-natural-number@^4.0.1:
   version "4.0.1"
   resolved "https://registry.yarnpkg.com/is-natural-number/-/is-natural-number-4.0.1.tgz#ab9d76e1db4ced51e35de0c72ebecf09f734cde8"
   integrity sha1-q5124dtM7VHjXeDHLr7PCfc0zeg=
 
+is-negative-zero@^2.0.0:
+  version "2.0.0"
+  resolved "https://registry.yarnpkg.com/is-negative-zero/-/is-negative-zero-2.0.0.tgz#9553b121b0fac28869da9ed459e20c7543788461"
+  integrity sha1-lVOxIbD6wohp2p7UWeIMdUN4hGE=
+
 is-npm@^4.0.0:
   version "4.0.0"
   resolved "https://registry.yarnpkg.com/is-npm/-/is-npm-4.0.0.tgz#c90dd8380696df87a7a6d823c20d0b12bbe3c84d"
 
+is-number-object@^1.0.3:
+  version "1.0.4"
+  resolved "https://registry.yarnpkg.com/is-number-object/-/is-number-object-1.0.4.tgz#36ac95e741cf18b283fc1ddf5e83da798e3ec197"
+  integrity sha512-zohwelOAur+5uXtk8O3GPQ1eAcu4ZX3UwxQhUlfFFMNpUd83gXgjbhJh6HmB6LUNV/ieOLQuDwJO3dWJosUeMw==
+
 is-number@^3.0.0:
   version "3.0.0"
   resolved "https://registry.yarnpkg.com/is-number/-/is-number-3.0.0.tgz#24fd6201a4782cf50561c810276afc7d12d71195"
@@ -3276,15 +3401,32 @@ is-regex@^1.0.5:
   dependencies:
     has "^1.0.3"
 
+is-regex@^1.1.1:
+  version "1.1.1"
+  resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.1.tgz#c6f98aacc546f6cec5468a07b7b153ab564a57b9"
+  integrity sha512-1+QkEcxiLlB7VEyFtyBg94e08OAsvq7FUBgApTq/w2ymCLyKJgDPsybBENVtA7XCQEgEXxKPonG+mvYRxh/LIg==
+  dependencies:
+    has-symbols "^1.0.1"
+
 is-retry-allowed@^1.1.0:
   version "1.2.0"
   resolved "https://registry.yarnpkg.com/is-retry-allowed/-/is-retry-allowed-1.2.0.tgz#d778488bd0a4666a3be8a1482b9f2baafedea8b4"
   integrity sha512-RUbUeKwvm3XG2VYamhJL1xFktgjvPzL0Hq8C+6yrWIswDy3BIXGqCxhxkc30N9jqK311gVU137K8Ei55/zVJRg==
 
+is-set@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-set/-/is-set-2.0.1.tgz#d1604afdab1724986d30091575f54945da7e5f43"
+  integrity sha512-eJEzOtVyenDs1TMzSQ3kU3K+E0GUS9sno+F0OBT97xsgcJsF9nXMBtkT9/kut5JEpM7oL7X/0qxR17K3mcwIAA==
+
 is-stream@^1.1.0:
   version "1.1.0"
   resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
 
+is-string@^1.0.4, is-string@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/is-string/-/is-string-1.0.5.tgz#40493ed198ef3ff477b8c7f92f644ec82a5cd3a6"
+  integrity sha512-buY6VNRjhQMiF1qWDouloZlQbRhDPCebwxSjxMjxgemYT46YMd2NR0/H+fBhEfWX4A/w9TBJ+ol+okqJKFE6vQ==
+
 is-symbol@^1.0.2:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/is-symbol/-/is-symbol-1.0.3.tgz#38e1014b9e6329be0de9d24a414fd7441ec61937"
@@ -3299,10 +3441,30 @@ is-type-of@^1.0.0:
     is-class-hotfix "~0.0.6"
     isstream "~0.1.2"
 
+is-typed-array@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/is-typed-array/-/is-typed-array-1.1.3.tgz#a4ff5a5e672e1a55f99c7f54e59597af5c1df04d"
+  integrity sha512-BSYUBOK/HJibQ30wWkWold5txYwMUXQct9YHAQJr8fSwvZoiglcqB0pd7vEN23+Tsi9IUEjztdOSzl4qLVYGTQ==
+  dependencies:
+    available-typed-arrays "^1.0.0"
+    es-abstract "^1.17.4"
+    foreach "^2.0.5"
+    has-symbols "^1.0.1"
+
 is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
 
+is-weakmap@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-weakmap/-/is-weakmap-2.0.1.tgz#5008b59bdc43b698201d18f62b37b2ca243e8cf2"
+  integrity sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==
+
+is-weakset@^2.0.1:
+  version "2.0.1"
+  resolved "https://registry.yarnpkg.com/is-weakset/-/is-weakset-2.0.1.tgz#e9a0af88dbd751589f5e50d80f4c98b780884f83"
+  integrity sha512-pi4vhbhVHGLxohUw7PhGsueT4vRGFoXhP7+RGN0jKIv9+8PWYCQTqtADngrxOm2g46hoH0+g8uZZBzMrvVGDmw==
+
 is-windows@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/is-windows/-/is-windows-1.0.2.tgz#d1850eb9791ecd18e6182ce12a30f396634bb19d"
@@ -3323,6 +3485,11 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
 
+isarray@^2.0.5:
+  version "2.0.5"
+  resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.5.tgz#8af1e4c1221244cc62459faf38940d4e644a5723"
+  integrity sha512-xHjhDr3cNBK0BzdUJSPXZntQUx/mwMS5Rw4A7lPJ90XGAO6ISP/ePDNuo0vhqOZU+UD5JoodwCAAoZQd3FeAKw==
+
 isbinaryfile@^4.0.6:
   version "4.0.6"
   resolved "https://registry.yarnpkg.com/isbinaryfile/-/isbinaryfile-4.0.6.tgz#edcb62b224e2b4710830b67498c8e4e5a4d2610b"
@@ -4668,6 +4835,19 @@ object-inspect@^1.7.0:
   version "1.7.0"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.7.0.tgz#f4f6bd181ad77f006b5ece60bd0b6f398ff74a67"
 
+object-inspect@^1.8.0:
+  version "1.8.0"
+  resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0"
+  integrity sha512-jLdtEOB112fORuypAyl/50VRVIBIdVQOSUUGQHzJ4xBSbit81zRarz7GThkEFZy1RceYrWYcPcBFPQwHyAc1gA==
+
+object-is@^1.1.3:
+  version "1.1.3"
+  resolved "https://registry.yarnpkg.com/object-is/-/object-is-1.1.3.tgz#2e3b9e65560137455ee3bd62aec4d90a2ea1cc81"
+  integrity sha512-teyqLvFWzLkq5B9ki8FVWA902UER2qkxmdA4nLf+wjOLAWgxzCWZNCxpDq9MvE8MmhWNr+I8w3BN49Vx36Y6Xg==
+  dependencies:
+    define-properties "^1.1.3"
+    es-abstract "^1.18.0-next.1"
+
 object-keys@^1.0.11, object-keys@^1.0.12, object-keys@^1.0.6, object-keys@^1.1.1:
   version "1.1.1"
   resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.1.1.tgz#1c47f272df277f3b1daf061677d9c82e2322c60e"
@@ -4687,6 +4867,16 @@ object.assign@^4.1.0:
     has-symbols "^1.0.0"
     object-keys "^1.0.11"
 
+object.assign@^4.1.1:
+  version "4.1.1"
+  resolved "https://registry.yarnpkg.com/object.assign/-/object.assign-4.1.1.tgz#303867a666cdd41936ecdedfb1f8f3e32a478cdd"
+  integrity sha512-VT/cxmx5yaoHSOTSyrCygIDFco+RsibY2NM0a4RdEeY/4KgqezwFtK1yr3U67xYhqJSlASm2pKhLVzPj2lr4bA==
+  dependencies:
+    define-properties "^1.1.3"
+    es-abstract "^1.18.0-next.0"
+    has-symbols "^1.0.1"
+    object-keys "^1.1.1"
+
 object.getownpropertydescriptors@^2.1.0:
   version "2.1.0"
   resolved "https://registry.yarnpkg.com/object.getownpropertydescriptors/-/object.getownpropertydescriptors-2.1.0.tgz#369bf1f9592d8ab89d712dced5cb81c7c5352649"
@@ -5374,6 +5564,19 @@ regex-not@^1.0.0, regex-not@^1.0.2:
     extend-shallow "^3.0.2"
     safe-regex "^1.1.0"
 
+regexp.prototype.flags@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/regexp.prototype.flags/-/regexp.prototype.flags-1.3.0.tgz#7aba89b3c13a64509dabcf3ca8d9fbb9bdf5cb75"
+  integrity sha512-2+Q0C5g951OlYlJz6yu5/M33IcsESLlLfsyIaLJaG4FA2r4yP8MvVMJUUP/fVBkSpbbbZlS5gynbEWLipiiXiQ==
+  dependencies:
+    define-properties "^1.1.3"
+    es-abstract "^1.17.0-next.1"
+
+regexparam@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/regexparam/-/regexparam-1.3.0.tgz#2fe42c93e32a40eff6235d635e0ffa344b92965f"
+  integrity sha512-6IQpFBv6e5vz1QAqI+V4k8P2e/3gRrqfCJ9FI+O1FLQTO+Uz6RXZEZOPmTJ6hlGj7gkERzY5BRCv09whKP96/g==
+
 regexpp@^2.0.1:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-2.0.1.tgz#8d19d31cf632482b589049f8281f93dbcba4d07f"
@@ -5692,6 +5895,14 @@ shellwords@^0.1.1:
   version "0.1.1"
   resolved "https://registry.yarnpkg.com/shellwords/-/shellwords-0.1.1.tgz#d6b9181c1a48d397324c84871efbcfc73fc0654b"
 
+side-channel@^1.0.3:
+  version "1.0.3"
+  resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.3.tgz#cdc46b057550bbab63706210838df5d4c19519c3"
+  integrity sha512-A6+ByhlLkksFoUepsGxfj5x1gTSrs+OydsRptUxeNCabQpCFUvcwIczgOigI8vhY/OJCnPnyE9rGiwgvr9cS1g==
+  dependencies:
+    es-abstract "^1.18.0-next.0"
+    object-inspect "^1.8.0"
+
 signal-exit@^3.0.0, signal-exit@^3.0.2:
   version "3.0.3"
   resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.3.tgz#a1410c2edd8f077b08b4e253c8eacfcaf057461c"
@@ -5981,7 +6192,7 @@ string-width@^4.0.0, string-width@^4.1.0, string-width@^4.2.0:
     is-fullwidth-code-point "^3.0.0"
     strip-ansi "^6.0.0"
 
-string.prototype.trimend@^1.0.0:
+string.prototype.trimend@^1.0.0, string.prototype.trimend@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/string.prototype.trimend/-/string.prototype.trimend-1.0.1.tgz#85812a6b847ac002270f5808146064c995fb6913"
   dependencies:
@@ -6004,7 +6215,7 @@ string.prototype.trimright@^2.1.1:
     es-abstract "^1.17.5"
     string.prototype.trimend "^1.0.0"
 
-string.prototype.trimstart@^1.0.0:
+string.prototype.trimstart@^1.0.0, string.prototype.trimstart@^1.0.1:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/string.prototype.trimstart/-/string.prototype.trimstart-1.0.1.tgz#14af6d9f34b053f7cfc89b72f8f2ee14b9039a54"
   dependencies:
@@ -6611,6 +6822,27 @@ whatwg-url@^7.0.0:
     tr46 "^1.0.1"
     webidl-conversions "^4.0.2"
 
+which-boxed-primitive@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/which-boxed-primitive/-/which-boxed-primitive-1.0.1.tgz#cbe8f838ebe91ba2471bb69e9edbda67ab5a5ec1"
+  integrity sha512-7BT4TwISdDGBgaemWU0N0OU7FeAEJ9Oo2P1PHRm/FCWoEi2VLWC9b6xvxAA3C/NMpxg3HXVgi0sMmGbNUbNepQ==
+  dependencies:
+    is-bigint "^1.0.0"
+    is-boolean-object "^1.0.0"
+    is-number-object "^1.0.3"
+    is-string "^1.0.4"
+    is-symbol "^1.0.2"
+
+which-collection@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/which-collection/-/which-collection-1.0.1.tgz#70eab71ebbbd2aefaf32f917082fc62cdcb70906"
+  integrity sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==
+  dependencies:
+    is-map "^2.0.1"
+    is-set "^2.0.1"
+    is-weakmap "^2.0.1"
+    is-weakset "^2.0.1"
+
 which-module@^2.0.0:
   version "2.0.0"
   resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a"
@@ -6620,6 +6852,18 @@ which-pm-runs@^1.0.0:
   resolved "https://registry.yarnpkg.com/which-pm-runs/-/which-pm-runs-1.0.0.tgz#670b3afbc552e0b55df6b7780ca74615f23ad1cb"
   integrity sha1-Zws6+8VS4LVd9rd4DKdGFfI60cs=
 
+which-typed-array@^1.1.2:
+  version "1.1.2"
+  resolved "https://registry.yarnpkg.com/which-typed-array/-/which-typed-array-1.1.2.tgz#e5f98e56bda93e3dac196b01d47c1156679c00b2"
+  integrity sha512-KT6okrd1tE6JdZAy3o2VhMoYPh3+J6EMZLyrxBQsZflI1QCZIxMrIYLkosd8Twf+YfknVIHmYQPgJt238p8dnQ==
+  dependencies:
+    available-typed-arrays "^1.0.2"
+    es-abstract "^1.17.5"
+    foreach "^2.0.5"
+    function-bind "^1.1.1"
+    has-symbols "^1.0.1"
+    is-typed-array "^1.1.3"
+
 which@^1.2.9, which@^1.3.0:
   version "1.3.1"
   resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a"

From f2b7d85b6e1145ad55545a25f0360e6d439a6aa6 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Wed, 7 Oct 2020 17:56:47 +0100
Subject: [PATCH 2/7] Fixing up middleware to handle uploads, views, records,
 automation runs and users.

---
 packages/server/src/api/controllers/auth.js   |  4 +-
 .../server/src/api/controllers/automation.js  |  2 +
 packages/server/src/api/routes/model.js       |  4 +-
 packages/server/src/api/routes/static.js      |  3 +-
 packages/server/src/api/routes/user.js        |  4 +-
 packages/server/src/api/routes/view.js        | 10 ++-
 packages/server/src/automations/index.js      | 25 +++++--
 packages/server/src/middleware/usageQuota.js  | 71 +++++++-----------
 .../server/src/utilities/fileProcessor.js     | 18 ++---
 packages/server/src/utilities/usageQuota.js   | 72 +++++++++++++++++++
 10 files changed, 147 insertions(+), 66 deletions(-)
 create mode 100644 packages/server/src/utilities/usageQuota.js

diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js
index b7a7c116b9..c16f76caca 100644
--- a/packages/server/src/api/controllers/auth.js
+++ b/packages/server/src/api/controllers/auth.js
@@ -3,7 +3,7 @@ const CouchDB = require("../../db")
 const ClientDb = require("../../db/clientDb")
 const bcrypt = require("../../utilities/bcrypt")
 const environment = require("../../environment")
-const { apiKeyTable } = require("../../db/dynamoClient")
+const { getAPIKey } = require("../../utilities/usageQuota")
 const { generateUserID } = require("../../db/utils")
 
 exports.authenticate = async ctx => {
@@ -55,7 +55,7 @@ exports.authenticate = async ctx => {
     }
     // if in cloud add the user api key
     if (environment.CLOUD) {
-      payload.apiKey = await apiKeyTable.get({ primary: ctx.user.appId })
+      payload.apiKey = getAPIKey(ctx.user.appId)
     }
 
     const token = jwt.sign(payload, ctx.config.jwtSecret, {
diff --git a/packages/server/src/api/controllers/automation.js b/packages/server/src/api/controllers/automation.js
index 5414f3878c..92391da7e9 100644
--- a/packages/server/src/api/controllers/automation.js
+++ b/packages/server/src/api/controllers/automation.js
@@ -33,6 +33,7 @@ function cleanAutomationInputs(automation) {
 exports.create = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
+  automation.appId = ctx.user.appId
 
   automation._id = generateAutomationID()
 
@@ -54,6 +55,7 @@ exports.create = async function(ctx) {
 exports.update = async function(ctx) {
   const db = new CouchDB(ctx.user.instanceId)
   let automation = ctx.request.body
+  automation.appId = ctx.user.appId
 
   automation = cleanAutomationInputs(automation)
   const response = await db.put(automation)
diff --git a/packages/server/src/api/routes/model.js b/packages/server/src/api/routes/model.js
index 3c667df520..fe782d4cf5 100644
--- a/packages/server/src/api/routes/model.js
+++ b/packages/server/src/api/routes/model.js
@@ -1,7 +1,6 @@
 const Router = require("@koa/router")
 const modelController = require("../controllers/model")
 const authorized = require("../../middleware/authorized")
-const usage = require("../../middleware/usageQuota")
 const { BUILDER, READ_MODEL } = require("../../utilities/accessLevels")
 
 const router = Router()
@@ -13,7 +12,7 @@ router
     authorized(READ_MODEL, ctx => ctx.params.id),
     modelController.find
   )
-  .post("/api/models", authorized(BUILDER), usage, modelController.save)
+  .post("/api/models", authorized(BUILDER), modelController.save)
   .post(
     "/api/models/csv/validate",
     authorized(BUILDER),
@@ -22,7 +21,6 @@ router
   .delete(
     "/api/models/:modelId/:revId",
     authorized(BUILDER),
-    usage,
     modelController.destroy
   )
 
diff --git a/packages/server/src/api/routes/static.js b/packages/server/src/api/routes/static.js
index aa136a3d15..5c33900eca 100644
--- a/packages/server/src/api/routes/static.js
+++ b/packages/server/src/api/routes/static.js
@@ -4,6 +4,7 @@ const { budibaseTempDir } = require("../../utilities/budibaseDir")
 const env = require("../../environment")
 const authorized = require("../../middleware/authorized")
 const { BUILDER } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -28,7 +29,7 @@ router
     authorized(BUILDER),
     controller.performLocalFileProcessing
   )
-  .post("/api/attachments/upload", controller.uploadFile)
+  .post("/api/attachments/upload", usage, controller.uploadFile)
   .get("/componentlibrary", controller.serveComponentLibrary)
   .get("/assets/:file*", controller.serveAppAsset)
   .get("/attachments/:file*", controller.serveAttachment)
diff --git a/packages/server/src/api/routes/user.js b/packages/server/src/api/routes/user.js
index 532943ea62..5289439e41 100644
--- a/packages/server/src/api/routes/user.js
+++ b/packages/server/src/api/routes/user.js
@@ -2,6 +2,7 @@ const Router = require("@koa/router")
 const controller = require("../controllers/user")
 const authorized = require("../../middleware/authorized")
 const { USER_MANAGEMENT, LIST_USERS } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -9,10 +10,11 @@ router
   .get("/api/users", authorized(LIST_USERS), controller.fetch)
   .get("/api/users/:username", authorized(USER_MANAGEMENT), controller.find)
   .put("/api/users/", authorized(USER_MANAGEMENT), controller.update)
-  .post("/api/users", authorized(USER_MANAGEMENT), controller.create)
+  .post("/api/users", authorized(USER_MANAGEMENT), usage, controller.create)
   .delete(
     "/api/users/:username",
     authorized(USER_MANAGEMENT),
+    usage,
     controller.destroy
   )
 
diff --git a/packages/server/src/api/routes/view.js b/packages/server/src/api/routes/view.js
index 2c88f6d19a..571e4494f1 100644
--- a/packages/server/src/api/routes/view.js
+++ b/packages/server/src/api/routes/view.js
@@ -3,6 +3,7 @@ const viewController = require("../controllers/view")
 const recordController = require("../controllers/record")
 const authorized = require("../../middleware/authorized")
 const { BUILDER, READ_VIEW } = require("../../utilities/accessLevels")
+const usage = require("../../middleware/usageQuota")
 
 const router = Router()
 
@@ -13,8 +14,13 @@ router
     recordController.fetchView
   )
   .get("/api/views", authorized(BUILDER), viewController.fetch)
-  .delete("/api/views/:viewName", authorized(BUILDER), viewController.destroy)
-  .post("/api/views", authorized(BUILDER), viewController.save)
+  .delete(
+    "/api/views/:viewName",
+    authorized(BUILDER),
+    usage,
+    viewController.destroy
+  )
+  .post("/api/views", authorized(BUILDER), usage, viewController.save)
   .post("/api/views/export", authorized(BUILDER), viewController.exportView)
   .get(
     "/api/views/export/download/:fileName",
diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js
index e419985ce2..882f24391b 100644
--- a/packages/server/src/automations/index.js
+++ b/packages/server/src/automations/index.js
@@ -3,6 +3,7 @@ const actions = require("./actions")
 const environment = require("../environment")
 const workerFarm = require("worker-farm")
 const singleThread = require("./thread")
+const { getAPIKey, update, Properties } = require("../utilities/usageQuota")
 
 let workers = workerFarm(require.resolve("./thread"))
 
@@ -18,16 +19,32 @@ function runWorker(job) {
   })
 }
 
+async function updateQuota(automation) {
+  const appId = automation.appId
+  const apiKey = await getAPIKey(appId)
+  // this will fail, causing automation to escape if limits reached
+  await update(apiKey, Properties.AUTOMATION, 1)
+}
+
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
 module.exports.init = function() {
   actions.init().then(() => {
     triggers.automationQueue.process(async job => {
-      if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
-        await runWorker(job)
-      } else {
-        await singleThread(job)
+      try {
+        if (environment.CLOUD) {
+          await updateQuota(job.data.automation)
+        }
+        if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
+          await runWorker(job)
+        } else {
+          await singleThread(job)
+        }
+      } catch (err) {
+        console.error(
+          `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
+        )
       }
     })
   })
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 1d119f25c8..74df19a015 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -1,9 +1,5 @@
 const CouchDB = require("../db")
-const environment = require("../environment")
-const { apiKeyTable } = require("../db/dynamoClient")
-
-// a normalised month in milliseconds
-const QUOTA_RESET = 2592000000
+const usageQuota = require("../utilities/usageQuota")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -12,36 +8,32 @@ const METHOD_MAP = {
 }
 
 const DOMAIN_MAP = {
-  models: "model",
-  records: "record",
+  records: usageQuota.Properties.RECORD,
+  upload: usageQuota.Properties.UPLOAD,
+  views: usageQuota.Properties.VIEW,
+  users: usageQuota.Properties.USER,
+  // this will not be updated by endpoint calls
+  // instead it will be updated by triggers
+  automationRuns: usageQuota.Properties.AUTOMATION,
 }
 
-function buildUpdateParams(key, property, usage) {
-  return {
-    primary: key,
-    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
-    expression: "ADD #quota.#prop :usage",
-    names: {
-      "#quota": "usageQuota",
-      "#prop": property,
-      "#limits": "limits",
-      "#quotaReset": "quotaReset",
-    },
-    values: {
-      ":usage": usage,
-      ":now": Date.now(),
-    },
+function getProperty(url) {
+  for (let domain of Object.keys(DOMAIN_MAP)) {
+    if (url.indexOf(domain) !== -1) {
+      return DOMAIN_MAP[domain]
+    }
   }
 }
 
 module.exports = async (ctx, next) => {
   const db = new CouchDB(ctx.user.instanceId)
-  const usage = METHOD_MAP[ctx.req.method]
-  const domainParts = ctx.req.url.split("/")
-  const property = DOMAIN_MAP[domainParts[domainParts.length - 1]]
+  let usage = METHOD_MAP[ctx.req.method]
+  const property = getProperty(ctx.req.url)
+  console.log(ctx.req.url)
   if (usage == null || property == null) {
     return next()
   }
+  console.log(`${usage} to ${property}`)
   // post request could be a save of a pre-existing entry
   if (ctx.request.body && ctx.request.body._id) {
     try {
@@ -52,26 +44,17 @@ module.exports = async (ctx, next) => {
       return
     }
   }
-  // don't try validate in builder
-  if (!environment.CLOUD) {
-    return next()
+  // update usage for uploads to be the total size
+  if (property === usageQuota.Properties.UPLOAD) {
+    const files =
+      ctx.request.files.file.length > 1
+        ? Array.from(ctx.request.files.file)
+        : [ctx.request.files.file]
+    usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
   try {
-    await apiKeyTable.update(buildUpdateParams(ctx.apiKey, property, usage))
+    await usageQuota.update(ctx.apiKey, property, usage)
   } catch (err) {
-    if (err.code !== "ConditionalCheckFailedException") {
-      // get the API key so we can check it
-      let apiKey = await apiKeyTable.get({ primary: ctx.apiKey })
-      // we have infact breached the reset period
-      if (apiKey && apiKey.quotaReset >= Date.now()) {
-        // update the quota reset period and reset the values for all properties
-        apiKey.quotaReset = Date.now() + QUOTA_RESET
-        for (let prop of Object.keys(apiKey.usageQuota)) {
-          apiKey.usageQuota[prop] = 0
-        }
-        await apiKeyTable.put({ item: apiKey })
-      }
-      ctx.throw(403, `Resource limits have been reached`)
-    }
+    ctx.throw(403, err)
   }
-}
\ No newline at end of file
+}
diff --git a/packages/server/src/utilities/fileProcessor.js b/packages/server/src/utilities/fileProcessor.js
index 3e580e9e37..734209733d 100644
--- a/packages/server/src/utilities/fileProcessor.js
+++ b/packages/server/src/utilities/fileProcessor.js
@@ -1,5 +1,5 @@
 const fs = require("fs")
-const sharp = require("sharp")
+// const sharp = require("sharp")
 const fsPromises = fs.promises
 
 const FORMATS = {
@@ -7,14 +7,14 @@ const FORMATS = {
 }
 
 async function processImage(file) {
-  const imgMeta = await sharp(file.path)
-    .resize(300)
-    .toFile(file.outputPath)
-
-  return {
-    ...file,
-    ...imgMeta,
-  }
+  // const imgMeta = await sharp(file.path)
+  //   .resize(300)
+  //   .toFile(file.outputPath)
+  //
+  // return {
+  //   ...file,
+  //   ...imgMeta,
+  // }
 }
 
 async function process(file) {
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
new file mode 100644
index 0000000000..7ba0005213
--- /dev/null
+++ b/packages/server/src/utilities/usageQuota.js
@@ -0,0 +1,72 @@
+const environment = require("../environment")
+const { apiKeyTable } = require("../db/dynamoClient")
+
+function buildUpdateParams(key, property, usage) {
+  return {
+    primary: key,
+    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
+    expression: "ADD #quota.#prop :usage",
+    names: {
+      "#quota": "usageQuota",
+      "#prop": property,
+      "#limits": "limits",
+      "#quotaReset": "quotaReset",
+    },
+    values: {
+      ":usage": usage,
+      ":now": Date.now(),
+    },
+  }
+}
+
+// a normalised month in milliseconds
+const QUOTA_RESET = 2592000000
+
+exports.Properties = {
+  RECORD: "records",
+  UPLOAD: "storage",
+  VIEW: "views",
+  USER: "users",
+  AUTOMATION: "automationRuns",
+}
+
+exports.getAPIKey = async appId => {
+  return apiKeyTable.get({ primary: appId })
+}
+
+/**
+ * Given a specified API key this will add to the usage object for the specified property.
+ * @param {string} apiKey The API key which is to be updated.
+ * @param {string} property The property which is to be added to (within the nested usageQuota object).
+ * @param {number} usage The amount (this can be negative) to adjust the number by.
+ * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
+ * also been reset after this call.
+ */
+exports.update = async (apiKey, property, usage) => {
+  // don't try validate in builder
+  if (!environment.CLOUD) {
+    return
+  }
+  try {
+    await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
+  } catch (err) {
+    if (err.code !== "ConditionalCheckFailedException") {
+      // get the API key so we can check it
+      let apiKey = await apiKeyTable.get({ primary: apiKey })
+      // we have infact breached the reset period
+      if (apiKey && apiKey.quotaReset >= Date.now()) {
+        // update the quota reset period and reset the values for all properties
+        apiKey.quotaReset = Date.now() + QUOTA_RESET
+        for (let prop of Object.keys(apiKey.usageQuota)) {
+          if (prop === property) {
+            apiKey.usageQuota[prop] = usage > 0 ? usage : 0
+          } else {
+            apiKey.usageQuota[prop] = 0
+          }
+        }
+        await apiKeyTable.put({ item: apiKey })
+      }
+      throw "Resource limits have been reached"
+    }
+  }
+}

From b400a06027677beeb78928f73d45e10c92615dc4 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 8 Oct 2020 17:34:41 +0100
Subject: [PATCH 3/7] Updates for API usage after testing against local Dynamo.

---
 .../server/scripts/createApiKeyAndAppId.js    | 55 +++++++++++++++++++
 packages/server/src/automations/index.js      |  9 +--
 .../src/automations/steps/createRecord.js     |  7 ++-
 .../src/automations/steps/createUser.js       |  7 ++-
 .../src/automations/steps/deleteRecord.js     |  7 ++-
 packages/server/src/automations/thread.js     |  1 +
 packages/server/src/db/dynamoClient.js        | 30 ++++++++--
 packages/server/src/middleware/authorized.js  | 12 ++--
 packages/server/src/middleware/usageQuota.js  |  7 ++-
 .../src/utilities/builder/setBuilderToken.js  |  4 +-
 packages/server/src/utilities/usageQuota.js   | 21 +++----
 11 files changed, 127 insertions(+), 33 deletions(-)
 create mode 100644 packages/server/scripts/createApiKeyAndAppId.js

diff --git a/packages/server/scripts/createApiKeyAndAppId.js b/packages/server/scripts/createApiKeyAndAppId.js
new file mode 100644
index 0000000000..f6a6c1fcee
--- /dev/null
+++ b/packages/server/scripts/createApiKeyAndAppId.js
@@ -0,0 +1,55 @@
+// THIS will create API Keys and App Ids input in a local Dynamo instance if it is running
+const dynamoClient = require("../src/db/dynamoClient")
+
+if (process.argv[2] == null || process.argv[3] == null) {
+  console.error(
+    "Inputs incorrect format, was expecting: node createApiKeyAndAppId.js <API_KEY> <APP_ID>"
+  )
+  process.exit(-1)
+}
+
+const FAKE_STRING = "fakestring"
+
+// set fake credentials for local dynamo to actually work
+process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
+process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
+dynamoClient.init("http://localhost:8333")
+
+async function run() {
+  await dynamoClient.apiKeyTable.put({
+    item: {
+      pk: process.argv[2],
+      accountId: FAKE_STRING,
+      trackingId: FAKE_STRING,
+      quotaReset: Date.now() + 2592000000,
+      usageQuota: {
+        automationRuns: 0,
+        records: 0,
+        storage: 0,
+        users: 0,
+        views: 0,
+      },
+      usageLimits: {
+        automationRuns: 10,
+        records: 10,
+        storage: 1000,
+        users: 10,
+        views: 10,
+      },
+    },
+  })
+  await dynamoClient.apiKeyTable.put({
+    item: {
+      pk: process.argv[3],
+      apiKey: process.argv[2],
+    },
+  })
+}
+
+run()
+  .then(() => {
+    console.log("Records should have been created.")
+  })
+  .catch(err => {
+    console.error("Cannot create records - " + err)
+  })
diff --git a/packages/server/src/automations/index.js b/packages/server/src/automations/index.js
index 882f24391b..f407e35a78 100644
--- a/packages/server/src/automations/index.js
+++ b/packages/server/src/automations/index.js
@@ -21,9 +21,10 @@ function runWorker(job) {
 
 async function updateQuota(automation) {
   const appId = automation.appId
-  const apiKey = await getAPIKey(appId)
+  const apiObj = await getAPIKey(appId)
   // this will fail, causing automation to escape if limits reached
-  await update(apiKey, Properties.AUTOMATION, 1)
+  await update(apiObj.apiKey, Properties.AUTOMATION, 1)
+  return apiObj.apiKey
 }
 
 /**
@@ -33,8 +34,8 @@ module.exports.init = function() {
   actions.init().then(() => {
     triggers.automationQueue.process(async job => {
       try {
-        if (environment.CLOUD) {
-          await updateQuota(job.data.automation)
+        if (environment.CLOUD && job.data.automation) {
+          job.data.automation.apiKey = await updateQuota(job.data.automation)
         }
         if (environment.BUDIBASE_ENVIRONMENT === "PRODUCTION") {
           await runWorker(job)
diff --git a/packages/server/src/automations/steps/createRecord.js b/packages/server/src/automations/steps/createRecord.js
index d0d6a36f5a..e268f218e2 100644
--- a/packages/server/src/automations/steps/createRecord.js
+++ b/packages/server/src/automations/steps/createRecord.js
@@ -1,5 +1,7 @@
 const recordController = require("../../api/controllers/record")
 const automationUtils = require("../automationUtils")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   name: "Create Row",
@@ -56,7 +58,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   // TODO: better logging of when actions are missed due to missing parameters
   if (inputs.record == null || inputs.record.modelId == null) {
     return
@@ -78,6 +80,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.RECORD, 1)
+    }
     await recordController.save(ctx)
     return {
       record: inputs.record,
diff --git a/packages/server/src/automations/steps/createUser.js b/packages/server/src/automations/steps/createUser.js
index de2b6ca1ad..f0bea286d7 100644
--- a/packages/server/src/automations/steps/createUser.js
+++ b/packages/server/src/automations/steps/createUser.js
@@ -1,5 +1,7 @@
 const accessLevels = require("../../utilities/accessLevels")
 const userController = require("../../api/controllers/user")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   description: "Create a new user",
@@ -56,7 +58,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   const { username, password, accessLevelId } = inputs
   const ctx = {
     user: {
@@ -68,6 +70,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.USER, 1)
+    }
     await userController.create(ctx)
     return {
       response: ctx.body,
diff --git a/packages/server/src/automations/steps/deleteRecord.js b/packages/server/src/automations/steps/deleteRecord.js
index 0a02099bd4..6126895da6 100644
--- a/packages/server/src/automations/steps/deleteRecord.js
+++ b/packages/server/src/automations/steps/deleteRecord.js
@@ -1,4 +1,6 @@
 const recordController = require("../../api/controllers/record")
+const environment = require("../../environment")
+const usage = require("../../utilities/usageQuota")
 
 module.exports.definition = {
   description: "Delete a row from your database",
@@ -48,7 +50,7 @@ module.exports.definition = {
   },
 }
 
-module.exports.run = async function({ inputs, instanceId }) {
+module.exports.run = async function({ inputs, instanceId, apiKey }) {
   // TODO: better logging of when actions are missed due to missing parameters
   if (inputs.id == null || inputs.revision == null) {
     return
@@ -63,6 +65,9 @@ module.exports.run = async function({ inputs, instanceId }) {
   }
 
   try {
+    if (environment.CLOUD) {
+      await usage.update(apiKey, usage.Properties.RECORD, -1)
+    }
     await recordController.destroy(ctx)
     return {
       response: ctx.body,
diff --git a/packages/server/src/automations/thread.js b/packages/server/src/automations/thread.js
index fa826afbe9..5362597cfd 100644
--- a/packages/server/src/automations/thread.js
+++ b/packages/server/src/automations/thread.js
@@ -62,6 +62,7 @@ class Orchestrator {
         const outputs = await stepFn({
           inputs: step.inputs,
           instanceId: this._instanceId,
+          apiKey: automation.apiKey,
         })
         if (step.stepId === FILTER_STEP_ID && !outputs.success) {
           break
diff --git a/packages/server/src/db/dynamoClient.js b/packages/server/src/db/dynamoClient.js
index 9f30b151da..6250486bf7 100644
--- a/packages/server/src/db/dynamoClient.js
+++ b/packages/server/src/db/dynamoClient.js
@@ -1,6 +1,8 @@
 let _ = require("lodash")
 let environment = require("../environment")
 
+const AWS_REGION = environment.AWS_REGION ? environment.AWS_REGION : "eu-west-1"
+
 const TableInfo = {
   API_KEYS: {
     name: "beta-api-key-table",
@@ -49,6 +51,7 @@ class Table {
     condition,
     names,
     values,
+    exists,
     otherProps,
   }) {
     let params = {
@@ -66,6 +69,13 @@ class Table {
     if (this._sort && sort) {
       params.Key[this._sort] = sort
     }
+    if (exists) {
+      params.ExpressionAttributeNames["#PRIMARY"] = this._primary
+      if (params.ConditionExpression) {
+        params.ConditionExpression += " AND "
+      }
+      params.ConditionExpression += "attribute_exists(#PRIMARY)"
+    }
     if (otherProps) {
       params = _.merge(params, otherProps)
     }
@@ -90,15 +100,17 @@ class Table {
   }
 }
 
-exports.init = () => {
-  if (!environment.CLOUD) {
-    return
-  }
+exports.init = endpoint => {
   let AWS = require("aws-sdk")
+  AWS.config.update({
+    region: AWS_REGION,
+  })
   let docClientParams = {
     correctClockSkew: true,
   }
-  if (environment.DYNAMO_ENDPOINT) {
+  if (endpoint) {
+    docClientParams.endpoint = endpoint
+  } else if (environment.DYNAMO_ENDPOINT) {
     docClientParams.endpoint = environment.DYNAMO_ENDPOINT
   }
   docClient = new AWS.DynamoDB.DocumentClient(docClientParams)
@@ -106,3 +118,11 @@ exports.init = () => {
 
 exports.apiKeyTable = new Table(TableInfo.API_KEYS)
 exports.userTable = new Table(TableInfo.USERS)
+
+if (environment.CLOUD) {
+  exports.init(`https://dynamodb.${AWS_REGION}.amazonaws.com`)
+} else {
+  process.env.AWS_ACCESS_KEY_ID = "KEY_ID"
+  process.env.AWS_SECRET_ACCESS_KEY = "SECRET_KEY"
+  exports.init("http://localhost:8333")
+}
diff --git a/packages/server/src/middleware/authorized.js b/packages/server/src/middleware/authorized.js
index b452d63cf5..4cce4c4670 100644
--- a/packages/server/src/middleware/authorized.js
+++ b/packages/server/src/middleware/authorized.js
@@ -16,8 +16,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
   }
 
   if (ctx.user.accessLevel._id === BUILDER_LEVEL_ID) {
-    await next()
-    return
+    return next()
   }
 
   if (permName === BUILDER) {
@@ -28,8 +27,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
   const permissionId = ({ name, itemId }) => name + (itemId ? `-${itemId}` : "")
 
   if (ctx.user.accessLevel._id === ADMIN_LEVEL_ID) {
-    await next()
-    return
+    return next()
   }
 
   const thisPermissionId = permissionId({
@@ -42,8 +40,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
     ctx.user.accessLevel._id === POWERUSER_LEVEL_ID &&
     !adminPermissions.map(permissionId).includes(thisPermissionId)
   ) {
-    await next()
-    return
+    return next()
   }
 
   if (
@@ -51,8 +48,7 @@ module.exports = (permName, getItemId) => async (ctx, next) => {
       .map(permissionId)
       .includes(thisPermissionId)
   ) {
-    await next()
-    return
+    return next()
   }
 
   ctx.throw(403, "Not Authorized")
diff --git a/packages/server/src/middleware/usageQuota.js b/packages/server/src/middleware/usageQuota.js
index 74df19a015..e82305dc12 100644
--- a/packages/server/src/middleware/usageQuota.js
+++ b/packages/server/src/middleware/usageQuota.js
@@ -1,5 +1,6 @@
 const CouchDB = require("../db")
 const usageQuota = require("../utilities/usageQuota")
+const environment = require("../environment")
 
 // currently only counting new writes and deletes
 const METHOD_MAP = {
@@ -29,11 +30,9 @@ module.exports = async (ctx, next) => {
   const db = new CouchDB(ctx.user.instanceId)
   let usage = METHOD_MAP[ctx.req.method]
   const property = getProperty(ctx.req.url)
-  console.log(ctx.req.url)
   if (usage == null || property == null) {
     return next()
   }
-  console.log(`${usage} to ${property}`)
   // post request could be a save of a pre-existing entry
   if (ctx.request.body && ctx.request.body._id) {
     try {
@@ -52,8 +51,12 @@ module.exports = async (ctx, next) => {
         : [ctx.request.files.file]
     usage = files.map(file => file.size).reduce((total, size) => total + size)
   }
+  if (!environment.CLOUD) {
+    return next()
+  }
   try {
     await usageQuota.update(ctx.apiKey, property, usage)
+    return next()
   } catch (err) {
     ctx.throw(403, err)
   }
diff --git a/packages/server/src/utilities/builder/setBuilderToken.js b/packages/server/src/utilities/builder/setBuilderToken.js
index 12622d5522..d43a9543e7 100644
--- a/packages/server/src/utilities/builder/setBuilderToken.js
+++ b/packages/server/src/utilities/builder/setBuilderToken.js
@@ -8,7 +8,9 @@ module.exports = (ctx, appId, instanceId) => {
     instanceId,
     appId,
   }
-
+  if (process.env.BUDIBASE_API_KEY) {
+    builderUser.apiKey = process.env.BUDIBASE_API_KEY
+  }
   const token = jwt.sign(builderUser, ctx.config.jwtSecret, {
     expiresIn: "30 days",
   })
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index 7ba0005213..f16fb6aba8 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -4,12 +4,12 @@ const { apiKeyTable } = require("../db/dynamoClient")
 function buildUpdateParams(key, property, usage) {
   return {
     primary: key,
-    condition: "#quota.#prop + :usage < #limits.model AND #quotaReset < :now",
+    condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now",
     expression: "ADD #quota.#prop :usage",
     names: {
       "#quota": "usageQuota",
       "#prop": property,
-      "#limits": "limits",
+      "#limits": "usageLimits",
       "#quotaReset": "quotaReset",
     },
     values: {
@@ -50,21 +50,22 @@ exports.update = async (apiKey, property, usage) => {
   try {
     await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
   } catch (err) {
-    if (err.code !== "ConditionalCheckFailedException") {
+    if (err.code === "ConditionalCheckFailedException") {
       // get the API key so we can check it
-      let apiKey = await apiKeyTable.get({ primary: apiKey })
+      const keyObj = await apiKeyTable.get({ primary: apiKey })
       // we have infact breached the reset period
-      if (apiKey && apiKey.quotaReset >= Date.now()) {
+      if (keyObj && keyObj.quotaReset <= Date.now()) {
         // update the quota reset period and reset the values for all properties
-        apiKey.quotaReset = Date.now() + QUOTA_RESET
-        for (let prop of Object.keys(apiKey.usageQuota)) {
+        keyObj.quotaReset = Date.now() + QUOTA_RESET
+        for (let prop of Object.keys(keyObj.usageQuota)) {
           if (prop === property) {
-            apiKey.usageQuota[prop] = usage > 0 ? usage : 0
+            keyObj.usageQuota[prop] = usage > 0 ? usage : 0
           } else {
-            apiKey.usageQuota[prop] = 0
+            keyObj.usageQuota[prop] = 0
           }
         }
-        await apiKeyTable.put({ item: apiKey })
+        await apiKeyTable.put({ item: keyObj })
+        return
       }
       throw "Resource limits have been reached"
     }

From 275bd64b1b8ed01e2de2f6f2483dc16fa99a2398 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Thu, 8 Oct 2020 18:36:31 +0100
Subject: [PATCH 4/7] Fixing linting issue.

---
 packages/server/src/utilities/fileProcessor.js | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/packages/server/src/utilities/fileProcessor.js b/packages/server/src/utilities/fileProcessor.js
index 734209733d..3e580e9e37 100644
--- a/packages/server/src/utilities/fileProcessor.js
+++ b/packages/server/src/utilities/fileProcessor.js
@@ -1,5 +1,5 @@
 const fs = require("fs")
-// const sharp = require("sharp")
+const sharp = require("sharp")
 const fsPromises = fs.promises
 
 const FORMATS = {
@@ -7,14 +7,14 @@ const FORMATS = {
 }
 
 async function processImage(file) {
-  // const imgMeta = await sharp(file.path)
-  //   .resize(300)
-  //   .toFile(file.outputPath)
-  //
-  // return {
-  //   ...file,
-  //   ...imgMeta,
-  // }
+  const imgMeta = await sharp(file.path)
+    .resize(300)
+    .toFile(file.outputPath)
+
+  return {
+    ...file,
+    ...imgMeta,
+  }
 }
 
 async function process(file) {

From 4ba636897bd4db53ef971b79ab9d7c461cc5f7d2 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 9 Oct 2020 11:32:25 +0100
Subject: [PATCH 5/7] Adding in default plan if the obj found does not have the
 required information.

---
 packages/server/src/utilities/usageQuota.js | 37 ++++++++++++++++++---
 1 file changed, 32 insertions(+), 5 deletions(-)

diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index f16fb6aba8..fdf8ea813c 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -1,10 +1,28 @@
 const environment = require("../environment")
 const { apiKeyTable } = require("../db/dynamoClient")
 
+const DEFAULT_USAGE = {
+  records: 0,
+  storage: 0,
+  views: 0,
+  automationRuns: 0,
+  users: 0,
+}
+
+const DEFAULT_PLAN = {
+  records: 1000,
+  // 1 GB
+  storage: 8589934592,
+  views: 10,
+  automationRuns: 100,
+  users: 10000,
+}
+
 function buildUpdateParams(key, property, usage) {
   return {
     primary: key,
-    condition: "#quota.#prop < #limits.#prop AND #quotaReset > :now",
+    condition:
+      "#quota.#prop < #limits.#prop AND #quotaReset > :now AND attribute_exists(#quota) AND attribute_exists(#limits)",
     expression: "ADD #quota.#prop :usage",
     names: {
       "#quota": "usageQuota",
@@ -19,8 +37,9 @@ function buildUpdateParams(key, property, usage) {
   }
 }
 
-// a normalised month in milliseconds
-const QUOTA_RESET = 2592000000
+function getNewQuotaReset() {
+  return Date.now() + 2592000000
+}
 
 exports.Properties = {
   RECORD: "records",
@@ -53,10 +72,18 @@ exports.update = async (apiKey, property, usage) => {
     if (err.code === "ConditionalCheckFailedException") {
       // get the API key so we can check it
       const keyObj = await apiKeyTable.get({ primary: apiKey })
+      // the usage quota or usage limits didn't exist
+      if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
+        keyObj.usageQuota = DEFAULT_USAGE
+        keyObj.usageLimits = DEFAULT_PLAN
+        keyObj.quotaReset = getNewQuotaReset()
+        await apiKeyTable.put({ item: keyObj })
+        return
+      }
       // we have infact breached the reset period
-      if (keyObj && keyObj.quotaReset <= Date.now()) {
+      else if (keyObj && keyObj.quotaReset <= Date.now()) {
         // update the quota reset period and reset the values for all properties
-        keyObj.quotaReset = Date.now() + QUOTA_RESET
+        keyObj.quotaReset = getNewQuotaReset()
         for (let prop of Object.keys(keyObj.usageQuota)) {
           if (prop === property) {
             keyObj.usageQuota[prop] = usage > 0 ? usage : 0

From 86102982a1e9b0d4c14fd9547e1d15acdb74cbde Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 9 Oct 2020 17:28:49 +0100
Subject: [PATCH 6/7] Making sure any errors that occur in usage quota are
 thrown up to stop endpoint from functioning.

---
 packages/server/src/utilities/usageQuota.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index fdf8ea813c..95fc76140f 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -94,7 +94,7 @@ exports.update = async (apiKey, property, usage) => {
         await apiKeyTable.put({ item: keyObj })
         return
       }
-      throw "Resource limits have been reached"
     }
+    throw "Resource limits have been reached"
   }
 }

From 73e0e37c668718435c85f35cf20680a71179c872 Mon Sep 17 00:00:00 2001
From: mike12345567 <me@michaeldrury.co.uk>
Date: Fri, 9 Oct 2020 20:44:35 +0100
Subject: [PATCH 7/7] Fixing some bugs with JWT creation not getting an API
 key.

---
 packages/server/src/api/controllers/auth.js | 2 +-
 packages/server/src/utilities/usageQuota.js | 7 +++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/packages/server/src/api/controllers/auth.js b/packages/server/src/api/controllers/auth.js
index c16f76caca..3f824524b0 100644
--- a/packages/server/src/api/controllers/auth.js
+++ b/packages/server/src/api/controllers/auth.js
@@ -55,7 +55,7 @@ exports.authenticate = async ctx => {
     }
     // if in cloud add the user api key
     if (environment.CLOUD) {
-      payload.apiKey = getAPIKey(ctx.user.appId)
+      payload.apiKey = await getAPIKey(ctx.user.appId)
     }
 
     const token = jwt.sign(payload, ctx.config.jwtSecret, {
diff --git a/packages/server/src/utilities/usageQuota.js b/packages/server/src/utilities/usageQuota.js
index 95fc76140f..11d4757398 100644
--- a/packages/server/src/utilities/usageQuota.js
+++ b/packages/server/src/utilities/usageQuota.js
@@ -69,13 +69,16 @@ exports.update = async (apiKey, property, usage) => {
   try {
     await apiKeyTable.update(buildUpdateParams(apiKey, property, usage))
   } catch (err) {
+    // conditional check means the condition failed, need to check why
     if (err.code === "ConditionalCheckFailedException") {
       // get the API key so we can check it
       const keyObj = await apiKeyTable.get({ primary: apiKey })
       // the usage quota or usage limits didn't exist
       if (keyObj && (keyObj.usageQuota == null || keyObj.usageLimits == null)) {
-        keyObj.usageQuota = DEFAULT_USAGE
-        keyObj.usageLimits = DEFAULT_PLAN
+        keyObj.usageQuota =
+          keyObj.usageQuota == null ? DEFAULT_USAGE : keyObj.usageQuota
+        keyObj.usageLimits =
+          keyObj.usageLimits == null ? DEFAULT_PLAN : keyObj.usageLimits
         keyObj.quotaReset = getNewQuotaReset()
         await apiKeyTable.put({ item: keyObj })
         return