diff --git a/cogs/admin.py b/cogs/admin.py
index 49ff777..9262230 100644
--- a/cogs/admin.py
+++ b/cogs/admin.py
@@ -299,8 +299,10 @@ guild=$2
                 )
             except UniqueViolationError:
                 # If it's already inserted, then nothing needs to be updated
-                # It just meansthis particular restriction is already set
+                # It just means this particular restriction is already set
                 pass
+            else:
+                ctx.bot.cache.add_restriction(ctx.guild, from_to, {"source": source, "destination": destination})
         elif overwrites:
             channel = overwrites.pop('channel')
             for target, setting in overwrites.items():
@@ -364,6 +366,7 @@ WHERE
     destination=$2 AND 
     from_to=$3 AND 
     guild=$4""", source, destination, arg2, ctx.guild.id)
+            ctx.bot.cache.remove_restriction(ctx.guild.id, arg2, {"source": source, "destination": destination})
 
         # If this isn't a blacklist/whitelist, then we are attempting to remove an overwrite
         else:
@@ -524,6 +527,8 @@ WHERE
             perm_value
         )
 
+        ctx.bot.cache.update_custom_permission(ctx.guild, cmd.qualified_name, perm_value)
+
         await ctx.send("I have just added your custom permissions; "
                        "you now need to have `{}` permissions to use the command `{}`".format(permission, command))
 
@@ -547,6 +552,8 @@ WHERE
             "DELETE FROM custom_permissions WHERE guild=$1 AND command=$2", ctx.guild.id, cmd.qualified_name
         )
 
+        ctx.bot.cache.update_custom_permission(ctx.guild, cmd.qualified_name, None)
+
         await ctx.send("I have just removed the custom permissions for {}!".format(cmd))
 
     @commands.command(aliases=['nick'])
diff --git a/cogs/config.py b/cogs/config.py
index 3e06229..8f2f882 100644
--- a/cogs/config.py
+++ b/cogs/config.py
@@ -279,10 +279,7 @@ class GuildConfiguration(commands.Cog):
         if setting.lower().strip() == "none":
             setting = None
 
-        result = await self._set_db_guild_opt("prefix", setting, ctx)
-        # We want to update our cache for prefixes
-        ctx.bot.cache.update_prefix(ctx.guild, setting)
-        return result
+        return await self._set_db_guild_opt("prefix", setting, ctx)
 
     async def _handle_set_default_alerts(self, ctx, setting):
         channel = await self._get_channel(ctx, setting)
diff --git a/utils/database.py b/utils/database.py
index 486026a..2a81a53 100644
--- a/utils/database.py
+++ b/utils/database.py
@@ -86,6 +86,17 @@ FROM
             from_restrictions.append(opt)
             self.restrictions[row['guild']][row['from_to']] = from_restrictions
 
+    def add_restriction(self, guild, from_to, restriction):
+        restrictions = self.restrictions[guild.id].get(from_to, [])
+        restrictions.append(restriction)
+        self.restrictions[guild.id][from_to] = restrictions
+
+    def remove_restriction(self, guild, from_to, restriction):
+        restrictions = self.restrictions[guild.id].get(from_to, [])
+        if restriction in restrictions:
+            restrictions.remove(restriction)
+            self.restrictions[guild.id][from_to] = restrictions
+
 
 class DB:
     def __init__(self):