fix #935

src/js/liquid-dict.js

@@ -1,7 +1,7 @@
 /*******************************************************************************
 
-    µMatrix - a Chromium browser extension to black/white list requests.
+    uMatrix - a Chromium browser extension to black/white list requests.
-    Copyright (C) 2014  Raymond Hill
+    Copyright (C) 2014-2018 Raymond Hill
 
     This program is free software: you can redistribute it and/or modify
     it under the terms of the GNU General Public License as published by
@@ -19,6 +19,8 @@
     Home: https://github.com/gorhill/uMatrix
 */
 
+'use strict';
+
 /******************************************************************************/
 
 µMatrix.LiquidDict = (function() {
@@ -26,55 +28,37 @@
 /******************************************************************************/
 
 var LiquidDict = function() {
-    this.dict = {};
+    this.dict = new Map();
-    this.count = 0;
+    this.reset();
-    this.duplicateCount = 0;
-    this.bucketCount = 0;
-    this.frozenBucketCount = 0;
-
-    // Somewhat arbitrary: I need to come up with hard data to know at which
-    // point binary search is better than indexOf.
-    this.cutoff = 500;
 };
 
 /******************************************************************************/
 
+// Somewhat arbitrary: I need to come up with hard data to know at which
+// point binary search is better than indexOf.
+
+LiquidDict.prototype.cutoff = 500;
+
+/******************************************************************************/
+
 var meltBucket = function(ldict, len, bucket) {
     ldict.frozenBucketCount -= 1;
-    var map = {};
+    if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) {
-    if ( bucket.charAt(0) === ' ' ) {
+        return new Set(bucket.trim().split(' '));
-        bucket.trim().split(' ').map(function(k) {
-            map[k] = true;
-        });
-    } else {
-        var offset = 0;
-        while ( offset < bucket.length ) {
-            map[bucket.substring(offset, len)] = true;
-            offset += len;
-        }
     }
-    return map;
+    let dict = new Set();
-};
+    let offset = 0;
-
+    while ( offset < bucket.length ) {
-/******************************************************************************/
+        dict.add(bucket.substring(offset, len));
-
+        offset += len;
-var melt = function(ldict) {
-    var buckets = ldict.dict;
-    var bucket;
-    for ( var key in buckets ) {
-        bucket = buckets[key];
-        if ( typeof bucket === 'string' ) {
-            buckets[key] = meltBucket(ldict, key.charCodeAt(0) & 0xFF, bucket);
-        }
     }
+    return dict;
 };
 
-/******************************************************************************/
-
 var freezeBucket = function(ldict, bucket) {
     ldict.frozenBucketCount += 1;
-    var words = Object.keys(bucket);
+    let words = Array.from(bucket);
-    var wordLen = words[0].length;
+    let wordLen = words[0].length;
     if ( wordLen * words.length < ldict.cutoff ) {
         return ' ' + words.join(' ') + ' ';
     }
@@ -91,43 +75,38 @@ var freezeBucket = function(ldict, bucket) {
 // helper function?
 
 LiquidDict.prototype.makeKey = function(word) {
-    var len = word.length;
+    let len = word.length;
-    if ( len > 255 ) {
+    if ( len > 255 ) { len = 255; }
-        len = 255;
+    let i = len >> 2;
-    }
+    return (word.charCodeAt(    0) & 0x03) << 14 |
-    var i = len >> 2;
+           (word.charCodeAt(    i) & 0x03) << 12 |
-    return String.fromCharCode(
+           (word.charCodeAt(  i+i) & 0x03) << 10 |
-        (word.charCodeAt(    0) & 0x03) << 14 |
+           (word.charCodeAt(i+i+i) & 0x03) <<  8 |
-        (word.charCodeAt(    i) & 0x03) << 12 |
+           len;
-        (word.charCodeAt(  i+i) & 0x03) << 10 |
-        (word.charCodeAt(i+i+i) & 0x03) <<  8 |
-        len
-    );
 };
 
 /******************************************************************************/
 
 LiquidDict.prototype.test = function(word) {
-    var key = this.makeKey(word);
+    let key = this.makeKey(word);
-    var bucket = this.dict[key];
+    let bucket = this.dict.get(key);
     if ( bucket === undefined ) {
         return false;
     }
     if ( typeof bucket === 'object' ) {
-        return bucket[word] !== undefined;
+        return bucket.has(word);
     }
-    if ( bucket.charAt(0) === ' ' ) {
+    if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) {
-        return bucket.indexOf(' ' + word + ' ') >= 0;
+        return bucket.indexOf(' ' + word + ' ') !== -1;
     }
     // binary search
-    var len = word.length;
+    let len = word.length;
-    var left = 0;
+    let left = 0;
     // http://jsperf.com/or-vs-floor/3
-    var right = ~~(bucket.length / len + 0.5);
+    let right = ~~(bucket.length / len + 0.5);
-    var i, needle;
     while ( left < right ) {
-        i = left + right >> 1;
+        let i = left + right >> 1;
-        needle = bucket.substr( len * i, len );
+        let needle = bucket.substr( len * i, len );
         if ( word < needle ) {
             right = i;
         } else if ( word > needle ) {
@@ -142,22 +121,21 @@ LiquidDict.prototype.test = function(word) {
 /******************************************************************************/
 
 LiquidDict.prototype.add = function(word) {
-    var key = this.makeKey(word);
+    let key = this.makeKey(word);
-    if ( key === undefined ) {
+    let bucket = this.dict.get(key);
-        return false;
-    }
-    var bucket = this.dict[key];
     if ( bucket === undefined ) {
-        this.dict[key] = bucket = {};
+        bucket = new Set();
-        this.bucketCount += 1;
+        this.dict.set(key, bucket);
-        bucket[word] = true;
+        bucket.add(word);
         this.count += 1;
         return true;
-    } else if ( typeof bucket === 'string' ) {
-        this.dict[key] = bucket = meltBucket(this, word.len, bucket);
     }
-    if ( bucket[word] === undefined ) {
+    if ( typeof bucket === 'string' ) {
-        bucket[word] = true;
+        bucket = meltBucket(this, word.len, bucket);
+        this.dict.set(key, bucket);
+    }
+    if ( bucket.has(word) === false ) {
+        bucket.add(word);
         this.count += 1;
         return true;
     }
@@ -168,12 +146,9 @@ LiquidDict.prototype.add = function(word) {
 /******************************************************************************/
 
 LiquidDict.prototype.freeze = function() {
-    var buckets = this.dict;
+    for ( let entry of this.dict ) {
-    var bucket;
+        if ( typeof entry[1] === 'object' ) {
-    for ( var key in buckets ) {
+            this.dict.set(entry[0], freezeBucket(this, entry[1]));
-        bucket = buckets[key];
-        if ( typeof bucket === 'object' ) {
-            buckets[key] = freezeBucket(this, bucket);
         }
     }
 };
@@ -181,15 +156,38 @@ LiquidDict.prototype.freeze = function() {
 /******************************************************************************/
 
 LiquidDict.prototype.reset = function() {
-    this.dict = {};
+    this.dict.clear();
     this.count = 0;
     this.duplicateCount = 0;
-    this.bucketCount = 0;
     this.frozenBucketCount = 0;
 };
 
 /******************************************************************************/
 
+let selfieVersion = 1;
+
+LiquidDict.prototype.toSelfie = function() {
+    this.freeze();
+    return {
+        version: selfieVersion,
+        count: this.count,
+        duplicateCount: this.duplicateCount,
+        frozenBucketCount: this.frozenBucketCount,
+        dict: Array.from(this.dict)
+    };
+};
+
+LiquidDict.prototype.fromSelfie = function(selfie) {
+    if ( selfie.version !== selfieVersion ) { return false; }
+    this.count = selfie.count;
+    this.duplicateCount = selfie.duplicateCount;
+    this.frozenBucketCount = selfie.frozenBucketCount;
+    this.dict = new Map(selfie.dict);
+    return true;
+};
+
+/******************************************************************************/
+
 return LiquidDict;
 
 /******************************************************************************/
@@ -199,4 +197,3 @@ return LiquidDict;
 /******************************************************************************/
 
 µMatrix.ubiquitousBlacklist = new µMatrix.LiquidDict();
-µMatrix.ubiquitousWhitelist = new µMatrix.LiquidDict();

src/js/storage.js

@@ -439,9 +439,12 @@
         callback = this.noopFunc;
     }
 
-    var loadHostsFilesEnd = function() {
+    var loadHostsFilesEnd = function(fromSelfie) {
-        µm.ubiquitousBlacklist.freeze();
+        if ( fromSelfie !== true ) {
-        vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) });
+            µm.ubiquitousBlacklist.freeze();
+            vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) });
+            µm.hostsFilesSelfie.create();
+        }
         vAPI.messaging.broadcast({ what: 'loadHostsFilesCompleted' });
         µm.getBytesInUse();
         callback();
@@ -472,7 +475,14 @@
         }
     };
 
-    this.getAvailableHostsFiles(loadHostsFilesStart);
+    var onSelfieReady = function(status) {
+        if ( status === true ) {
+            return loadHostsFilesEnd(true);
+        }
+        µm.getAvailableHostsFiles(loadHostsFilesStart);
+    };
+
+    this.hostsFilesSelfie.load(onSelfieReady);
 };
 
 /******************************************************************************/
@@ -647,6 +657,9 @@
             'selectedHostsFiles',
             'externalHostsFiles'
         );
+        if ( hostsChanged ) {
+            µm.hostsFilesSelfie.destroy();
+        }
         let recipesChanged = applyAssetSelection(
             metadata,
             details.recipes,
@@ -657,7 +670,6 @@
             µm.recipeManager.reset();
             µm.loadRecipes(true);
         }
-
         if ( typeof callback === 'function' ) {
             callback({
                 hostsChanged: hostsChanged,
@@ -680,7 +692,50 @@
 
 /******************************************************************************/
 
+µMatrix.hostsFilesSelfie = (function() {
+    let timer;
+
+    return {
+        create: function() {
+            this.cancel();
+            timer = vAPI.setTimeout(
+                function() {
+                    timer = undefined;
+                    vAPI.cacheStorage.set({
+                        hostsFilesSelfie: µMatrix.ubiquitousBlacklist.toSelfie()
+                    });
+                },
+                120000
+            );
+        },
+        destroy: function() {
+            this.cancel();
+            vAPI.cacheStorage.remove('hostsFilesSelfie');
+        },
+        load: function(callback) {
+            this.cancel();
+            vAPI.cacheStorage.get('hostsFilesSelfie', function(bin) {
+                callback(
+                    bin instanceof Object &&
+                    bin.hostsFilesSelfie instanceof Object &&
+                    µMatrix.ubiquitousBlacklist.fromSelfie(bin.hostsFilesSelfie)
+                );
+            });
+        },
+        cancel: function() {
+            if ( timer !== undefined ) {
+                clearTimeout(timer);
+            }
+            timer = undefined;
+        }
+    };
+})();
+
+/******************************************************************************/
+
 µMatrix.loadPublicSuffixList = function(callback) {
+    let µm = this;
+
     if ( typeof callback !== 'function' ) {
         callback = this.noopFunc;
     }
@@ -688,15 +743,64 @@
     var applyPublicSuffixList = function(details) {
         if ( !details.error ) {
             publicSuffixList.parse(details.content, punycode.toASCII);
+            µm.publicSuffixListSelfie.create();
         }
         callback();
     };
 
-    this.assets.get(this.pslAssetKey, applyPublicSuffixList);
+    let onSelfieReady = function(status) {
+        if ( status === true ) {
+            return callback();
+        }
+        µm.assets.get(µm.pslAssetKey, applyPublicSuffixList);
+    };
+
+    this.publicSuffixListSelfie.load(onSelfieReady);
 };
 
 /******************************************************************************/
 
+µMatrix.publicSuffixListSelfie = (function() {
+    let timer;
+
+    return {
+        create: function() {
+            this.cancel();
+            timer = vAPI.setTimeout(
+                function() {
+                    timer = undefined;
+                    vAPI.cacheStorage.set({
+                        publicSuffixListSelfie: publicSuffixList.toSelfie()
+                    });
+                },
+                60000
+            );
+        },
+        destroy: function() {
+            this.cancel();
+            vAPI.cacheStorage.remove('publicSuffixListSelfie');
+        },
+        load: function(callback) {
+            this.cancel();
+            vAPI.cacheStorage.get('publicSuffixListSelfie', function(bin) {
+                callback(
+                    bin instanceof Object &&
+                    bin.publicSuffixListSelfie instanceof Object &&
+                    publicSuffixList.fromSelfie(bin.publicSuffixListSelfie)
+                );
+            });
+        },
+        cancel: function() {
+            if ( timer !== undefined ) {
+                clearTimeout(timer);
+            }
+            timer = undefined;
+        }
+    };
+})();
+
+/******************************************************************************/
+
 µMatrix.scheduleAssetUpdater = (function() {
     var timer, next = 0;
     return function(updateDelay) {
@@ -727,9 +831,10 @@
 /******************************************************************************/
 
 µMatrix.assetObserver = function(topic, details) {
+    let µmus = this.userSettings;
+
     // Do not update filter list if not in use.
     if ( topic === 'before-asset-updated' ) {
-        let µmus = this.userSettings;
         if (
             details.type === 'internal' ||
             details.type === 'filters' &&
@@ -743,6 +848,14 @@
     }
 
     if ( topic === 'after-asset-updated' ) {
+        if (
+            details.type === 'filters' &&
+            µmus.selectedHostsFiles.indexOf(details.assetKey) !== -1
+        ) {
+            this.hostsFilesSelfie.destroy();
+        } else if ( details.assetKey === this.pslAssetKey ) {
+            this.publicSuffixListSelfie.destroy();
+        }
         vAPI.messaging.broadcast({
             what: 'assetUpdated',
             key: details.assetKey,
@@ -766,7 +879,7 @@
         if (
             this.arraysIntersect(
                 details.assetKeys,
-                this.userSettings.selectedRecipeFiles
+                µmus.selectedRecipeFiles
             )
         ) {
             this.loadRecipes(true);
@@ -774,12 +887,12 @@
         if (
             this.arraysIntersect(
                 details.assetKeys,
-                this.userSettings.selectedHostsFiles
+                µmus.selectedHostsFiles
             )
         ) {
             this.loadHostsFiles();
         }
-        if ( this.userSettings.autoUpdate ) {
+        if ( µmus.autoUpdate ) {
             this.scheduleAssetUpdater(25200000);
         } else {
             this.scheduleAssetUpdater(0);

src/lib/publicsuffixlist.js

@@ -37,9 +37,8 @@
 
 /******************************************************************************/
 
-var exceptions = {};
+var exceptions = new Map();
-var rules = {};
+var rules = new Map();
-var selfieMagic = 'iscjsfsaolnm';
 
 // This value dictate how the search will be performed:
 //    < this.cutoffLength = indexOf()
@@ -92,9 +91,8 @@ function getPublicSuffix(hostname) {
     }
     // Since we slice down the hostname with each pass, the first match
     // is the longest, so no need to find all the matching rules.
-    var pos;
     while ( true ) {
-        pos = hostname.indexOf('.');
+        let pos = hostname.indexOf('.');
         if ( pos < 0 ) {
             return hostname;
         }
@@ -118,17 +116,17 @@ function getPublicSuffix(hostname) {
 
 function search(store, hostname) {
     // Extract TLD
-    var pos = hostname.lastIndexOf('.');
+    let tld, remainder;
-    var tld, remainder;
+    let pos = hostname.lastIndexOf('.');
-    if ( pos < 0 ) {
+    if ( pos === -1 ) {
         tld = hostname;
         remainder = hostname;
     } else {
         tld = hostname.slice(pos + 1);
         remainder = hostname.slice(0, pos);
     }
-    var substore = store[tld];
+    let substore = store.get(tld);
-    if ( !substore ) {
+    if ( substore === undefined ) {
         return false;
     }
     // If substore is a string, use indexOf()
@@ -136,17 +134,19 @@ function search(store, hostname) {
         return substore.indexOf(' ' + remainder + ' ') >= 0;
     }
     // It is an array: use binary search.
-    var l = remainder.length;
+    let l = remainder.length;
-    var haystack = substore[l];
+    if ( l >= substore.length ) {
-    if ( !haystack ) {
         return false;
     }
-    var left = 0;
+    let haystack = substore[l];
-    var right = Math.floor(haystack.length / l + 0.5);
+    if ( haystack === null ) {
-    var i, needle;
+        return false;
+    }
+    let left = 0;
+    let right = Math.floor(haystack.length / l + 0.5);
     while ( left < right ) {
-        i = left + right >> 1;
+        let i = left + right >> 1;
-        needle = haystack.substr( l * i, l );
+        let needle = haystack.substr(l*i, l);
         if ( remainder < needle ) {
             right = i;
         } else if ( remainder > needle ) {
@@ -168,22 +168,21 @@ function search(store, hostname) {
 // Suggestion: use <https://github.com/bestiejs/punycode.js> it's quite good.
 
 function parse(text, toAscii) {
-    exceptions = {};
+    exceptions = new Map();
-    rules = {};
+    rules = new Map();
 
-    var lineBeg = 0, lineEnd;
+    let lineBeg = 0;
-    var textEnd = text.length;
+    let textEnd = text.length;
-    var line, store, pos, tld;
 
     while ( lineBeg < textEnd ) {
-        lineEnd = text.indexOf('\n', lineBeg);
+        let lineEnd = text.indexOf('\n', lineBeg);
         if ( lineEnd < 0 ) {
             lineEnd = text.indexOf('\r', lineBeg);
             if ( lineEnd < 0 ) {
                 lineEnd = textEnd;
             }
         }
-        line = text.slice(lineBeg, lineEnd).trim();
+        let line = text.slice(lineBeg, lineEnd).trim();
         lineBeg = lineEnd + 1;
 
         if ( line.length === 0 ) {
@@ -191,18 +190,19 @@ function parse(text, toAscii) {
         }
 
         // Ignore comments
-        pos = line.indexOf('//');
+        let pos = line.indexOf('//');
-        if ( pos >= 0 ) {
+        if ( pos !== -1 ) {
             line = line.slice(0, pos);
         }
 
         // Ignore surrounding whitespaces
         line = line.trim();
-        if ( !line ) {
+        if ( line.length === 0 ) {
             continue;
         }
 
         // Is this an exception rule?
+        let store;
         if ( line.charAt(0) === '!' ) {
             store = exceptions;
             line = line.slice(1);
@@ -220,8 +220,9 @@ function parse(text, toAscii) {
         line = line.toLowerCase();
 
         // Extract TLD
+        let tld;
         pos = line.lastIndexOf('.');
-        if ( pos < 0 ) {
+        if ( pos === -1 ) {
             tld = line;
         } else {
             tld = line.slice(pos + 1);
@@ -229,13 +230,15 @@ function parse(text, toAscii) {
         }
 
         // Store suffix using tld as key
-        if ( !store.hasOwnProperty(tld) ) {
+        let substore = store.get(tld);
-            store[tld] = [];
+        if ( substore === undefined ) {
+            store.set(tld, substore = []);
         }
         if ( line ) {
-            store[tld].push(line);
+            substore.push(line);
         }
     }
+
     crystallize(exceptions);
     crystallize(rules);
 
@@ -248,69 +251,81 @@ function parse(text, toAscii) {
 // for future look up.
 
 function crystallize(store) {
-    var suffixes, suffix, i, l;
+    for ( let entry of store ) {
+        let tld = entry[0];
+        let suffixes = entry[1];
 
-    for ( var tld in store ) {
-        if ( !store.hasOwnProperty(tld) ) {
-            continue;
-        }
-        suffixes = store[tld].join(' ');
         // No suffix
-        if ( !suffixes ) {
+        if ( suffixes.length === 0 ) {
-            store[tld] = '';
+            store.set(tld, '');
             continue;
         }
+
         // Concatenated list of suffixes less than cutoff length:
         //   Store as string, lookup using indexOf()
-        if ( suffixes.length < cutoffLength ) {
+        let s = suffixes.join(' ');
-            store[tld] = ' ' + suffixes + ' ';
+        if ( s.length < cutoffLength ) {
+            store.set(tld, ' ' + s + ' ');
             continue;
         }
+
         // Concatenated list of suffixes greater or equal to cutoff length
         //   Store as array keyed on suffix length, lookup using binary search.
         // I borrowed the idea to key on string length here:
         //   http://ejohn.org/blog/dictionary-lookups-in-javascript/#comment-392072
-
+        let buckets = [];
-        i = store[tld].length;
+        for ( let suffix of suffixes ) {
-        suffixes = [];
+            let l = suffix.length;
-        while ( i-- ) {
+            if ( buckets.length <= l ) {
-            suffix = store[tld][i];
+                extendArray(buckets, l);
-            l = suffix.length;
-            if ( !suffixes[l] ) {
-                suffixes[l] = [];
             }
-            suffixes[l].push(suffix);
+            if ( buckets[l] === null ) {
+                buckets[l] = [];
+            }
+            buckets[l].push(suffix);
         }
-        l = suffixes.length;
+        for ( let i = 0; i < buckets.length; i++ ) {
-        while ( l-- ) {
+            let bucket = buckets[i];
-            if ( suffixes[l] ) {
+            if ( bucket !== null ) {
-                suffixes[l] = suffixes[l].sort().join('');
+                buckets[i] = bucket.sort().join('');
             }
         }
-        store[tld] = suffixes;
+        store.set(tld, buckets);
     }
+
     return store;
 }
 
+let extendArray = function(aa, rb) {
+    for ( let i = aa.length; i <= rb; i++ ) {
+        aa.push(null);
+    }
+};
+
 /******************************************************************************/
 
-function toSelfie() {
+let selfieMagic = 3;
+
+let toSelfie = function() {
     return {
         magic: selfieMagic,
-        rules: rules,
+        rules: Array.from(rules),
-        exceptions: exceptions
+        exceptions: Array.from(exceptions)
     };
-}
+};
 
-function fromSelfie(selfie) {
+let fromSelfie = function(selfie) {
-    if ( typeof selfie !== 'object' || typeof selfie.magic !== 'string' || selfie.magic !== selfieMagic ) {
+    if (
+        selfie instanceof Object === false ||
+        selfie.magic !== selfieMagic
+    ) {
         return false;
     }
-    rules = selfie.rules;
+    rules = new Map(selfie.rules);
-    exceptions = selfie.exceptions;
+    exceptions = new Map(selfie.exceptions);
     callListeners(onChangedListeners);
     return true;
-}
+};
 
 /******************************************************************************/