CodeMirror/uMatrix
1
0
Fork 0
mirror of https://github.com/gorhill/uMatrix.git synced 2024-05-20 12:13:19 +12:00
Code Issues Releases Wiki Activity

fix #935

Browse source
This commit is contained in:
Raymond Hill 2018-01-30 18:30:26 -05:00
parent b8c4eadc7a
commit e9e5aa295c
No known key found for this signature in database
GPG key ID: 25E1490B761470C2
3 changed files with 283 additions and 158 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

163
src/js/liquid-dict.js
View file

@ -1,7 +1,7 @@
/*******************************************************************************
µMatrix - a Chromium browser extension to black/white list requests.
Copyright (C) 2014 Raymond Hill
uMatrix - a Chromium browser extension to black/white list requests.
Copyright (C) 2014-2018 Raymond Hill
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
@ -19,6 +19,8 @@
Home: https://github.com/gorhill/uMatrix
*/
'use strict';
/******************************************************************************/
µMatrix.LiquidDict = (function() {
@ -26,55 +28,37 @@
/******************************************************************************/
var LiquidDict = function() {
this.dict = {};
this.count = 0;
this.duplicateCount = 0;
this.bucketCount = 0;
this.frozenBucketCount = 0;
// Somewhat arbitrary: I need to come up with hard data to know at which
// point binary search is better than indexOf.
this.cutoff = 500;
this.dict = new Map();
this.reset();
};
/******************************************************************************/
// Somewhat arbitrary: I need to come up with hard data to know at which
// point binary search is better than indexOf.
LiquidDict.prototype.cutoff = 500;
/******************************************************************************/
var meltBucket = function(ldict, len, bucket) {
ldict.frozenBucketCount -= 1;
var map = {};
if ( bucket.charAt(0) === ' ' ) {
bucket.trim().split(' ').map(function(k) {
map[k] = true;
});
} else {
var offset = 0;
while ( offset < bucket.length ) {
map[bucket.substring(offset, len)] = true;
offset += len;
}
if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) {
return new Set(bucket.trim().split(' '));
}
return map;
};
/******************************************************************************/
var melt = function(ldict) {
var buckets = ldict.dict;
var bucket;
for ( var key in buckets ) {
bucket = buckets[key];
if ( typeof bucket === 'string' ) {
buckets[key] = meltBucket(ldict, key.charCodeAt(0) & 0xFF, bucket);
}
let dict = new Set();
let offset = 0;
while ( offset < bucket.length ) {
dict.add(bucket.substring(offset, len));
offset += len;
}
return dict;
};
/******************************************************************************/
var freezeBucket = function(ldict, bucket) {
ldict.frozenBucketCount += 1;
var words = Object.keys(bucket);
var wordLen = words[0].length;
let words = Array.from(bucket);
let wordLen = words[0].length;
if ( wordLen * words.length < ldict.cutoff ) {
return ' ' + words.join(' ') + ' ';
}
@ -91,43 +75,38 @@ var freezeBucket = function(ldict, bucket) {
// helper function?
LiquidDict.prototype.makeKey = function(word) {
var len = word.length;
if ( len > 255 ) {
len = 255;
}
var i = len >> 2;
return String.fromCharCode(
(word.charCodeAt( 0) & 0x03) << 14 |
(word.charCodeAt( i) & 0x03) << 12 |
(word.charCodeAt( i+i) & 0x03) << 10 |
(word.charCodeAt(i+i+i) & 0x03) << 8 |
len
);
let len = word.length;
if ( len > 255 ) { len = 255; }
let i = len >> 2;
return (word.charCodeAt( 0) & 0x03) << 14 |
(word.charCodeAt( i) & 0x03) << 12 |
(word.charCodeAt( i+i) & 0x03) << 10 |
(word.charCodeAt(i+i+i) & 0x03) << 8 |
len;
};
/******************************************************************************/
LiquidDict.prototype.test = function(word) {
var key = this.makeKey(word);
var bucket = this.dict[key];
let key = this.makeKey(word);
let bucket = this.dict.get(key);
if ( bucket === undefined ) {
return false;
}
if ( typeof bucket === 'object' ) {
return bucket[word] !== undefined;
return bucket.has(word);
}
if ( bucket.charAt(0) === ' ' ) {
return bucket.indexOf(' ' + word + ' ') >= 0;
if ( bucket.charCodeAt(0) === 0x20 /* ' ' */ ) {
return bucket.indexOf(' ' + word + ' ') !== -1;
}
// binary search
var len = word.length;
var left = 0;
let len = word.length;
let left = 0;
// http://jsperf.com/or-vs-floor/3
var right = ~~(bucket.length / len + 0.5);
var i, needle;
let right = ~~(bucket.length / len + 0.5);
while ( left < right ) {
i = left + right >> 1;
needle = bucket.substr( len * i, len );
let i = left + right >> 1;
let needle = bucket.substr( len * i, len );
if ( word < needle ) {
right = i;
} else if ( word > needle ) {
@ -142,22 +121,21 @@ LiquidDict.prototype.test = function(word) {
/******************************************************************************/
LiquidDict.prototype.add = function(word) {
var key = this.makeKey(word);
if ( key === undefined ) {
return false;
}
var bucket = this.dict[key];
let key = this.makeKey(word);
let bucket = this.dict.get(key);
if ( bucket === undefined ) {
this.dict[key] = bucket = {};
this.bucketCount += 1;
bucket[word] = true;
bucket = new Set();
this.dict.set(key, bucket);
bucket.add(word);
this.count += 1;
return true;
} else if ( typeof bucket === 'string' ) {
this.dict[key] = bucket = meltBucket(this, word.len, bucket);
}
if ( bucket[word] === undefined ) {
bucket[word] = true;
if ( typeof bucket === 'string' ) {
bucket = meltBucket(this, word.len, bucket);
this.dict.set(key, bucket);
}
if ( bucket.has(word) === false ) {
bucket.add(word);
this.count += 1;
return true;
}
@ -168,12 +146,9 @@ LiquidDict.prototype.add = function(word) {
/******************************************************************************/
LiquidDict.prototype.freeze = function() {
var buckets = this.dict;
var bucket;
for ( var key in buckets ) {
bucket = buckets[key];
if ( typeof bucket === 'object' ) {
buckets[key] = freezeBucket(this, bucket);
for ( let entry of this.dict ) {
if ( typeof entry[1] === 'object' ) {
this.dict.set(entry[0], freezeBucket(this, entry[1]));
}
}
};
@ -181,15 +156,38 @@ LiquidDict.prototype.freeze = function() {
/******************************************************************************/
LiquidDict.prototype.reset = function() {
this.dict = {};
this.dict.clear();
this.count = 0;
this.duplicateCount = 0;
this.bucketCount = 0;
this.frozenBucketCount = 0;
};
/******************************************************************************/
let selfieVersion = 1;
LiquidDict.prototype.toSelfie = function() {
this.freeze();
return {
version: selfieVersion,
count: this.count,
duplicateCount: this.duplicateCount,
frozenBucketCount: this.frozenBucketCount,
dict: Array.from(this.dict)
};
};
LiquidDict.prototype.fromSelfie = function(selfie) {
if ( selfie.version !== selfieVersion ) { return false; }
this.count = selfie.count;
this.duplicateCount = selfie.duplicateCount;
this.frozenBucketCount = selfie.frozenBucketCount;
this.dict = new Map(selfie.dict);
return true;
};
/******************************************************************************/
return LiquidDict;
/******************************************************************************/
@ -199,4 +197,3 @@ return LiquidDict;
/******************************************************************************/
µMatrix.ubiquitousBlacklist = new µMatrix.LiquidDict();
µMatrix.ubiquitousWhitelist = new µMatrix.LiquidDict();

133
src/js/storage.js
View file

@ -439,9 +439,12 @@
callback = this.noopFunc;
}
var loadHostsFilesEnd = function() {
µm.ubiquitousBlacklist.freeze();
vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) });
var loadHostsFilesEnd = function(fromSelfie) {
if ( fromSelfie !== true ) {
µm.ubiquitousBlacklist.freeze();
vAPI.storage.set({ liveHostsFiles: Array.from(µm.liveHostsFiles) });
µm.hostsFilesSelfie.create();
}
vAPI.messaging.broadcast({ what: 'loadHostsFilesCompleted' });
µm.getBytesInUse();
callback();
@ -472,7 +475,14 @@
}
};
this.getAvailableHostsFiles(loadHostsFilesStart);
var onSelfieReady = function(status) {
if ( status === true ) {
return loadHostsFilesEnd(true);
}
µm.getAvailableHostsFiles(loadHostsFilesStart);
};
this.hostsFilesSelfie.load(onSelfieReady);
};
/******************************************************************************/
@ -647,6 +657,9 @@
'selectedHostsFiles',
'externalHostsFiles'
);
if ( hostsChanged ) {
µm.hostsFilesSelfie.destroy();
}
let recipesChanged = applyAssetSelection(
metadata,
details.recipes,
@ -657,7 +670,6 @@
µm.recipeManager.reset();
µm.loadRecipes(true);
}
if ( typeof callback === 'function' ) {
callback({
hostsChanged: hostsChanged,
@ -680,7 +692,50 @@
/******************************************************************************/
µMatrix.hostsFilesSelfie = (function() {
let timer;
return {
create: function() {
this.cancel();
timer = vAPI.setTimeout(
function() {
timer = undefined;
vAPI.cacheStorage.set({
hostsFilesSelfie: µMatrix.ubiquitousBlacklist.toSelfie()
});
},
120000
);
},
destroy: function() {
this.cancel();
vAPI.cacheStorage.remove('hostsFilesSelfie');
},
load: function(callback) {
this.cancel();
vAPI.cacheStorage.get('hostsFilesSelfie', function(bin) {
callback(
bin instanceof Object &&
bin.hostsFilesSelfie instanceof Object &&
µMatrix.ubiquitousBlacklist.fromSelfie(bin.hostsFilesSelfie)
);
});
},
cancel: function() {
if ( timer !== undefined ) {
clearTimeout(timer);
}
timer = undefined;
}
};
})();
/******************************************************************************/
µMatrix.loadPublicSuffixList = function(callback) {
let µm = this;
if ( typeof callback !== 'function' ) {
callback = this.noopFunc;
}
@ -688,15 +743,64 @@
var applyPublicSuffixList = function(details) {
if ( !details.error ) {
publicSuffixList.parse(details.content, punycode.toASCII);
µm.publicSuffixListSelfie.create();
}
callback();
};
this.assets.get(this.pslAssetKey, applyPublicSuffixList);
let onSelfieReady = function(status) {
if ( status === true ) {
return callback();
}
µm.assets.get(µm.pslAssetKey, applyPublicSuffixList);
};
this.publicSuffixListSelfie.load(onSelfieReady);
};
/******************************************************************************/
µMatrix.publicSuffixListSelfie = (function() {
let timer;
return {
create: function() {
this.cancel();
timer = vAPI.setTimeout(
function() {
timer = undefined;
vAPI.cacheStorage.set({
publicSuffixListSelfie: publicSuffixList.toSelfie()
});
},
60000
);
},
destroy: function() {
this.cancel();
vAPI.cacheStorage.remove('publicSuffixListSelfie');
},
load: function(callback) {
this.cancel();
vAPI.cacheStorage.get('publicSuffixListSelfie', function(bin) {
callback(
bin instanceof Object &&
bin.publicSuffixListSelfie instanceof Object &&
publicSuffixList.fromSelfie(bin.publicSuffixListSelfie)
);
});
},
cancel: function() {
if ( timer !== undefined ) {
clearTimeout(timer);
}
timer = undefined;
}
};
})();
/******************************************************************************/
µMatrix.scheduleAssetUpdater = (function() {
var timer, next = 0;
return function(updateDelay) {
@ -727,9 +831,10 @@
/******************************************************************************/
µMatrix.assetObserver = function(topic, details) {
let µmus = this.userSettings;
// Do not update filter list if not in use.
if ( topic === 'before-asset-updated' ) {
let µmus = this.userSettings;
if (
details.type === 'internal' ||
details.type === 'filters' &&
@ -743,6 +848,14 @@
}
if ( topic === 'after-asset-updated' ) {
if (
details.type === 'filters' &&
µmus.selectedHostsFiles.indexOf(details.assetKey) !== -1
) {
this.hostsFilesSelfie.destroy();
} else if ( details.assetKey === this.pslAssetKey ) {
this.publicSuffixListSelfie.destroy();
}
vAPI.messaging.broadcast({
what: 'assetUpdated',
key: details.assetKey,
@ -766,7 +879,7 @@
if (
this.arraysIntersect(
details.assetKeys,
this.userSettings.selectedRecipeFiles
µmus.selectedRecipeFiles
)
) {
this.loadRecipes(true);
@ -774,12 +887,12 @@
if (
this.arraysIntersect(
details.assetKeys,
this.userSettings.selectedHostsFiles
µmus.selectedHostsFiles
)
) {
this.loadHostsFiles();
}
if ( this.userSettings.autoUpdate ) {
if ( µmus.autoUpdate ) {
this.scheduleAssetUpdater(25200000);
} else {
this.scheduleAssetUpdater(0);

145
src/lib/publicsuffixlist.js
View file

@ -37,9 +37,8 @@
/******************************************************************************/
var exceptions = {};
var rules = {};
var selfieMagic = 'iscjsfsaolnm';
var exceptions = new Map();
var rules = new Map();
// This value dictate how the search will be performed:
// < this.cutoffLength = indexOf()
@ -92,9 +91,8 @@ function getPublicSuffix(hostname) {
}
// Since we slice down the hostname with each pass, the first match
// is the longest, so no need to find all the matching rules.
var pos;
while ( true ) {
pos = hostname.indexOf('.');
let pos = hostname.indexOf('.');
if ( pos < 0 ) {
return hostname;
}
@ -118,17 +116,17 @@ function getPublicSuffix(hostname) {
function search(store, hostname) {
// Extract TLD
var pos = hostname.lastIndexOf('.');
var tld, remainder;
if ( pos < 0 ) {
let tld, remainder;
let pos = hostname.lastIndexOf('.');
if ( pos === -1 ) {
tld = hostname;
remainder = hostname;
} else {
tld = hostname.slice(pos + 1);
remainder = hostname.slice(0, pos);
}
var substore = store[tld];
if ( !substore ) {
let substore = store.get(tld);
if ( substore === undefined ) {
return false;
}
// If substore is a string, use indexOf()
@ -136,17 +134,19 @@ function search(store, hostname) {
return substore.indexOf(' ' + remainder + ' ') >= 0;
}
// It is an array: use binary search.
var l = remainder.length;
var haystack = substore[l];
if ( !haystack ) {
let l = remainder.length;
if ( l >= substore.length ) {
return false;
}
var left = 0;
var right = Math.floor(haystack.length / l + 0.5);
var i, needle;
let haystack = substore[l];
if ( haystack === null ) {
return false;
}
let left = 0;
let right = Math.floor(haystack.length / l + 0.5);
while ( left < right ) {
i = left + right >> 1;
needle = haystack.substr( l * i, l );
let i = left + right >> 1;
let needle = haystack.substr(l*i, l);
if ( remainder < needle ) {
right = i;
} else if ( remainder > needle ) {
@ -168,22 +168,21 @@ function search(store, hostname) {
// Suggestion: use <https://github.com/bestiejs/punycode.js> it's quite good.
function parse(text, toAscii) {
exceptions = {};
rules = {};
exceptions = new Map();
rules = new Map();
var lineBeg = 0, lineEnd;
var textEnd = text.length;
var line, store, pos, tld;
let lineBeg = 0;
let textEnd = text.length;
while ( lineBeg < textEnd ) {
lineEnd = text.indexOf('\n', lineBeg);
let lineEnd = text.indexOf('\n', lineBeg);
if ( lineEnd < 0 ) {
lineEnd = text.indexOf('\r', lineBeg);
if ( lineEnd < 0 ) {
lineEnd = textEnd;
}
}
line = text.slice(lineBeg, lineEnd).trim();
let line = text.slice(lineBeg, lineEnd).trim();
lineBeg = lineEnd + 1;
if ( line.length === 0 ) {
@ -191,18 +190,19 @@ function parse(text, toAscii) {
}
// Ignore comments
pos = line.indexOf('//');
if ( pos >= 0 ) {
let pos = line.indexOf('//');
if ( pos !== -1 ) {
line = line.slice(0, pos);
}
// Ignore surrounding whitespaces
line = line.trim();
if ( !line ) {
if ( line.length === 0 ) {
continue;
}
// Is this an exception rule?
let store;
if ( line.charAt(0) === '!' ) {
store = exceptions;
line = line.slice(1);
@ -220,8 +220,9 @@ function parse(text, toAscii) {
line = line.toLowerCase();
// Extract TLD
let tld;
pos = line.lastIndexOf('.');
if ( pos < 0 ) {
if ( pos === -1 ) {
tld = line;
} else {
tld = line.slice(pos + 1);
@ -229,13 +230,15 @@ function parse(text, toAscii) {
}
// Store suffix using tld as key
if ( !store.hasOwnProperty(tld) ) {
store[tld] = [];
let substore = store.get(tld);
if ( substore === undefined ) {
store.set(tld, substore = []);
}
if ( line ) {
store[tld].push(line);
substore.push(line);
}
}
crystallize(exceptions);
crystallize(rules);
@ -248,69 +251,81 @@ function parse(text, toAscii) {
// for future look up.
function crystallize(store) {
var suffixes, suffix, i, l;
for ( let entry of store ) {
let tld = entry[0];
let suffixes = entry[1];
for ( var tld in store ) {
if ( !store.hasOwnProperty(tld) ) {
continue;
}
suffixes = store[tld].join(' ');
// No suffix
if ( !suffixes ) {
store[tld] = '';
if ( suffixes.length === 0 ) {
store.set(tld, '');
continue;
}
// Concatenated list of suffixes less than cutoff length:
// Store as string, lookup using indexOf()
if ( suffixes.length < cutoffLength ) {
store[tld] = ' ' + suffixes + ' ';
let s = suffixes.join(' ');
if ( s.length < cutoffLength ) {
store.set(tld, ' ' + s + ' ');
continue;
}
// Concatenated list of suffixes greater or equal to cutoff length
// Store as array keyed on suffix length, lookup using binary search.
// I borrowed the idea to key on string length here:
// http://ejohn.org/blog/dictionary-lookups-in-javascript/#comment-392072
i = store[tld].length;
suffixes = [];
while ( i-- ) {
suffix = store[tld][i];
l = suffix.length;
if ( !suffixes[l] ) {
suffixes[l] = [];
let buckets = [];
for ( let suffix of suffixes ) {
let l = suffix.length;
if ( buckets.length <= l ) {
extendArray(buckets, l);
}
suffixes[l].push(suffix);
if ( buckets[l] === null ) {
buckets[l] = [];
}
buckets[l].push(suffix);
}
l = suffixes.length;
while ( l-- ) {
if ( suffixes[l] ) {
suffixes[l] = suffixes[l].sort().join('');
for ( let i = 0; i < buckets.length; i++ ) {
let bucket = buckets[i];
if ( bucket !== null ) {
buckets[i] = bucket.sort().join('');
}
}
store[tld] = suffixes;
store.set(tld, buckets);
}
return store;
}
let extendArray = function(aa, rb) {
for ( let i = aa.length; i <= rb; i++ ) {
aa.push(null);
}
};
/******************************************************************************/
function toSelfie() {
let selfieMagic = 3;
let toSelfie = function() {
return {
magic: selfieMagic,
rules: rules,
exceptions: exceptions
rules: Array.from(rules),
exceptions: Array.from(exceptions)
};
}
};
function fromSelfie(selfie) {
if ( typeof selfie !== 'object' || typeof selfie.magic !== 'string' || selfie.magic !== selfieMagic ) {
let fromSelfie = function(selfie) {
if (
selfie instanceof Object === false ||
selfie.magic !== selfieMagic
) {
return false;
}
rules = selfie.rules;
exceptions = selfie.exceptions;
rules = new Map(selfie.rules);
exceptions = new Map(selfie.exceptions);
callListeners(onChangedListeners);
return true;
}
};
/******************************************************************************/