From c2d7096500c11b72888b4d901b64185f9cc7b9d1 Mon Sep 17 00:00:00 2001 From: gorhill Date: Wed, 29 Mar 2017 09:12:12 -0400 Subject: [PATCH] refactor assets management; working webextension version --- .jshintrc | 4 +- assets/assets.json | 75 + platform/chromium/manifest.json | 3 +- platform/chromium/polyfill.js | 234 +++ platform/chromium/vapi-background.js | 86 +- platform/chromium/vapi-common.js | 2 +- platform/firefox/polyfill.js | 95 ++ platform/firefox/vapi-background.js | 2 + platform/webext/manifest.json | 61 + platform/webext/polyfill.js | 30 + src/_locales/en/messages.json | 4 +- src/background.html | 2 +- src/css/common.css | 37 + src/css/hosts-files.css | 222 ++- src/css/popup.css | 4 +- src/hosts-files.html | 43 +- src/js/asset-viewer.js | 10 +- src/js/assets.js | 2001 +++++++++----------------- src/js/background.js | 14 +- src/js/hosts-files.js | 521 +++---- src/js/messaging.js | 38 +- src/js/popup.js | 2 +- src/js/start.js | 29 +- src/js/storage.js | 469 +++--- src/js/traffic.js | 23 +- src/js/utils.js | 68 +- src/popup.html | 54 +- tools/make-firefox-meta.py | 0 tools/make-firefox.sh | 2 +- tools/make-webext-meta.py | 29 + tools/make-webext.sh | 32 + 31 files changed, 2083 insertions(+), 2113 deletions(-) create mode 100644 assets/assets.json create mode 100644 platform/chromium/polyfill.js create mode 100644 platform/firefox/polyfill.js create mode 100644 platform/webext/manifest.json create mode 100644 platform/webext/polyfill.js mode change 100644 => 100755 tools/make-firefox-meta.py create mode 100755 tools/make-webext-meta.py create mode 100755 tools/make-webext.sh diff --git a/.jshintrc b/.jshintrc index 4abac7c..e4e255b 100644 --- a/.jshintrc +++ b/.jshintrc @@ -5,9 +5,9 @@ "strict": "global", "globals": { "self": false, - "vAPI": false, "chrome": false, - "safari": false, + "vAPI": false, + "µMatrix": false, "Components": false // global variable in Firefox } } diff --git a/assets/assets.json b/assets/assets.json new file mode 100644 index 0000000..e896366 --- /dev/null +++ b/assets/assets.json @@ -0,0 +1,75 @@ +{ + "assets.json": { + "content": "internal", + "updateAfter": 13, + "contentURL": [ + "https://raw.githubusercontent.com/gorhill/uMatrix/master/assets/assets.json", + "assets/assets.json" + ] + }, + "public_suffix_list.dat": { + "content": "internal", + "updateAfter": 19, + "contentURL": [ + "https://publicsuffix.org/list/public_suffix_list.dat", + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat" + ] + }, + "malware-0": { + "content": "filters", + "group": "malware", + "title": "Malware Domain List", + "contentURL": [ + "https://www.malwaredomainlist.com/hostslist/hosts.txt", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt" + ] + }, + "malware-1": { + "content": "filters", + "group": "malware", + "title": "Malware domains", + "contentURL": [ + "https://mirror.cedia.org.ec/malwaredomains/justdomains", + "https://mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt" + ], + "supportURL": "http://www.malwaredomains.com/" + }, + "dpollock-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "title": "Dan Pollock’s hosts file", + "contentURL": "http://someonewhocares.org/hosts/hosts", + "supportURL": "http://someonewhocares.org/hosts/" + }, + "hphosts": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "title": "hpHosts’ Ad and tracking servers", + "contentURL": "https://hosts-file.net/.%5Cad_servers.txt", + "supportURL": "https://hosts-file.net/" + }, + "mvps-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 11, + "title": "MVPS HOSTS", + "contentURL": "http://winhelp2002.mvps.org/hosts.txt", + "supportURL": "http://winhelp2002.mvps.org/" + }, + "plowe-0": { + "content": "filters", + "group": "multipurpose", + "updateAfter": 13, + "title": "Peter Lowe’s Ad and tracking server list", + "contentURL": [ + "https://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=1&mimetype=plaintext", + "assets/thirdparties/pgl.yoyo.org/as/serverlist", + "assets/thirdparties/pgl.yoyo.org/as/serverlist.txt" + ], + "supportURL": "https://pgl.yoyo.org/adservers/" + } +} diff --git a/platform/chromium/manifest.json b/platform/chromium/manifest.json index 8646f8d..56dbea2 100644 --- a/platform/chromium/manifest.json +++ b/platform/chromium/manifest.json @@ -78,7 +78,6 @@ "webNavigation", "webRequest", "webRequestBlocking", - "http://*/*", - "https://*/*" + "" ] } diff --git a/platform/chromium/polyfill.js b/platform/chromium/polyfill.js new file mode 100644 index 0000000..1ee5a83 --- /dev/null +++ b/platform/chromium/polyfill.js @@ -0,0 +1,234 @@ +/******************************************************************************* + + uMatrix - a browser extension to block requests. + Copyright (C) 2017 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uMatrix + + This file has been originally imported from: + https://github.com/gorhill/uBlock/tree/master/platform/chromium + +*/ + +// For background page or non-background pages + +/* exported objectAssign */ + +'use strict'; + +/******************************************************************************/ +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1067 +// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/String/startsWith +// Firefox 17/Chromium 41 supports `startsWith`. + +if ( String.prototype.startsWith instanceof Function === false ) { + String.prototype.startsWith = function(needle, pos) { + if ( typeof pos !== 'number' ) { + pos = 0; + } + return this.lastIndexOf(needle, pos) === pos; + }; +} + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1067 +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/String/endsWith +// Firefox 17/Chromium 41 supports `endsWith`. + +if ( String.prototype.endsWith instanceof Function === false ) { + String.prototype.endsWith = function(needle, pos) { + if ( typeof pos !== 'number' ) { + pos = this.length; + } + pos -= needle.length; + return this.indexOf(needle, pos) === pos; + }; +} + +/******************************************************************************/ + +// As per MDN, Object.assign appeared first in Chromium 45. +// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign#Browser_compatibility + +var objectAssign = Object.assign || function(target, source) { + var keys = Object.keys(source); + for ( var i = 0, n = keys.length, key; i < n; i++ ) { + key = keys[i]; + target[key] = source[key]; + } + return target; +}; + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1070 +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Browser_compatibility +// This polyfill is designed to fulfill *only* what uBlock Origin needs -- this +// is not an accurate API of the real Set() type. + +if ( self.Set instanceof Function === false ) { + self.Set = function(iter) { + this.clear(); + if ( Array.isArray(iter) ) { + for ( var i = 0, n = iter.length; i < n; i++ ) { + this.add(iter[i]); + } + return; + } + }; + + self.Set.polyfill = true; + + self.Set.prototype.clear = function() { + this._set = Object.create(null); + this.size = 0; + // Iterator stuff + this._values = undefined; + this._i = undefined; + this.value = undefined; + this.done = true; + }; + + self.Set.prototype.add = function(k) { + if ( this._set[k] === undefined ) { + this._set[k] = true; + this.size += 1; + } + return this; + }; + + self.Set.prototype.delete = function(k) { + if ( this._set[k] !== undefined ) { + delete this._set[k]; + this.size -= 1; + return true; + } + return false; + }; + + self.Set.prototype.has = function(k) { + return this._set[k] !== undefined; + }; + + self.Set.prototype.next = function() { + if ( this._i < this.size ) { + this.value = this._values[this._i++]; + } else { + this._values = undefined; + this.value = undefined; + this.done = true; + } + return this; + }; + + self.Set.prototype.values = function() { + this._values = Object.keys(this._set); + this._i = 0; + this.value = undefined; + this.done = false; + return this; + }; +} + +/******************************************************************************/ + +// https://github.com/gorhill/uBlock/issues/1070 +// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Set#Browser_compatibility +// This polyfill is designed to fulfill *only* what uBlock Origin needs -- this +// is not an accurate API of the real Map() type. + +if ( self.Map instanceof Function === false ) { + self.Map = function(iter) { + this.clear(); + if ( Array.isArray(iter) ) { + for ( var i = 0, n = iter.length, entry; i < n; i++ ) { + entry = iter[i]; + this.set(entry[0], entry[1]); + } + return; + } + }; + + self.Map.polyfill = true; + + self.Map.prototype.clear = function() { + this._map = Object.create(null); + this.size = 0; + // Iterator stuff + this._keys = undefined; + this._i = undefined; + this.value = undefined; + this.done = true; + }; + + self.Map.prototype.delete = function(k) { + if ( this._map[k] !== undefined ) { + delete this._map[k]; + this.size -= 1; + return true; + } + return false; + }; + + self.Map.prototype.entries = function() { + this._keys = Object.keys(this._map); + this._i = 0; + this.value = [ undefined, undefined ]; + this.done = false; + return this; + }; + + self.Map.prototype.get = function(k) { + return this._map[k]; + }; + + self.Map.prototype.has = function(k) { + return this._map[k] !== undefined; + }; + + self.Map.prototype.next = function() { + if ( this._i < this.size ) { + var key = this._keys[this._i++]; + this.value[0] = key; + this.value[1] = this._map[key]; + } else { + this._keys = undefined; + this.value = undefined; + this.done = true; + } + return this; + }; + + self.Map.prototype.set = function(k, v) { + if ( v !== undefined ) { + if ( this._map[k] === undefined ) { + this.size += 1; + } + this._map[k] = v; + } else { + if ( this._map[k] !== undefined ) { + this.size -= 1; + } + delete this._map[k]; + } + return this; + }; +} + +/******************************************************************************/ diff --git a/platform/chromium/vapi-background.js b/platform/chromium/vapi-background.js index 52ae5c0..6d9c3e9 100644 --- a/platform/chromium/vapi-background.js +++ b/platform/chromium/vapi-background.js @@ -1,7 +1,8 @@ /******************************************************************************* uMatrix - a browser extension to block requests. - Copyright (C) 2014-2016 The uBlock authors + Copyright (C) 2014-2017 The uBlock Origin authors + Copyright (C) 2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -52,14 +53,16 @@ chrome.privacy.network.networkPredictionEnabled.set({ // Tell Chromium to allow all javascript: µMatrix will control whether // javascript execute through `Content-Policy-Directive` and webRequest. // https://github.com/gorhill/httpswitchboard/issues/74 -chrome.contentSettings.javascript.set({ - primaryPattern: 'https://*/*', - setting: 'allow' -}); -chrome.contentSettings.javascript.set({ - primaryPattern: 'http://*/*', - setting: 'allow' -}); +if ( chrome.contentSettings instanceof Object ) { + chrome.contentSettings.javascript.set({ + primaryPattern: 'https://*/*', + setting: 'allow' + }); + chrome.contentSettings.javascript.set({ + primaryPattern: 'http://*/*', + setting: 'allow' + }); +} /******************************************************************************/ @@ -107,6 +110,7 @@ vAPI.app.restart = function() { // chrome.storage.local.get(null, function(bin){ console.debug('%o', bin); }); vAPI.storage = chrome.storage.local; +vAPI.cacheStorage = chrome.storage.local; /******************************************************************************/ @@ -583,8 +587,9 @@ vAPI.net = {}; /******************************************************************************/ vAPI.net.registerListeners = function() { - var µm = µMatrix; - var httpRequestHeadersJunkyard = []; + var µm = µMatrix, + reNetworkURL = /^(?:https?|wss?):\/\//, + httpRequestHeadersJunkyard = []; // Abstraction layer to deal with request headers // >>>>>>>> @@ -657,11 +662,24 @@ vAPI.net.registerListeners = function() { // Normalizing request types // >>>>>>>> + var extToTypeMap = new Map([ + ['eot','font'],['otf','font'],['svg','font'],['ttf','font'],['woff','font'],['woff2','font'], + ['mp3','media'],['mp4','media'],['webm','media'], + ['gif','image'],['ico','image'],['jpeg','image'],['jpg','image'],['png','image'],['webp','image'] + ]); + var normalizeRequestDetails = function(details) { details.tabId = details.tabId.toString(); + var type = details.type; + + if ( type === 'imageset' ) { + details.type = 'image'; + return; + } + // The rest of the function code is to normalize request type - if ( details.type !== 'other' ) { + if ( type !== 'other' ) { return; } @@ -672,26 +690,11 @@ vAPI.net.registerListeners = function() { } } + // Try to map known "extension" part of URL to request type. var path = µm.URI.pathFromURI(details.url), pos = path.indexOf('.', path.length - 6); - - // https://github.com/chrisaljoudi/uBlock/issues/862 - // If no transposition possible, transpose to `object` as per - // Chromium bug 410382 (see below) - if ( pos === -1 ) { - return; - } - - var needle = path.slice(pos) + '.'; - if ( '.eot.ttf.otf.svg.woff.woff2.'.indexOf(needle) !== -1 ) { - details.type = 'font'; - return; - } - // Still need this because often behind-the-scene requests are wrongly - // categorized as 'other' - if ( '.ico.png.gif.jpg.jpeg.mp3.mp4.webm.webp.'.indexOf(needle) !== -1 ) { - details.type = 'image'; - return; + if ( pos !== -1 && (type = extToTypeMap.get(path.slice(pos + 1))) ) { + details.type = type; } }; // <<<<<<<< @@ -700,12 +703,13 @@ vAPI.net.registerListeners = function() { // Network event handlers // >>>>>>>> var onBeforeRequestClient = this.onBeforeRequest.callback; - var onBeforeRequest = function(details) { - normalizeRequestDetails(details); - return onBeforeRequestClient(details); - }; chrome.webRequest.onBeforeRequest.addListener( - onBeforeRequest, + function(details) { + if ( reNetworkURL.test(details.url) ) { + normalizeRequestDetails(details); + return onBeforeRequestClient(details); + } + }, //function(details) { // quickProfiler.start('onBeforeRequest'); // var r = onBeforeRequest(details); @@ -713,8 +717,8 @@ vAPI.net.registerListeners = function() { // return r; //}, { - 'urls': this.onBeforeRequest.urls || [''], - 'types': this.onBeforeRequest.types || [] + 'urls': this.onBeforeRequest.urls || [ '' ], + 'types': this.onBeforeRequest.types || undefined }, this.onBeforeRequest.extra ); @@ -735,8 +739,8 @@ vAPI.net.registerListeners = function() { chrome.webRequest.onBeforeSendHeaders.addListener( onBeforeSendHeaders, { - 'urls': this.onBeforeSendHeaders.urls || [''], - 'types': this.onBeforeSendHeaders.types || [] + 'urls': this.onBeforeSendHeaders.urls || [ '' ], + 'types': this.onBeforeSendHeaders.types || undefined }, this.onBeforeSendHeaders.extra ); @@ -749,8 +753,8 @@ vAPI.net.registerListeners = function() { chrome.webRequest.onHeadersReceived.addListener( onHeadersReceived, { - 'urls': this.onHeadersReceived.urls || [''], - 'types': this.onHeadersReceived.types || [] + 'urls': this.onHeadersReceived.urls || [ '' ], + 'types': this.onHeadersReceived.types || undefined }, this.onHeadersReceived.extra ); diff --git a/platform/chromium/vapi-common.js b/platform/chromium/vapi-common.js index 56b192e..f857e14 100644 --- a/platform/chromium/vapi-common.js +++ b/platform/chromium/vapi-common.js @@ -88,7 +88,7 @@ vAPI.closePopup = function() { // This storage is optional, but it is nice to have, for a more polished user // experience. -vAPI.localStorage = window.localStorage; +vAPI.localStorage = self.localStorage; /******************************************************************************/ diff --git a/platform/firefox/polyfill.js b/platform/firefox/polyfill.js new file mode 100644 index 0000000..7007309 --- /dev/null +++ b/platform/firefox/polyfill.js @@ -0,0 +1,95 @@ +/******************************************************************************* + + uMatrix - a browser extension to block requests. + Copyright (C) 2017 Raymond Hill + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uMatrix + + This file has been originally imported from: + https://github.com/gorhill/uBlock/tree/master/platform/chromium + +*/ + +// For background page or non-background pages + +/* exported objectAssign */ + +'use strict'; + +/******************************************************************************/ +/******************************************************************************/ + +// As per MDN, Object.assign appeared first in Firefox 34. +// https://developer.mozilla.org/en/docs/Web/JavaScript/Reference/Global_Objects/Object/assign#Browser_compatibility + +var objectAssign = Object.assign || function(target, source) { + var keys = Object.keys(source); + for ( var i = 0, n = keys.length, key; i < n; i++ ) { + key = keys[i]; + target[key] = source[key]; + } + return target; +}; + +/******************************************************************************/ + +// Patching for Pale Moon which does not implement ES6 Set/Map. +// Test for non-ES6 Set/Map: check if property `iterator` is present. +// The code is strictly to satisfy uBO's core, not to be an accurate +// implementation of ES6. + +if ( self.Set.prototype.iterator instanceof Function ) { + //console.log('Patching non-ES6 Set() to be more ES6-like.'); + self.Set.prototype._values = self.Set.prototype.values; + self.Set.prototype.values = function() { + this._valueIter = this._values(); + this.value = undefined; + this.done = false; + return this; + }; + self.Set.prototype.next = function() { + try { + this.value = this._valueIter.next(); + } catch (ex) { + this._valueIter = undefined; + this.value = undefined; + this.done = true; + } + return this; + }; +} + +if ( self.Map.prototype.iterator instanceof Function ) { + //console.log('Patching non-ES6 Map() to be more ES6-like.'); + self.Map.prototype._entries = self.Map.prototype.entries; + self.Map.prototype.entries = function() { + this._entryIter = this._entries(); + this.value = undefined; + this.done = false; + return this; + }; + self.Map.prototype.next = function() { + try { + this.value = this._entryIter.next(); + } catch (ex) { + this._entryIter = undefined; + this.value = undefined; + this.done = true; + } + return this; + }; +} + diff --git a/platform/firefox/vapi-background.js b/platform/firefox/vapi-background.js index daa5a38..f520b01 100644 --- a/platform/firefox/vapi-background.js +++ b/platform/firefox/vapi-background.js @@ -580,6 +580,8 @@ vAPI.storage = (function() { return api; })(); +vAPI.cacheStorage = vAPI.storage; + /******************************************************************************/ // This must be executed/setup early. diff --git a/platform/webext/manifest.json b/platform/webext/manifest.json new file mode 100644 index 0000000..7c638e4 --- /dev/null +++ b/platform/webext/manifest.json @@ -0,0 +1,61 @@ +{ + "applications": { + "gecko": { + "id": "uMatrix@raymondhill.net", + "strict_min_version": "53.0a1" + } + }, + "author": "Raymond Hill", + "background": { + "page": "background.html" + }, + "browser_action": { + "browser_style": false, + "default_icon": { + "19": "img/browsericons/icon19.png", + "38": "img/browsericons/icon38.png" + }, + "default_title": "uMatrix", + "default_popup": "popup.html" + }, + "content_scripts": [ + { + "matches": ["http://*/*", "https://*/*"], + "js": ["js/vapi-client.js", "js/contentscript-start.js"], + "run_at": "document_start", + "all_frames": true + }, + { + "matches": ["http://*/*", "https://*/*"], + "js": ["js/contentscript-end.js"], + "run_at": "document_end", + "all_frames": true + } + ], + "default_locale": "en", + "description": "__MSG_extShortDesc__", + "icons": { + "16": "img/icon_16.png", + "128": "img/icon_128.png" + }, + "manifest_version": 2, + "minimum_chrome_version": "26.0", + "name": "uMatrix", + "options_ui": { + "page": "options_ui.html" + }, + "permissions": [ + "browsingData", + "contentSettings", + "cookies", + "privacy", + "storage", + "tabs", + "webNavigation", + "webRequest", + "webRequestBlocking", + "" + ], + "short_name": "uMatrix", + "version": "0.9.9" +} diff --git a/platform/webext/polyfill.js b/platform/webext/polyfill.js new file mode 100644 index 0000000..78e57a9 --- /dev/null +++ b/platform/webext/polyfill.js @@ -0,0 +1,30 @@ +/******************************************************************************* + + uBlock Origin - a browser extension to block requests. + Copyright (C) 2016 The uBlock Origin authors + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see {http://www.gnu.org/licenses/}. + + Home: https://github.com/gorhill/uBlock +*/ + +// For background page or non-background pages + +'use strict'; + +/******************************************************************************/ + +var objectAssign = Object.assign; + +/******************************************************************************/ diff --git a/src/_locales/en/messages.json b/src/_locales/en/messages.json index 83bd95a..ae0ac16 100644 --- a/src/_locales/en/messages.json +++ b/src/_locales/en/messages.json @@ -50,11 +50,11 @@ "description": "HAS TO FIT IN MATRIX HEADER!" }, "imagePrettyName": { - "message": "media", + "message": "image", "description": "HAS TO FIT IN MATRIX HEADER!" }, "pluginPrettyName": { - "message": "plugin", + "message": "media", "description": "HAS TO FIT IN MATRIX HEADER!" }, "scriptPrettyName": { diff --git a/src/background.html b/src/background.html index 06b789b..de437e1 100644 --- a/src/background.html +++ b/src/background.html @@ -5,9 +5,9 @@ uMatrix + - diff --git a/src/css/common.css b/src/css/common.css index 018a0d3..e3d3139 100644 --- a/src/css/common.css +++ b/src/css/common.css @@ -80,3 +80,40 @@ body[dir="ltr"] .tip-anchor-right[data-i18n-tip]:hover:after, body[dir="rtl"] .tip-anchor-left[data-i18n-tip]:hover:after { right: -3vw; } +button.custom { + padding: 0.6em 1em; + border: 1px solid transparent; + border-color: #ccc #ccc #bbb #bbb; + border-radius: 3px; + background-color: hsl(216, 0%, 75%); + background-image: linear-gradient(#f2f2f2, #dddddd); + background-repeat: repeat-x; + color: #000; + opacity: 0.8; + } +button.custom:hover { + opacity: 1.0; + } +button.custom.important { + padding: 0.6em 1em; + border: 1px solid transparent; + border-color: #ffcc7f #ffcc7f hsl(36, 100%, 73%); + border-radius: 3px; + background-color: hsl(36, 100%, 75%); + background-image: linear-gradient(#ffdca8, #ffcc7f); + background-repeat: repeat-x; + color: #222; + opacity: 0.8; + } +button.custom.important:hover { + opacity: 1.0; + } +button.custom.disabled, +button.custom[disabled] { + border-color: #ddd #ddd hsl(36, 0%, 85%); + background-color: hsl(36, 0%, 72%); + background-image: linear-gradient(#f2f2f2, #dddddd); + color: #666; + opacity: 0.6; + pointer-events: none; + } diff --git a/src/css/hosts-files.css b/src/css/hosts-files.css index 4bebf8c..656535a 100644 --- a/src/css/hosts-files.css +++ b/src/css/hosts-files.css @@ -1,3 +1,13 @@ +@keyframes spin { + 0% { transform: rotate(0deg); -webkit-transform: rotate(0deg); } + 12.5% { transform: rotate(45deg); -webkit-transform: rotate(45deg); } + 25% { transform: rotate(90deg); -webkit-transform: rotate(90deg); } + 37.5% { transform: rotate(135deg); -webkit-transform: rotate(135deg); } + 50% { transform: rotate(180deg); -webkit-transform: rotate(180deg); } + 67.5% { transform: rotate(225deg); -webkit-transform: rotate(225deg); } + 75% { transform: rotate(270deg); -webkit-transform: rotate(270deg); } + 87.5% { transform: rotate(315deg); -webkit-transform: rotate(315deg); } + } ul { padding: 0; list-style-type: none; @@ -25,85 +35,96 @@ body[dir="rtl"] li.listEntry { margin-right: 1em; } li.listEntry > * { + margin-right: 0.5em; + text-indent: 0; unicode-bidi: embed; } li.listEntry > a:nth-of-type(2) { font-size: 13px; opacity: 0.5; } +li.listEntry.toRemove > input[type="checkbox"] { + visibility: hidden; + } +li.listEntry.toRemove > a.content { + text-decoration: line-through; + } +li.listEntry > .fa { + color: inherit; + display: none; + font-size: 110%; + opacity: 0.5; + vertical-align: baseline; + } +li.listEntry > a.fa:hover { + opacity: 1; + } +li.listEntry.support > a.support { + display: inline-block; + } +li.listEntry > a.remove, +li.listEntry > a.remove:visited { + color: darkred; + } +li.listEntry.external > a.remove { + display: inline-block; + } +li.listEntry.mustread > a.mustread { + display: inline-block; + } +li.listEntry.mustread > a.mustread:hover { + color: mediumblue; + } +li.listEntry > .counts { + display: none; + font-size: smaller; +} +li.listEntry > input[type="checkbox"]:checked ~ .counts { + display: inline; +} +li.listEntry span.status { + color: #444; + cursor: default; + display: none; +} +li.listEntry span.status:hover { + opacity: 1; + } +li.listEntry span.unsecure { + color: darkred; + } +li.listEntry.unsecure > input[type="checkbox"]:checked ~ span.unsecure { + display: inline-block; + } +li.listEntry span.failed { + color: darkred; + } +li.listEntry.failed span.failed { + display: inline-block; + } +li.listEntry span.cache { + cursor: pointer; + } +li.listEntry.cached:not(.obsolete) > input[type="checkbox"]:checked ~ span.cache { + display: inline-block; + } +li.listEntry span.obsolete { + color: hsl(36, 100%, 40%); + } +body:not(.updating) li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.obsolete { + display: inline-block; + } +li.listEntry span.updating { + transform-origin: 50% 46%; + } +body.updating li.listEntry.obsolete > input[type="checkbox"]:checked ~ span.updating { + animation: spin 1s step-start infinite; + display: inline-block; + } .dim { opacity: 0.5; } -/* I designed the button with: http://charliepark.org/bootstrap_buttons/ */ -button.custom { - padding: 5px; - border: 1px solid transparent; - border-color: #80b3ff #80b3ff hsl(216, 100%, 75%); - border-radius: 3px; - background-color: hsl(216, 100%, 75%); - background-image: linear-gradient(#a8cbff, #80b3ff); - background-repeat: repeat-x; - color: #222; - cursor: pointer; - opacity: 0.8; - } -button.custom.disabled { - border-color: #dddddd #dddddd hsl(36, 0%, 85%); - background-color: hsl(36, 0%, 72%); - background-image: linear-gradient(#f2f2f2, #dddddd); - color: #aaa; - pointer-events: none; - } -button.custom:hover { - opacity: 1.0; - } -button.custom.reloadAll:not(.disabled) { - border-color: #ffcc7f #ffcc7f hsl(36, 100%, 73%); - background-color: hsl(36, 100%, 75%); - background-image: linear-gradient(#ffdca8, #ffcc7f); - } -#buttonApply { - display: initial; - padding: 1em 1em; - position: fixed; - top: 1em; - } -body[dir="ltr"] #buttonApply { - right: 1em; - } -body[dir="rtl"] #buttonApply { - left: 1em; - } -#buttonApply.disabled { - display: none; - } -span.status { - border: 1px solid transparent; - color: #444; - display: inline-block; - font-size: smaller; - line-height: 1; - margin: 0 0 0 0.5em; - opacity: 0.8; - padding: 1px 2px; -} -span.unsecure { - background-color: hsl(0, 100%, 88%); - border-color: hsl(0, 100%, 83%); - } -span.purge { - border-color: #ddd; - background-color: #eee; - cursor: pointer; - } -span.purge:hover { - opacity: 1; - } -span.obsolete { - border-color: hsl(36, 100%, 73%); - background-color: hsl(36, 100%, 75%); - } -#externalListsDiv { +#externalLists { margin: 2em auto 0 auto; } body[dir="ltr"] #externalListsDiv { @@ -119,64 +140,3 @@ body[dir="rtl"] #externalListsDiv { height: 12em; white-space: pre; } -body #busyOverlay { - background-color: transparent; - bottom: 0; - cursor: wait; - display: none; - left: 0; - position: fixed; - right: 0; - top: 0; - z-index: 1000; - } -body.busy #busyOverlay { - display: block; - } -#busyOverlay > div:nth-of-type(1) { - background-color: white; - bottom: 0; - left: 0; - opacity: 0.75; - position: absolute; - right: 0; - top: 0; - } -#busyOverlay > div:nth-of-type(2) { - background-color: #eee; - border: 1px solid transparent; - border-color: #80b3ff #80b3ff hsl(216, 100%, 75%); - border-radius: 3px; - box-sizing: border-box; - height: 3em; - left: 10%; - position: absolute; - bottom: 75%; - width: 80%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(1) { - background-color: hsl(216, 100%, 75%); - background-image: linear-gradient(#a8cbff, #80b3ff); - background-repeat: repeat-x; - border: 0; - box-sizing: border-box; - color: #222; - height: 100%; - left: 0; - padding: 0; - position: absolute; - width: 25%; - } -#busyOverlay > div:nth-of-type(2) > div:nth-of-type(2) { - background-color: transparent; - border: 0; - box-sizing: border-box; - height: 100%; - left: 0; - line-height: 3em; - overflow: hidden; - position: absolute; - text-align: center; - top: 0; - width: 100%; - } diff --git a/src/css/popup.css b/src/css/popup.css index c124343..7efb15c 100644 --- a/src/css/popup.css +++ b/src/css/popup.css @@ -74,7 +74,9 @@ body[dir="rtl"] #gotoDashboard > span:last-of-type { padding: 02px 0; text-align: center; } - +#toolbarContainer { + position: relative; +} .toolbar { border: 0; display: inline-block; diff --git a/src/hosts-files.html b/src/hosts-files.html index c7d9f58..775f840 100644 --- a/src/hosts-files.html +++ b/src/hosts-files.html @@ -11,43 +11,38 @@

+

+ + + +

  • - - -

  • +
-
-
+

-

-
- -
-
- -
diff --git a/src/js/asset-viewer.js b/src/js/asset-viewer.js index aaeda50..1eea1b0 100644 --- a/src/js/asset-viewer.js +++ b/src/js/asset-viewer.js @@ -1,7 +1,7 @@ /******************************************************************************* - µMatrix - a Chromium browser extension to block requests. - Copyright (C) 2014 Raymond Hill + uMatrix - a Chromium browser extension to block requests. + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,14 +19,14 @@ Home: https://github.com/gorhill/uMatrix */ -/* global vAPI, uDom */ +/* global uDom */ + +'use strict'; /******************************************************************************/ (function() { -'use strict'; - /******************************************************************************/ var messager = vAPI.messaging.channel('asset-viewer.js'); diff --git a/src/js/assets.js b/src/js/assets.js index a4a13c1..a0e2411 100644 --- a/src/js/assets.js +++ b/src/js/assets.js @@ -19,298 +19,90 @@ Home: https://github.com/gorhill/uMatrix */ -/* global vAPI, µMatrix, YaMD5 */ - -/******************************************************************************* - -File system structure: - assets - umatrix - ... - thirdparties - ... - user - filters.txt - ... - -*/ - -/******************************************************************************/ - -// Low-level asset files manager - -µMatrix.assets = (function() { - 'use strict'; /******************************************************************************/ -var oneSecond = 1000; -var oneMinute = 60 * oneSecond; -var oneHour = 60 * oneMinute; -var oneDay = 24 * oneHour; +µMatrix.assets = (function() { /******************************************************************************/ -var projectRepositoryRoot = µMatrix.projectServerRoot; -var nullFunc = function() {}; -var reIsExternalPath = /^[a-z]+:\/\//; -var reIsUserPath = /^assets\/user\//; -var reIsCachePath = /^cache:\/\//; -var lastRepoMetaTimestamp = 0; -var lastRepoMetaIsRemote = false; -var refreshRepoMetaPeriod = 5 * oneHour; -var errorCantConnectTo = vAPI.i18n('errorCantConnectTo'); -var xhrTimeout = vAPI.localStorage.getItem('xhrTimeout') || 30000; +var reIsExternalPath = /^(?:[a-z-]+):\/\//, + reIsUserAsset = /^user-/, + errorCantConnectTo = vAPI.i18n('errorCantConnectTo'), + noopfunc = function(){}; -var exports = { - autoUpdate: true, - autoUpdateDelay: 4 * oneDay, - - // https://github.com/chrisaljoudi/uBlock/issues/426 - remoteFetchBarrier: 0 +var api = { }; /******************************************************************************/ -var AssetEntry = function() { - this.localChecksum = ''; - this.repoChecksum = ''; - this.expireTimestamp = 0; +var observers = []; + +api.addObserver = function(observer) { + if ( observers.indexOf(observer) === -1 ) { + observers.push(observer); + } }; -var RepoMetadata = function() { - this.entries = {}; - this.waiting = []; +api.removeObserver = function(observer) { + var pos; + while ( (pos = observers.indexOf(observer)) !== -1 ) { + observers.splice(pos, 1); + } }; -var repoMetadata = null; - -// We need these to persist beyond repoMetaData -var homeURLs = {}; - -/******************************************************************************/ - -var stringIsNotEmpty = function(s) { - return typeof s === 'string' && s !== ''; -}; - -/******************************************************************************/ - -var cacheIsObsolete = function(t) { - return typeof t !== 'number' || (Date.now() - t) >= exports.autoUpdateDelay; -}; - -/******************************************************************************/ - -var cachedAssetsManager = (function() { - var exports = {}; - var entries = null; - var cachedAssetPathPrefix = 'cached_asset_content://'; - - var getEntries = function(callback) { - if ( entries !== null ) { - callback(entries); - return; +var fireNotification = function(topic, details) { + var result; + for ( var i = 0; i < observers.length; i++ ) { + if ( observers[i](topic, details) === false ) { + result = false; } - // Flush cached non-user assets if these are from a prior version. - // https://github.com/gorhill/httpswitchboard/issues/212 - var onLastVersionRead = function(store) { - var currentVersion = vAPI.app.version; - var lastVersion = store.extensionLastVersion || '0.0.0.0'; - if ( currentVersion !== lastVersion ) { - vAPI.storage.set({ 'extensionLastVersion': currentVersion }); - } - callback(entries); - }; - var onLoaded = function(bin) { - // https://github.com/gorhill/httpswitchboard/issues/381 - // Maybe the index was requested multiple times and already - // fetched by one of the occurrences. - if ( entries === null ) { - var lastError = vAPI.lastError(); - if ( lastError ) { - console.error( - 'µMatrix> cachedAssetsManager> getEntries():', - lastError.message - ); - } - entries = bin.cached_asset_entries || {}; - } - vAPI.storage.get('extensionLastVersion', onLastVersionRead); - }; - vAPI.storage.get('cached_asset_entries', onLoaded); - }; - exports.entries = getEntries; - - exports.load = function(path, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - 'path': path, - 'content': '' - }; - var cachedContentPath = cachedAssetPathPrefix + path; - var onLoaded = function(bin) { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µMatrix> cachedAssetsManager.load():', details.error); - cbError(details); - return; - } - // Not sure how this can happen, but I've seen it happen. It could - // be because the save occurred while I was stepping in the code - // though, which means it would not occur during normal operation. - // Still, just to be safe. - if ( stringIsNotEmpty(bin[cachedContentPath]) === false ) { - exports.remove(path); - details.error = 'Error: not found'; - cbError(details); - return; - } - details.content = bin[cachedContentPath]; - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries[path] === undefined ) { - details.error = 'Error: not found'; - cbError(details); - return; - } - vAPI.storage.get(cachedContentPath, onLoaded); - }; - getEntries(onEntries); - }; - - exports.save = function(path, content, cbSuccess, cbError) { - cbSuccess = cbSuccess || nullFunc; - cbError = cbError || cbSuccess; - var details = { - path: path, - content: content - }; - if ( content === '' ) { - exports.remove(path); - cbSuccess(details); - return; - } - var cachedContentPath = cachedAssetPathPrefix + path; - var bin = {}; - bin[cachedContentPath] = content; - var removedItems = []; - var onSaved = function() { - var lastError = vAPI.lastError(); - if ( lastError ) { - details.error = 'Error: ' + lastError.message; - console.error('µMatrix> cachedAssetsManager.save():', details.error); - cbError(details); - return; - } - // Saving over an existing item must be seen as removing an - // existing item and adding a new one. - if ( typeof exports.onRemovedListener === 'function' ) { - exports.onRemovedListener(removedItems); - } - cbSuccess(details); - }; - var onEntries = function(entries) { - if ( entries.hasOwnProperty(path) ) { - removedItems.push(path); - } - entries[path] = Date.now(); - bin.cached_asset_entries = entries; - vAPI.storage.set(bin, onSaved); - }; - getEntries(onEntries); - }; - - exports.remove = function(pattern, before) { - var onEntries = function(entries) { - var keystoRemove = []; - var removedItems = []; - var paths = Object.keys(entries); - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( typeof pattern === 'string' && path !== pattern ) { - continue; - } - if ( pattern instanceof RegExp && !pattern.test(path) ) { - continue; - } - if ( typeof before === 'number' && entries[path] >= before ) { - continue; - } - removedItems.push(path); - keystoRemove.push(cachedAssetPathPrefix + path); - delete entries[path]; - } - if ( keystoRemove.length ) { - vAPI.storage.remove(keystoRemove); - vAPI.storage.set({ 'cached_asset_entries': entries }); - if ( typeof exports.onRemovedListener === 'function' ) { - exports.onRemovedListener(removedItems); - } - } - }; - getEntries(onEntries); - }; - - exports.removeAll = function(callback) { - var onEntries = function() { - // Careful! do not remove 'assets/user/' - exports.remove(/^https?:\/\/[a-z0-9]+/); - exports.remove(/^assets\/(umatrix|thirdparties)\//); - exports.remove(/^cache:\/\//); - exports.remove('assets/checksums.txt'); - if ( typeof callback === 'function' ) { - callback(null); - } - }; - getEntries(onEntries); - }; - - exports.rmrf = function() { - exports.remove(/./); - }; - - exports.onRemovedListener = null; - - return exports; -})(); + } + return result; +}; /******************************************************************************/ -var getTextFileFromURL = function(url, onLoad, onError) { - // console.log('µMatrix.assets/getTextFileFromURL("%s"):', url); +api.fetchText = function(url, onLoad, onError) { + var actualUrl = reIsExternalPath.test(url) ? url : vAPI.getURL(url); + + if ( typeof onError !== 'function' ) { + onError = onLoad; + } // https://github.com/gorhill/uMatrix/issues/15 var onResponseReceived = function() { this.onload = this.onerror = this.ontimeout = null; // xhr for local files gives status 0, but actually succeeds - var status = this.status || 200; - if ( status < 200 || status >= 300 ) { - return onError.call(this); + var details = { + url: url, + content: '', + statusCode: this.status || 200, + statusText: this.statusText || '' + }; + if ( details.statusCode < 200 || details.statusCode >= 300 ) { + return onError.call(null, details); } // consider an empty result to be an error if ( stringIsNotEmpty(this.responseText) === false ) { - return onError.call(this); + return onError.call(null, details); } // we never download anything else than plain text: discard if response // appears to be a HTML document: could happen when server serves // some kind of error page I suppose var text = this.responseText.trim(); - if ( text.charAt(0) === '<' && text.slice(-1) === '>' ) { - return onError.call(this); + if ( text.startsWith('<') && text.endsWith('>') ) { + return onError.call(null, details); } - return onLoad.call(this); + details.content = this.responseText; + return onLoad.call(null, details); }; var onErrorReceived = function() { this.onload = this.onerror = this.ontimeout = null; - onError.call(this); + µMatrix.logger.writeOne('', 'error', errorCantConnectTo.replace('{{msg}}', actualUrl)); + onError.call(null, { url: url, content: '' }); }; // Be ready for thrown exceptions: @@ -318,8 +110,8 @@ var getTextFileFromURL = function(url, onLoad, onError) { // `file:///` on Chromium 40 results in an exception being thrown. var xhr = new XMLHttpRequest(); try { - xhr.open('get', url, true); - xhr.timeout = xhrTimeout; + xhr.open('get', actualUrl, true); + xhr.timeout = 30000; xhr.onload = onResponseReceived; xhr.onerror = onErrorReceived; xhr.ontimeout = onErrorReceived; @@ -330,1154 +122,791 @@ var getTextFileFromURL = function(url, onLoad, onError) { } }; -/******************************************************************************/ +/******************************************************************************* -var updateLocalChecksums = function() { - var localChecksums = []; - var entries = repoMetadata.entries; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entry = entries[path]; - if ( entry.localChecksum !== '' ) { - localChecksums.push(entry.localChecksum + ' ' + path); - } - } - cachedAssetsManager.save('assets/checksums.txt', localChecksums.join('\n')); + TODO(seamless migration): + This block of code will be removed when I am confident all users have + moved to a version of uBO which does not require the old way of caching + assets. + + api.listKeyAliases: a map of old asset keys to new asset keys. + + migrate(): to seamlessly migrate the old cache manager to the new one: + - attempt to preserve and move content of cached assets to new locations; + - removes all traces of now obsolete cache manager entries in cacheStorage. + + This code will typically execute only once, when the newer version of uBO + is first installed and executed. + +**/ + +api.listKeyAliases = { + "assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat": "public_suffix_list.dat", + "assets/thirdparties/hosts-file.net/ad-servers": "hphosts", + "assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt": "malware-0", + "assets/thirdparties/mirror1.malwaredomains.com/files/justdomains": "malware-1", + "assets/thirdparties/pgl.yoyo.org/as/serverlist": "plowe-0", + "assets/thirdparties/someonewhocares.org/hosts/hosts": "dpollock-0", + "assets/thirdparties/winhelp2002.mvps.org/hosts.txt": "mvps-0" }; -/******************************************************************************/ +var migrate = function(callback) { + var entries, + moveCount = 0, + toRemove = []; -// Gather meta data of all assets. + var countdown = function(change) { + moveCount -= (change || 0); + if ( moveCount !== 0 ) { return; } + vAPI.cacheStorage.remove(toRemove); + saveAssetCacheRegistry(); + callback(); + }; -var getRepoMetadata = function(callback) { - callback = callback || nullFunc; - - // https://github.com/chrisaljoudi/uBlock/issues/515 - // Handle re-entrancy here, i.e. we MUST NOT tamper with the waiting list - // of callers, if any, except to add one at the end of the list. - if ( repoMetadata !== null && repoMetadata.waiting.length !== 0 ) { - repoMetadata.waiting.push(callback); - return; - } - - if ( exports.remoteFetchBarrier === 0 && lastRepoMetaIsRemote === false ) { - lastRepoMetaTimestamp = 0; - } - if ( (Date.now() - lastRepoMetaTimestamp) >= refreshRepoMetaPeriod ) { - repoMetadata = null; - } - if ( repoMetadata !== null ) { - callback(repoMetadata); - return; - } - - lastRepoMetaTimestamp = Date.now(); - lastRepoMetaIsRemote = exports.remoteFetchBarrier === 0; - - var localChecksums; - var repoChecksums; - - var checksumsReceived = function() { - if ( localChecksums === undefined || repoChecksums === undefined ) { - return; - } - // Remove from cache assets which no longer exist in the repo - var entries = repoMetadata.entries; - var checksumsChanged = false; - var entry; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; + var onContentRead = function(oldKey, newKey, bin) { + var content = bin && bin['cached_asset_content://' + oldKey] || undefined; + if ( content ) { + assetCacheRegistry[newKey] = { + readTime: Date.now(), + writeTime: entries[oldKey] + }; + if ( reIsExternalPath.test(oldKey) ) { + assetCacheRegistry[newKey].remoteURL = oldKey; } - entry = entries[path]; - // If repo checksums could not be fetched, assume no change - if ( repoChecksums === '' ) { - entry.repoChecksum = entry.localChecksum; + bin = {}; + bin['cache/' + newKey] = content; + vAPI.cacheStorage.set(bin); + } + countdown(1); + }; + + var onEntries = function(bin) { + entries = bin && bin.cached_asset_entries; + if ( !entries ) { return callback(); } + if ( bin && bin.assetCacheRegistry ) { + assetCacheRegistry = bin.assetCacheRegistry; + } + var aliases = api.listKeyAliases; + for ( var oldKey in entries ) { + if ( oldKey.endsWith('assets/user/filters.txt') ) { continue; } + var newKey = aliases[oldKey]; + if ( !newKey && /^https?:\/\//.test(oldKey) ) { + newKey = oldKey; } - if ( entry.repoChecksum !== '' || entry.localChecksum === '' ) { - continue; + if ( newKey ) { + vAPI.cacheStorage.get( + 'cached_asset_content://' + oldKey, + onContentRead.bind(null, oldKey, newKey) + ); + moveCount += 1; } - checksumsChanged = true; - cachedAssetsManager.remove(path); - entry.localChecksum = ''; + toRemove.push('cached_asset_content://' + oldKey); } - if ( checksumsChanged ) { - updateLocalChecksums(); - } - // Notify all waiting callers - // https://github.com/chrisaljoudi/uBlock/issues/515 - // VERY IMPORTANT: because of re-entrancy, we MUST: - // - process the waiting callers in a FIFO manner - // - not cache repoMetadata.waiting.length, we MUST use the live - // value, because it can change while looping - // - not change the waiting list until they are all processed - for ( var i = 0; i < repoMetadata.waiting.length; i++ ) { - repoMetadata.waiting[i](repoMetadata); - } - repoMetadata.waiting.length = 0; + toRemove.push('cached_asset_entries', 'extensionLastVersion'); + countdown(); }; - var validateChecksums = function(details) { - if ( details.error || details.content === '' ) { - return ''; - } - if ( /^(?:[0-9a-f]{32}\s+\S+(?:\s+|$))+/.test(details.content) === false ) { - return ''; - } - return details.content; - }; - - var parseChecksums = function(text, which) { - var entries = repoMetadata.entries; - var lines = text.split(/\n+/); - var i = lines.length; - var fields, assetPath; - while ( i-- ) { - fields = lines[i].trim().split(/\s+/); - if ( fields.length !== 2 ) { - continue; - } - assetPath = fields[1]; - if ( entries[assetPath] === undefined ) { - entries[assetPath] = new AssetEntry(); - } - entries[assetPath][which + 'Checksum'] = fields[0]; - } - }; - - var onLocalChecksumsLoaded = function(details) { - if ( (localChecksums = validateChecksums(details)) ) { - parseChecksums(localChecksums, 'local'); - } - checksumsReceived(); - }; - - var onRepoChecksumsLoaded = function(details) { - if ( (repoChecksums = validateChecksums(details)) ) { - parseChecksums(repoChecksums, 'repo'); - } - checksumsReceived(); - }; - - repoMetadata = new RepoMetadata(); - repoMetadata.waiting.push(callback); - readRepoFile('assets/checksums.txt', onRepoChecksumsLoaded); - readLocalFile('assets/checksums.txt', onLocalChecksumsLoaded); -}; - -// https://www.youtube.com/watch?v=-t3WYfgM4x8 - -/******************************************************************************/ - -exports.setHomeURL = function(path, homeURL) { - if ( typeof homeURL !== 'string' || homeURL === '' ) { - return; - } - homeURLs[path] = homeURL; -}; - -/******************************************************************************/ - -// Get a local asset, do not look-up repo or remote location if local asset -// is not found. - -var readLocalFile = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; - - var onInstallFileLoaded = function() { - //console.log('µMatrix> readLocalFile("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µMatrix> readLocalFile("%s") / onInstallFileError()', path); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µMatrix> readLocalFile("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function(details) { - //console.error('µMatrix> readLocalFile("%s") / onCachedContentError()', path); - if ( reIsExternalPath.test(path) ) { - reportBack('', 'Error: asset not found'); - return; - } - // It's ok for user data to not be found - if ( reIsUserPath.test(path) ) { - reportBack(''); - return; - } - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; - - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); -}; - -// https://www.youtube.com/watch?v=r9KVpuFPtHc - -/******************************************************************************/ - -// Get the repository copy of a built-in asset. - -var readRepoFile = function(path, callback) { - // https://github.com/chrisaljoudi/uBlock/issues/426 - if ( exports.remoteFetchBarrier !== 0 ) { - readLocalFile(path, callback); - return; - } - - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content, - 'error': err - }; - callback(details); - }; - - var repositoryURL = projectRepositoryRoot + path; - - var onRepoFileLoaded = function() { - //console.log('µMatrix> readRepoFile("%s") / onRepoFileLoaded()', path); - // https://github.com/gorhill/httpswitchboard/issues/263 - if ( this.status === 200 ) { - reportBack(this.responseText); - } else { - reportBack('', 'Error: ' + this.statusText); - } - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - reportBack('', 'Error'); - }; - - // 'umatrix=...' is to skip browser cache - getTextFileFromURL( - repositoryURL + '?umatrix=' + Date.now(), - onRepoFileLoaded, - onRepoFileError + vAPI.cacheStorage.get( + [ 'cached_asset_entries', 'assetCacheRegistry' ], + onEntries ); }; -/******************************************************************************/ +/******************************************************************************* -// An asset from an external source with a copy shipped with the extension: -// Path --> starts with 'assets/(thirdparties|umatrix)/', with a home URL -// External --> -// Repository --> has checksum (to detect need for update only) -// Cache --> has expiration timestamp (in cache) -// Local --> install time version + The purpose of the asset source registry is to keep key detail information + about an asset: + - Where to load it from: this may consist of one or more URLs, either local + or remote. + - After how many days an asset should be deemed obsolete -- i.e. in need of + an update. + - The origin and type of an asset. + - The last time an asset was registered. -var readRepoCopyAsset = function(path, callback) { - var assetEntry; - var homeURL = homeURLs[path]; +**/ - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; +var assetSourceRegistryStatus, + assetSourceRegistry = Object.create(null); - var updateChecksum = function() { - if ( assetEntry !== undefined && assetEntry.repoChecksum !== assetEntry.localChecksum ) { - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); - } - }; - - var onInstallFileLoaded = function() { - //console.log('µMatrix> readRepoCopyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µMatrix> readRepoCopyAsset("%s") / onInstallFileError():', path, this.statusText); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µMatrix> readRepoCopyAsset("%s") / onCacheFileLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function(details) { - //console.log('µMatrix> readRepoCopyAsset("%s") / onCacheFileError()', path); - getTextFileFromURL(vAPI.getURL(details.path), onInstallFileLoaded, onInstallFileError); - }; - - var repositoryURL = projectRepositoryRoot + path; - var repositoryURLSkipCache = repositoryURL + '?umatrix=' + Date.now(); - - var onRepoFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µMatrix> readRepoCopyAsset("%s") / onRepoFileLoaded("%s"): error', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µMatrix> readRepoCopyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onHomeFileLoaded = function() { - if ( stringIsNotEmpty(this.responseText) === false ) { - console.error('µMatrix> readRepoCopyAsset("%s") / onHomeFileLoaded("%s"): no response', path, homeURL); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); - } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - } - return; - } - //console.log('µMatrix> readRepoCopyAsset("%s") / onHomeFileLoaded("%s")', path, homeURL); - updateChecksum(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onHomeFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', homeURL)); - // Fetch from repo only if obsolescence was due to repo checksum - if ( assetEntry.localChecksum !== assetEntry.repoChecksum ) { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); +var registerAssetSource = function(assetKey, dict) { + var entry = assetSourceRegistry[assetKey] || {}; + for ( var prop in dict ) { + if ( dict.hasOwnProperty(prop) === false ) { continue; } + if ( dict[prop] === undefined ) { + delete entry[prop]; } else { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + entry[prop] = dict[prop]; } - }; - - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Auto-update enabled AND (not in cache OR in cache but obsolete) - var timestamp = entries[path]; - var inCache = typeof timestamp === 'number'; - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && stringIsNotEmpty(homeURL) - ) { - if ( inCache === false || cacheIsObsolete(timestamp) ) { - //console.log('µMatrix> readRepoCopyAsset("%s") / onCacheMetaReady(): not cached or obsolete', path); - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - return; + } + var contentURL = dict.contentURL; + if ( contentURL !== undefined ) { + if ( typeof contentURL === 'string' ) { + contentURL = entry.contentURL = [ contentURL ]; + } else if ( Array.isArray(contentURL) === false ) { + contentURL = entry.contentURL = []; + } + var remoteURLCount = 0; + for ( var i = 0; i < contentURL.length; i++ ) { + if ( reIsExternalPath.test(contentURL[i]) ) { + remoteURLCount += 1; } } + entry.hasLocalURL = remoteURLCount !== contentURL.length; + entry.hasRemoteURL = remoteURLCount !== 0; + } else if ( entry.contentURL === undefined ) { + entry.contentURL = []; + } + if ( typeof entry.updateAfter !== 'number' ) { + entry.updateAfter = 13; + } + if ( entry.submitter ) { + entry.submitTime = Date.now(); // To detect stale entries + } + assetSourceRegistry[assetKey] = entry; +}; - // In cache - if ( inCache ) { - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; +var unregisterAssetSource = function(assetKey) { + assetCacheRemove(assetKey); + delete assetSourceRegistry[assetKey]; +}; + +var saveAssetSourceRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetSourceRegistry: assetSourceRegistry }); + }; + return function(lazily) { + if ( timer !== undefined ) { + clearTimeout(timer); } + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; +})(); - // Not in cache - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); +var updateAssetSourceRegistry = function(json, silent) { + var newDict; + try { + newDict = JSON.parse(json); + } catch (ex) { + } + if ( newDict instanceof Object === false ) { return; } + + var oldDict = assetSourceRegistry, + assetKey; + + // Remove obsolete entries (only those which were built-in). + for ( assetKey in oldDict ) { + if ( + newDict[assetKey] === undefined && + oldDict[assetKey].submitter === undefined + ) { + unregisterAssetSource(assetKey); + } + } + // Add/update existing entries. Notify of new asset sources. + for ( assetKey in newDict ) { + if ( oldDict[assetKey] === undefined && !silent ) { + fireNotification( + 'builtin-asset-source-added', + { assetKey: assetKey, entry: newDict[assetKey] } + ); + } + registerAssetSource(assetKey, newDict[assetKey]); + } + saveAssetSourceRegistry(); +}; + +var getAssetSourceRegistry = function(callback) { + // Already loaded. + if ( assetSourceRegistryStatus === 'ready' ) { + callback(assetSourceRegistry); + return; + } + + // Being loaded. + if ( Array.isArray(assetSourceRegistryStatus) ) { + assetSourceRegistryStatus.push(callback); + return; + } + + // Not loaded: load it. + assetSourceRegistryStatus = [ callback ]; + + var registryReady = function() { + var callers = assetSourceRegistryStatus; + assetSourceRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetSourceRegistry); + } }; - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } - - // Repo copy changed: fetch from home URL - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µMatrix> readRepoCopyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - if ( stringIsNotEmpty(homeURL) ) { - getTextFileFromURL(homeURL, onHomeFileLoaded, onHomeFileError); - } else { - getTextFileFromURL(repositoryURLSkipCache, onRepoFileLoaded, onRepoFileError); + // First-install case. + var createRegistry = function() { + api.fetchText( + µMatrix.assetsBootstrapLocation || 'assets/assets.json', + function(details) { + updateAssetSourceRegistry(details.content, true); + registryReady(); } + ); + }; + + vAPI.cacheStorage.get('assetSourceRegistry', function(bin) { + if ( !bin || !bin.assetSourceRegistry ) { + createRegistry(); return; } - - // Load from cache - cachedAssetsManager.entries(onCacheMetaReady); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -// https://www.youtube.com/watch?v=uvUW4ozs7pY - -/******************************************************************************/ - -// An important asset shipped with the extension -- typically small, or -// doesn't change often: -// Path --> starts with 'assets/(thirdparties|umatrix)/', without a home URL -// Repository --> has checksum (to detect need for update and corruption) -// Cache --> whatever from above -// Local --> install time version - -var readRepoOnlyAsset = function(path, callback) { - - var assetEntry; - - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; - } - callback(details); - }; - - var onInstallFileLoaded = function() { - //console.log('µMatrix> readRepoOnlyAsset("%s") / onInstallFileLoaded()', path); - reportBack(this.responseText); - }; - - var onInstallFileError = function() { - console.error('µMatrix> readRepoOnlyAsset("%s") / onInstallFileError()', path); - reportBack('', 'Error'); - }; - - var onCachedContentLoaded = function(details) { - //console.log('µMatrix> readRepoOnlyAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - //console.log('µMatrix> readRepoOnlyAsset("%s") / onCachedContentError()', path); - getTextFileFromURL(vAPI.getURL(path), onInstallFileLoaded, onInstallFileError); - }; - - var repositoryURL = projectRepositoryRoot + path + '?umatrix=' + Date.now(); - - var onRepoFileLoaded = function() { - if ( typeof this.responseText !== 'string' ) { - console.error('µMatrix> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): no response', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - if ( YaMD5.hashStr(this.responseText) !== assetEntry.repoChecksum ) { - console.error('µMatrix> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s"): bad md5 checksum', path, repositoryURL); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - return; - } - //console.log('µMatrix> readRepoOnlyAsset("%s") / onRepoFileLoaded("%s")', path, repositoryURL); - assetEntry.localChecksum = assetEntry.repoChecksum; - updateLocalChecksums(); - cachedAssetsManager.save(path, this.responseText, callback); - }; - - var onRepoFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', repositoryURL)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onRepoMetaReady = function(meta) { - assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - reportBack('', 'Error: asset not found'); - return; - } - - // Asset added or changed: load from repo URL and then cache result - if ( - exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - assetEntry.localChecksum !== assetEntry.repoChecksum - ) { - //console.log('µMatrix> readRepoOnlyAsset("%s") / onRepoMetaReady(): repo has newer version', path); - getTextFileFromURL(repositoryURL, onRepoFileLoaded, onRepoFileError); - return; - } - - // Load from cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - getRepoMetadata(onRepoMetaReady); -}; - -/******************************************************************************/ - -// Asset doesn't exist. Just for symmetry purpose. - -var readNilAsset = function(path, callback) { - callback({ - 'path': path, - 'content': '', - 'error': 'Error: asset not found' + assetSourceRegistry = bin.assetSourceRegistry; + registryReady(); }); }; -/******************************************************************************/ +api.registerAssetSource = function(assetKey, details) { + getAssetSourceRegistry(function() { + registerAssetSource(assetKey, details); + saveAssetSourceRegistry(true); + }); +}; -// An external asset: -// Path --> starts with 'http' -// External --> https://..., http://... -// Cache --> has expiration timestamp (in cache) +api.unregisterAssetSource = function(assetKey) { + getAssetSourceRegistry(function() { + unregisterAssetSource(assetKey); + saveAssetSourceRegistry(true); + }); +}; -var readExternalAsset = function(path, callback) { - var reportBack = function(content, err) { - var details = { - 'path': path, - 'content': content - }; - if ( err ) { - details.error = err; +/******************************************************************************* + + The purpose of the asset cache registry is to keep track of all assets + which have been persisted into the local cache. + +**/ + +var assetCacheRegistryStatus, + assetCacheRegistryStartTime = Date.now(), + assetCacheRegistry = {}; + +var getAssetCacheRegistry = function(callback) { + // Already loaded. + if ( assetCacheRegistryStatus === 'ready' ) { + callback(assetCacheRegistry); + return; + } + + // Being loaded. + if ( Array.isArray(assetCacheRegistryStatus) ) { + assetCacheRegistryStatus.push(callback); + return; + } + + // Not loaded: load it. + assetCacheRegistryStatus = [ callback ]; + + var registryReady = function() { + var callers = assetCacheRegistryStatus; + assetCacheRegistryStatus = 'ready'; + var fn; + while ( (fn = callers.shift()) ) { + fn(assetCacheRegistry); } + }; + + var migrationDone = function() { + vAPI.cacheStorage.get('assetCacheRegistry', function(bin) { + if ( bin && bin.assetCacheRegistry ) { + assetCacheRegistry = bin.assetCacheRegistry; + } + registryReady(); + }); + }; + + migrate(migrationDone); +}; + +var saveAssetCacheRegistry = (function() { + var timer; + var save = function() { + timer = undefined; + vAPI.cacheStorage.set({ assetCacheRegistry: assetCacheRegistry }); + }; + return function(lazily) { + if ( timer !== undefined ) { clearTimeout(timer); } + if ( lazily ) { + timer = vAPI.setTimeout(save, 500); + } else { + save(); + } + }; +})(); + +var assetCacheRead = function(assetKey, callback) { + var internalKey = 'cache/' + assetKey; + + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { details.error = err; } callback(details); }; - var onCachedContentLoaded = function(details) { - //console.log('µMatrix> readExternalAsset("%s") / onCachedContentLoaded()', path); - reportBack(details.content); - }; - - var onCachedContentError = function() { - console.error('µMatrix> readExternalAsset("%s") / onCachedContentError()', path); - reportBack('', 'Error'); - }; - - var onExternalFileLoaded = function() { - // https://github.com/chrisaljoudi/uBlock/issues/708 - // A successful download should never return an empty file: turn this - // into an error condition. - if ( stringIsNotEmpty(this.responseText) === false ) { - onExternalFileError(); - return; + var onAssetRead = function(bin) { + if ( !bin || !bin[internalKey] ) { + return reportBack('', 'E_NOTFOUND'); } - //console.log('µMatrix> readExternalAsset("%s") / onExternalFileLoaded1()', path); - cachedAssetsManager.save(path, this.responseText); - reportBack(this.responseText); - }; - - var onExternalFileError = function() { - console.error(errorCantConnectTo.replace('{{url}}', path)); - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); - }; - - var onCacheMetaReady = function(entries) { - // Fetch from remote if: - // - Not in cache OR - // - // - Auto-update enabled AND in cache but obsolete - var timestamp = entries[path]; - var notInCache = typeof timestamp !== 'number'; - var updateCache = exports.remoteFetchBarrier === 0 && - exports.autoUpdate && - cacheIsObsolete(timestamp); - if ( notInCache || updateCache ) { - getTextFileFromURL(path, onExternalFileLoaded, onExternalFileError); - return; + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + return reportBack('', 'E_NOTFOUND'); } - - // In cache - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); + entry.readTime = Date.now(); + saveAssetCacheRegistry(true); + reportBack(bin[internalKey]); }; - cachedAssetsManager.entries(onCacheMetaReady); + var onReady = function() { + vAPI.cacheStorage.get(internalKey, onAssetRead); + }; + + getAssetCacheRegistry(onReady); }; -/******************************************************************************/ - -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved - -var readUserAsset = function(path, callback) { - var onCachedContentLoaded = function(details) { - //console.log('µMatrix.assets/readUserAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; - - var onCachedContentError = function() { - //console.log('µMatrix.assets/readUserAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); - }; - - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); -}; - -/******************************************************************************/ - -// Asset available only from the cache. -// Cache data: -// Path --> starts with 'cache://' -// Cache --> whatever - -var readCacheAsset = function(path, callback) { - var onCachedContentLoaded = function(details) { - //console.log('µMatrix.assets/readCacheAsset("%s")/onCachedContentLoaded()', path); - callback({ 'path': path, 'content': details.content }); - }; - - var onCachedContentError = function() { - //console.log('µMatrix.assets/readCacheAsset("%s")/onCachedContentError()', path); - callback({ 'path': path, 'content': '' }); - }; - - cachedAssetsManager.load(path, onCachedContentLoaded, onCachedContentError); -}; - -/******************************************************************************/ - -// Assets -// -// A copy of an asset from an external source shipped with the extension: -// Path --> starts with 'assets/(thirdparties|umatrix)/', with a home URL -// External --> -// Repository --> has checksum (to detect obsolescence) -// Cache --> has expiration timestamp (to detect obsolescence) -// Local --> install time version -// -// An important asset shipped with the extension (usually small, or doesn't -// change often): -// Path --> starts with 'assets/(thirdparties|umatrix)/', without a home URL -// Repository --> has checksum (to detect obsolescence or data corruption) -// Cache --> whatever from above -// Local --> install time version -// -// An external filter list: -// Path --> starts with 'http' -// External --> -// Cache --> has expiration timestamp (to detect obsolescence) -// -// User data: -// Path --> starts with 'assets/user/' -// Cache --> whatever user saved -// -// When a checksum is present, it is used to determine whether the asset -// needs to be updated. -// When an expiration timestamp is present, it is used to determine whether -// the asset needs to be updated. -// -// If no update required, an asset if first fetched from the cache. If the -// asset is not cached it is fetched from the closest location: local for -// an asset shipped with the extension, external for an asset not shipped -// with the extension. - -exports.get = function(path, callback) { - - if ( reIsUserPath.test(path) ) { - readUserAsset(path, callback); - return; +var assetCacheWrite = function(assetKey, details, callback) { + var internalKey = 'cache/' + assetKey; + var content = ''; + if ( typeof details === 'string' ) { + content = details; + } else if ( details instanceof Object ) { + content = details.content || ''; } - if ( reIsCachePath.test(path) ) { - readCacheAsset(path, callback); - return; + if ( content === '' ) { + return assetCacheRemove(assetKey, callback); } - if ( reIsExternalPath.test(path) ) { - readExternalAsset(path, callback); - return; - } - - var onRepoMetaReady = function(meta) { - var assetEntry = meta.entries[path]; - - // Asset doesn't exist - if ( assetEntry === undefined ) { - readNilAsset(path, callback); - return; + var reportBack = function(content) { + var details = { assetKey: assetKey, content: content }; + if ( typeof callback === 'function' ) { + callback(details); } - - // Asset is repo copy of external content - if ( - homeURLs.hasOwnProperty(path) && - stringIsNotEmpty(homeURLs[path]) - ) { - readRepoCopyAsset(path, callback); - return; - } - - // Asset is repo only - readRepoOnlyAsset(path, callback); + fireNotification('after-asset-updated', details); }; - getRepoMetadata(onRepoMetaReady); + var onReady = function() { + var entry = assetCacheRegistry[assetKey]; + if ( entry === undefined ) { + entry = assetCacheRegistry[assetKey] = {}; + } + entry.writeTime = entry.readTime = Date.now(); + if ( details instanceof Object && typeof details.url === 'string' ) { + entry.remoteURL = details.url; + } + var bin = { assetCacheRegistry: assetCacheRegistry }; + bin[internalKey] = content; + vAPI.cacheStorage.set(bin); + reportBack(content); + }; + getAssetCacheRegistry(onReady); }; -// https://www.youtube.com/watch?v=98y0Q7nLGWk - -/******************************************************************************/ - -exports.getLocal = readLocalFile; - -/******************************************************************************/ - -exports.put = function(path, content, callback) { - cachedAssetsManager.save(path, content, callback); -}; - -/******************************************************************************/ - -exports.rmrf = function() { - cachedAssetsManager.rmrf(); -}; - -/******************************************************************************/ - -exports.metadata = function(callback) { - var out = {}; - - // https://github.com/chrisaljoudi/uBlock/issues/186 - // We need to check cache obsolescence when both cache and repo meta data - // has been gathered. - var checkCacheObsolescence = function() { - var entry; - for ( var path in out ) { - if ( out.hasOwnProperty(path) === false ) { +var assetCacheRemove = function(pattern, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + removedEntries = [], + removedContent = []; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp && !pattern.test(assetKey) ) { continue; } - entry = out[path]; - entry.cacheObsolete = - homeURLs.hasOwnProperty(path) && - stringIsNotEmpty(homeURLs[path]) && - cacheIsObsolete(entry.lastModified || 0); + if ( typeof pattern === 'string' && assetKey !== pattern ) { + continue; + } + removedEntries.push(assetKey); + removedContent.push('cache/' + assetKey); + delete cacheDict[assetKey]; + } + if ( removedContent.length !== 0 ) { + vAPI.cacheStorage.remove(removedContent); + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); + } + if ( typeof callback === 'function' ) { + callback(); + } + for ( var i = 0; i < removedEntries.length; i++ ) { + fireNotification('after-asset-updated', { assetKey: removedEntries[i] }); } - callback(out); }; - var onRepoMetaReady = function(meta) { - var entries = meta.entries; - var entryRepo, entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; + getAssetCacheRegistry(onReady); +}; + +var assetCacheMarkAsDirty = function(pattern, exclude, callback) { + var onReady = function() { + var cacheDict = assetCacheRegistry, + cacheEntry, + mustSave = false; + for ( var assetKey in cacheDict ) { + if ( pattern instanceof RegExp ) { + if ( pattern.test(assetKey) === false ) { continue; } + } else if ( typeof pattern === 'string' ) { + if ( assetKey !== pattern ) { continue; } + } else if ( Array.isArray(pattern) ) { + if ( pattern.indexOf(assetKey) === -1 ) { continue; } } - entryRepo = entries[path]; - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; + if ( exclude instanceof RegExp ) { + if ( exclude.test(assetKey) ) { continue; } + } else if ( typeof exclude === 'string' ) { + if ( assetKey === exclude ) { continue; } + } else if ( Array.isArray(exclude) ) { + if ( exclude.indexOf(assetKey) !== -1 ) { continue; } } - entryOut.localChecksum = entryRepo.localChecksum; - entryOut.repoChecksum = entryRepo.repoChecksum; - entryOut.homeURL = homeURLs[path] || ''; - entryOut.repoObsolete = entryOut.localChecksum !== entryOut.repoChecksum; + cacheEntry = cacheDict[assetKey]; + if ( !cacheEntry.writeTime ) { continue; } + cacheDict[assetKey].writeTime = 0; + mustSave = true; } - checkCacheObsolescence(); - }; - - var onCacheMetaReady = function(entries) { - var entryOut; - for ( var path in entries ) { - if ( entries.hasOwnProperty(path) === false ) { - continue; - } - entryOut = out[path]; - if ( entryOut === undefined ) { - entryOut = out[path] = {}; - } - entryOut.lastModified = entries[path]; - // User data is not literally cache data - if ( reIsUserPath.test(path) ) { - continue; - } - entryOut.cached = true; - if ( reIsExternalPath.test(path) ) { - entryOut.homeURL = path; - } - } - getRepoMetadata(onRepoMetaReady); - }; - - cachedAssetsManager.entries(onCacheMetaReady); -}; - -/******************************************************************************/ - -exports.purge = function(pattern, before) { - cachedAssetsManager.remove(pattern, before); -}; - -exports.purgeAll = function(callback) { - cachedAssetsManager.removeAll(callback); -}; - -/******************************************************************************/ - -exports.onAssetCacheRemoved = { - addListener: function(callback) { - cachedAssetsManager.onRemovedListener = callback || null; - } -}; - -/******************************************************************************/ - -return exports; - -})(); - -/******************************************************************************/ -/******************************************************************************/ - -µMatrix.assetUpdater = (function() { - -'use strict'; - -/******************************************************************************/ - -var µm = µMatrix; - -var updateDaemonTimer = null; -var autoUpdateDaemonTimerPeriod = 11 * 60 * 1000; // 11 minutes -var manualUpdateDaemonTimerPeriod = 5 * 1000; // 5 seconds - -var updateCycleFirstPeriod = 7 * 60 * 1000; // 7 minutes -var updateCycleNextPeriod = 11 * 60 * 60 * 1000; // 11 hours -var updateCycleTime = 0; - -var toUpdate = {}; -var toUpdateCount = 0; -var updated = {}; -var updatedCount = 0; -var metadata = null; - -var onStartListener = null; -var onCompletedListener = null; -var onAssetUpdatedListener = null; - -var exports = { - manualUpdate: false, - manualUpdateProgress: { - value: 0, - text: null - } -}; - -/******************************************************************************/ - -var onOneUpdated = function(details) { - // Resource fetched, we can safely restart the daemon. - scheduleUpdateDaemon(); - - var path = details.path; - if ( details.error ) { - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount)); - //console.debug('µMatrix.assetUpdater/onOneUpdated: "%s" failed', path); - return; - } - - //console.debug('µMatrix.assetUpdater/onOneUpdated: "%s"', path); - updated[path] = true; - updatedCount += 1; - - if ( typeof onAssetUpdatedListener === 'function' ) { - onAssetUpdatedListener(details); - } - - manualUpdateNotify(false, updatedCount / (updatedCount + toUpdateCount + 1)); -}; - -/******************************************************************************/ - -var updateOne = function() { - // Because this can be called from outside the daemon's main loop - µm.assets.autoUpdate = µm.userSettings.autoUpdate || exports.manualUpdate; - - var metaEntry; - var updatingCount = 0; - var updatingText = null; - - for ( var path in toUpdate ) { - if ( toUpdate.hasOwnProperty(path) === false ) { - continue; - } - if ( toUpdate[path] !== true ) { - continue; - } - toUpdate[path] = false; - toUpdateCount -= 1; - if ( metadata.hasOwnProperty(path) === false ) { - continue; - } - metaEntry = metadata[path]; - if ( !metaEntry.cacheObsolete && !metaEntry.repoObsolete ) { - continue; - } - - // Will restart the update daemon once the resource is received: the - // fetching of a resource may take some time, possibly beyond the - // next scheduled daemon cycle, so this ensure the daemon won't do - // anything else before the resource is fetched (or times out). - suspendUpdateDaemon(); - - //console.debug('µMatrix.assetUpdater/updateOne: assets.get("%s")', path); - µm.assets.get(path, onOneUpdated); - updatingCount = 1; - updatingText = metaEntry.homeURL || path; - break; - } - - manualUpdateNotify( - false, - (updatedCount + updatingCount/2) / (updatedCount + toUpdateCount + updatingCount + 1), - updatingText - ); -}; - -/******************************************************************************/ - -// Update one asset, fetch metadata if not done yet. - -var safeUpdateOne = function() { - if ( metadata !== null ) { - updateOne(); - return; - } - - // Because this can be called from outside the daemon's main loop - µm.assets.autoUpdate = µm.userSettings.autoUpdate || exports.manualUpdate; - - var onMetadataReady = function(response) { - scheduleUpdateDaemon(); - metadata = response; - updateOne(); - }; - - suspendUpdateDaemon(); - µm.assets.metadata(onMetadataReady); -}; - -/******************************************************************************/ - -var safeStartListener = function(callback) { - // Because this can be called from outside the daemon's main loop - µm.assets.autoUpdate = µm.userSettings.autoUpdate || exports.manualUpdate; - - var onStartListenerDone = function(assets) { - scheduleUpdateDaemon(); - assets = assets || {}; - for ( var path in assets ) { - if ( assets.hasOwnProperty(path) === false ) { - continue; - } - if ( toUpdate.hasOwnProperty(path) ) { - continue; - } - //console.debug('assets.js > µMatrix.assetUpdater/safeStartListener: "%s"', path); - toUpdate[path] = true; - toUpdateCount += 1; + if ( mustSave ) { + var bin = { assetCacheRegistry: assetCacheRegistry }; + vAPI.cacheStorage.set(bin); } if ( typeof callback === 'function' ) { callback(); } }; - - if ( typeof onStartListener === 'function' ) { - suspendUpdateDaemon(); - onStartListener(onStartListenerDone); - } else { - onStartListenerDone(null); + if ( typeof exclude === 'function' ) { + callback = exclude; + exclude = undefined; } + getAssetCacheRegistry(onReady); }; /******************************************************************************/ -var updateDaemon = function() { - updateDaemonTimer = null; - scheduleUpdateDaemon(); +var stringIsNotEmpty = function(s) { + return typeof s === 'string' && s !== ''; +}; - µm.assets.autoUpdate = µm.userSettings.autoUpdate || exports.manualUpdate; +/******************************************************************************/ - if ( µm.assets.autoUpdate !== true ) { - return; +api.get = function(assetKey, options, callback) { + if ( typeof options === 'function' ) { + callback = options; + options = {}; + } else if ( typeof callback !== 'function' ) { + callback = noopfunc; } - // Start an update cycle? - if ( updateCycleTime !== 0 ) { - if ( Date.now() >= updateCycleTime ) { - //console.debug('µMatrix.assetUpdater/updateDaemon: update cycle started'); - reset(); - safeStartListener(); + var assetDetails = {}, + contentURLs, + contentURL; + + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; } - return; - } + callback(details); + }; - // Any asset to update? - if ( toUpdateCount !== 0 ) { - safeUpdateOne(); - return; - } - // Nothing left to update + var onContentNotLoaded = function() { + var isExternal; + while ( (contentURL = contentURLs.shift()) ) { + isExternal = reIsExternalPath.test(contentURL); + if ( isExternal === false || assetDetails.hasLocalURL !== true ) { + break; + } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + api.fetchText(contentURL, onContentLoaded, onContentNotLoaded); + }; - // In case of manual update, fire progress notifications - manualUpdateNotify(true, 1, ''); - - // If anything was updated, notify listener - if ( updatedCount !== 0 ) { - if ( typeof onCompletedListener === 'function' ) { - //console.debug('µMatrix.assetUpdater/updateDaemon: update cycle completed'); - onCompletedListener({ - updated: JSON.parse(JSON.stringify(updated)), // give callee its own safe copy - updatedCount: updatedCount + var onContentLoaded = function(details) { + if ( stringIsNotEmpty(details.content) === false ) { + onContentNotLoaded(); + return; + } + if ( reIsExternalPath.test(contentURL) && options.dontCache !== true ) { + assetCacheWrite(assetKey, { + content: details.content, + url: contentURL }); } - } + reportBack(details.content); + }; - // Schedule next update cycle - if ( updateCycleTime === 0 ) { - reset(); - //console.debug('µMatrix.assetUpdater/updateDaemon: update cycle re-scheduled'); - updateCycleTime = Date.now() + updateCycleNextPeriod; - } + var onCachedContentLoaded = function(details) { + if ( details.content !== '' ) { + return reportBack(details.content); + } + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + onContentNotLoaded(); + }); + }; + + assetCacheRead(assetKey, onCachedContentLoaded); }; /******************************************************************************/ -var scheduleUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - } - updateDaemonTimer = vAPI.setTimeout( - updateDaemon, - exports.manualUpdate ? manualUpdateDaemonTimerPeriod : autoUpdateDaemonTimerPeriod - ); -}; +var getRemote = function(assetKey, callback) { + var assetDetails = {}, + contentURLs, + contentURL; -var suspendUpdateDaemon = function() { - if ( updateDaemonTimer !== null ) { - clearTimeout(updateDaemonTimer); - updateDaemonTimer = null; - } -}; + var reportBack = function(content, err) { + var details = { assetKey: assetKey, content: content }; + if ( err ) { + details.error = assetDetails.lastError = err; + } else { + assetDetails.lastError = undefined; + } + callback(details); + }; -scheduleUpdateDaemon(); + var onRemoteContentLoaded = function(details) { + if ( stringIsNotEmpty(details.content) === false ) { + registerAssetSource(assetKey, { error: { time: Date.now(), error: 'No content' } }); + tryLoading(); + return; + } + assetCacheWrite(assetKey, { + content: details.content, + url: contentURL + }); + registerAssetSource(assetKey, { error: undefined }); + reportBack(details.content); + }; -/******************************************************************************/ + var onRemoteContentError = function(details) { + var text = details.statusText; + if ( details.statusCode === 0 ) { + text = 'network error'; + } + registerAssetSource(assetKey, { error: { time: Date.now(), error: text } }); + tryLoading(); + }; -var reset = function() { - toUpdate = {}; - toUpdateCount = 0; - updated = {}; - updatedCount = 0; - updateCycleTime = 0; - metadata = null; + var tryLoading = function() { + while ( (contentURL = contentURLs.shift()) ) { + if ( reIsExternalPath.test(contentURL) ) { break; } + } + if ( !contentURL ) { + return reportBack('', 'E_NOTFOUND'); + } + api.fetchText(contentURL, onRemoteContentLoaded, onRemoteContentError); + }; + + getAssetSourceRegistry(function(registry) { + assetDetails = registry[assetKey] || {}; + if ( typeof assetDetails.contentURL === 'string' ) { + contentURLs = [ assetDetails.contentURL ]; + } else if ( Array.isArray(assetDetails.contentURL) ) { + contentURLs = assetDetails.contentURL.slice(0); + } else { + contentURLs = []; + } + tryLoading(); + }); }; /******************************************************************************/ -var manualUpdateNotify = function(done, value, text) { - if ( exports.manualUpdate === false ) { - return; - } +api.put = function(assetKey, content, callback) { + assetCacheWrite(assetKey, content, callback); +}; - exports.manualUpdate = !done; - exports.manualUpdateProgress.value = value || 0; - if ( typeof text === 'string' ) { - exports.manualUpdateProgress.text = text; - } +/******************************************************************************/ - vAPI.messaging.broadcast({ - what: 'forceUpdateAssetsProgress', - done: !exports.manualUpdate, - progress: exports.manualUpdateProgress, - updatedCount: updatedCount +api.metadata = function(callback) { + var assetRegistryReady = false, + cacheRegistryReady = false; + + var onReady = function() { + var assetDict = JSON.parse(JSON.stringify(assetSourceRegistry)), + cacheDict = assetCacheRegistry, + assetEntry, cacheEntry, + now = Date.now(), obsoleteAfter; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry ) { + assetEntry.cached = true; + assetEntry.writeTime = cacheEntry.writeTime; + obsoleteAfter = cacheEntry.writeTime + assetEntry.updateAfter * 86400000; + assetEntry.obsolete = obsoleteAfter < now; + assetEntry.remoteURL = cacheEntry.remoteURL; + } else { + assetEntry.writeTime = 0; + obsoleteAfter = 0; + assetEntry.obsolete = true; + } + } + callback(assetDict); + }; + + getAssetSourceRegistry(function() { + assetRegistryReady = true; + if ( cacheRegistryReady ) { onReady(); } }); - // When manually updating, whatever launched the manual update is - // responsible to launch a reload of the filter lists. - if ( exports.manualUpdate !== true ) { - reset(); - } + getAssetCacheRegistry(function() { + cacheRegistryReady = true; + if ( assetRegistryReady ) { onReady(); } + }); }; /******************************************************************************/ -// Manual update: just a matter of forcing the update daemon to work on a -// tighter schedule. +api.purge = assetCacheMarkAsDirty; -exports.force = function() { - if ( exports.manualUpdate ) { - return; - } +api.remove = function(pattern, callback) { + assetCacheRemove(pattern, callback); +}; - reset(); +api.rmrf = function() { + assetCacheRemove(/./); +}; - exports.manualUpdate = true; +/******************************************************************************/ - var onStartListenerDone = function() { - if ( toUpdateCount === 0 ) { - updateCycleTime = Date.now() + updateCycleNextPeriod; - manualUpdateNotify(true, 1); - } else { - manualUpdateNotify(false, 0); - safeUpdateOne(); +// Asset updater area. +var updaterStatus, + updaterTimer, + updaterAssetDelayDefault = 120000, + updaterAssetDelay = updaterAssetDelayDefault, + updaterUpdated = [], + updaterFetched = new Set(); + +var updateFirst = function() { + updaterStatus = 'updating'; + updaterFetched.clear(); + updaterUpdated = []; + fireNotification('before-assets-updated'); + updateNext(); +}; + +var updateNext = function() { + var assetDict, cacheDict; + + // This will remove a cached asset when it's no longer in use. + var garbageCollectOne = function(assetKey) { + var cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && cacheEntry.readTime < assetCacheRegistryStartTime ) { + assetCacheRemove(assetKey); } }; - safeStartListener(onStartListenerDone); -}; - -/******************************************************************************/ - -exports.onStart = { - addListener: function(callback) { - onStartListener = callback || null; - if ( typeof onStartListener === 'function' ) { - updateCycleTime = Date.now() + updateCycleFirstPeriod; + var findOne = function() { + var now = Date.now(), + assetEntry, cacheEntry; + for ( var assetKey in assetDict ) { + assetEntry = assetDict[assetKey]; + if ( assetEntry.hasRemoteURL !== true ) { continue; } + if ( updaterFetched.has(assetKey) ) { continue; } + cacheEntry = cacheDict[assetKey]; + if ( cacheEntry && (cacheEntry.writeTime + assetEntry.updateAfter * 86400000) > now ) { + continue; + } + if ( fireNotification('before-asset-updated', { assetKey: assetKey }) !== false ) { + return assetKey; + } + garbageCollectOne(assetKey); } + }; + + var updatedOne = function(details) { + if ( details.content !== '' ) { + updaterUpdated.push(details.assetKey); + if ( details.assetKey === 'assets.json' ) { + updateAssetSourceRegistry(details.content); + } + } else { + fireNotification('asset-update-failed', { assetKey: details.assetKey }); + } + if ( findOne() !== undefined ) { + vAPI.setTimeout(updateNext, updaterAssetDelay); + } else { + updateDone(); + } + }; + + var updateOne = function() { + var assetKey = findOne(); + if ( assetKey === undefined ) { + return updateDone(); + } + updaterFetched.add(assetKey); + getRemote(assetKey, updatedOne); + }; + + getAssetSourceRegistry(function(dict) { + assetDict = dict; + if ( !cacheDict ) { return; } + updateOne(); + }); + + getAssetCacheRegistry(function(dict) { + cacheDict = dict; + if ( !assetDict ) { return; } + updateOne(); + }); +}; + +var updateDone = function() { + var assetKeys = updaterUpdated.slice(0); + updaterFetched.clear(); + updaterUpdated = []; + updaterStatus = undefined; + updaterAssetDelay = updaterAssetDelayDefault; + fireNotification('after-assets-updated', { assetKeys: assetKeys }); +}; + +api.updateStart = function(details) { + var oldUpdateDelay = updaterAssetDelay, + newUpdateDelay = details.delay || updaterAssetDelayDefault; + updaterAssetDelay = Math.min(oldUpdateDelay, newUpdateDelay); + if ( updaterStatus !== undefined ) { + if ( newUpdateDelay < oldUpdateDelay ) { + clearTimeout(updaterTimer); + updaterTimer = vAPI.setTimeout(updateNext, updaterAssetDelay); + } + return; + } + updateFirst(); +}; + +api.updateStop = function() { + if ( updaterTimer ) { + clearTimeout(updaterTimer); + updaterTimer = undefined; + } + if ( updaterStatus !== undefined ) { + updateDone(); } }; /******************************************************************************/ -exports.onAssetUpdated = { - addListener: function(callback) { - onAssetUpdatedListener = callback || null; - } -}; +return api; /******************************************************************************/ -exports.onCompleted = { - addListener: function(callback) { - onCompletedListener = callback || null; - } -}; - -/******************************************************************************/ - -// Typically called when an update has been forced. - -exports.restart = function() { - reset(); - updateCycleTime = Date.now() + updateCycleNextPeriod; -}; - -/******************************************************************************/ - -return exports; - })(); /******************************************************************************/ diff --git a/src/js/background.js b/src/js/background.js index 6e7bc34..0376346 100644 --- a/src/js/background.js +++ b/src/js/background.js @@ -1,7 +1,7 @@ /******************************************************************************* µMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,11 +19,11 @@ Home: https://github.com/gorhill/uMatrix */ -/* global chrome */ +'use strict'; /******************************************************************************/ -var µMatrix = (function() { +var µMatrix = (function() { // jshint ignore:line /******************************************************************************/ @@ -127,12 +127,8 @@ return { updateAssetsEvery: 11 * oneDay + 1 * oneHour + 1 * oneMinute + 1 * oneSecond, firstUpdateAfter: 11 * oneMinute, nextUpdateAfter: 11 * oneHour, - projectServerRoot: 'https://raw.githubusercontent.com/gorhill/umatrix/master/', - pslPath: 'assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat', - - // permanent hosts files - permanentHostsFiles: { - }, + assetsBootstrapLocation: 'assets/assets.json', + pslAssetKey: 'public_suffix_list.dat', // list of live hosts files liveHostsFiles: { diff --git a/src/js/hosts-files.js b/src/js/hosts-files.js index 37612a6..a1f3dfe 100644 --- a/src/js/hosts-files.js +++ b/src/js/hosts-files.js @@ -1,7 +1,7 @@ /******************************************************************************* - µMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014-2015 Raymond Hill + uMatrix - a Chromium browser extension to black/white list requests. + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,37 +19,34 @@ Home: https://github.com/gorhill/uMatrix */ -/* global vAPI, uDom */ - -/******************************************************************************/ - -(function() { +/* global uDom */ 'use strict'; /******************************************************************************/ -var listDetails = {}; -var externalHostsFiles = ''; -var cacheWasPurged = false; -var needUpdate = false; -var hasCachedContent = false; +(function() { + +/******************************************************************************/ + +var listDetails = {}, + lastUpdateTemplateString = vAPI.i18n('hostsFilesLastUpdate'), + hostsFilesSettingsHash, + reValidExternalList = /[a-z-]+:\/\/\S*\/\S+/; /******************************************************************************/ var onMessage = function(msg) { switch ( msg.what ) { + case 'assetUpdated': + updateAssetStatus(msg); + break; + case 'assetsUpdated': + document.body.classList.remove('updating'); + break; case 'loadHostsFilesCompleted': renderHostsFiles(); break; - - case 'forceUpdateAssetsProgress': - renderBusyOverlay(true, msg.progress); - if ( msg.done ) { - messager.send({ what: 'reloadHostsFiles' }); - } - break; - default: break; } @@ -65,115 +62,120 @@ var renderNumber = function(value) { /******************************************************************************/ -// TODO: get rid of background page dependencies +var renderHostsFiles = function(soft) { + var listEntryTemplate = uDom('#templates .listEntry'), + listStatsTemplate = vAPI.i18n('hostsFilesPerFileStats'), + renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString, + reExternalHostFile = /^https?:/; -var renderHostsFiles = function() { - var listEntryTemplate = uDom('#templates .listEntry'); - var listStatsTemplate = vAPI.i18n('hostsFilesPerFileStats'); - var lastUpdateString = vAPI.i18n('hostsFilesLastUpdate'); - var renderElapsedTimeToString = vAPI.i18n.renderElapsedTimeToString; - var reExternalHostFile = /^https?:/; - - // Assemble a pretty blacklist name if possible + // Assemble a pretty list name if possible var listNameFromListKey = function(listKey) { var list = listDetails.current[listKey] || listDetails.available[listKey]; var listTitle = list ? list.title : ''; - if ( listTitle === '' ) { - return listKey; - } + if ( listTitle === '' ) { return listKey; } return listTitle; }; - var liFromListEntry = function(listKey) { - var elem, text; - var entry = listDetails.available[listKey]; - var li = listEntryTemplate.clone(); - - if ( entry.off !== true ) { - li.descendants('input').attr('checked', ''); + var liFromListEntry = function(listKey, li) { + var entry = listDetails.available[listKey], + elem; + if ( !li ) { + li = listEntryTemplate.clone().nodeAt(0); } - - elem = li.descendants('a:nth-of-type(1)'); - elem.attr('href', encodeURI(listKey)); - elem.text(listNameFromListKey(listKey) + '\u200E'); - - elem = li.descendants('a:nth-of-type(2)'); - if ( entry.homeDomain ) { - elem.attr('href', 'http://' + encodeURI(entry.homeHostname)); - elem.text('(' + entry.homeDomain + ')'); - elem.css('display', ''); - } - - elem = li.descendants('span:nth-of-type(1)'); - text = listStatsTemplate - .replace('{{used}}', renderNumber(!entry.off && !isNaN(+entry.entryUsedCount) ? entry.entryUsedCount : 0)) - .replace('{{total}}', !isNaN(+entry.entryCount) ? renderNumber(entry.entryCount) : '?'); - elem.text(text); - - // https://github.com/gorhill/uBlock/issues/78 - // Badge for non-secure connection - var remoteURL = listKey; - if ( remoteURL.lastIndexOf('http:', 0) !== 0 ) { - remoteURL = entry.homeURL || ''; - } - if ( remoteURL.lastIndexOf('http:', 0) === 0 ) { - li.descendants('span.status.unsecure').css('display', ''); - } - - // https://github.com/chrisaljoudi/uBlock/issues/104 - var asset = listDetails.cache[listKey] || {}; - - // Badge for update status - if ( entry.off !== true ) { - if ( asset.repoObsolete ) { - li.descendants('span.status.new').css('display', ''); - needUpdate = true; - } else if ( asset.cacheObsolete ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; - } else if ( entry.external && !asset.cached ) { - li.descendants('span.status.obsolete').css('display', ''); - needUpdate = true; + if ( li.getAttribute('data-listkey') !== listKey ) { + li.setAttribute('data-listkey', listKey); + elem = li.querySelector('input[type="checkbox"]'); + elem.checked = entry.off !== true; + elem = li.querySelector('a:nth-of-type(1)'); + elem.setAttribute('href', 'asset-viewer.html?url=' + encodeURI(listKey)); + elem.setAttribute('type', 'text/html'); + elem.textContent = listNameFromListKey(listKey); + li.classList.remove('toRemove'); + if ( entry.supportName ) { + li.classList.add('support'); + elem = li.querySelector('a.support'); + elem.setAttribute('href', entry.supportURL); + elem.setAttribute('title', entry.supportName); + } else { + li.classList.remove('support'); + } + if ( entry.external ) { + li.classList.add('external'); + } else { + li.classList.remove('external'); + } + if ( entry.instructionURL ) { + li.classList.add('mustread'); + elem = li.querySelector('a.mustread'); + elem.setAttribute('href', entry.instructionURL); + } else { + li.classList.remove('mustread'); } } - - // In cache - if ( asset.cached ) { - elem = li.descendants('span.status.purge'); - elem.css('display', ''); - elem.attr('title', lastUpdateString.replace('{{ago}}', renderElapsedTimeToString(asset.lastModified))); - hasCachedContent = true; + // https://github.com/gorhill/uBlock/issues/1429 + if ( !soft ) { + elem = li.querySelector('input[type="checkbox"]'); + elem.checked = entry.off !== true; } + elem = li.querySelector('span.counts'); + var text = ''; + if ( !isNaN(+entry.entryUsedCount) && !isNaN(+entry.entryCount) ) { + text = listStatsTemplate + .replace('{{used}}', renderNumber(entry.off ? 0 : entry.entryUsedCount)) + .replace('{{total}}', renderNumber(entry.entryCount)); + } + elem.textContent = text; + // https://github.com/chrisaljoudi/uBlock/issues/104 + var asset = listDetails.cache[listKey] || {}; + var remoteURL = asset.remoteURL; + li.classList.toggle( + 'unsecure', + typeof remoteURL === 'string' && remoteURL.lastIndexOf('http:', 0) === 0 + ); + li.classList.toggle('failed', asset.error !== undefined); + li.classList.toggle('obsolete', asset.obsolete === true); + li.classList.toggle('cached', asset.cached === true && asset.writeTime > 0); + if ( asset.cached ) { + li.querySelector('.status.cache').setAttribute( + 'title', + lastUpdateTemplateString.replace('{{ago}}', renderElapsedTimeToString(asset.writeTime)) + ); + } + li.classList.remove('discard'); return li; }; var onListsReceived = function(details) { // Before all, set context vars listDetails = details; - needUpdate = false; - hasCachedContent = false; - var availableLists = details.available; - var listKeys = Object.keys(details.available); + // Incremental rendering: this will allow us to easily discard unused + // DOM list entries. + uDom('#lists .listEntry').addClass('discard'); + + var availableLists = details.available, + listKeys = Object.keys(details.available); // Sort works this way: // - Send /^https?:/ items at the end (custom hosts file URL) listKeys.sort(function(a, b) { - var ta = availableLists[a].title || a; - var tb = availableLists[b].title || b; + var ta = availableLists[a].title || a, + tb = availableLists[b].title || b; if ( reExternalHostFile.test(ta) === reExternalHostFile.test(tb) ) { return ta.localeCompare(tb); } - if ( reExternalHostFile.test(tb) ) { - return -1; - } - return 1; + return reExternalHostFile.test(tb) ? -1 : 1; }); - var ulList = uDom('#lists').empty(); + + var ulList = document.querySelector('#lists'); for ( var i = 0; i < listKeys.length; i++ ) { - ulList.append(liFromListEntry(listKeys[i])); + var liEntry = liFromListEntry(listKeys[i], ulList.children[i]); + if ( liEntry.parentElement === null ) { + ulList.appendChild(liEntry); + } } + uDom('#lists .listEntry.discard').remove(); uDom('#listsOfBlockedHostsPrompt').text( vAPI.i18n('hostsFilesStats').replace( '{{blockedHostnameCount}}', @@ -182,8 +184,10 @@ var renderHostsFiles = function() { ); uDom('#autoUpdate').prop('checked', listDetails.autoUpdate === true); + if ( !soft ) { + hostsFilesSettingsHash = hashFromCurrentFromSettings(); + } renderWidgets(); - renderBusyOverlay(details.manualUpdate, details.manualUpdateProgress); }; messager.send({ what: 'getLists' }, onListsReceived); @@ -191,198 +195,163 @@ var renderHostsFiles = function() { /******************************************************************************/ -// Progress must be normalized to [0, 1], or can be undefined. - -var renderBusyOverlay = function(state, progress) { - progress = progress || {}; - var showProgress = typeof progress.value === 'number'; - if ( showProgress ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:first-child').css( - 'width', - (progress.value * 100).toFixed(1) + '%' - ); - var text = progress.text || ''; - if ( text !== '' ) { - uDom('#busyOverlay > div:nth-of-type(2) > div:last-child').text(text); - } - } - uDom('#busyOverlay > div:nth-of-type(2)').css('display', showProgress ? '' : 'none'); - uDom('body').toggleClass('busy', !!state); -}; - -/******************************************************************************/ - -// This is to give a visual hint that the selection of blacklists has changed. - var renderWidgets = function() { - uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); - uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged()); - uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent); + uDom('#buttonUpdate').toggleClass('disabled', document.querySelector('body:not(.updating) #lists .listEntry.obsolete > input[type="checkbox"]:checked') === null); + uDom('#buttonPurgeAll').toggleClass('disabled', document.querySelector('#lists .listEntry.cached') === null); + uDom('#buttonApply').toggleClass('disabled', hostsFilesSettingsHash === hashFromCurrentFromSettings()); }; /******************************************************************************/ -// Return whether selection of lists changed. - -var listsSelectionChanged = function() { - if ( cacheWasPurged ) { - return true; +var updateAssetStatus = function(details) { + var li = document.querySelector('#lists .listEntry[data-listkey="' + details.key + '"]'); + if ( li === null ) { return; } + li.classList.toggle('failed', !!details.failed); + li.classList.toggle('obsolete', !details.cached); + li.classList.toggle('cached', !!details.cached); + if ( details.cached ) { + li.querySelector('.status.cache').setAttribute( + 'title', + lastUpdateTemplateString.replace( + '{{ago}}', + vAPI.i18n.renderElapsedTimeToString(Date.now()) + ) + ); } - var availableLists = listDetails.available; - var currentLists = listDetails.current; - var location, availableOff, currentOff; - // This check existing entries - for ( location in availableLists ) { - if ( availableLists.hasOwnProperty(location) === false ) { - continue; - } - availableOff = availableLists[location].off === true; - currentOff = currentLists[location] === undefined || currentLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; + renderWidgets(); +}; + +/******************************************************************************* + + Compute a hash from all the settings affecting how filter lists are loaded + in memory. + +**/ + +var hashFromCurrentFromSettings = function() { + var hash = [], + listHash = [], + listEntries = document.querySelectorAll('#lists .listEntry[data-listkey]:not(.toRemove)'), + liEntry, + i = listEntries.length; + while ( i-- ) { + liEntry = listEntries[i]; + if ( liEntry.querySelector('input[type="checkbox"]:checked') !== null ) { + listHash.push(liEntry.getAttribute('data-listkey')); } } - // This check removed entries - for ( location in currentLists ) { - if ( currentLists.hasOwnProperty(location) === false ) { - continue; - } - currentOff = currentLists[location].off === true; - availableOff = availableLists[location] === undefined || availableLists[location].off === true; - if ( availableOff !== currentOff ) { - return true; - } + hash.push( + listHash.sort().join(), + reValidExternalList.test(document.getElementById('externalHostsFiles').value), + document.querySelector('#lists .listEntry.toRemove') !== null + ); + return hash.join(); +}; + +/******************************************************************************/ + +var onHostsFilesSettingsChanged = function() { + renderWidgets(); +}; + +/******************************************************************************/ + +var onRemoveExternalHostsFile = function(ev) { + var liEntry = uDom(this).ancestors('[data-listkey]'), + listKey = liEntry.attr('data-listkey'); + if ( listKey ) { + liEntry.toggleClass('toRemove'); + renderWidgets(); } - return false; -}; - -/******************************************************************************/ - -// Return whether content need update. - -var listsContentChanged = function() { - return needUpdate; -}; - -/******************************************************************************/ - -// This is to give a visual hint that the selection of blacklists has changed. - -var updateWidgets = function() { - uDom('#buttonApply').toggleClass('disabled', !listsSelectionChanged()); - uDom('#buttonUpdate').toggleClass('disabled', !listsContentChanged()); - uDom('#buttonPurgeAll').toggleClass('disabled', !hasCachedContent); - uDom('body').toggleClass('busy', false); -}; - -/******************************************************************************/ - -var onListCheckboxChanged = function() { - var href = uDom(this).parent().descendants('a').first().attr('href'); - if ( typeof href !== 'string' ) { - return; - } - if ( listDetails.available[href] === undefined ) { - return; - } - listDetails.available[href].off = !this.checked; - updateWidgets(); -}; - -/******************************************************************************/ - -var onListLinkClicked = function(ev) { - messager.send({ - what: 'gotoExtensionURL', - url: 'asset-viewer.html?url=' + uDom(this).attr('href') - }); ev.preventDefault(); }; /******************************************************************************/ var onPurgeClicked = function() { - var button = uDom(this); - var li = button.parent(); - var href = li.descendants('a').first().attr('href'); - if ( !href ) { - return; - } - messager.send({ what: 'purgeCache', path: href }); - button.remove(); - if ( li.descendants('input').first().prop('checked') ) { - cacheWasPurged = true; - updateWidgets(); + var button = uDom(this), + liEntry = button.ancestors('[data-listkey]'), + listKey = liEntry.attr('data-listkey'); + if ( !listKey ) { return; } + + messager.send({ what: 'purgeCache', assetKey: listKey }); + liEntry.addClass('obsolete'); + liEntry.removeClass('cached'); + + if ( liEntry.descendants('input').first().prop('checked') ) { + renderWidgets(); } }; /******************************************************************************/ var selectHostsFiles = function(callback) { - var switches = []; - var lis = uDom('#lists .listEntry'), li; - var i = lis.length; + // Hosts files to select + var toSelect = [], + liEntries = document.querySelectorAll('#lists .listEntry[data-listkey]:not(.toRemove)'), + i = liEntries.length, + liEntry; while ( i-- ) { - li = lis.at(i); - switches.push({ - location: li.descendants('a').attr('href'), - off: li.descendants('input').prop('checked') === false - }); + liEntry = liEntries[i]; + if ( liEntry.querySelector('input[type="checkbox"]:checked') !== null ) { + toSelect.push(liEntry.getAttribute('data-listkey')); + } } + // External hosts files to remove + var toRemove = []; + liEntries = document.querySelectorAll('#lists .listEntry.toRemove[data-listkey]'); + i = liEntries.length; + while ( i-- ) { + toRemove.push(liEntries[i].getAttribute('data-listkey')); + } + + // External hosts files to import + var externalListsElem = document.getElementById('externalHostsFiles'), + toImport = externalListsElem.value.trim(); + externalListsElem.value = ''; + messager.send({ - what: 'selectHostsFiles', - switches: switches - }, callback); + what: 'selectHostsFiles', + toSelect: toSelect, + toImport: toImport, + toRemove: toRemove + }, + callback + ); + + hostsFilesSettingsHash = hashFromCurrentFromSettings(); }; /******************************************************************************/ var buttonApplyHandler = function() { uDom('#buttonApply').removeClass('enabled'); - - renderBusyOverlay(true); - - var onSelectionDone = function() { + selectHostsFiles(function() { messager.send({ what: 'reloadHostsFiles' }); - }; - - selectHostsFiles(onSelectionDone); - - cacheWasPurged = false; + }); + renderWidgets(); }; /******************************************************************************/ var buttonUpdateHandler = function() { uDom('#buttonUpdate').removeClass('enabled'); - - if ( needUpdate ) { - renderBusyOverlay(true); - - var onSelectionDone = function() { - messager.send({ what: 'forceUpdateAssets' }); - }; - - selectHostsFiles(onSelectionDone); - - cacheWasPurged = false; - } + selectHostsFiles(function() { + document.body.classList.add('updating'); + messager.send({ what: 'forceUpdateAssets' }); + renderWidgets(); + }); + renderWidgets(); }; /******************************************************************************/ var buttonPurgeAllHandler = function() { uDom('#buttonPurgeAll').removeClass('enabled'); - - renderBusyOverlay(true); - - var onCompleted = function() { - cacheWasPurged = true; - renderHostsFiles(); - }; - - messager.send({ what: 'purgeAllCaches' }, onCompleted); + messager.send({ what: 'purgeAllCaches' }, function() { + renderHostsFiles(true); + }); }; /******************************************************************************/ @@ -397,52 +366,16 @@ var autoUpdateCheckboxChanged = function() { /******************************************************************************/ -var renderExternalLists = function() { - var onReceived = function(details) { - uDom('#externalHostsFiles').val(details); - externalHostsFiles = details; - }; - messager.send({ what: 'userSettings', name: 'externalHostsFiles' }, onReceived); -}; +uDom('#autoUpdate').on('change', autoUpdateCheckboxChanged); +uDom('#buttonApply').on('click', buttonApplyHandler); +uDom('#buttonUpdate').on('click', buttonUpdateHandler); +uDom('#buttonPurgeAll').on('click', buttonPurgeAllHandler); +uDom('#lists').on('change', '.listEntry > input', onHostsFilesSettingsChanged); +uDom('#lists').on('click', '.listEntry > a.remove', onRemoveExternalHostsFile); +uDom('#lists').on('click', 'span.cache', onPurgeClicked); +uDom('#externalHostsFiles').on('input', onHostsFilesSettingsChanged); -/******************************************************************************/ - -var externalListsChangeHandler = function() { - uDom('#externalListsParse').prop( - 'disabled', - this.value.trim() === externalHostsFiles - ); -}; - -/******************************************************************************/ - -var externalListsApplyHandler = function() { - externalHostsFiles = uDom('#externalHostsFiles').val(); - messager.send({ - what: 'userSettings', - name: 'externalHostsFiles', - value: externalHostsFiles - }); - renderHostsFiles(); - uDom('#externalListsParse').prop('disabled', true); -}; - -/******************************************************************************/ - -uDom.onLoad(function() { - uDom('#autoUpdate').on('change', autoUpdateCheckboxChanged); - uDom('#buttonApply').on('click', buttonApplyHandler); - uDom('#buttonUpdate').on('click', buttonUpdateHandler); - uDom('#buttonPurgeAll').on('click', buttonPurgeAllHandler); - uDom('#lists').on('change', '.listEntry > input', onListCheckboxChanged); - uDom('#lists').on('click', '.listEntry > a:nth-of-type(1)', onListLinkClicked); - uDom('#lists').on('click', 'span.purge', onPurgeClicked); - uDom('#externalHostsFiles').on('input', externalListsChangeHandler); - uDom('#externalListsParse').on('click', externalListsApplyHandler); - - renderHostsFiles(); - renderExternalLists(); -}); +renderHostsFiles(); /******************************************************************************/ diff --git a/src/js/messaging.js b/src/js/messaging.js index f14157d..4d5bbd2 100644 --- a/src/js/messaging.js +++ b/src/js/messaging.js @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uMatrix */ -/* global µMatrix, vAPI */ +'use strict'; /******************************************************************************/ /******************************************************************************/ @@ -28,8 +28,6 @@ (function() { -'use strict'; - var µm = µMatrix; /******************************************************************************/ @@ -40,7 +38,12 @@ function onMessage(request, sender, callback) { // Async switch ( request.what ) { case 'getAssetContent': - return µm.assets.getLocal(request.url, callback); + µm.assets.get(request.url, { dontCache: true }, callback); + return; + + case 'selectHostsFiles': + µm.selectHostsFiles(request, callback); + return; default: break; @@ -55,7 +58,8 @@ function onMessage(request, sender, callback) { break; case 'forceUpdateAssets': - µm.assetUpdater.force(); + µm.scheduleAssetUpdater(0); + µm.assets.updateStart({ delay: 2000 }); break; case 'getUserSettings': @@ -82,10 +86,6 @@ function onMessage(request, sender, callback) { µm.reloadHostsFiles(); break; - case 'selectHostsFiles': - µm.selectHostsFiles(request.switches); - break; - case 'userSettings': if ( request.hasOwnProperty('value') === false ) { request.value = undefined; @@ -571,8 +571,6 @@ vAPI.messaging.listen('contentscript-end.js', onMessage); (function() { -'use strict'; - /******************************************************************************/ var µm = µMatrix; @@ -800,8 +798,6 @@ var getLists = function(callback) { var onMetadataReady = function(entries) { r.cache = entries; prepEntries(r.cache); - r.manualUpdate = µm.assetUpdater.manualUpdate; - r.manualUpdateProgress = µm.assetUpdater.manualUpdateProgress; callback(r); }; var onAvailableHostsFilesReady = function(lists) { @@ -822,9 +818,6 @@ var onMessage = function(request, sender, callback) { case 'getLists': return getLists(callback); - case 'purgeAllCaches': - return µm.assets.purgeAll(callback); - default: break; } @@ -834,7 +827,16 @@ var onMessage = function(request, sender, callback) { switch ( request.what ) { case 'purgeCache': - µm.assets.purge(request.path); + µm.assets.purge(request.assetKey); + µm.assets.remove('compiled/' + request.assetKey); + break; + + case 'purgeAllCaches': + if ( request.hard ) { + µm.assets.remove(/./); + } else { + µm.assets.purge(/./, 'public_suffix_list.dat'); + } break; default: @@ -944,8 +946,6 @@ vAPI.messaging.listen('about.js', onMessage); (function() { -'use strict'; - /******************************************************************************/ var µm = µMatrix; diff --git a/src/js/popup.js b/src/js/popup.js index 0e04cd6..013e2c6 100644 --- a/src/js/popup.js +++ b/src/js/popup.js @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uMatrix */ -/* global punycode, vAPI, uDom */ +/* global punycode, uDom */ /* jshint esnext: true, bitwise: false */ 'use strict'; diff --git a/src/js/start.js b/src/js/start.js index 257bc0a..3d361fc 100644 --- a/src/js/start.js +++ b/src/js/start.js @@ -1,7 +1,7 @@ /******************************************************************************* µMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -89,22 +89,11 @@ var rwLocalUserSettings = { /******************************************************************************/ -// Important: raise barrier to remote fetching: we do not want resources to -// be pulled from remote server at start up time. - -µm.assets.remoteFetchBarrier += 1; - -/******************************************************************************/ - var onAllDone = function() { µm.webRequest.start(); - // https://github.com/chrisaljoudi/uBlock/issues/184 - // Check for updates not too far in the future. - µm.assetUpdater.onStart.addListener(µm.updateStartHandler.bind(µm)); - µm.assetUpdater.onCompleted.addListener(µm.updateCompleteHandler.bind(µm)); - µm.assetUpdater.onAssetUpdated.addListener(µm.assetUpdatedHandler.bind(µm)); - µm.assets.onAssetCacheRemoved.addListener(µm.assetCacheRemovedHandler.bind(µm)); + µm.assets.addObserver(µm.assetObserver.bind(µm)); + µm.scheduleAssetUpdater(µm.userSettings.autoUpdate ? 7 * 60 * 1000 : 0); for ( var key in defaultLocalUserSettings ) { if ( defaultLocalUserSettings.hasOwnProperty(key) === false ) { @@ -118,9 +107,7 @@ var onAllDone = function() { } } - vAPI.cloud.start([ - 'myRulesPane' - ]); + vAPI.cloud.start([ 'myRulesPane' ]); }; var onTabsReady = function(tabs) { @@ -135,12 +122,6 @@ var onTabsReady = function(tabs) { onAllDone(); }; -var onHostsFilesLoaded = function() { - // Important: remove barrier to remote fetching, this was useful only - // for launch time. - µm.assets.remoteFetchBarrier -= 1; -}; - var onUserSettingsLoaded = function() { // Version 0.9.0.0 // Remove obsolete user settings which may have been loaded. @@ -154,7 +135,7 @@ var onUserSettingsLoaded = function() { delete µm.userSettings.subframeColor; delete µm.userSettings.subframeOpacity; - µm.loadHostsFiles(onHostsFilesLoaded); + µm.loadHostsFiles(); }; var onPSLReady = function() { diff --git a/src/js/storage.js b/src/js/storage.js index 7fac8b7..dae286f 100644 --- a/src/js/storage.js +++ b/src/js/storage.js @@ -19,7 +19,9 @@ Home: https://github.com/gorhill/uMatrix */ -/* global chrome, µMatrix, punycode, publicSuffixList */ +/* global objectAssign, punycode, publicSuffixList */ + +'use strict'; /******************************************************************************/ @@ -28,7 +30,12 @@ var getBytesInUseHandler = function(bytesInUse) { µm.storageUsed = bytesInUse; }; - vAPI.storage.getBytesInUse(null, getBytesInUseHandler); + // Not all WebExtension implementations support getBytesInUse(). + if ( typeof vAPI.storage.getBytesInUse === 'function' ) { + vAPI.storage.getBytesInUse(null, getBytesInUseHandler); + } else { + µm.storageUsed = undefined; + } }; /******************************************************************************/ @@ -89,65 +96,84 @@ /******************************************************************************/ -µMatrix.getAvailableHostsFiles = function(callback) { - var availableHostsFiles = {}; - var redirections = {}; - var µm = this; +µMatrix.listKeysFromCustomHostsFiles = function(raw) { + var out = new Set(), + reIgnore = /^[!#]/, + reValid = /^[a-z-]+:\/\/\S+/, + lineIter = new this.utils.LineIterator(raw), + location; + while ( lineIter.eot() === false ) { + location = lineIter.next().trim(); + if ( reIgnore.test(location) || !reValid.test(location) ) { continue; } + out.add(location); + } + return this.utils.setToArray(out); +}; - var fixLocation = function(location) { - // https://github.com/chrisaljoudi/uBlock/issues/418 - // We now support built-in external filter lists - if ( /^https?:/.test(location) === false ) { - location = 'assets/thirdparties/' + location; - } - return location; - }; +/******************************************************************************/ + +µMatrix.getAvailableHostsFiles = function(callback) { + var µm = this, + availableHostsFiles = {}; + + // Custom filter lists. + var importedListKeys = this.listKeysFromCustomHostsFiles(µm.userSettings.externalHostsFiles), + i = importedListKeys.length, + listKey, entry; + while ( i-- ) { + listKey = importedListKeys[i]; + entry = { + content: 'filters', + contentURL: listKey, + external: true, + submitter: 'user', + title: listKey + }; + availableHostsFiles[listKey] = entry; + this.assets.registerAssetSource(listKey, entry); + } // selected lists - var onSelectedHostsFilesLoaded = function(store) { - var lists = store.liveHostsFiles; - var locations = Object.keys(lists); - var oldLocation, newLocation; - var availableEntry, storedEntry; - - while ( (oldLocation = locations.pop()) ) { - newLocation = redirections[oldLocation] || oldLocation; - availableEntry = availableHostsFiles[newLocation]; - if ( availableEntry === undefined ) { - continue; + var onSelectedHostsFilesLoaded = function(bin) { + // Now get user's selection of lists + for ( var assetKey in bin.liveHostsFiles ) { + var availableEntry = availableHostsFiles[assetKey]; + if ( availableEntry === undefined ) { continue; } + var liveEntry = bin.liveHostsFiles[assetKey]; + availableEntry.off = liveEntry.off || false; + if ( liveEntry.entryCount !== undefined ) { + availableEntry.entryCount = liveEntry.entryCount; } - storedEntry = lists[oldLocation] || {}; - availableEntry.off = storedEntry.off || false; - µm.assets.setHomeURL(newLocation, availableEntry.homeURL || ''); - if ( storedEntry.entryCount !== undefined ) { - availableEntry.entryCount = storedEntry.entryCount; - } - if ( storedEntry.entryUsedCount !== undefined ) { - availableEntry.entryUsedCount = storedEntry.entryUsedCount; + if ( liveEntry.entryUsedCount !== undefined ) { + availableEntry.entryUsedCount = liveEntry.entryUsedCount; } // This may happen if the list name was pulled from the list content - if ( availableEntry.title === '' && storedEntry.title !== undefined ) { - availableEntry.title = storedEntry.title; + if ( availableEntry.title === '' && liveEntry.title !== undefined ) { + availableEntry.title = liveEntry.title; } } + + // Remove unreferenced imported filter lists. + var dict = new Set(importedListKeys); + for ( assetKey in availableHostsFiles ) { + var entry = availableHostsFiles[assetKey]; + if ( entry.submitter !== 'user' ) { continue; } + if ( dict.has(assetKey) ) { continue; } + delete availableHostsFiles[assetKey]; + µm.assets.unregisterAssetSource(assetKey); + µm.assets.remove(assetKey); + } + callback(availableHostsFiles); }; // built-in lists - var onBuiltinHostsFilesLoaded = function(details) { - var location, locations; - try { - locations = JSON.parse(details.content); - } catch (e) { - locations = {}; - } - var hostsFileEntry; - for ( location in locations ) { - if ( locations.hasOwnProperty(location) === false ) { - continue; - } - hostsFileEntry = locations[location]; - availableHostsFiles[fixLocation(location)] = hostsFileEntry; + var onBuiltinHostsFilesLoaded = function(entries) { + for ( var assetKey in entries ) { + if ( entries.hasOwnProperty(assetKey) === false ) { continue; } + entry = entries[assetKey]; + if ( entry.content !== 'filters' ) { continue; } + availableHostsFiles[assetKey] = objectAssign({}, entry); } // Now get user's selection of lists @@ -157,37 +183,7 @@ ); }; - // permanent hosts files - var location; - var lists = this.permanentHostsFiles; - for ( location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - availableHostsFiles[location] = lists[location]; - } - - // custom lists - var c; - var locations = this.userSettings.externalHostsFiles.split('\n'); - for ( var i = 0; i < locations.length; i++ ) { - location = locations[i].trim(); - c = location.charAt(0); - if ( location === '' || c === '!' || c === '#' ) { - continue; - } - // Coarse validation - if ( /[^0-9A-Za-z!*'();:@&=+$,\/?%#\[\]_.~-]/.test(location) ) { - continue; - } - availableHostsFiles[location] = { - title: location, - external: true - }; - } - - // get built-in block lists. - this.assets.get('assets/umatrix/hosts-files.json', onBuiltinHostsFilesLoaded); + this.assets.metadata(onBuiltinHostsFilesLoaded); }; /******************************************************************************/ @@ -245,8 +241,6 @@ /******************************************************************************/ µMatrix.mergeHostsFile = function(details) { - //console.log('storage.js > mergeHostsFile from "%s": "%s..."', details.path, details.content.slice(0, 40)); - var usedCount = this.ubiquitousBlacklist.count; var duplicateCount = this.ubiquitousBlacklist.duplicateCount; @@ -255,7 +249,7 @@ usedCount = this.ubiquitousBlacklist.count - usedCount; duplicateCount = this.ubiquitousBlacklist.duplicateCount - duplicateCount; - var hostsFilesMeta = this.liveHostsFiles[details.path]; + var hostsFilesMeta = this.liveHostsFiles[details.assetKey]; hostsFilesMeta.entryCount = usedCount + duplicateCount; hostsFilesMeta.entryUsedCount = usedCount; }; @@ -263,8 +257,6 @@ /******************************************************************************/ µMatrix.mergeHostsFileContent = function(rawText) { - //console.log('storage.js > mergeHostsFileContent from "%s": "%s..."', details.path, details.content.slice(0, 40)); - var rawEnd = rawText.length; var ubiquitousBlacklist = this.ubiquitousBlacklist; var reLocalhost = /(^|\s)(localhost\.localdomain|localhost|local|broadcasthost|0\.0\.0\.0|127\.0\.0\.1|::1|fe80::1%lo0)(?=\s|$)/g; @@ -307,7 +299,6 @@ // For example, when a filter contains whitespace characters, or // whatever else outside the range of printable ascii characters. if ( matches[0] !== line ) { - //console.error('"%s": "%s" !== "%s"', details.path, matches[0], line); continue; } @@ -324,30 +315,97 @@ // `switches` contains the filter lists for which the switch must be revisited. -µMatrix.selectHostsFiles = function(switches) { - switches = switches || {}; +µMatrix.selectHostsFiles = function(details, callback) { + var µm = this, + externalHostsFiles = this.userSettings.externalHostsFiles, + i, n, assetKey; - // Only the lists referenced by the switches are touched. - var liveHostsFiles = this.liveHostsFiles; - var entry, state, location; - var i = switches.length; - while ( i-- ) { - entry = switches[i]; - state = entry.off === true; - location = entry.location; - if ( liveHostsFiles.hasOwnProperty(location) === false ) { - if ( state !== true ) { - liveHostsFiles[location] = { off: state }; + // Hosts file to select + if ( Array.isArray(details.toSelect) ) { + for ( assetKey in this.liveHostsFiles ) { + if ( this.liveHostsFiles.hasOwnProperty(assetKey) === false ) { + continue; + } + if ( details.toSelect.indexOf(assetKey) !== -1 ) { + this.liveHostsFiles[assetKey].off = false; + } else if ( details.merge !== true ) { + this.liveHostsFiles[assetKey].off = true; } - continue; } - if ( liveHostsFiles[location].off === state ) { - continue; - } - liveHostsFiles[location].off = state; } - vAPI.storage.set({ 'liveHostsFiles': liveHostsFiles }); + // Imported hosts files to remove + if ( Array.isArray(details.toRemove) ) { + var removeURLFromHaystack = function(haystack, needle) { + return haystack.replace( + new RegExp( + '(^|\\n)' + + needle.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + + '(\\n|$)', 'g'), + '\n' + ).trim(); + }; + for ( i = 0, n = details.toRemove.length; i < n; i++ ) { + assetKey = details.toRemove[i]; + delete this.liveHostsFiles[assetKey]; + externalHostsFiles = removeURLFromHaystack(externalHostsFiles, assetKey); + this.assets.remove(assetKey); + } + } + + // Hosts file to import + if ( typeof details.toImport === 'string' ) { + // https://github.com/gorhill/uBlock/issues/1181 + // Try mapping the URL of an imported filter list to the assetKey of an + // existing stock list. + var assetKeyFromURL = function(url) { + var needle = url.replace(/^https?:/, ''); + var assets = µm.liveHostsFiles, asset; + for ( var assetKey in assets ) { + asset = assets[assetKey]; + if ( asset.content !== 'filters' ) { continue; } + if ( typeof asset.contentURL === 'string' ) { + if ( asset.contentURL.endsWith(needle) ) { return assetKey; } + continue; + } + if ( Array.isArray(asset.contentURL) === false ) { continue; } + for ( i = 0, n = asset.contentURL.length; i < n; i++ ) { + if ( asset.contentURL[i].endsWith(needle) ) { + return assetKey; + } + } + } + return url; + }; + var importedSet = new Set(this.listKeysFromCustomHostsFiles(externalHostsFiles)), + toImportSet = new Set(this.listKeysFromCustomHostsFiles(details.toImport)), + iter = toImportSet.values(); + for (;;) { + var entry = iter.next(); + if ( entry.done ) { break; } + if ( importedSet.has(entry.value) ) { continue; } + assetKey = assetKeyFromURL(entry.value); + if ( assetKey === entry.value ) { + importedSet.add(entry.value); + } + this.liveHostsFiles[assetKey] = { + content: 'filters', + contentURL: [ assetKey ], + title: assetKey + }; + } + externalHostsFiles = this.utils.setToArray(importedSet).sort().join('\n'); + } + + if ( externalHostsFiles !== this.userSettings.externalHostsFiles ) { + this.userSettings.externalHostsFiles = externalHostsFiles; + vAPI.storage.set({ externalHostsFiles: externalHostsFiles }); + } + vAPI.storage.set({ 'liveHostsFiles': this.liveHostsFiles }); + + if ( typeof callback === 'function' ) { + callback(); + } }; /******************************************************************************/ @@ -356,16 +414,7 @@ // revisited. µMatrix.reloadHostsFiles = function() { - var µm = this; - - // We are just reloading the filter lists: we do not want assets to update. - this.assets.autoUpdate = false; - - var onHostsFilesReady = function() { - µm.assets.autoUpdate = µm.userSettings.autoUpdate; - }; - - this.loadHostsFiles(onHostsFilesReady); + this.loadHostsFiles(); }; /******************************************************************************/ @@ -382,119 +431,95 @@ callback(); }; - this.assets.get(this.pslPath, applyPublicSuffixList); + this.assets.get(this.pslAssetKey, applyPublicSuffixList); }; /******************************************************************************/ -µMatrix.updateStartHandler = function(callback) { - var µm = this; - var onListsReady = function(lists) { - var assets = {}; - for ( var location in lists ) { - if ( lists.hasOwnProperty(location) === false ) { - continue; - } - if ( lists[location].off ) { - continue; - } - assets[location] = true; +µMatrix.scheduleAssetUpdater = (function() { + var timer, next = 0; + return function(updateDelay) { + if ( timer ) { + clearTimeout(timer); + timer = undefined; } - assets[µm.pslPath] = true; - callback(assets); - }; - - this.getAvailableHostsFiles(onListsReady); -}; - -/******************************************************************************/ - -µMatrix.assetUpdatedHandler = function(details) { - var path = details.path || ''; - - if ( path !== '' ) { - this.logger.writeOne('', 'info', vAPI.i18n('loggerEntryAssetUpdated').replace('{{value}}', path)); - } - - if ( this.liveHostsFiles.hasOwnProperty(path) === false ) { - return; - } - var entry = this.liveHostsFiles[path]; - if ( entry.off ) { - return; - } - // Compile the list while we have the raw version in memory - //console.debug('µMatrix.getCompiledFilterList/onRawListLoaded: compiling "%s"', path); - //this.assets.put( - // this.getCompiledFilterListPath(path), - // this.compileFilters(details.content) - //); -}; - -/******************************************************************************/ - -µMatrix.updateCompleteHandler = function(details) { - var µm = this; - - var updatedCount = details.updatedCount; - if ( updatedCount === 0 ) { - return; - } - - // Assets are supposed to have been all updated, prevent fetching from - // remote servers. - µm.assets.remoteFetchBarrier += 1; - - var onFiltersReady = function() { - µm.assets.remoteFetchBarrier -= 1; - }; - - var onPSLReady = function() { - if ( updatedCount !== 0 ) { - //console.debug('storage.js > µMatrix.updateCompleteHandler: reloading filter lists'); - µm.loadHostsFiles(onFiltersReady); - } else { - onFiltersReady(); - } - }; - - if ( details.hasOwnProperty(this.pslPath) ) { - //console.debug('storage.js > µMatrix.updateCompleteHandler: reloading PSL'); - this.loadPublicSuffixList(onPSLReady); - updatedCount -= 1; - } else { - onPSLReady(); - } -}; - -/******************************************************************************/ - -µMatrix.assetCacheRemovedHandler = (function() { - var barrier = false; - - var handler = function(paths) { - if ( barrier ) { + if ( updateDelay === 0 ) { + next = 0; return; } - barrier = true; - var i = paths.length; - var path; - while ( i-- ) { - path = paths[i]; - if ( this.liveHostsFiles.hasOwnProperty(path) ) { - //console.debug('µMatrix.assetCacheRemovedHandler: decompiling "%s"', path); - //this.purgeCompiledFilterList(path); - continue; - } - if ( path === this.pslPath ) { - //console.debug('µMatrix.assetCacheRemovedHandler: decompiling "%s"', path); - //this.assets.purge('cache://compiled-publicsuffixlist'); - continue; + var now = Date.now(); + // Use the new schedule if and only if it is earlier than the previous + // one. + if ( next !== 0 ) { + updateDelay = Math.min(updateDelay, Math.max(next - now, 0)); + } + next = now + updateDelay; + timer = vAPI.setTimeout(function() { + timer = undefined; + next = 0; + µMatrix.assets.updateStart({ delay: 120000 }); + }, updateDelay); + }; +})(); + +/******************************************************************************/ + +µMatrix.assetObserver = function(topic, details) { + // Do not update filter list if not in use. + if ( topic === 'before-asset-updated' ) { + if ( + this.liveHostsFiles.hasOwnProperty(details.assetKey) === false || + this.liveHostsFiles[details.assetKey].off === true + ) { + return false; + } + return; + } + + if ( topic === 'after-asset-updated' ) { + vAPI.messaging.broadcast({ + what: 'assetUpdated', + key: details.assetKey, + cached: true + }); + return; + } + + // Update failed. + if ( topic === 'asset-update-failed' ) { + vAPI.messaging.broadcast({ + what: 'assetUpdated', + key: details.assetKey, + failed: true + }); + return; + } + + // Reload all filter lists if needed. + if ( topic === 'after-assets-updated' ) { + if ( details.assetKeys.length !== 0 ) { + this.loadHostsFiles(); + } + if ( this.userSettings.autoUpdate ) { + this.scheduleAssetUpdater(25200000); + } else { + this.scheduleAssetUpdater(0); + } + vAPI.messaging.broadcast({ + what: 'assetsUpdated', + assetKeys: details.assetKeys + }); + return; + } + + // New asset source became available, if it's a filter list, should we + // auto-select it? + if ( topic === 'builtin-asset-source-added' ) { + if ( details.entry.content === 'filters' ) { + if ( details.entry.off !== true ) { + this.saveSelectedFilterLists([ details.assetKey ], true); } } - //this.destroySelfie(); - barrier = false; - }; - - return handler; -})(); + return; + } +}; diff --git a/src/js/traffic.js b/src/js/traffic.js index 9625bcb..5f1a7c9 100644 --- a/src/js/traffic.js +++ b/src/js/traffic.js @@ -1,7 +1,7 @@ /******************************************************************************* uMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014-2016 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -449,44 +449,33 @@ var requestTypeNormalizer = { 'font' : 'css', 'image' : 'image', 'main_frame' : 'doc', + 'media' : 'plugin', 'object' : 'plugin', 'other' : 'other', 'ping' : 'ping', 'script' : 'script', 'stylesheet' : 'css', 'sub_frame' : 'frame', + 'websocket' : 'xhr', 'xmlhttprequest': 'xhr' }; /******************************************************************************/ vAPI.net.onBeforeRequest = { - urls: [ - "http://*/*", - "https://*/*" - ], extra: [ 'blocking' ], callback: onBeforeRequestHandler }; vAPI.net.onBeforeSendHeaders = { - urls: [ - "http://*/*", - "https://*/*" - ], + urls: [ 'http://*/*', 'https://*/*' ], extra: [ 'blocking', 'requestHeaders' ], callback: onBeforeSendHeadersHandler }; vAPI.net.onHeadersReceived = { - urls: [ - "http://*/*", - "https://*/*" - ], - types: [ - "main_frame", - "sub_frame" - ], + urls: [ 'http://*/*', 'https://*/*' ], + types: [ 'main_frame', 'sub_frame' ], extra: [ 'blocking', 'responseHeaders' ], callback: onHeadersReceived }; diff --git a/src/js/utils.js b/src/js/utils.js index 6d63458..38bbcae 100644 --- a/src/js/utils.js +++ b/src/js/utils.js @@ -1,7 +1,7 @@ /******************************************************************************* µMatrix - a Chromium browser extension to black/white list requests. - Copyright (C) 2014 Raymond Hill + Copyright (C) 2014-2017 Raymond Hill This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -19,7 +19,7 @@ Home: https://github.com/gorhill/uMatrix */ -/* global chrome, µMatrix */ +'use strict'; /******************************************************************************/ @@ -45,9 +45,71 @@ var gotoExtensionURL = function(url) { /******************************************************************************/ +var LineIterator = function(text, offset) { + this.text = text; + this.textLen = this.text.length; + this.offset = offset || 0; +}; + +LineIterator.prototype.next = function() { + var lineEnd = this.text.indexOf('\n', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.text.indexOf('\r', this.offset); + if ( lineEnd === -1 ) { + lineEnd = this.textLen; + } + } + var line = this.text.slice(this.offset, lineEnd); + this.offset = lineEnd + 1; + return line; +}; + +LineIterator.prototype.rewind = function() { + if ( this.offset <= 1 ) { + this.offset = 0; + return; + } + var lineEnd = this.text.lastIndexOf('\n', this.offset - 2); + if ( lineEnd !== -1 ) { + this.offset = lineEnd + 1; + } else { + lineEnd = this.text.lastIndexOf('\r', this.offset - 2); + this.offset = lineEnd !== -1 ? lineEnd + 1 : 0; + } +}; + +LineIterator.prototype.eot = function() { + return this.offset >= this.textLen; +}; + +/******************************************************************************/ + +var setToArray = typeof Array.from === 'function' + ? Array.from + : function(dict) { + var out = [], + entries = dict.values(), + entry; + for (;;) { + entry = entries.next(); + if ( entry.done ) { break; } + out.push(entry.value); + } + return out; + }; + +var setFromArray = function(arr) { + return new Set(arr); +}; + +/******************************************************************************/ + return { gotoURL: gotoURL, - gotoExtensionURL: gotoExtensionURL + gotoExtensionURL: gotoExtensionURL, + LineIterator: LineIterator, + setToArray: setToArray, + setFromArray: setFromArray }; /******************************************************************************/ diff --git a/src/popup.html b/src/popup.html index 67d859f..42c95ee 100644 --- a/src/popup.html +++ b/src/popup.html @@ -19,39 +19,39 @@
uMatrix -
- -
-
    -
  • ∗ -
  • -
  • -
-
-
- -
- +
+
+
-
    -
  • -
  • -
  • +
      +
    • ∗ +
    • +
+ +
+ +
+
    +
  • +
  • +
  • +
+
+
+
+ + + + +
+
+ +
- - - -
- -
- - -
-
diff --git a/tools/make-firefox-meta.py b/tools/make-firefox-meta.py old mode 100644 new mode 100755 diff --git a/tools/make-firefox.sh b/tools/make-firefox.sh index a00f0e1..93ec0c7 100755 --- a/tools/make-firefox.sh +++ b/tools/make-firefox.sh @@ -19,7 +19,7 @@ cp -R src/* $DES/ mv $DES/img/icon_128.png $DES/icon.png cp platform/firefox/css/* $DES/css/ -cp platform/firefox/vapi-*.js $DES/js/ +cp platform/firefox/*.js $DES/js/ cp platform/firefox/bootstrap.js $DES/ cp platform/firefox/frame*.js $DES/ cp -R platform/chromium/img $DES/ diff --git a/tools/make-webext-meta.py b/tools/make-webext-meta.py new file mode 100755 index 0000000..15df315 --- /dev/null +++ b/tools/make-webext-meta.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 + +import os +import json +import sys + +if len(sys.argv) == 1 or not sys.argv[1]: + raise SystemExit('Build dir missing.') + +proj_dir = os.path.join(os.path.split(os.path.abspath(__file__))[0], '..') +build_dir = os.path.abspath(sys.argv[1]) + +# Import version number from chromium platform +chromium_manifest = {} +webext_manifest = {} + +chromium_manifest_file = os.path.join(proj_dir, 'platform', 'chromium', 'manifest.json') +with open(chromium_manifest_file) as f1: + chromium_manifest = json.load(f1) + +webext_manifest_file = os.path.join(build_dir, 'manifest.json') +with open(webext_manifest_file) as f2: + webext_manifest = json.load(f2) + +webext_manifest['version'] = chromium_manifest['version'] + +with open(webext_manifest_file, 'w') as f2: + json.dump(webext_manifest, f2, indent=2, separators=(',', ': '), sort_keys=True) + f2.write('\n') diff --git a/tools/make-webext.sh b/tools/make-webext.sh new file mode 100755 index 0000000..326c59e --- /dev/null +++ b/tools/make-webext.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# +# This script assumes a linux environment + +echo "*** uMatrix.webext: Creating web store package" +echo "*** uMatrix.webext: Copying files" + +DES=dist/build/uMatrix.webext +rm -rf $DES +mkdir -p $DES + +cp -R ./assets $DES/ +cp -R ./src/* $DES/ +cp -R $DES/_locales/nb $DES/_locales/no +cp platform/chromium/*.html $DES/ +cp platform/webext/polyfill.js $DES/js/ +cp platform/chromium/*.js $DES/js/ +cp -R platform/chromium/img/* $DES/img/ +cp platform/webext/manifest.json $DES/ +cp LICENSE.txt $DES/ + +echo "*** uMatrix.webext: Generating meta..." +python tools/make-webext-meta.py $DES/ + +if [ "$1" = all ]; then + echo "*** uMatrix.webext: Creating package..." + pushd $DES > /dev/null + zip ../$(basename $DES).zip -qr * + popd > /dev/null +fi + +echo "*** uMatrix.webext: Package done."