Remove obsolete assets

Related issue:
- https://github.com/vtriolet/writings/blob/main/posts/2021/ublock_origin_and_umatrix_denial_of_service.adoc

assets/assets.json

@@ -16,25 +16,6 @@
 			"assets/thirdparties/publicsuffix.org/list/effective_tld_names.dat"
 		]
 	},
-	"malware-0": {
-		"content": "filters",
-		"title": "Malware Domain List",
-		"contentURL": [
-			"https://www.malwaredomainlist.com/hostslist/hosts.txt",
-			"assets/thirdparties/www.malwaredomainlist.com/hostslist/hosts.txt"
-		]
-	},
-	"malware-1": {
-		"content": "filters",
-		"title": "Malware domains",
-		"contentURL": [
-			"https://mirror.cedia.org.ec/malwaredomains/justdomains",
-			"https://mirror1.malwaredomains.com/files/justdomains",
-			"assets/thirdparties/mirror1.malwaredomains.com/files/justdomains",
-			"assets/thirdparties/mirror1.malwaredomains.com/files/justdomains.txt"
-		],
-		"supportURL": "http://www.malwaredomains.com/"
-	},
 	"dpollock-0": {
 		"content": "filters",
 		"updateAfter": 11,
@@ -45,25 +26,15 @@
 		],
 		"supportURL": "https://someonewhocares.org/hosts/"
 	},
-	"hphosts": {
-		"content": "filters",
-		"updateAfter": 11,
-		"title": "hpHosts’ Ad and tracking servers",
-		"contentURL": [
-			"https://hosts-file.net/.%5Cad_servers.txt",
-			"assets/thirdparties/hosts-file.net/ad_servers.txt"
-		],
-		"supportURL": "https://hosts-file.net/"
-	},
 	"mvps-0": {
 		"content": "filters",
 		"updateAfter": 11,
 		"title": "MVPS HOSTS",
 		"contentURL": [
-			"http://winhelp2002.mvps.org/hosts.txt",
+			"https://winhelp2002.mvps.org/hosts.txt",
 			"assets/thirdparties/winhelp2002.mvps.org/hosts.txt"
 		],
-		"supportURL": "http://winhelp2002.mvps.org/"
+		"supportURL": "https://winhelp2002.mvps.org/"
 	},
 	"plowe-0": {
 		"content": "filters",

dist/chromium/publish-beta.py

@@ -39,7 +39,7 @@ if not re.search('^\d+\.\d+\.\d+(b|rc)\d+$', version):
 
 cs_extension_id = 'eckgcipdkhcfghnmincccnhpdmnbefki'
 tmpdir = tempfile.TemporaryDirectory()
-raw_zip_filename = 'uMatrix.chromium.zip'
+raw_zip_filename = 'uMatrix_'+ version + '.chromium.zip'
 raw_zip_filepath = os.path.join(tmpdir.name, raw_zip_filename)
 github_owner = 'gorhill'
 github_repo = 'uMatrix'

dist/firefox/publish-signed-beta.py

@@ -61,10 +61,10 @@ if match.group(2):
 
 extension_id = 'uMatrix@raymondhill.net'
 tmpdir = tempfile.TemporaryDirectory()
-raw_xpi_filename = 'uMatrix.firefox.xpi'
+raw_xpi_filename = 'uMatrix_' + tag_version + '.firefox.xpi'
 raw_xpi_filepath = os.path.join(tmpdir.name, raw_xpi_filename)
 unsigned_xpi_filepath = os.path.join(tmpdir.name, 'uMatrix.firefox.unsigned.xpi')
-signed_xpi_filename = 'uMatrix.firefox.signed.xpi'
+signed_xpi_filename = 'uMatrix_' + tag_version + '.firefox.signed.xpi'
 signed_xpi_filepath = os.path.join(tmpdir.name, signed_xpi_filename)
 github_owner = 'gorhill'
 github_repo = 'uMatrix'
@@ -173,19 +173,32 @@ with zipfile.ZipFile(raw_xpi_filepath, 'r') as zipin:
 # - https://addons-server.readthedocs.io/en/latest/topics/api/signing.html
 #
 
-print('Ask AMO to sign self-hosted xpi package...')
+amo_api_key = ''
-with open(unsigned_xpi_filepath, 'rb') as f:
+amo_secret = ''
-    amo_api_key = input_secret('AMO API key', 'amo_api_key')
+
-    amo_secret = input_secret('AMO API secret', 'amo_secret')
+def get_jwt_auth():
+    global amo_api_key
+    if amo_api_key == '':
+        amo_api_key = input_secret('AMO API key', 'amo_api_key')
+    global amo_secret
+    if amo_secret == '':
+        amo_secret = input_secret('AMO API secret', 'amo_secret')
     amo_nonce = os.urandom(8).hex()
     jwt_payload = {
         'iss': amo_api_key,
         'jti': amo_nonce,
         'iat': datetime.datetime.utcnow(),
-        'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=180),
+        'exp': datetime.datetime.utcnow() + datetime.timedelta(seconds=15),
     }
-    jwt_auth = 'JWT ' + jwt.encode(jwt_payload, amo_secret).decode()
+    return 'JWT ' + jwt.encode(jwt_payload, amo_secret).decode()
-    headers = { 'Authorization': jwt_auth, }
+
+print('Ask AMO to sign self-hosted xpi package...')
+with open(unsigned_xpi_filepath, 'rb') as f:
+    # https://blog.mozilla.org/addons/2019/11/11/security-improvements-in-amo-upload-tools/
+    #   "We recommend allowing up to 15 minutes."
+    interval = 60                   # check every 60 seconds
+    countdown = 15 * 60 / interval  # for at most 15 minutes
+    headers = { 'Authorization': get_jwt_auth(), }
     data = { 'channel': 'unlisted' }
     files = { 'upload': f, }
     signing_url = 'https://addons.mozilla.org/api/v3/addons/{0}/versions/{1}/'.format(extension_id, ext_version)
@@ -201,11 +214,6 @@ with open(unsigned_xpi_filepath, 'rb') as f:
     print('Waiting for AMO to process the request to sign the self-hosted xpi package...')
     # Wait for signed package to be ready
     signing_check_url = signing_request_response['url']
-    # TODO: use real time instead
-    # https://blog.mozilla.org/addons/2019/11/11/security-improvements-in-amo-upload-tools/
-    #   "We recommend allowing up to 15 minutes."
-    interval = 30                   # check every 30 seconds
-    countdown = 15 * 60 / interval  # for at most 15 minutes
     while True:
         sys.stdout.write('.')
         sys.stdout.flush()
@@ -214,15 +222,18 @@ with open(unsigned_xpi_filepath, 'rb') as f:
         if countdown <= 0:
             print('Error: AMO signing timed out')
             exit(1)
+        headers = { 'Authorization': get_jwt_auth(), }
         response = requests.get(signing_check_url, headers=headers)
         if response.status_code != 200:
             print('Error: AMO signing failed -- server error {0}'.format(response.status_code))
+            print(response.text)
             exit(1)
         signing_check_response = response.json()
         if not signing_check_response['processed']:
             continue
         if not signing_check_response['valid']:
             print('Error: AMO validation failed')
+            print(response.text)
             exit(1)
         if not signing_check_response['files'] or len(signing_check_response['files']) == 0:
             continue
@@ -239,6 +250,7 @@ with open(unsigned_xpi_filepath, 'rb') as f:
         response = requests.get(download_url, headers=headers)
         if response.status_code != 200:
             print('Error: Download signed package failed -- server error {0}'.format(response.status_code))
+            print(response.text)
             exit(1)
         with open(signed_xpi_filepath, 'wb') as f:
             f.write(response.content)

dist/firefox/updates.json

@@ -3,10 +3,10 @@
   "uMatrix@raymondhill.net": {
    "updates": [
     {
-     "version": "1.4.1.2",
+     "version": "1.4.1.6",
      "browser_specific_settings": { "gecko": { "strict_min_version": "56" } },
-     "update_info_url": "https://github.com/gorhill/uMatrix/releases/tag/1.4.1b2",
+     "update_info_url": "https://github.com/gorhill/uMatrix/releases/tag/1.4.1b6",
-     "update_link": "https://github.com/gorhill/uMatrix/releases/download/1.4.1b2/uMatrix_1.4.1b2.firefox.signed.xpi"
+     "update_link": "https://github.com/gorhill/uMatrix/releases/download/1.4.1b6/uMatrix_1.4.1b6.firefox.signed.xpi"
     }
    ]
   }

dist/version

@@ -1 +1 @@
-1.4.1.4
+1.4.3.0

src/_locales/pt_BR/messages.json

@@ -72,7 +72,7 @@
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "fetchPrettyName": {
-        "message": "obter",
+        "message": "fetch",
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "framePrettyName": {
@@ -328,11 +328,11 @@
         "description": "Placeholder string for logger output filtering input field"
     },
     "loggerPopupPanelTip": {
-        "message": "Alternar o painel do popup",
+        "message": "Alternar o painel do pop-up",
         "description": "Tooltip for the popup panel button in the logger page"
     },
     "loggerInfoTip": {
-        "message": "uBlock Origin wiki: O Registro",
+        "message": "uBlock Origin wiki: Registros",
         "description": "Tooltip for the top-right info label in the logger page"
     },
     "loggerClearTip": {
@@ -628,7 +628,7 @@
         "description": ""
     },
     "hostsFilesAutoUpdatePrompt": {
-        "message": "Autoatualizar recursos",
+        "message": "Auto-atualizar recursos",
         "description": ""
     },
     "hostsFilesUpdateNow": {

src/_locales/ru/messages.json

@@ -72,7 +72,7 @@
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "fetchPrettyName": {
-        "message": "получить",
+        "message": "fetch",
         "description": "HAS TO FIT IN MATRIX HEADER!"
     },
     "framePrettyName": {

src/js/background.js

@@ -57,6 +57,8 @@ const rawSettingsDefault = {
     assetFetchBypassBrowserCache: false,
     assetFetchTimeout: 30,
     autoUpdateAssetFetchPeriod: 120,
+    cacheStorageAPI: 'unset',
+    cacheStorageCompression: true,
     cnameIgnoreList: 'unset',
     cnameIgnore1stParty: true,
     cnameIgnoreExceptions: true,
@@ -124,7 +126,7 @@ const rawSettingsDefault = {
             '</body></html>'
         ].join(''),
     framePlaceholderBackground: 'default',
-    suspendTabsUntilReady: false
+    suspendTabsUntilReady: 'unset'
 };
 
 /******************************************************************************/

src/js/cachestorage.js

@@ -119,7 +119,7 @@
                     disconnect();
                 },
                 Math.max(
-                    µMatrix.hiddenSettings.autoUpdateAssetFetchPeriod * 2 * 1000,
+                    µMatrix.rawSettings.autoUpdateAssetFetchPeriod * 2 * 1000,
                     180000
                 )
             );
@@ -296,7 +296,7 @@
             const promises = [ getDb() ];
             const entries = [];
             const dontCompress =
-                µMatrix.hiddenSettings.cacheStorageCompression !== true;
+                µMatrix.rawSettings.cacheStorageCompression !== true;
             const handleEncodingResult = result => {
                 entries.push({ key: result.key, value: result.data });
             };

src/js/codemirror/mode/raw-settings.js

@@ -23,7 +23,7 @@
 
 'use strict';
 
-CodeMirror.defineMode("raw-settings", function() {
+CodeMirror.defineMode('raw-settings', function() {
     return {
         token: function(stream) {
             if ( stream.sol() ) {

src/js/contentscript-no-inline-script.js

@@ -26,7 +26,7 @@
 // https://github.com/hackademix/noscript/commit/6e80d3f130773fc9a9123c5c4c2e97d63e90fa2a
 
 (function() {
-    let html = document.documentElement;
+    const html = document.documentElement;
     if ( html instanceof HTMLElement === false ) { return; }
 
     let meta;
@@ -45,8 +45,8 @@
     //
     // The CSP directive is enforced as soon as the meta tag is inserted:
     // > Enforce the policy policy.
-    let head = document.head,
+    const head = document.head;
-        parent = head;
+    let parent = head;
     if ( parent === null ) {
         parent = document.createElement('head');
         html.appendChild(parent);

src/js/main-blocked.js

@@ -87,7 +87,7 @@ uDom('.what').text(details.url);
         return s;
     };
 
-    const renderParams = function(parentNode, rawURL) {
+    const renderParams = function(parentNode, rawURL, depth = 0) {
         const a = document.createElement('a');
         a.href = rawURL;
         if ( a.search.length === 0 ) { return false; }
@@ -109,9 +109,9 @@ uDom('.what').text(details.url);
             const name = safeDecodeURIComponent(param.slice(0, pos));
             const value = safeDecodeURIComponent(param.slice(pos + 1));
             const li = liFromParam(name, value);
-            if ( reURL.test(value) ) {
+            if ( depth < 2 && reURL.test(value) ) {
                 const ul = document.createElement('ul');
-                renderParams(ul, value);
+                renderParams(ul, value, depth + 1);
                 li.appendChild(ul);
             }
             parentNode.appendChild(li);

src/js/popup.js

@@ -77,6 +77,11 @@ const resizePopup = (( ) => {
 
     // The purpose of `xobserver` is to initiate the resize handler only
     // when the popup panel is actually visible.
+
+    // https://github.com/uBlockOrigin/uMatrix-issues/issues/235
+    if ( self.IntersectionObserver instanceof Object === false ) {
+        return ( ) => { };
+    }
     let xobserver = new IntersectionObserver(intersections => {
         if ( intersections.length === 0 ) { return; }
         if ( intersections[0].isIntersecting === false ) { return; }

src/js/start.js

@@ -32,6 +32,11 @@
     ]);
     log.info(`User settings ready ${Date.now()-vAPI.T0} ms after launch`);
 
+    const cacheBackend = await µm.cacheStorage.select(
+        µm.rawSettings.cacheStorageAPI
+    );
+    log.info(`Backend storage for cache will be ${cacheBackend}`);
+
     const shouldWASM =
         vAPI.canWASM === true &&
         µm.rawSettings.disableWebAssembly !== true;

src/js/storage.js

@@ -496,9 +496,19 @@ self.addEventListener('rawSettingsChanged', ( ) => {
     const addedCount = this.ubiquitousBlacklistRef.addedCount;
     const addCount = this.ubiquitousBlacklistRef.addCount;
 
-    this.mergeHostsFileContent(details.content);
+    // https://www.reddit.com/r/uMatrix/comments/ftebgz/
+    //   Be ready to deal with a removed asset.
+
+    if ( typeof details.content === 'string' && details.content !== '' ) {
+        this.mergeHostsFileContent(details.content);
+    }
 
     const hostsFileMeta = this.liveHostsFiles.get(details.assetKey);
+    if ( hostsFileMeta === undefined ) {
+        this.liveHostsFiles.delete(details.assetKey);
+        return;
+    }
+
     hostsFileMeta.entryCount =
         this.ubiquitousBlacklistRef.addCount - addCount;
     hostsFileMeta.entryUsedCount =

src/js/tab.js

@@ -55,7 +55,7 @@ const µm = µMatrix;
     //   its APIs, but since this is not the case, uMatrix inherits the duty to
     //   make it seamless on its side.
     if ( pageURL.startsWith('wyciwyg:') ) {
-        let match = /^wyciwyg:\/\/\d+\//.exec(pageURL);
+        const match = /^wyciwyg:\/\/\d+\//.exec(pageURL);
         if ( match !== null ) {
             pageURL = pageURL.slice(match[0].length);
         }
@@ -64,18 +64,18 @@ const µm = µMatrix;
     // If the URL is that of our "blocked page" document, return the URL of
     // the blocked page.
     if ( pageURL.startsWith(vAPI.getURL('main-blocked.html')) ) {
-        let matches = /main-blocked\.html\?details=([^&]+)/.exec(pageURL);
+        const parsedURL = new URL(pageURL);
-        if ( matches && matches.length === 2 ) {
+        const details = parsedURL.searchParams.get('details');
+        if ( details ) {
             try {
-                let details = JSON.parse(atob(matches[1]));
+                pageURL = JSON.parse(decodeURIComponent(details)).url;
-                pageURL = details.url;
+            } catch (ex) {
-            } catch (e) {
             }
         }
     }
 
-    let uri = this.URI.set(pageURL);
+    const uri = this.URI.set(pageURL);
-    let scheme = uri.scheme;
+    const scheme = uri.scheme;
     if ( scheme === 'https' || scheme === 'http' ) {
         return uri.normalizedURI();
     }

tools/make-assets.sh

@@ -14,11 +14,14 @@ cp ./assets/assets.json $DES/
 if [ -n "${TRAVIS_TAG}" ]; then
   pushd .. > /dev/null
   git clone --depth 1 https://github.com/uBlockOrigin/uAssets.git
+  git checkout 84dc2761abb4193bb34290aa6d90266610f735f6
   popd > /dev/null
 fi
 
 mkdir $DES/thirdparties
-cp -R ../uAssets/thirdparties/hosts-file.net             $DES/thirdparties/
+pushd ../uAssets
+git checkout 84dc2761abb4193bb34290aa6d90266610f735f6
+popd
 cp -R ../uAssets/thirdparties/mirror1.malwaredomains.com $DES/thirdparties/
 cp -R ../uAssets/thirdparties/pgl.yoyo.org               $DES/thirdparties/
 cp -R ../uAssets/thirdparties/publicsuffix.org           $DES/thirdparties/
@@ -27,5 +30,8 @@ cp -R ../uAssets/thirdparties/winhelp2002.mvps.org       $DES/thirdparties/
 cp -R ../uAssets/thirdparties/www.malwaredomainlist.com  $DES/thirdparties/
 mkdir $DES/umatrix
 cp -R ../uAssets/recipes/*                               $DES/umatrix/
+pushd ../uAssets
+git checkout master
+popd
 
 echo "done."