Merge pull request #493 from ttimasdf/feat-ogtitle

Feature: add og:title metadata as alternative title

archivebox/extractors/title.py

@@ -1,6 +1,7 @@
 __package__ = 'archivebox.extractors'
 
 import re
+from html.parser import HTMLParser
 from pathlib import Path
 from typing import Optional
 
@@ -24,11 +25,32 @@ from ..config import (
 from ..logging_util import TimedProgress
 
 
-HTML_TITLE_REGEX = re.compile(
-    r'<title.*?>'                      # start matching text after <title> tag
-    r'(.[^<>]+)',                      # get everything up to these symbols
-    re.IGNORECASE | re.MULTILINE | re.DOTALL | re.UNICODE,
-)
+class TitleParser(HTMLParser):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.title_tag = ""
+        self.title_og = ""
+        self.inside_title_tag = False
+
+    @property
+    def title(self):
+        return self.title_tag or self.title_og or None
+
+    def handle_starttag(self, tag, attrs):
+        if tag.lower() == "title" and not self.title_tag:
+            self.inside_title_tag = True
+        elif tag.lower() == "meta" and not self.title_og:
+            attrs = dict(attrs)
+            if attrs.get("property") == "og:title" and attrs.get("content"):
+                self.title_og = attrs.get("content")
+
+    def handle_data(self, data):
+        if self.inside_title_tag and data:
+            self.title_tag += data.strip()
+    
+    def handle_endtag(self, tag):
+        if tag.lower() == "title":
+            self.inside_title_tag = False
 
 
 @enforce_types
@@ -62,8 +84,9 @@ def save_title(link: Link, out_dir: Optional[Path]=None, timeout: int=TIMEOUT) -
     timer = TimedProgress(timeout, prefix='      ')
     try:
         html = download_url(link.url, timeout=timeout)
-        match = re.search(HTML_TITLE_REGEX, html)
-        output = htmldecode(match.group(1).strip()) if match else None
+        parser = TitleParser()
+        parser.feed(html)
+        output = parser.title
         if output:
             if not link.title or len(output) >= len(link.title):
                 Snapshot.objects.filter(url=link.url, timestamp=link.timestamp).update(title=output)