f6cf35a45d
`item.find(p)` returns either an `ElementTree.Element` or `None`. The [lambda on line 24][lambda] coerces the return value to a bool, which is `False` if the `<link>` element has no children (see [`ElementTree.py` line 207][etbooldef]), so the lambda returns `None`. Further, returning a `Link` with `url=None` violates [an assertion in `index/schema.py`][assertion], which crashes the `archivebox add` command. [lambda]:3d54b1321b/archivebox/parsers/pinboard_rss.py (L24)[etbooldef]:3d8993a744/Lib/xml/etree/ElementTree.py (L207)[assertion]:3d54b1321b/archivebox/index/schema.py (L165)
58 lines
1.7 KiB
Python
58 lines
1.7 KiB
Python
__package__ = 'archivebox.parsers'
|
|
|
|
|
|
from typing import IO, Iterable
|
|
from datetime import datetime, timezone
|
|
|
|
from xml.etree import ElementTree
|
|
|
|
from ..index.schema import Link
|
|
from ..util import (
|
|
htmldecode,
|
|
enforce_types,
|
|
)
|
|
|
|
|
|
@enforce_types
|
|
def parse_pinboard_rss_export(rss_file: IO[str], **_kwargs) -> Iterable[Link]:
|
|
"""Parse Pinboard RSS feed files into links"""
|
|
|
|
rss_file.seek(0)
|
|
root = ElementTree.parse(rss_file).getroot()
|
|
items = root.findall("{http://purl.org/rss/1.0/}item")
|
|
for item in items:
|
|
find = lambda p: item.find(p).text.strip() if item.find(p) is not None else None # type: ignore
|
|
|
|
url = find("{http://purl.org/rss/1.0/}link")
|
|
tags = find("{http://purl.org/dc/elements/1.1/}subject")
|
|
title = find("{http://purl.org/rss/1.0/}title")
|
|
ts_str = find("{http://purl.org/dc/elements/1.1/}date")
|
|
|
|
if url is None:
|
|
# Yielding a Link with no URL will
|
|
# crash on a URL validation assertion
|
|
continue
|
|
|
|
# Pinboard includes a colon in its date stamp timezone offsets, which
|
|
# Python can't parse. Remove it:
|
|
if ts_str and ts_str[-3:-2] == ":":
|
|
ts_str = ts_str[:-3]+ts_str[-2:]
|
|
|
|
if ts_str:
|
|
time = datetime.strptime(ts_str, "%Y-%m-%dT%H:%M:%S%z")
|
|
else:
|
|
time = datetime.now(timezone.utc)
|
|
|
|
yield Link(
|
|
url=htmldecode(url),
|
|
timestamp=str(time.timestamp()),
|
|
title=htmldecode(title) or None,
|
|
tags=htmldecode(tags) or None,
|
|
sources=[rss_file.name],
|
|
)
|
|
|
|
|
|
KEY = 'pinboard_rss'
|
|
NAME = 'Pinboard RSS'
|
|
PARSER = parse_pinboard_rss_export
|