69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
|
"""
|
||
|
|
Everything related to parsing links from input sources.
|
||
|
|
|
||
|
|
For a list of supported services, see the README.md.
|
||
|
|
For examples of supported import formats see tests/.
|
||
|
|
"""
|
||
|
|
|
||
|
|
__package__ = 'archivebox.parsers'
|
||
|
|
|
||
|
|
|
||
|
|
from typing import Tuple, List
|
||
|
|
|
||
|
|
from ..config import TIMEOUT
|
||
|
|
from ..util import (
|
||
|
|
check_url_parsing_invariants,
|
||
|
|
TimedProgress,
|
||
|
|
Link,
|
||
|
|
enforce_types,
|
||
|
|
)
|
||
|
|
from .pocket_html import parse_pocket_html_export
|
||
|
|
from .pinboard_rss import parse_pinboard_rss_export
|
||
|
|
from .shaarli_rss import parse_shaarli_rss_export
|
||
|
|
from .medium_rss import parse_medium_rss_export
|
||
|
|
from .netscape_html import parse_netscape_html_export
|
||
|
|
from .generic_rss import parse_generic_rss_export
|
||
|
|
from .generic_json import parse_generic_json_export
|
||
|
|
from .generic_txt import parse_generic_txt_export
|
||
|
|
|
||
|
|
|
||
|
|
@enforce_types
|
||
|
|
def parse_links(source_file: str) -> Tuple[List[Link], str]:
|
||
|
|
"""parse a list of URLs with their metadata from an
|
||
|
|
RSS feed, bookmarks export, or text file
|
||
|
|
"""
|
||
|
|
|
||
|
|
check_url_parsing_invariants()
|
||
|
|
PARSERS = (
|
||
|
|
# Specialized parsers
|
||
|
|
('Pocket HTML', parse_pocket_html_export),
|
||
|
|
('Pinboard RSS', parse_pinboard_rss_export),
|
||
|
|
('Shaarli RSS', parse_shaarli_rss_export),
|
||
|
|
('Medium RSS', parse_medium_rss_export),
|
||
|
|
|
||
|
|
# General parsers
|
||
|
|
('Netscape HTML', parse_netscape_html_export),
|
||
|
|
('Generic RSS', parse_generic_rss_export),
|
||
|
|
('Generic JSON', parse_generic_json_export),
|
||
|
|
|
||
|
|
# Fallback parser
|
||
|
|
('Plain Text', parse_generic_txt_export),
|
||
|
|
)
|
||
|
|
timer = TimedProgress(TIMEOUT * 4)
|
||
|
|
with open(source_file, 'r', encoding='utf-8') as file:
|
||
|
|
for parser_name, parser_func in PARSERS:
|
||
|
|
try:
|
||
|
|
links = list(parser_func(file))
|
||
|
|
if links:
|
||
|
|
timer.end()
|
||
|
|
return links, parser_name
|
||
|
|
except Exception as err: # noqa
|
||
|
|
# Parsers are tried one by one down the list, and the first one
|
||
|
|
# that succeeds is used. To see why a certain parser was not used
|
||
|
|
# due to error or format incompatibility, uncomment this line:
|
||
|
|
# print('[!] Parser {} failed: {} {}'.format(parser_name, err.__class__.__name__, err))
|
||
|
|
pass
|
||
|
|
|
||
|
|
timer.end()
|
||
|
|
return [], 'Failed to parse'
|