CodeMirror/ArchiveBox
1
0
Fork 0
mirror of synced 2024-06-30 20:10:35 +12:00
Code Issues Projects Releases Wiki Activity

Merge pull request #96 from fgtham/fix_issue_74

Browse source 
fix unstable sorting between consecutive runs
This commit is contained in:
Nick Sweeting 2018-09-19 21:55:36 -04:00 committed by GitHub
parent 8a23358fc8 5450afd18b
commit d46caf604d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 4 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
archiver/links.py
View file

@ -34,6 +34,7 @@ Link {
import datetime
from html import unescape
from collections import OrderedDict
from util import (
domain,
@ -87,7 +88,7 @@ def uniquefied_links(sorted_links):
ensures that all non-duplicate links have monotonically increasing timestamps
"""
unique_urls = {}
unique_urls = OrderedDict()
lower = lambda url: url.lower().strip()
without_www = lambda url: url.replace('://www.', '://', 1)
@ -100,7 +101,7 @@ def uniquefied_links(sorted_links):
link = merge_links(unique_urls[fuzzy_url], link)
unique_urls[fuzzy_url] = link
unique_timestamps = {}
unique_timestamps = OrderedDict()
for link in unique_urls.values():
link['timestamp'] = lowest_uniq_timestamp(unique_timestamps, link['timestamp'])
unique_timestamps[link['timestamp']] = link
@ -108,7 +109,7 @@ def uniquefied_links(sorted_links):
return unique_timestamps.values()
def sorted_links(links):
sort_func = lambda link: (link['timestamp'], link['url'])
sort_func = lambda link: (link['timestamp'].split('.', 1)[0], link['url'])
return sorted(links, key=sort_func, reverse=True)
def links_after_timestamp(links, timestamp=None):