From b64f5080257711465e2c752cda8c56d3bacc8e51 Mon Sep 17 00:00:00 2001 From: Serene-Arc Date: Wed, 4 Jan 2023 19:37:02 +1000 Subject: [PATCH] Conform path length to filename scheme restriction --- bdfr/file_name_formatter.py | 16 +++++++++++----- tests/test_file_name_formatter.py | 24 +++++++++++++++++------- 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/bdfr/file_name_formatter.py b/bdfr/file_name_formatter.py index 0330336..dd04fad 100644 --- a/bdfr/file_name_formatter.py +++ b/bdfr/file_name_formatter.py @@ -27,6 +27,8 @@ class FileNameFormatter: "title", "upvotes", ) + WINDOWS_MAX_PATH_LENGTH = 260 + LINUX_MAX_PATH_LENGTH = 4096 def __init__( self, @@ -41,6 +43,10 @@ class FileNameFormatter: self.directory_format_string: list[str] = directory_format_string.split("/") self.time_format_string = time_format_string self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None + if self.restiction_scheme == "windows": + self.max_path = self.WINDOWS_MAX_PATH_LENGTH + else: + self.max_path = self.find_max_path_length() def _format_name(self, submission: Union[Comment, Submission], format_string: str) -> str: if isinstance(submission, Submission): @@ -63,6 +69,7 @@ class FileNameFormatter: if platform.system() == "Windows": result = FileNameFormatter._format_for_windows(result) elif self.restiction_scheme == "windows": + logger.debug("Forcing Windows-compatible filenames") result = FileNameFormatter._format_for_windows(result) return result @@ -135,14 +142,13 @@ class FileNameFormatter: raise BulkDownloaderException(f"Could not determine path name: {subfolder}, {index}, {resource.extension}") return file_path - @staticmethod - def limit_file_name_length(filename: str, ending: str, root: Path) -> Path: + def limit_file_name_length(self, filename: str, ending: str, root: Path) -> Path: root = root.resolve().expanduser() possible_id = re.search(r"((?:_\w{6})?$)", filename) if possible_id: ending = possible_id.group(1) + ending filename = filename[: possible_id.start()] - max_path = FileNameFormatter.find_max_path_length() + max_path = self.max_path max_file_part_length_chars = 255 - len(ending) max_file_part_length_bytes = 255 - len(ending.encode("utf-8")) max_path_length = max_path - len(ending) - len(str(root)) - 1 @@ -166,9 +172,9 @@ class FileNameFormatter: return int(subprocess.check_output(["getconf", "PATH_MAX", "/"])) except (ValueError, subprocess.CalledProcessError, OSError): if platform.system() == "Windows": - return 260 + return FileNameFormatter.WINDOWS_MAX_PATH_LENGTH else: - return 4096 + return FileNameFormatter.LINUX_MAX_PATH_LENGTH def format_resource_paths( self, diff --git a/tests/test_file_name_formatter.py b/tests/test_file_name_formatter.py index 5f94e5f..fb34a53 100644 --- a/tests/test_file_name_formatter.py +++ b/tests/test_file_name_formatter.py @@ -33,6 +33,12 @@ def submission() -> MagicMock: return test +@pytest.fixture() +def test_formatter() -> FileNameFormatter: + out = FileNameFormatter("{TITLE}", "", "ISO") + return out + + def check_valid_windows_path(test_string: str): return test_string == FileNameFormatter._format_for_windows(test_string) @@ -231,8 +237,8 @@ def test_format_multiple_resources(): ("😍💕✨" * 100, "_1.png"), ), ) -def test_limit_filename_length(test_filename: str, test_ending: str): - result = FileNameFormatter.limit_file_name_length(test_filename, test_ending, Path(".")) +def test_limit_filename_length(test_filename: str, test_ending: str, test_formatter: FileNameFormatter): + result = test_formatter.limit_file_name_length(test_filename, test_ending, Path(".")) assert len(result.name) <= 255 assert len(result.name.encode("utf-8")) <= 255 assert len(str(result)) <= FileNameFormatter.find_max_path_length() @@ -253,8 +259,10 @@ def test_limit_filename_length(test_filename: str, test_ending: str): ("😍💕✨" * 100 + "_aaa1aa", "_1.png", "_aaa1aa_1.png"), ), ) -def test_preserve_id_append_when_shortening(test_filename: str, test_ending: str, expected_end: str): - result = FileNameFormatter.limit_file_name_length(test_filename, test_ending, Path(".")) +def test_preserve_id_append_when_shortening( + test_filename: str, test_ending: str, expected_end: str, test_formatter: FileNameFormatter +): + result = test_formatter.limit_file_name_length(test_filename, test_ending, Path(".")) assert len(result.name) <= 255 assert len(result.name.encode("utf-8")) <= 255 assert result.name.endswith(expected_end) @@ -284,8 +292,8 @@ def test_shorten_filename_real(submission: MagicMock, tmp_path: Path): ("a" * 500, "_bbbbbb.jpg"), ), ) -def test_shorten_path(test_name: str, test_ending: str, tmp_path: Path): - result = FileNameFormatter.limit_file_name_length(test_name, test_ending, tmp_path) +def test_shorten_path(test_name: str, test_ending: str, tmp_path: Path, test_formatter: FileNameFormatter): + result = test_formatter.limit_file_name_length(test_name, test_ending, tmp_path) assert len(str(result.name)) <= 255 assert len(str(result.name).encode("UTF-8")) <= 255 assert len(str(result.name).encode("cp1252")) <= 255 @@ -482,7 +490,9 @@ def test_get_max_path_length(): def test_windows_max_path(tmp_path: Path): with unittest.mock.patch("platform.system", return_value="Windows"): with unittest.mock.patch("bdfr.file_name_formatter.FileNameFormatter.find_max_path_length", return_value=260): - result = FileNameFormatter.limit_file_name_length("test" * 100, "_1.png", tmp_path) + mock = MagicMock() + mock.max_path = 260 + result = FileNameFormatter.limit_file_name_length(mock, "test" * 100, "_1.png", tmp_path) assert len(str(result)) <= 260 assert len(result.name) <= (260 - len(str(tmp_path)))