Conform path length to filename scheme restriction
This commit is contained in:
parent
8c57dc2283
commit
b64f508025
|
@ -27,6 +27,8 @@ class FileNameFormatter:
|
||||||
"title",
|
"title",
|
||||||
"upvotes",
|
"upvotes",
|
||||||
)
|
)
|
||||||
|
WINDOWS_MAX_PATH_LENGTH = 260
|
||||||
|
LINUX_MAX_PATH_LENGTH = 4096
|
||||||
|
|
||||||
def __init__(
|
def __init__(
|
||||||
self,
|
self,
|
||||||
|
@ -41,6 +43,10 @@ class FileNameFormatter:
|
||||||
self.directory_format_string: list[str] = directory_format_string.split("/")
|
self.directory_format_string: list[str] = directory_format_string.split("/")
|
||||||
self.time_format_string = time_format_string
|
self.time_format_string = time_format_string
|
||||||
self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None
|
self.restiction_scheme = restriction_scheme.lower().strip() if restriction_scheme else None
|
||||||
|
if self.restiction_scheme == "windows":
|
||||||
|
self.max_path = self.WINDOWS_MAX_PATH_LENGTH
|
||||||
|
else:
|
||||||
|
self.max_path = self.find_max_path_length()
|
||||||
|
|
||||||
def _format_name(self, submission: Union[Comment, Submission], format_string: str) -> str:
|
def _format_name(self, submission: Union[Comment, Submission], format_string: str) -> str:
|
||||||
if isinstance(submission, Submission):
|
if isinstance(submission, Submission):
|
||||||
|
@ -63,6 +69,7 @@ class FileNameFormatter:
|
||||||
if platform.system() == "Windows":
|
if platform.system() == "Windows":
|
||||||
result = FileNameFormatter._format_for_windows(result)
|
result = FileNameFormatter._format_for_windows(result)
|
||||||
elif self.restiction_scheme == "windows":
|
elif self.restiction_scheme == "windows":
|
||||||
|
logger.debug("Forcing Windows-compatible filenames")
|
||||||
result = FileNameFormatter._format_for_windows(result)
|
result = FileNameFormatter._format_for_windows(result)
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
@ -135,14 +142,13 @@ class FileNameFormatter:
|
||||||
raise BulkDownloaderException(f"Could not determine path name: {subfolder}, {index}, {resource.extension}")
|
raise BulkDownloaderException(f"Could not determine path name: {subfolder}, {index}, {resource.extension}")
|
||||||
return file_path
|
return file_path
|
||||||
|
|
||||||
@staticmethod
|
def limit_file_name_length(self, filename: str, ending: str, root: Path) -> Path:
|
||||||
def limit_file_name_length(filename: str, ending: str, root: Path) -> Path:
|
|
||||||
root = root.resolve().expanduser()
|
root = root.resolve().expanduser()
|
||||||
possible_id = re.search(r"((?:_\w{6})?$)", filename)
|
possible_id = re.search(r"((?:_\w{6})?$)", filename)
|
||||||
if possible_id:
|
if possible_id:
|
||||||
ending = possible_id.group(1) + ending
|
ending = possible_id.group(1) + ending
|
||||||
filename = filename[: possible_id.start()]
|
filename = filename[: possible_id.start()]
|
||||||
max_path = FileNameFormatter.find_max_path_length()
|
max_path = self.max_path
|
||||||
max_file_part_length_chars = 255 - len(ending)
|
max_file_part_length_chars = 255 - len(ending)
|
||||||
max_file_part_length_bytes = 255 - len(ending.encode("utf-8"))
|
max_file_part_length_bytes = 255 - len(ending.encode("utf-8"))
|
||||||
max_path_length = max_path - len(ending) - len(str(root)) - 1
|
max_path_length = max_path - len(ending) - len(str(root)) - 1
|
||||||
|
@ -166,9 +172,9 @@ class FileNameFormatter:
|
||||||
return int(subprocess.check_output(["getconf", "PATH_MAX", "/"]))
|
return int(subprocess.check_output(["getconf", "PATH_MAX", "/"]))
|
||||||
except (ValueError, subprocess.CalledProcessError, OSError):
|
except (ValueError, subprocess.CalledProcessError, OSError):
|
||||||
if platform.system() == "Windows":
|
if platform.system() == "Windows":
|
||||||
return 260
|
return FileNameFormatter.WINDOWS_MAX_PATH_LENGTH
|
||||||
else:
|
else:
|
||||||
return 4096
|
return FileNameFormatter.LINUX_MAX_PATH_LENGTH
|
||||||
|
|
||||||
def format_resource_paths(
|
def format_resource_paths(
|
||||||
self,
|
self,
|
||||||
|
|
|
@ -33,6 +33,12 @@ def submission() -> MagicMock:
|
||||||
return test
|
return test
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture()
|
||||||
|
def test_formatter() -> FileNameFormatter:
|
||||||
|
out = FileNameFormatter("{TITLE}", "", "ISO")
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
def check_valid_windows_path(test_string: str):
|
def check_valid_windows_path(test_string: str):
|
||||||
return test_string == FileNameFormatter._format_for_windows(test_string)
|
return test_string == FileNameFormatter._format_for_windows(test_string)
|
||||||
|
|
||||||
|
@ -231,8 +237,8 @@ def test_format_multiple_resources():
|
||||||
("😍💕✨" * 100, "_1.png"),
|
("😍💕✨" * 100, "_1.png"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_limit_filename_length(test_filename: str, test_ending: str):
|
def test_limit_filename_length(test_filename: str, test_ending: str, test_formatter: FileNameFormatter):
|
||||||
result = FileNameFormatter.limit_file_name_length(test_filename, test_ending, Path("."))
|
result = test_formatter.limit_file_name_length(test_filename, test_ending, Path("."))
|
||||||
assert len(result.name) <= 255
|
assert len(result.name) <= 255
|
||||||
assert len(result.name.encode("utf-8")) <= 255
|
assert len(result.name.encode("utf-8")) <= 255
|
||||||
assert len(str(result)) <= FileNameFormatter.find_max_path_length()
|
assert len(str(result)) <= FileNameFormatter.find_max_path_length()
|
||||||
|
@ -253,8 +259,10 @@ def test_limit_filename_length(test_filename: str, test_ending: str):
|
||||||
("😍💕✨" * 100 + "_aaa1aa", "_1.png", "_aaa1aa_1.png"),
|
("😍💕✨" * 100 + "_aaa1aa", "_1.png", "_aaa1aa_1.png"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_preserve_id_append_when_shortening(test_filename: str, test_ending: str, expected_end: str):
|
def test_preserve_id_append_when_shortening(
|
||||||
result = FileNameFormatter.limit_file_name_length(test_filename, test_ending, Path("."))
|
test_filename: str, test_ending: str, expected_end: str, test_formatter: FileNameFormatter
|
||||||
|
):
|
||||||
|
result = test_formatter.limit_file_name_length(test_filename, test_ending, Path("."))
|
||||||
assert len(result.name) <= 255
|
assert len(result.name) <= 255
|
||||||
assert len(result.name.encode("utf-8")) <= 255
|
assert len(result.name.encode("utf-8")) <= 255
|
||||||
assert result.name.endswith(expected_end)
|
assert result.name.endswith(expected_end)
|
||||||
|
@ -284,8 +292,8 @@ def test_shorten_filename_real(submission: MagicMock, tmp_path: Path):
|
||||||
("a" * 500, "_bbbbbb.jpg"),
|
("a" * 500, "_bbbbbb.jpg"),
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
def test_shorten_path(test_name: str, test_ending: str, tmp_path: Path):
|
def test_shorten_path(test_name: str, test_ending: str, tmp_path: Path, test_formatter: FileNameFormatter):
|
||||||
result = FileNameFormatter.limit_file_name_length(test_name, test_ending, tmp_path)
|
result = test_formatter.limit_file_name_length(test_name, test_ending, tmp_path)
|
||||||
assert len(str(result.name)) <= 255
|
assert len(str(result.name)) <= 255
|
||||||
assert len(str(result.name).encode("UTF-8")) <= 255
|
assert len(str(result.name).encode("UTF-8")) <= 255
|
||||||
assert len(str(result.name).encode("cp1252")) <= 255
|
assert len(str(result.name).encode("cp1252")) <= 255
|
||||||
|
@ -482,7 +490,9 @@ def test_get_max_path_length():
|
||||||
def test_windows_max_path(tmp_path: Path):
|
def test_windows_max_path(tmp_path: Path):
|
||||||
with unittest.mock.patch("platform.system", return_value="Windows"):
|
with unittest.mock.patch("platform.system", return_value="Windows"):
|
||||||
with unittest.mock.patch("bdfr.file_name_formatter.FileNameFormatter.find_max_path_length", return_value=260):
|
with unittest.mock.patch("bdfr.file_name_formatter.FileNameFormatter.find_max_path_length", return_value=260):
|
||||||
result = FileNameFormatter.limit_file_name_length("test" * 100, "_1.png", tmp_path)
|
mock = MagicMock()
|
||||||
|
mock.max_path = 260
|
||||||
|
result = FileNameFormatter.limit_file_name_length(mock, "test" * 100, "_1.png", tmp_path)
|
||||||
assert len(str(result)) <= 260
|
assert len(str(result)) <= 260
|
||||||
assert len(result.name) <= (260 - len(str(tmp_path)))
|
assert len(result.name) <= (260 - len(str(tmp_path)))
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue