From 520440986bcd54274e21327bda34694cd078c336 Mon Sep 17 00:00:00 2001 From: Jon Bogaty Date: Mon, 24 Aug 2026 12:23:07 -0500 Subject: [PATCH] fix: require HTTPS for remote file reads --- docs/core/workflows.md | 2 ++ .../extended-data/src/extended_data/io/files.py | 15 +++++++++------ .../tests/core/test_file_data_type.py | 9 ++++++++- 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/docs/core/workflows.md b/docs/core/workflows.md index cdcdbfc..f3b9293 100644 --- a/docs/core/workflows.md +++ b/docs/core/workflows.md @@ -16,6 +16,8 @@ print(loaded["service"]["name"].upper_first()) ``` `DataFile` keeps source labels and metadata promoted and redacted before they enter workflow step names or result metadata. +Remote reads accept HTTPS URLs only; clear-text and unsupported URL schemes are +rejected before a request is created. ## DataWorkflow diff --git a/packages/extended-data/src/extended_data/io/files.py b/packages/extended-data/src/extended_data/io/files.py index 40a4664..dde16c3 100644 --- a/packages/extended-data/src/extended_data/io/files.py +++ b/packages/extended-data/src/extended_data/io/files.py @@ -12,6 +12,7 @@ from dataclasses import dataclass, field from pathlib import Path from typing import TYPE_CHECKING, Any, TypeAlias, cast +from urllib.parse import urlsplit import validators @@ -411,13 +412,13 @@ def is_url(path: str) -> bool: """Check if a string is a valid and safe URL. Uses the validators library for robust URL validation, - restricted to HTTP/HTTPS schemes only. + restricted to HTTPS URLs only so remote file reads are encrypted in transit. Args: path (str): The string to check. Returns: - bool: True if the string is a valid HTTP/HTTPS URL. + bool: True if the string is a valid HTTPS URL. """ if not path: return False @@ -425,8 +426,7 @@ def is_url(path: str) -> bool: result = validators.url(path) if result is not True: return False - # Additional check: only allow http/https schemes - return path.startswith(("http://", "https://")) + return urlsplit(path).scheme == "https" def read_file( @@ -455,11 +455,14 @@ def read_file( Raises: urllib.error.URLError: If the URL cannot be accessed. - ValueError: If the URL scheme is not allowed (only http/https permitted). + ValueError: If the URL scheme is not allowed (only HTTPS is permitted). """ path_str = str(file_path) - # Handle URLs (is_url already validates HTTP/HTTPS only) + if "://" in path_str and not is_url(path_str): + raise ValueError("Remote file URLs must use HTTPS") + + # Handle URLs (is_url already validates HTTPS only). if is_url(path_str): headers = headers or {} request = urllib.request.Request(path_str, headers=dict(headers)) diff --git a/packages/extended-data/tests/core/test_file_data_type.py b/packages/extended-data/tests/core/test_file_data_type.py index feadbae..7be0fb5 100644 --- a/packages/extended-data/tests/core/test_file_data_type.py +++ b/packages/extended-data/tests/core/test_file_data_type.py @@ -302,7 +302,7 @@ def test_file_path_rel_to_root(file_path: FilePath, expected_rel_to_root: str) - @pytest.mark.parametrize( ("path", "expected"), [ - ("http://example.com/file.txt", True), + ("http://example.com/file.txt", False), ("https://example.com/file.txt", True), ("/path/to/file.txt", False), ("relative/path.txt", False), @@ -323,6 +323,13 @@ def test_is_url(path: str, expected: bool) -> None: assert is_url(path) == expected +@pytest.mark.parametrize("url", ["http://example.com/data.txt", "ftp://example.com/data.txt"]) +def test_read_file_rejects_unencrypted_or_unsupported_urls(url: str) -> None: + """Fail closed before a non-HTTPS URL can reach a network request.""" + with pytest.raises(ValueError, match="must use HTTPS"): + read_file(url) + + def test_resolve_local_path_absolute() -> None: """Tests resolving an absolute path.