Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/core/workflows.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ print(loaded["service"]["name"].upper_first())
```

`DataFile` keeps source labels and metadata promoted and redacted before they enter workflow step names or result metadata.
Remote reads accept HTTPS URLs only; clear-text and unsupported URL schemes are
rejected before a request is created.

## DataWorkflow

Expand Down
15 changes: 9 additions & 6 deletions packages/extended-data/src/extended_data/io/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from dataclasses import dataclass, field
from pathlib import Path
from typing import TYPE_CHECKING, Any, TypeAlias, cast
from urllib.parse import urlsplit

import validators

Expand Down Expand Up @@ -411,22 +412,21 @@ def is_url(path: str) -> bool:
"""Check if a string is a valid and safe URL.

Uses the validators library for robust URL validation,
restricted to HTTP/HTTPS schemes only.
restricted to HTTPS URLs only so remote file reads are encrypted in transit.

Args:
path (str): The string to check.

Returns:
bool: True if the string is a valid HTTP/HTTPS URL.
bool: True if the string is a valid HTTPS URL.
"""
if not path:
return False
# validators.url returns True for valid URLs, ValidationError otherwise
result = validators.url(path)
if result is not True:
return False
# Additional check: only allow http/https schemes
return path.startswith(("http://", "https://"))
return urlsplit(path).scheme == "https"


def read_file(
Expand Down Expand Up @@ -455,11 +455,14 @@ def read_file(

Raises:
urllib.error.URLError: If the URL cannot be accessed.
ValueError: If the URL scheme is not allowed (only http/https permitted).
ValueError: If the URL scheme is not allowed (only HTTPS is permitted).
"""
path_str = str(file_path)

# Handle URLs (is_url already validates HTTP/HTTPS only)
if "://" in path_str and not is_url(path_str):
raise ValueError("Remote file URLs must use HTTPS")
Comment on lines +462 to +463

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Reject clear-text redirect targets

When an accepted HTTPS endpoint responds with a redirect whose Location uses http://, the default urllib.request.urlopen redirect handler creates and follows that request without re-running this guard; it also carries the supplied custom headers into the redirected request. Consequently, both the response and potentially sensitive headers can still travel over clear text despite the newly documented HTTPS-only boundary. Use an opener/redirect handler that rejects every non-HTTPS redirect target.

AGENTS.md reference: AGENTS.md:L77-L77

Useful? React with 👍 / 👎.

Comment on lines +462 to +463

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 Badge Preserve Windows paths containing doubled separators

On Windows, callers can validly pass a string such as C://Users/me/config.json, which Path normally resolves as an absolute local path. The new substring check instead treats any :// as evidence of a remote URL; because is_url rejects the c scheme, read_file now raises ValueError before local-path resolution. Detect an actual URI scheme without misclassifying drive-letter paths.

Useful? React with 👍 / 👎.


# Handle URLs (is_url already validates HTTPS only).
if is_url(path_str):
headers = headers or {}
request = urllib.request.Request(path_str, headers=dict(headers))
Expand Down
9 changes: 8 additions & 1 deletion packages/extended-data/tests/core/test_file_data_type.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ def test_file_path_rel_to_root(file_path: FilePath, expected_rel_to_root: str) -
@pytest.mark.parametrize(
("path", "expected"),
[
("http://example.com/file.txt", True),
("http://example.com/file.txt", False),
("https://example.com/file.txt", True),
("/path/to/file.txt", False),
("relative/path.txt", False),
Expand All @@ -323,6 +323,13 @@ def test_is_url(path: str, expected: bool) -> None:
assert is_url(path) == expected


@pytest.mark.parametrize("url", ["http://example.com/data.txt", "ftp://example.com/data.txt"])
def test_read_file_rejects_unencrypted_or_unsupported_urls(url: str) -> None:
"""Fail closed before a non-HTTPS URL can reach a network request."""
with pytest.raises(ValueError, match="must use HTTPS"):
read_file(url)


def test_resolve_local_path_absolute() -> None:
"""Tests resolving an absolute path.

Expand Down