From 002cbb031959612c7c3367833cb70b03a8d35c53 Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sat, 22 Aug 2026 23:50:48 +0800 Subject: [PATCH 1/2] fix: recognize stdlib modules across all supported Python versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stdlib-safety check keyed off sys.stdlib_module_names of the interpreter running the scan. A pickle scanned in CI on one Python version is often loaded on another, so modules added in newer releases were flagged as unsafe by older scanners, and modules removed in newer releases (imp, asynchat, smtpd, ...) passed scans only to resolve to a malicious PyPI squatter — or fail outright — on the target. Replace the single-version list with a checked-in union of top-level stdlib module names across every supported Python (3.10-3.14), merged at import time with the running interpreter's own names so a scanner can never miss names from a release newer than this table. The residual risk inherent to any static union — a removed module name squatted by a malicious PyPI distribution on interpreters where it is no longer standard — is documented in the module docstring, and scanners that know exactly which interpreter will unpickle the file can narrow the effective set with use_stdlib_of(*versions). scripts/generate_stdlib_names.py regenerates the table; run it when the support matrix changes: uv run --no-project python scripts/generate_stdlib_names.py Fixes #311 --- fickling/fickle.py | 9 +- fickling/stdlib_names.py | 78 +++++++++++++++++ scripts/generate_stdlib_names.py | 146 +++++++++++++++++++++++++++++++ test/test_stdlib_names.py | 87 ++++++++++++++++++ 4 files changed, 316 insertions(+), 4 deletions(-) create mode 100644 fickling/stdlib_names.py create mode 100755 scripts/generate_stdlib_names.py create mode 100644 test/test_stdlib_names.py diff --git a/fickling/fickle.py b/fickling/fickle.py index c71b7f1..fbf46b1 100644 --- a/fickling/fickle.py +++ b/fickling/fickle.py @@ -21,6 +21,7 @@ overload, ) +from fickling import stdlib_names from fickling.exception import ExpansionAttackError, ResourceExhaustionError, WrongMethodError T = TypeVar("T") @@ -56,8 +57,6 @@ def __post_init__(self) -> None: GenericSequence = Sequence[T] make_constant = ast.Constant -BUILTIN_STDLIB_MODULE_NAMES: frozenset[str] = sys.stdlib_module_names - OPCODES_BY_NAME: dict[str, type[Opcode]] = {} OPCODE_INFO_BY_NAME: dict[str, OpcodeInfo] = {opcode.name: opcode for opcode in opcodes} @@ -279,7 +278,9 @@ def __post_init__(self) -> None: def is_std_module(module_name: str) -> bool: - return module_name.partition(".")[0] in BUILTIN_STDLIB_MODULE_NAMES + # Looked up through the module so runtime retargeting via + # stdlib_names.use_stdlib_of() takes effect (issue #311). + return module_name.partition(".")[0] in stdlib_names.STDLIB_MODULE_NAMES def is_private_or_dunder_stdlib_module(name: str) -> bool: @@ -287,7 +288,7 @@ def is_private_or_dunder_stdlib_module(name: str) -> bool: `__future__`, …): internal or non-public, so it should never appear in a pickle. Benign exceptions are handled in _is_allowed_private_import. """ - return name.startswith("_") and name in BUILTIN_STDLIB_MODULE_NAMES + return name.startswith("_") and name in stdlib_names.STDLIB_MODULE_NAMES def import_name_components(node: ast.Import | ast.ImportFrom) -> Iterator[str]: diff --git a/fickling/stdlib_names.py b/fickling/stdlib_names.py new file mode 100644 index 0000000..a0928f2 --- /dev/null +++ b/fickling/stdlib_names.py @@ -0,0 +1,78 @@ +"""Top-level standard-library module names across supported Pythons. + +Generated by scripts/generate_stdlib_names.py — do not edit by hand; +regenerate instead. Last generated 2026-08-22 15:35 UTC. + +Why a checked-in table instead of ``sys.stdlib_module_names`` alone: the +pickle being scanned is not necessarily loaded by the interpreter doing +the scanning (imagine a CI job gating artifacts for developer machines). +A module that exists in *any* supported Python version must therefore be +treated as potentially-standard, or benign pickles get flagged whenever +the scanner runs on a different version than the loader. + +Default semantics: ``STDLIB_MODULE_NAMES`` is the union of every listed +version plus the running interpreter's own names, so a scanner can never +miss names from a Python newer than this table. The residual risk of +this union — a removed module name (``imp``, ``asynchat``, …) squatted +by a malicious PyPI distribution on newer interpreters — is inherent to +static analysis without knowing the target version; callers who know +exactly which interpreter will unpickle may narrow the set with +:func:`use_stdlib_of`. +""" + +from __future__ import annotations + +import sys + +__all__ = [ + "PYTHON_VERSIONS", + "STDLIB_MODULE_NAMES", + "STDLIB_MODULE_NAMES_BY_VERSION", + "use_stdlib_of", +] + + + +PYTHON_VERSIONS = ['3.10', '3.11', '3.12', '3.13', '3.14'] + +STDLIB_MODULE_NAMES_BY_VERSION: dict[str, frozenset[str]] = { + '3.10': frozenset({'__future__', '_abc', '_aix_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bootsubprocess', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_compat_pickle', '_compression', '_contextvars', '_crypt', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_io', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_msi', '_multibytecodec', '_multiprocessing', '_opcode', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydecimal', '_pyio', '_queue', '_random', '_scproxy', '_sha1', '_sha256', '_sha3', '_sha512', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_symtable', '_thread', '_threading_local', '_tkinter', '_tracemalloc', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_zoneinfo', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast', 'asynchat', 'asyncio', 'asyncore', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'binhex', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'distutils', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'imp', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'ossaudiodev', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.11': frozenset({'__future__', '_abc', '_aix_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bootsubprocess', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_compat_pickle', '_compression', '_contextvars', '_crypt', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_io', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_msi', '_multibytecodec', '_multiprocessing', '_opcode', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydecimal', '_pyio', '_queue', '_random', '_scproxy', '_sha1', '_sha256', '_sha3', '_sha512', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_symtable', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_zoneinfo', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast', 'asynchat', 'asyncio', 'asyncore', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'distutils', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'imp', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'ossaudiodev', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.12': frozenset({'__future__', '_abc', '_aix_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_compat_pickle', '_compression', '_contextvars', '_crypt', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_io', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_msi', '_multibytecodec', '_multiprocessing', '_opcode', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_queue', '_random', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_symtable', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'ossaudiodev', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.13': frozenset({'__future__', '_abc', '_aix_support', '_android_support', '_apple_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_colorize', '_compat_pickle', '_compression', '_contextvars', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_interpchannels', '_interpqueues', '_interpreters', '_io', '_ios_support', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_multibytecodec', '_multiprocessing', '_opcode', '_opcode_metadata', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_pyrepl', '_queue', '_random', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_suggestions', '_symtable', '_sysconfig', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', 'abc', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msvcrt', 'multiprocessing', 'netrc', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'socket', 'socketserver', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.14': frozenset({'__future__', '_abc', '_aix_support', '_android_support', '_apple_support', '_ast', '_ast_unparse', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_colorize', '_compat_pickle', '_contextvars', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_hmac', '_imp', '_interpchannels', '_interpqueues', '_interpreters', '_io', '_ios_support', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_multibytecodec', '_multiprocessing', '_opcode', '_opcode_metadata', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_py_warnings', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_pyrepl', '_queue', '_random', '_remote_debugging', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_suggestions', '_symtable', '_sysconfig', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_types', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', '_zstd', 'abc', 'annotationlib', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'compression', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msvcrt', 'multiprocessing', 'netrc', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'socket', 'socketserver', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), +} + +def _with_running_interpreter() -> frozenset[str]: + """Union the table with the running interpreter's own stdlib names.""" + return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names + + +STDLIB_MODULE_NAMES_UNION: frozenset[str] = frozenset().union( + *STDLIB_MODULE_NAMES_BY_VERSION.values() +) +STDLIB_MODULE_NAMES: frozenset[str] = _with_running_interpreter() + + +def use_stdlib_of(*versions: str) -> frozenset[str]: + """Restrict the effective stdlib name set to specific Python versions. + + For scanners that know exactly which interpreter will unpickle the + target file, narrowing avoids treating removed-and-possibly-squatted + names as standard. Pass no arguments to restore the default union. + Unknown versions raise ``KeyError``. + """ + global STDLIB_MODULE_NAMES + if not versions: + STDLIB_MODULE_NAMES = _with_running_interpreter() + else: + unknown = [v for v in versions if v not in STDLIB_MODULE_NAMES_BY_VERSION] + if unknown: + raise KeyError( + f"Unsupported Python versions {unknown}; " + f"known: {sorted(STDLIB_MODULE_NAMES_BY_VERSION)}" + ) + STDLIB_MODULE_NAMES = frozenset( + name for v in versions for name in STDLIB_MODULE_NAMES_BY_VERSION[v] + ) + return STDLIB_MODULE_NAMES diff --git a/scripts/generate_stdlib_names.py b/scripts/generate_stdlib_names.py new file mode 100755 index 0000000..66cf4f0 --- /dev/null +++ b/scripts/generate_stdlib_names.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +"""Regenerate fickling/stdlib_names.py. + +The checked-in table must cover the union of top-level stdlib module names +across every Python version fickling supports, because the scanner cannot +know in advance which interpreter will load the pickle it is analyzing (a +CI job may scan a pickle that later executes on a developer's machine). + +Run from the repository root with uv available: + + uv run --no-project python scripts/generate_stdlib_names.py + +The script shells out to each supported interpreter via ``uv`` and merges +their ``sys.stdlib_module_names`` into per-version frozensets plus the +union that ships as the default. +""" + +from __future__ import annotations + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SUPPORTED_PYTHONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] + +OUTPUT = Path(__file__).parent.parent / "fickling" / "stdlib_names.py" + +HEADER = '''"""Top-level standard-library module names across supported Pythons. + +Generated by scripts/generate_stdlib_names.py — do not edit by hand; +regenerate instead. Last generated {generated}. + +Why a checked-in table instead of ``sys.stdlib_module_names`` alone: the +pickle being scanned is not necessarily loaded by the interpreter doing +the scanning (imagine a CI job gating artifacts for developer machines). +A module that exists in *any* supported Python version must therefore be +treated as potentially-standard, or benign pickles get flagged whenever +the scanner runs on a different version than the loader. + +Default semantics: ``STDLIB_MODULE_NAMES`` is the union of every listed +version plus the running interpreter's own names, so a scanner can never +miss names from a Python newer than this table. The residual risk of +this union — a removed module name (``imp``, ``asynchat``, …) squatted +by a malicious PyPI distribution on newer interpreters — is inherent to +static analysis without knowing the target version; callers who know +exactly which interpreter will unpickle may narrow the set with +:func:`use_stdlib_of`. +""" + +from __future__ import annotations + +import sys + +__all__ = [ + "PYTHON_VERSIONS", + "STDLIB_MODULE_NAMES", + "STDLIB_MODULE_NAMES_BY_VERSION", + "use_stdlib_of", +] + + +''' + + +def collect(version: str) -> list[str]: + result = subprocess.run( + [ + "uv", + "run", + "--no-project", + "-p", + version, + "python", + "-c", + "import json, sys; print(json.dumps(sorted(sys.stdlib_module_names)))", + ], + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def main() -> None: + by_version: dict[str, list[str]] = {} + for version in SUPPORTED_PYTHONS: + by_version[version] = collect(version) + + union: set[str] = set() + for names in by_version.values(): + union |= set(names) + + lines = [HEADER.format(generated=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))] + lines.append(f"PYTHON_VERSIONS = {SUPPORTED_PYTHONS!r}\n") + lines.append("STDLIB_MODULE_NAMES_BY_VERSION: dict[str, frozenset[str]] = {") + for version in SUPPORTED_PYTHONS: + names = ", ".join(repr(n) for n in by_version[version]) + lines.append(f" {version!r}: frozenset({{{names}}}),") + lines.append("}\n") + + text = "\n".join(lines) + # Append runtime augmentation and override API verbatim (not regenerated). + text += ''' +def _with_running_interpreter() -> frozenset[str]: + """Union the table with the running interpreter's own stdlib names.""" + return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names + + +STDLIB_MODULE_NAMES_UNION: frozenset[str] = frozenset().union( + *STDLIB_MODULE_NAMES_BY_VERSION.values() +) +STDLIB_MODULE_NAMES: frozenset[str] = _with_running_interpreter() + + +def use_stdlib_of(*versions: str) -> frozenset[str]: + """Restrict the effective stdlib name set to specific Python versions. + + For scanners that know exactly which interpreter will unpickle the + target file, narrowing avoids treating removed-and-possibly-squatted + names as standard. Pass no arguments to restore the default union. + Unknown versions raise ``KeyError``. + """ + global STDLIB_MODULE_NAMES + if not versions: + STDLIB_MODULE_NAMES = _with_running_interpreter() + else: + unknown = [v for v in versions if v not in STDLIB_MODULE_NAMES_BY_VERSION] + if unknown: + raise KeyError( + f"Unsupported Python versions {unknown}; " + f"known: {sorted(STDLIB_MODULE_NAMES_BY_VERSION)}" + ) + STDLIB_MODULE_NAMES = frozenset( + name for v in versions for name in STDLIB_MODULE_NAMES_BY_VERSION[v] + ) + return STDLIB_MODULE_NAMES +''' + + OUTPUT.write_text(text) + print(f"wrote {OUTPUT}") + print(f"union size: {len(union)}") + + +if __name__ == "__main__": + main() diff --git a/test/test_stdlib_names.py b/test/test_stdlib_names.py new file mode 100644 index 0000000..c41abf6 --- /dev/null +++ b/test/test_stdlib_names.py @@ -0,0 +1,87 @@ +import sys +from unittest import TestCase + +from fickling import stdlib_names +from fickling.fickle import is_private_or_dunder_stdlib_module, is_std_module + + +class TestStdlibNameUnion(TestCase): + def test_removed_modules_still_recognized(self): + """Modules removed from recent Pythons must still count as stdlib when + the scanner runs on an interpreter that no longer ships them (issue + #311): a pickle importing `asynchat` is benign on 3.10/3.11 targets, + and the scanner cannot assume its own version matches the loader's. + """ + for removed in ("asynchat", "asyncore", "imp", "smtpd", "distutils"): + self.assertTrue(is_std_module(removed), removed) + + def test_future_modules_always_recognized(self): + """Names that only exist in newer supported versions must be treated + as stdlib even when this interpreter predates them.""" + for added in ("tomllib", "graphlib", "annotationlib", "compression"): + self.assertTrue( + is_std_module(added), + f"{added} missing from union (scanner: {sys.version_info[:2]})", + ) + + def test_nonstdlib_names_still_flagged(self): + self.assertFalse(is_std_module("definitely_not_a_module_xyz")) + self.assertFalse(is_std_module("os_malicious_squat")) + + def test_top_level_component_semantics(self): + """Only the top-level package decides; submodules inherit (unchanged + pre-existing behavior).""" + self.assertTrue(is_std_module("os.path")) + self.assertFalse(is_std_module("evil.os")) + + def test_dunder_and_private_modules(self): + self.assertTrue(is_private_or_dunder_stdlib_module("_socket")) + self.assertTrue(is_private_or_dunder_stdlib_module("__future__")) + self.assertFalse(is_private_or_dunder_stdlib_module("socket")) + self.assertFalse(is_private_or_dunder_stdlib_module("_not_in_stdlib_at_all")) + + def test_per_version_tables_are_real_data(self): + # Sanity-check provenance: every version table contains `sys` and + # `pickle`, and version-specific additions/removals are where CPython + # put them. + for names in stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION.values(): + self.assertIn("sys", names) + self.assertIn("pickle", names) + v310 = stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION["3.10"] + v311 = stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION["3.11"] + v314 = stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION["3.14"] + self.assertIn("imp", v310) + self.assertNotIn("imp", v314) + self.assertNotIn("tomllib", v310) + self.assertIn("tomllib", v311) + self.assertIn("annotationlib", v314) + + def test_union_covers_every_table(self): + union = stdlib_names.STDLIB_MODULE_NAMES_UNION + for version, names in stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION.items(): + self.assertTrue( + names <= union, + f"{version} has names missing from the union", + ) + + def test_running_interpreter_names_included(self): + self.assertTrue( + set(sys.stdlib_module_names) <= stdlib_names.STDLIB_MODULE_NAMES + ) + + def test_use_stdlib_of_narrows_and_restores(self): + try: + narrowed = stdlib_names.use_stdlib_of("3.10") + self.assertIn("imp", narrowed) + self.assertNotIn("tomllib", narrowed) + # The narrowing is visible through fickling.fickle predicates. + self.assertTrue(is_std_module("imp")) + self.assertFalse(is_std_module("tomllib")) + finally: + restored = stdlib_names.use_stdlib_of() + self.assertIn("tomllib", restored) + self.assertTrue(is_std_module("tomllib")) + + def test_use_stdlib_of_rejects_unknown_versions(self): + with self.assertRaises(KeyError): + stdlib_names.use_stdlib_of("9.9") From 2c8ac5b9c45629248fda352d56f22712424a291a Mon Sep 17 00:00:00 2001 From: fei <204683769+feiiiiii5@users.noreply.github.com> Date: Sun, 23 Aug 2026 00:10:34 +0800 Subject: [PATCH 2/2] feat: derive shadowable-stdlib set from generated per-version tables MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Supersedes the hand-curated static lists with data captured from real interpreters: - stdlib_names.py now also emits SHADOWED_STDLIB_MODULE_NAMES — public top-level names present in any supported release but absent from the newest one (3.14), i.e. the removable dead batteries plus names like binhex/spwd/nis the manual list missed. - ShadowedStdlibImports analysis flags imports of these names as SUSPICIOUS instead of trusting them as plain stdlib; they are also excluded from likely_safe_imports. - SHADOWED_STDLIB_IMPORT_ALLOWLIST stays as the user extension point. - use_stdlib_of() targeting now silences shadow warnings for modules the selected target genuinely ships. Regenerate via scripts/generate_stdlib_names.py when the support matrix changes. --- fickling/analysis.py | 18 +++++++++ fickling/fickle.py | 47 ++++++++++++++++++++++++ fickling/stdlib_names.py | 49 ++++++++++++++++++++----- scripts/generate_stdlib_names.py | 57 ++++++++++++++++++++++++----- test/test_stdlib_names.py | 63 ++++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+), 18 deletions(-) diff --git a/fickling/analysis.py b/fickling/analysis.py index 8466c79..d5d780a 100644 --- a/fickling/analysis.py +++ b/fickling/analysis.py @@ -301,6 +301,24 @@ def analyze(self, context: AnalysisContext) -> Iterator[AnalysisResult]: ) +class ShadowedStdlibImports(Analysis): + def analyze(self, context: AnalysisContext) -> Iterator[AnalysisResult]: + for node in context.pickled.shadowed_stdlib_imports(): + shortened = context.shorten_code(node) + if context.mark_reported(shortened): + yield AnalysisResult( + Severity.SUSPICIOUS, + f"`{shortened}` imports a module that was removed from the " + "Python standard library in a recent version; on modern " + "interpreters this name resolves to a third-party package " + "of the same name, which can execute arbitrary code. " + "Treat as unsafe unless explicitly allowlisted " + "(fickling.fickle.SHADOWED_STDLIB_IMPORT_ALLOWLIST)", + "ShadowedStdlibImports", + trigger=shortened, + ) + + class UnsafeImportsML(Analysis): # ML-specific unsafe modules only; general-purpose modules (os, subprocess, # socket, pickle, etc.) are already covered by fickle.py's UNSAFE_IMPORTS diff --git a/fickling/fickle.py b/fickling/fickle.py index fbf46b1..d0875da 100644 --- a/fickling/fickle.py +++ b/fickling/fickle.py @@ -291,6 +291,25 @@ def is_private_or_dunder_stdlib_module(name: str) -> bool: return name.startswith("_") and name in stdlib_names.STDLIB_MODULE_NAMES +# User extension point: add top-level module names here (e.g. `distutils` for +# a scanner that knows its targets run setuptools' shim) to treat imports of +# removed-and-shadowable stdlib names as plain stdlib again — use only when +# you know no third-party package can shadow them on the loading interpreter. +SHADOWED_STDLIB_IMPORT_ALLOWLIST: frozenset[str] = frozenset() + + +def is_shadowed_stdlib_module(module_name: str) -> bool: + """True for names stdlib somewhere in the support matrix but removed from + recent releases, where third-party PyPI packages of the same name can + shadow them (see fickling.stdlib_names.SHADOWED_STDLIB_MODULE_NAMES). + """ + root = module_name.partition(".")[0] + return ( + root in stdlib_names.current_shadowed_names() + and root not in SHADOWED_STDLIB_IMPORT_ALLOWLIST + ) + + def import_name_components(node: ast.Import | ast.ImportFrom) -> Iterator[str]: """Yield every dotted component of an import's module and imported names. @@ -690,6 +709,7 @@ def _process_import(self, node: ast.Import | ast.ImportFrom): isinstance(node, ast.ImportFrom) and node.module is not None and is_std_module(node.module) + and not is_shadowed_stdlib_module(node.module) and ( not any(is_private_or_dunder_stdlib_module(c) for c in node.module.split(".")) or _is_allowed_private_import(node) @@ -1345,6 +1365,33 @@ def private_stdlib_imports(self) -> Iterator[ast.Import | ast.ImportFrom]: if any(is_private_or_dunder_stdlib_module(c) for c in import_name_components(node)): yield node + def shadowed_stdlib_imports(self) -> Iterator[ast.Import | ast.ImportFrom]: + """Imports of modules removed from the stdlib in a recent version. + + These names can be shadowed by third-party PyPI packages on modern + interpreters, so they are reported separately (ShadowedStdlibImports) + instead of being treated as plain stdlib. + """ + for node in self.properties.imports: + if isinstance(node, ast.ImportFrom): + if node.module is None: + continue + root = node.module.partition(".")[0] + if ( + root in stdlib_names.current_shadowed_names() + and root not in SHADOWED_STDLIB_IMPORT_ALLOWLIST + ): + yield node + else: + for alias in node.names: + root = alias.name.partition(".")[0] + if ( + root in stdlib_names.current_shadowed_names() + and root not in SHADOWED_STDLIB_IMPORT_ALLOWLIST + ): + yield node + break + @property def ast(self) -> ast.Module: if self._ast is None: diff --git a/fickling/stdlib_names.py b/fickling/stdlib_names.py index a0928f2..fa43a45 100644 --- a/fickling/stdlib_names.py +++ b/fickling/stdlib_names.py @@ -1,7 +1,7 @@ """Top-level standard-library module names across supported Pythons. Generated by scripts/generate_stdlib_names.py — do not edit by hand; -regenerate instead. Last generated 2026-08-22 15:35 UTC. +regenerate instead. Last generated 2026-08-22 16:02 UTC. Why a checked-in table instead of ``sys.stdlib_module_names`` alone: the pickle being scanned is not necessarily loaded by the interpreter doing @@ -28,6 +28,7 @@ "PYTHON_VERSIONS", "STDLIB_MODULE_NAMES", "STDLIB_MODULE_NAMES_BY_VERSION", + "current_shadowed_names", "use_stdlib_of", ] @@ -43,15 +44,43 @@ '3.14': frozenset({'__future__', '_abc', '_aix_support', '_android_support', '_apple_support', '_ast', '_ast_unparse', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_colorize', '_compat_pickle', '_contextvars', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_hmac', '_imp', '_interpchannels', '_interpqueues', '_interpreters', '_io', '_ios_support', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_multibytecodec', '_multiprocessing', '_opcode', '_opcode_metadata', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_py_warnings', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_pyrepl', '_queue', '_random', '_remote_debugging', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_suggestions', '_symtable', '_sysconfig', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_types', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', '_zstd', 'abc', 'annotationlib', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'compression', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msvcrt', 'multiprocessing', 'netrc', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'socket', 'socketserver', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), } -def _with_running_interpreter() -> frozenset[str]: - """Union the table with the running interpreter's own stdlib names.""" - return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names +# Names stdlib somewhere in the support matrix but absent from the +# newest listed release (3.14): removable dead batteries plus +# future-only additions. On interpreters lacking them these names can +# be shadowed by third-party PyPI packages, so imports are surfaced by +# the ShadowedStdlibImports analysis instead of being trusted. +SHADOWED_STDLIB_MODULE_NAMES: frozenset[str] = frozenset({'aifc', 'asynchat', 'asyncore', 'audioop', 'binhex', 'cgi', 'cgitb', 'chunk', 'crypt', 'distutils', 'imghdr', 'imp', 'lib2to3', 'mailcap', 'msilib', 'nis', 'nntplib', 'ossaudiodev', 'pipes', 'smtpd', 'sndhdr', 'spwd', 'sunau', 'telnetlib', 'uu', 'xdrlib'}) + +_SELECTED_VERSIONS: tuple[str, ...] | None = None STDLIB_MODULE_NAMES_UNION: frozenset[str] = frozenset().union( *STDLIB_MODULE_NAMES_BY_VERSION.values() ) -STDLIB_MODULE_NAMES: frozenset[str] = _with_running_interpreter() + + +def _compute_default_stdlib() -> frozenset[str]: + """Union the table with the running interpreter's own stdlib names.""" + return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names + + +STDLIB_MODULE_NAMES: frozenset[str] = _compute_default_stdlib() + + +def current_shadowed_names() -> frozenset[str]: + """Names that are not stdlib under the currently selected target(s). + + Default (no :func:`use_stdlib_of` selection): names absent from the + newest supported release. With a selection: names absent from *every* + selected version, so known-good targets stop warning about names they + genuinely ship. + """ + if _SELECTED_VERSIONS is None: + return SHADOWED_STDLIB_MODULE_NAMES + present: set[str] = set() + for version in _SELECTED_VERSIONS: + present |= STDLIB_MODULE_NAMES_BY_VERSION[version] + return SHADOWED_STDLIB_MODULE_NAMES - present def use_stdlib_of(*versions: str) -> frozenset[str]: @@ -62,9 +91,10 @@ def use_stdlib_of(*versions: str) -> frozenset[str]: names as standard. Pass no arguments to restore the default union. Unknown versions raise ``KeyError``. """ - global STDLIB_MODULE_NAMES + global STDLIB_MODULE_NAMES, _SELECTED_VERSIONS if not versions: - STDLIB_MODULE_NAMES = _with_running_interpreter() + _SELECTED_VERSIONS = None + STDLIB_MODULE_NAMES = _compute_default_stdlib() else: unknown = [v for v in versions if v not in STDLIB_MODULE_NAMES_BY_VERSION] if unknown: @@ -72,7 +102,8 @@ def use_stdlib_of(*versions: str) -> frozenset[str]: f"Unsupported Python versions {unknown}; " f"known: {sorted(STDLIB_MODULE_NAMES_BY_VERSION)}" ) - STDLIB_MODULE_NAMES = frozenset( - name for v in versions for name in STDLIB_MODULE_NAMES_BY_VERSION[v] + _SELECTED_VERSIONS = tuple(versions) + STDLIB_MODULE_NAMES = frozenset().union( + *(STDLIB_MODULE_NAMES_BY_VERSION[v] for v in versions) ) return STDLIB_MODULE_NAMES diff --git a/scripts/generate_stdlib_names.py b/scripts/generate_stdlib_names.py index 66cf4f0..1318bf3 100755 --- a/scripts/generate_stdlib_names.py +++ b/scripts/generate_stdlib_names.py @@ -56,6 +56,7 @@ "PYTHON_VERSIONS", "STDLIB_MODULE_NAMES", "STDLIB_MODULE_NAMES_BY_VERSION", + "current_shadowed_names", "use_stdlib_of", ] @@ -98,19 +99,55 @@ def main() -> None: names = ", ".join(repr(n) for n in by_version[version]) lines.append(f" {version!r}: frozenset({{{names}}}),") lines.append("}\n") + latest = SUPPORTED_PYTHONS[-1] + # Only public top-level names: underscore-prefixed entries differ across + # platform builds without being real removals, and imports of private + # stdlib modules are already surfaced separately. + shadowed = sorted( + n for n in (union - set(by_version[latest])) if not n.startswith("_") + ) + lines.append( + "# Names stdlib somewhere in the support matrix but absent from the\n" + f"# newest listed release ({latest}): removable dead batteries plus\n" + "# future-only additions. On interpreters lacking them these names can\n" + "# be shadowed by third-party PyPI packages, so imports are surfaced by\n" + "# the ShadowedStdlibImports analysis instead of being trusted.\n" + ) + names = ", ".join(repr(n) for n in shadowed) + lines.append(f"SHADOWED_STDLIB_MODULE_NAMES: frozenset[str] = frozenset({{{names}}})\n") text = "\n".join(lines) # Append runtime augmentation and override API verbatim (not regenerated). text += ''' -def _with_running_interpreter() -> frozenset[str]: - """Union the table with the running interpreter's own stdlib names.""" - return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names - +_SELECTED_VERSIONS: tuple[str, ...] | None = None STDLIB_MODULE_NAMES_UNION: frozenset[str] = frozenset().union( *STDLIB_MODULE_NAMES_BY_VERSION.values() ) -STDLIB_MODULE_NAMES: frozenset[str] = _with_running_interpreter() + + +def _compute_default_stdlib() -> frozenset[str]: + """Union the table with the running interpreter's own stdlib names.""" + return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names + + +STDLIB_MODULE_NAMES: frozenset[str] = _compute_default_stdlib() + + +def current_shadowed_names() -> frozenset[str]: + """Names that are not stdlib under the currently selected target(s). + + Default (no :func:`use_stdlib_of` selection): names absent from the + newest supported release. With a selection: names absent from *every* + selected version, so known-good targets stop warning about names they + genuinely ship. + """ + if _SELECTED_VERSIONS is None: + return SHADOWED_STDLIB_MODULE_NAMES + present: set[str] = set() + for version in _SELECTED_VERSIONS: + present |= STDLIB_MODULE_NAMES_BY_VERSION[version] + return SHADOWED_STDLIB_MODULE_NAMES - present def use_stdlib_of(*versions: str) -> frozenset[str]: @@ -121,9 +158,10 @@ def use_stdlib_of(*versions: str) -> frozenset[str]: names as standard. Pass no arguments to restore the default union. Unknown versions raise ``KeyError``. """ - global STDLIB_MODULE_NAMES + global STDLIB_MODULE_NAMES, _SELECTED_VERSIONS if not versions: - STDLIB_MODULE_NAMES = _with_running_interpreter() + _SELECTED_VERSIONS = None + STDLIB_MODULE_NAMES = _compute_default_stdlib() else: unknown = [v for v in versions if v not in STDLIB_MODULE_NAMES_BY_VERSION] if unknown: @@ -131,8 +169,9 @@ def use_stdlib_of(*versions: str) -> frozenset[str]: f"Unsupported Python versions {unknown}; " f"known: {sorted(STDLIB_MODULE_NAMES_BY_VERSION)}" ) - STDLIB_MODULE_NAMES = frozenset( - name for v in versions for name in STDLIB_MODULE_NAMES_BY_VERSION[v] + _SELECTED_VERSIONS = tuple(versions) + STDLIB_MODULE_NAMES = frozenset().union( + *(STDLIB_MODULE_NAMES_BY_VERSION[v] for v in versions) ) return STDLIB_MODULE_NAMES ''' diff --git a/test/test_stdlib_names.py b/test/test_stdlib_names.py index c41abf6..24c327f 100644 --- a/test/test_stdlib_names.py +++ b/test/test_stdlib_names.py @@ -85,3 +85,66 @@ def test_use_stdlib_of_narrows_and_restores(self): def test_use_stdlib_of_rejects_unknown_versions(self): with self.assertRaises(KeyError): stdlib_names.use_stdlib_of("9.9") + + +class TestShadowedStdlibImports(TestCase): + @staticmethod + def _pickled_importing(module: str): + import fickling.fickle as op + from fickling.fickle import Pickled + + return Pickled( + [ + op.Proto.create(4), + op.ShortBinUnicode(module), + op.ShortBinUnicode("attr"), + op.StackGlobal(), + op.Stop(), + ] + ) + + def test_removed_name_import_is_shadow_flagged(self): + from fickling.analysis import Severity, check_safety + + result = check_safety(self._pickled_importing("asynchat")) + self.assertEqual(result.severity, Severity.SUSPICIOUS) + + def test_live_stdlib_name_not_shadow_flagged(self): + from fickling.analysis import Severity, check_safety + + result = check_safety(self._pickled_importing("collections")) + self.assertEqual(result.severity, Severity.LIKELY_SAFE) + + def test_current_target_ships_removed_module(self): + """Narrowing to a version that still ships the module silences the + shadow warning.""" + from fickling.analysis import Severity, check_safety + from fickling.fickle import is_shadowed_stdlib_module + + try: + stdlib_names.use_stdlib_of("3.10") + self.assertFalse(is_shadowed_stdlib_module("asynchat")) + self.assertEqual( + check_safety(self._pickled_importing("asynchat")).severity, + Severity.LIKELY_SAFE, + ) + finally: + stdlib_names.use_stdlib_of() + self.assertTrue(is_shadowed_stdlib_module("asynchat")) + + def test_allowlist_restores_benign_classification(self): + import fickling.fickle as fickle_module + from fickling.analysis import Severity, check_safety + + try: + # chunk is shadowable but not on the hard UNSAFE_IMPORTS list, + # so the allowlist alone flips it back to benign. + fickle_module.SHADOWED_STDLIB_IMPORT_ALLOWLIST = frozenset({"chunk"}) + self.assertFalse(fickle_module.is_shadowed_stdlib_module("chunk")) + self.assertTrue(fickle_module.is_shadowed_stdlib_module("imp")) + self.assertEqual( + check_safety(self._pickled_importing("chunk")).severity, + Severity.LIKELY_SAFE, + ) + finally: + fickle_module.SHADOWED_STDLIB_IMPORT_ALLOWLIST = frozenset()