diff --git a/fickling/analysis.py b/fickling/analysis.py index 8466c79..d5d780a 100644 --- a/fickling/analysis.py +++ b/fickling/analysis.py @@ -301,6 +301,24 @@ def analyze(self, context: AnalysisContext) -> Iterator[AnalysisResult]: ) +class ShadowedStdlibImports(Analysis): + def analyze(self, context: AnalysisContext) -> Iterator[AnalysisResult]: + for node in context.pickled.shadowed_stdlib_imports(): + shortened = context.shorten_code(node) + if context.mark_reported(shortened): + yield AnalysisResult( + Severity.SUSPICIOUS, + f"`{shortened}` imports a module that was removed from the " + "Python standard library in a recent version; on modern " + "interpreters this name resolves to a third-party package " + "of the same name, which can execute arbitrary code. " + "Treat as unsafe unless explicitly allowlisted " + "(fickling.fickle.SHADOWED_STDLIB_IMPORT_ALLOWLIST)", + "ShadowedStdlibImports", + trigger=shortened, + ) + + class UnsafeImportsML(Analysis): # ML-specific unsafe modules only; general-purpose modules (os, subprocess, # socket, pickle, etc.) are already covered by fickle.py's UNSAFE_IMPORTS diff --git a/fickling/fickle.py b/fickling/fickle.py index c71b7f1..d0875da 100644 --- a/fickling/fickle.py +++ b/fickling/fickle.py @@ -21,6 +21,7 @@ overload, ) +from fickling import stdlib_names from fickling.exception import ExpansionAttackError, ResourceExhaustionError, WrongMethodError T = TypeVar("T") @@ -56,8 +57,6 @@ def __post_init__(self) -> None: GenericSequence = Sequence[T] make_constant = ast.Constant -BUILTIN_STDLIB_MODULE_NAMES: frozenset[str] = sys.stdlib_module_names - OPCODES_BY_NAME: dict[str, type[Opcode]] = {} OPCODE_INFO_BY_NAME: dict[str, OpcodeInfo] = {opcode.name: opcode for opcode in opcodes} @@ -279,7 +278,9 @@ def __post_init__(self) -> None: def is_std_module(module_name: str) -> bool: - return module_name.partition(".")[0] in BUILTIN_STDLIB_MODULE_NAMES + # Looked up through the module so runtime retargeting via + # stdlib_names.use_stdlib_of() takes effect (issue #311). + return module_name.partition(".")[0] in stdlib_names.STDLIB_MODULE_NAMES def is_private_or_dunder_stdlib_module(name: str) -> bool: @@ -287,7 +288,26 @@ def is_private_or_dunder_stdlib_module(name: str) -> bool: `__future__`, …): internal or non-public, so it should never appear in a pickle. Benign exceptions are handled in _is_allowed_private_import. """ - return name.startswith("_") and name in BUILTIN_STDLIB_MODULE_NAMES + return name.startswith("_") and name in stdlib_names.STDLIB_MODULE_NAMES + + +# User extension point: add top-level module names here (e.g. `distutils` for +# a scanner that knows its targets run setuptools' shim) to treat imports of +# removed-and-shadowable stdlib names as plain stdlib again — use only when +# you know no third-party package can shadow them on the loading interpreter. +SHADOWED_STDLIB_IMPORT_ALLOWLIST: frozenset[str] = frozenset() + + +def is_shadowed_stdlib_module(module_name: str) -> bool: + """True for names stdlib somewhere in the support matrix but removed from + recent releases, where third-party PyPI packages of the same name can + shadow them (see fickling.stdlib_names.SHADOWED_STDLIB_MODULE_NAMES). + """ + root = module_name.partition(".")[0] + return ( + root in stdlib_names.current_shadowed_names() + and root not in SHADOWED_STDLIB_IMPORT_ALLOWLIST + ) def import_name_components(node: ast.Import | ast.ImportFrom) -> Iterator[str]: @@ -689,6 +709,7 @@ def _process_import(self, node: ast.Import | ast.ImportFrom): isinstance(node, ast.ImportFrom) and node.module is not None and is_std_module(node.module) + and not is_shadowed_stdlib_module(node.module) and ( not any(is_private_or_dunder_stdlib_module(c) for c in node.module.split(".")) or _is_allowed_private_import(node) @@ -1344,6 +1365,33 @@ def private_stdlib_imports(self) -> Iterator[ast.Import | ast.ImportFrom]: if any(is_private_or_dunder_stdlib_module(c) for c in import_name_components(node)): yield node + def shadowed_stdlib_imports(self) -> Iterator[ast.Import | ast.ImportFrom]: + """Imports of modules removed from the stdlib in a recent version. + + These names can be shadowed by third-party PyPI packages on modern + interpreters, so they are reported separately (ShadowedStdlibImports) + instead of being treated as plain stdlib. + """ + for node in self.properties.imports: + if isinstance(node, ast.ImportFrom): + if node.module is None: + continue + root = node.module.partition(".")[0] + if ( + root in stdlib_names.current_shadowed_names() + and root not in SHADOWED_STDLIB_IMPORT_ALLOWLIST + ): + yield node + else: + for alias in node.names: + root = alias.name.partition(".")[0] + if ( + root in stdlib_names.current_shadowed_names() + and root not in SHADOWED_STDLIB_IMPORT_ALLOWLIST + ): + yield node + break + @property def ast(self) -> ast.Module: if self._ast is None: diff --git a/fickling/stdlib_names.py b/fickling/stdlib_names.py new file mode 100644 index 0000000..fa43a45 --- /dev/null +++ b/fickling/stdlib_names.py @@ -0,0 +1,109 @@ +"""Top-level standard-library module names across supported Pythons. + +Generated by scripts/generate_stdlib_names.py — do not edit by hand; +regenerate instead. Last generated 2026-08-22 16:02 UTC. + +Why a checked-in table instead of ``sys.stdlib_module_names`` alone: the +pickle being scanned is not necessarily loaded by the interpreter doing +the scanning (imagine a CI job gating artifacts for developer machines). +A module that exists in *any* supported Python version must therefore be +treated as potentially-standard, or benign pickles get flagged whenever +the scanner runs on a different version than the loader. + +Default semantics: ``STDLIB_MODULE_NAMES`` is the union of every listed +version plus the running interpreter's own names, so a scanner can never +miss names from a Python newer than this table. The residual risk of +this union — a removed module name (``imp``, ``asynchat``, …) squatted +by a malicious PyPI distribution on newer interpreters — is inherent to +static analysis without knowing the target version; callers who know +exactly which interpreter will unpickle may narrow the set with +:func:`use_stdlib_of`. +""" + +from __future__ import annotations + +import sys + +__all__ = [ + "PYTHON_VERSIONS", + "STDLIB_MODULE_NAMES", + "STDLIB_MODULE_NAMES_BY_VERSION", + "current_shadowed_names", + "use_stdlib_of", +] + + + +PYTHON_VERSIONS = ['3.10', '3.11', '3.12', '3.13', '3.14'] + +STDLIB_MODULE_NAMES_BY_VERSION: dict[str, frozenset[str]] = { + '3.10': frozenset({'__future__', '_abc', '_aix_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bootsubprocess', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_compat_pickle', '_compression', '_contextvars', '_crypt', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_io', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_msi', '_multibytecodec', '_multiprocessing', '_opcode', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydecimal', '_pyio', '_queue', '_random', '_scproxy', '_sha1', '_sha256', '_sha3', '_sha512', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_symtable', '_thread', '_threading_local', '_tkinter', '_tracemalloc', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_zoneinfo', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast', 'asynchat', 'asyncio', 'asyncore', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'binhex', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'distutils', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'imp', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'ossaudiodev', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.11': frozenset({'__future__', '_abc', '_aix_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bootsubprocess', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_compat_pickle', '_compression', '_contextvars', '_crypt', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_io', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_msi', '_multibytecodec', '_multiprocessing', '_opcode', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydecimal', '_pyio', '_queue', '_random', '_scproxy', '_sha1', '_sha256', '_sha3', '_sha512', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_symtable', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_zoneinfo', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast', 'asynchat', 'asyncio', 'asyncore', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'distutils', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'imp', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'ossaudiodev', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtpd', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.12': frozenset({'__future__', '_abc', '_aix_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_compat_pickle', '_compression', '_contextvars', '_crypt', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_io', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_msi', '_multibytecodec', '_multiprocessing', '_opcode', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_queue', '_random', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_symtable', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', 'abc', 'aifc', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'audioop', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cgi', 'cgitb', 'chunk', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'crypt', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'imghdr', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'lib2to3', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'mailcap', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msilib', 'msvcrt', 'multiprocessing', 'netrc', 'nis', 'nntplib', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'ossaudiodev', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pipes', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'sndhdr', 'socket', 'socketserver', 'spwd', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'sunau', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'telnetlib', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uu', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xdrlib', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.13': frozenset({'__future__', '_abc', '_aix_support', '_android_support', '_apple_support', '_ast', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_colorize', '_compat_pickle', '_compression', '_contextvars', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_imp', '_interpchannels', '_interpqueues', '_interpreters', '_io', '_ios_support', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_multibytecodec', '_multiprocessing', '_opcode', '_opcode_metadata', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_pyrepl', '_queue', '_random', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_suggestions', '_symtable', '_sysconfig', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', 'abc', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msvcrt', 'multiprocessing', 'netrc', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'socket', 'socketserver', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), + '3.14': frozenset({'__future__', '_abc', '_aix_support', '_android_support', '_apple_support', '_ast', '_ast_unparse', '_asyncio', '_bisect', '_blake2', '_bz2', '_codecs', '_codecs_cn', '_codecs_hk', '_codecs_iso2022', '_codecs_jp', '_codecs_kr', '_codecs_tw', '_collections', '_collections_abc', '_colorize', '_compat_pickle', '_contextvars', '_csv', '_ctypes', '_curses', '_curses_panel', '_datetime', '_dbm', '_decimal', '_elementtree', '_frozen_importlib', '_frozen_importlib_external', '_functools', '_gdbm', '_hashlib', '_heapq', '_hmac', '_imp', '_interpchannels', '_interpqueues', '_interpreters', '_io', '_ios_support', '_json', '_locale', '_lsprof', '_lzma', '_markupbase', '_md5', '_multibytecodec', '_multiprocessing', '_opcode', '_opcode_metadata', '_operator', '_osx_support', '_overlapped', '_pickle', '_posixshmem', '_posixsubprocess', '_py_abc', '_py_warnings', '_pydatetime', '_pydecimal', '_pyio', '_pylong', '_pyrepl', '_queue', '_random', '_remote_debugging', '_scproxy', '_sha1', '_sha2', '_sha3', '_signal', '_sitebuiltins', '_socket', '_sqlite3', '_sre', '_ssl', '_stat', '_statistics', '_string', '_strptime', '_struct', '_suggestions', '_symtable', '_sysconfig', '_thread', '_threading_local', '_tkinter', '_tokenize', '_tracemalloc', '_types', '_typing', '_uuid', '_warnings', '_weakref', '_weakrefset', '_winapi', '_wmi', '_zoneinfo', '_zstd', 'abc', 'annotationlib', 'antigravity', 'argparse', 'array', 'ast', 'asyncio', 'atexit', 'base64', 'bdb', 'binascii', 'bisect', 'builtins', 'bz2', 'cProfile', 'calendar', 'cmath', 'cmd', 'code', 'codecs', 'codeop', 'collections', 'colorsys', 'compileall', 'compression', 'concurrent', 'configparser', 'contextlib', 'contextvars', 'copy', 'copyreg', 'csv', 'ctypes', 'curses', 'dataclasses', 'datetime', 'dbm', 'decimal', 'difflib', 'dis', 'doctest', 'email', 'encodings', 'ensurepip', 'enum', 'errno', 'faulthandler', 'fcntl', 'filecmp', 'fileinput', 'fnmatch', 'fractions', 'ftplib', 'functools', 'gc', 'genericpath', 'getopt', 'getpass', 'gettext', 'glob', 'graphlib', 'grp', 'gzip', 'hashlib', 'heapq', 'hmac', 'html', 'http', 'idlelib', 'imaplib', 'importlib', 'inspect', 'io', 'ipaddress', 'itertools', 'json', 'keyword', 'linecache', 'locale', 'logging', 'lzma', 'mailbox', 'marshal', 'math', 'mimetypes', 'mmap', 'modulefinder', 'msvcrt', 'multiprocessing', 'netrc', 'nt', 'ntpath', 'nturl2path', 'numbers', 'opcode', 'operator', 'optparse', 'os', 'pathlib', 'pdb', 'pickle', 'pickletools', 'pkgutil', 'platform', 'plistlib', 'poplib', 'posix', 'posixpath', 'pprint', 'profile', 'pstats', 'pty', 'pwd', 'py_compile', 'pyclbr', 'pydoc', 'pydoc_data', 'pyexpat', 'queue', 'quopri', 'random', 're', 'readline', 'reprlib', 'resource', 'rlcompleter', 'runpy', 'sched', 'secrets', 'select', 'selectors', 'shelve', 'shlex', 'shutil', 'signal', 'site', 'smtplib', 'socket', 'socketserver', 'sqlite3', 'sre_compile', 'sre_constants', 'sre_parse', 'ssl', 'stat', 'statistics', 'string', 'stringprep', 'struct', 'subprocess', 'symtable', 'sys', 'sysconfig', 'syslog', 'tabnanny', 'tarfile', 'tempfile', 'termios', 'textwrap', 'this', 'threading', 'time', 'timeit', 'tkinter', 'token', 'tokenize', 'tomllib', 'trace', 'traceback', 'tracemalloc', 'tty', 'turtle', 'turtledemo', 'types', 'typing', 'unicodedata', 'unittest', 'urllib', 'uuid', 'venv', 'warnings', 'wave', 'weakref', 'webbrowser', 'winreg', 'winsound', 'wsgiref', 'xml', 'xmlrpc', 'zipapp', 'zipfile', 'zipimport', 'zlib', 'zoneinfo'}), +} + +# Names stdlib somewhere in the support matrix but absent from the +# newest listed release (3.14): removable dead batteries plus +# future-only additions. On interpreters lacking them these names can +# be shadowed by third-party PyPI packages, so imports are surfaced by +# the ShadowedStdlibImports analysis instead of being trusted. + +SHADOWED_STDLIB_MODULE_NAMES: frozenset[str] = frozenset({'aifc', 'asynchat', 'asyncore', 'audioop', 'binhex', 'cgi', 'cgitb', 'chunk', 'crypt', 'distutils', 'imghdr', 'imp', 'lib2to3', 'mailcap', 'msilib', 'nis', 'nntplib', 'ossaudiodev', 'pipes', 'smtpd', 'sndhdr', 'spwd', 'sunau', 'telnetlib', 'uu', 'xdrlib'}) + +_SELECTED_VERSIONS: tuple[str, ...] | None = None + +STDLIB_MODULE_NAMES_UNION: frozenset[str] = frozenset().union( + *STDLIB_MODULE_NAMES_BY_VERSION.values() +) + + +def _compute_default_stdlib() -> frozenset[str]: + """Union the table with the running interpreter's own stdlib names.""" + return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names + + +STDLIB_MODULE_NAMES: frozenset[str] = _compute_default_stdlib() + + +def current_shadowed_names() -> frozenset[str]: + """Names that are not stdlib under the currently selected target(s). + + Default (no :func:`use_stdlib_of` selection): names absent from the + newest supported release. With a selection: names absent from *every* + selected version, so known-good targets stop warning about names they + genuinely ship. + """ + if _SELECTED_VERSIONS is None: + return SHADOWED_STDLIB_MODULE_NAMES + present: set[str] = set() + for version in _SELECTED_VERSIONS: + present |= STDLIB_MODULE_NAMES_BY_VERSION[version] + return SHADOWED_STDLIB_MODULE_NAMES - present + + +def use_stdlib_of(*versions: str) -> frozenset[str]: + """Restrict the effective stdlib name set to specific Python versions. + + For scanners that know exactly which interpreter will unpickle the + target file, narrowing avoids treating removed-and-possibly-squatted + names as standard. Pass no arguments to restore the default union. + Unknown versions raise ``KeyError``. + """ + global STDLIB_MODULE_NAMES, _SELECTED_VERSIONS + if not versions: + _SELECTED_VERSIONS = None + STDLIB_MODULE_NAMES = _compute_default_stdlib() + else: + unknown = [v for v in versions if v not in STDLIB_MODULE_NAMES_BY_VERSION] + if unknown: + raise KeyError( + f"Unsupported Python versions {unknown}; " + f"known: {sorted(STDLIB_MODULE_NAMES_BY_VERSION)}" + ) + _SELECTED_VERSIONS = tuple(versions) + STDLIB_MODULE_NAMES = frozenset().union( + *(STDLIB_MODULE_NAMES_BY_VERSION[v] for v in versions) + ) + return STDLIB_MODULE_NAMES diff --git a/scripts/generate_stdlib_names.py b/scripts/generate_stdlib_names.py new file mode 100755 index 0000000..1318bf3 --- /dev/null +++ b/scripts/generate_stdlib_names.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python +"""Regenerate fickling/stdlib_names.py. + +The checked-in table must cover the union of top-level stdlib module names +across every Python version fickling supports, because the scanner cannot +know in advance which interpreter will load the pickle it is analyzing (a +CI job may scan a pickle that later executes on a developer's machine). + +Run from the repository root with uv available: + + uv run --no-project python scripts/generate_stdlib_names.py + +The script shells out to each supported interpreter via ``uv`` and merges +their ``sys.stdlib_module_names`` into per-version frozensets plus the +union that ships as the default. +""" + +from __future__ import annotations + +import json +import subprocess +from datetime import datetime, timezone +from pathlib import Path + +SUPPORTED_PYTHONS = ["3.10", "3.11", "3.12", "3.13", "3.14"] + +OUTPUT = Path(__file__).parent.parent / "fickling" / "stdlib_names.py" + +HEADER = '''"""Top-level standard-library module names across supported Pythons. + +Generated by scripts/generate_stdlib_names.py — do not edit by hand; +regenerate instead. Last generated {generated}. + +Why a checked-in table instead of ``sys.stdlib_module_names`` alone: the +pickle being scanned is not necessarily loaded by the interpreter doing +the scanning (imagine a CI job gating artifacts for developer machines). +A module that exists in *any* supported Python version must therefore be +treated as potentially-standard, or benign pickles get flagged whenever +the scanner runs on a different version than the loader. + +Default semantics: ``STDLIB_MODULE_NAMES`` is the union of every listed +version plus the running interpreter's own names, so a scanner can never +miss names from a Python newer than this table. The residual risk of +this union — a removed module name (``imp``, ``asynchat``, …) squatted +by a malicious PyPI distribution on newer interpreters — is inherent to +static analysis without knowing the target version; callers who know +exactly which interpreter will unpickle may narrow the set with +:func:`use_stdlib_of`. +""" + +from __future__ import annotations + +import sys + +__all__ = [ + "PYTHON_VERSIONS", + "STDLIB_MODULE_NAMES", + "STDLIB_MODULE_NAMES_BY_VERSION", + "current_shadowed_names", + "use_stdlib_of", +] + + +''' + + +def collect(version: str) -> list[str]: + result = subprocess.run( + [ + "uv", + "run", + "--no-project", + "-p", + version, + "python", + "-c", + "import json, sys; print(json.dumps(sorted(sys.stdlib_module_names)))", + ], + capture_output=True, + text=True, + check=True, + ) + return json.loads(result.stdout) + + +def main() -> None: + by_version: dict[str, list[str]] = {} + for version in SUPPORTED_PYTHONS: + by_version[version] = collect(version) + + union: set[str] = set() + for names in by_version.values(): + union |= set(names) + + lines = [HEADER.format(generated=datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC"))] + lines.append(f"PYTHON_VERSIONS = {SUPPORTED_PYTHONS!r}\n") + lines.append("STDLIB_MODULE_NAMES_BY_VERSION: dict[str, frozenset[str]] = {") + for version in SUPPORTED_PYTHONS: + names = ", ".join(repr(n) for n in by_version[version]) + lines.append(f" {version!r}: frozenset({{{names}}}),") + lines.append("}\n") + latest = SUPPORTED_PYTHONS[-1] + # Only public top-level names: underscore-prefixed entries differ across + # platform builds without being real removals, and imports of private + # stdlib modules are already surfaced separately. + shadowed = sorted( + n for n in (union - set(by_version[latest])) if not n.startswith("_") + ) + lines.append( + "# Names stdlib somewhere in the support matrix but absent from the\n" + f"# newest listed release ({latest}): removable dead batteries plus\n" + "# future-only additions. On interpreters lacking them these names can\n" + "# be shadowed by third-party PyPI packages, so imports are surfaced by\n" + "# the ShadowedStdlibImports analysis instead of being trusted.\n" + ) + names = ", ".join(repr(n) for n in shadowed) + lines.append(f"SHADOWED_STDLIB_MODULE_NAMES: frozenset[str] = frozenset({{{names}}})\n") + + text = "\n".join(lines) + # Append runtime augmentation and override API verbatim (not regenerated). + text += ''' +_SELECTED_VERSIONS: tuple[str, ...] | None = None + +STDLIB_MODULE_NAMES_UNION: frozenset[str] = frozenset().union( + *STDLIB_MODULE_NAMES_BY_VERSION.values() +) + + +def _compute_default_stdlib() -> frozenset[str]: + """Union the table with the running interpreter's own stdlib names.""" + return STDLIB_MODULE_NAMES_UNION | sys.stdlib_module_names + + +STDLIB_MODULE_NAMES: frozenset[str] = _compute_default_stdlib() + + +def current_shadowed_names() -> frozenset[str]: + """Names that are not stdlib under the currently selected target(s). + + Default (no :func:`use_stdlib_of` selection): names absent from the + newest supported release. With a selection: names absent from *every* + selected version, so known-good targets stop warning about names they + genuinely ship. + """ + if _SELECTED_VERSIONS is None: + return SHADOWED_STDLIB_MODULE_NAMES + present: set[str] = set() + for version in _SELECTED_VERSIONS: + present |= STDLIB_MODULE_NAMES_BY_VERSION[version] + return SHADOWED_STDLIB_MODULE_NAMES - present + + +def use_stdlib_of(*versions: str) -> frozenset[str]: + """Restrict the effective stdlib name set to specific Python versions. + + For scanners that know exactly which interpreter will unpickle the + target file, narrowing avoids treating removed-and-possibly-squatted + names as standard. Pass no arguments to restore the default union. + Unknown versions raise ``KeyError``. + """ + global STDLIB_MODULE_NAMES, _SELECTED_VERSIONS + if not versions: + _SELECTED_VERSIONS = None + STDLIB_MODULE_NAMES = _compute_default_stdlib() + else: + unknown = [v for v in versions if v not in STDLIB_MODULE_NAMES_BY_VERSION] + if unknown: + raise KeyError( + f"Unsupported Python versions {unknown}; " + f"known: {sorted(STDLIB_MODULE_NAMES_BY_VERSION)}" + ) + _SELECTED_VERSIONS = tuple(versions) + STDLIB_MODULE_NAMES = frozenset().union( + *(STDLIB_MODULE_NAMES_BY_VERSION[v] for v in versions) + ) + return STDLIB_MODULE_NAMES +''' + + OUTPUT.write_text(text) + print(f"wrote {OUTPUT}") + print(f"union size: {len(union)}") + + +if __name__ == "__main__": + main() diff --git a/test/test_stdlib_names.py b/test/test_stdlib_names.py new file mode 100644 index 0000000..24c327f --- /dev/null +++ b/test/test_stdlib_names.py @@ -0,0 +1,150 @@ +import sys +from unittest import TestCase + +from fickling import stdlib_names +from fickling.fickle import is_private_or_dunder_stdlib_module, is_std_module + + +class TestStdlibNameUnion(TestCase): + def test_removed_modules_still_recognized(self): + """Modules removed from recent Pythons must still count as stdlib when + the scanner runs on an interpreter that no longer ships them (issue + #311): a pickle importing `asynchat` is benign on 3.10/3.11 targets, + and the scanner cannot assume its own version matches the loader's. + """ + for removed in ("asynchat", "asyncore", "imp", "smtpd", "distutils"): + self.assertTrue(is_std_module(removed), removed) + + def test_future_modules_always_recognized(self): + """Names that only exist in newer supported versions must be treated + as stdlib even when this interpreter predates them.""" + for added in ("tomllib", "graphlib", "annotationlib", "compression"): + self.assertTrue( + is_std_module(added), + f"{added} missing from union (scanner: {sys.version_info[:2]})", + ) + + def test_nonstdlib_names_still_flagged(self): + self.assertFalse(is_std_module("definitely_not_a_module_xyz")) + self.assertFalse(is_std_module("os_malicious_squat")) + + def test_top_level_component_semantics(self): + """Only the top-level package decides; submodules inherit (unchanged + pre-existing behavior).""" + self.assertTrue(is_std_module("os.path")) + self.assertFalse(is_std_module("evil.os")) + + def test_dunder_and_private_modules(self): + self.assertTrue(is_private_or_dunder_stdlib_module("_socket")) + self.assertTrue(is_private_or_dunder_stdlib_module("__future__")) + self.assertFalse(is_private_or_dunder_stdlib_module("socket")) + self.assertFalse(is_private_or_dunder_stdlib_module("_not_in_stdlib_at_all")) + + def test_per_version_tables_are_real_data(self): + # Sanity-check provenance: every version table contains `sys` and + # `pickle`, and version-specific additions/removals are where CPython + # put them. + for names in stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION.values(): + self.assertIn("sys", names) + self.assertIn("pickle", names) + v310 = stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION["3.10"] + v311 = stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION["3.11"] + v314 = stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION["3.14"] + self.assertIn("imp", v310) + self.assertNotIn("imp", v314) + self.assertNotIn("tomllib", v310) + self.assertIn("tomllib", v311) + self.assertIn("annotationlib", v314) + + def test_union_covers_every_table(self): + union = stdlib_names.STDLIB_MODULE_NAMES_UNION + for version, names in stdlib_names.STDLIB_MODULE_NAMES_BY_VERSION.items(): + self.assertTrue( + names <= union, + f"{version} has names missing from the union", + ) + + def test_running_interpreter_names_included(self): + self.assertTrue( + set(sys.stdlib_module_names) <= stdlib_names.STDLIB_MODULE_NAMES + ) + + def test_use_stdlib_of_narrows_and_restores(self): + try: + narrowed = stdlib_names.use_stdlib_of("3.10") + self.assertIn("imp", narrowed) + self.assertNotIn("tomllib", narrowed) + # The narrowing is visible through fickling.fickle predicates. + self.assertTrue(is_std_module("imp")) + self.assertFalse(is_std_module("tomllib")) + finally: + restored = stdlib_names.use_stdlib_of() + self.assertIn("tomllib", restored) + self.assertTrue(is_std_module("tomllib")) + + def test_use_stdlib_of_rejects_unknown_versions(self): + with self.assertRaises(KeyError): + stdlib_names.use_stdlib_of("9.9") + + +class TestShadowedStdlibImports(TestCase): + @staticmethod + def _pickled_importing(module: str): + import fickling.fickle as op + from fickling.fickle import Pickled + + return Pickled( + [ + op.Proto.create(4), + op.ShortBinUnicode(module), + op.ShortBinUnicode("attr"), + op.StackGlobal(), + op.Stop(), + ] + ) + + def test_removed_name_import_is_shadow_flagged(self): + from fickling.analysis import Severity, check_safety + + result = check_safety(self._pickled_importing("asynchat")) + self.assertEqual(result.severity, Severity.SUSPICIOUS) + + def test_live_stdlib_name_not_shadow_flagged(self): + from fickling.analysis import Severity, check_safety + + result = check_safety(self._pickled_importing("collections")) + self.assertEqual(result.severity, Severity.LIKELY_SAFE) + + def test_current_target_ships_removed_module(self): + """Narrowing to a version that still ships the module silences the + shadow warning.""" + from fickling.analysis import Severity, check_safety + from fickling.fickle import is_shadowed_stdlib_module + + try: + stdlib_names.use_stdlib_of("3.10") + self.assertFalse(is_shadowed_stdlib_module("asynchat")) + self.assertEqual( + check_safety(self._pickled_importing("asynchat")).severity, + Severity.LIKELY_SAFE, + ) + finally: + stdlib_names.use_stdlib_of() + self.assertTrue(is_shadowed_stdlib_module("asynchat")) + + def test_allowlist_restores_benign_classification(self): + import fickling.fickle as fickle_module + from fickling.analysis import Severity, check_safety + + try: + # chunk is shadowable but not on the hard UNSAFE_IMPORTS list, + # so the allowlist alone flips it back to benign. + fickle_module.SHADOWED_STDLIB_IMPORT_ALLOWLIST = frozenset({"chunk"}) + self.assertFalse(fickle_module.is_shadowed_stdlib_module("chunk")) + self.assertTrue(fickle_module.is_shadowed_stdlib_module("imp")) + self.assertEqual( + check_safety(self._pickled_importing("chunk")).severity, + Severity.LIKELY_SAFE, + ) + finally: + fickle_module.SHADOWED_STDLIB_IMPORT_ALLOWLIST = frozenset()