From 7f0b4c940b2e9e04ab7ac64053c3aa1f71490913 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Fri, 1 May 2026 12:27:11 +0100 Subject: [PATCH 1/3] Use WeakKeyDictionary for WRITE_LOCKS to prevent file object leaks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit File objects registered in WRITE_LOCKS were never released, causing a memory leak in long-running processes that open many log files (e.g., task executors creating a per-task BytesLogger or WriteLogger). WeakKeyDictionary stores keys as weak references, so entries expire automatically when the last strong reference to a file object is dropped — no manual cleanup needed. Closes #806 (no need to expose it, we tidy it up correctly ourselves) --- CHANGELOG.md | 6 ++++++ src/structlog/_output.py | 5 ++++- tests/test_output.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f177a6bc..5cc501ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,6 +27,12 @@ You can find our backwards-compatibility policy [here](https://github.com/hynek/ [#802](https://github.com/hynek/structlog/pull/802) +### Fixed + +- `WRITE_LOCKS` now uses a `weakref.WeakKeyDictionary` instead of a plain `dict`. + File objects used as keys are now released automatically when all application references are dropped, preventing a memory leak in long-running processes that open many log files (e.g., task executors that create a per-task `BytesLogger` or `WriteLogger`). + + ### Changed - `structlog.dev.ConsoleRenderer` does not warn anymore when the `exception` key has a rendered value despite having a fancy formatter configured. diff --git a/src/structlog/_output.py b/src/structlog/_output.py index b1612de6..bfa3ef40 100644 --- a/src/structlog/_output.py +++ b/src/structlog/_output.py @@ -12,13 +12,16 @@ import copy import sys import threading +import weakref from pickle import PicklingError from sys import stderr, stdout from typing import IO, Any, BinaryIO, TextIO -WRITE_LOCKS: dict[IO[Any], threading.Lock] = {} +WRITE_LOCKS: weakref.WeakKeyDictionary[IO[Any], threading.Lock] = ( + weakref.WeakKeyDictionary() +) def _get_lock_for_file(file: IO[Any]) -> threading.Lock: diff --git a/tests/test_output.py b/tests/test_output.py index 8553aad9..e2d598e1 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -4,6 +4,7 @@ # repository for complete details. import copy +import gc import pickle from io import BytesIO, StringIO @@ -23,6 +24,33 @@ from .helpers import stdlib_log_methods +@pytest.mark.parametrize( + ("logger_cls", "mode"), + [(PrintLogger, "w"), (WriteLogger, "w"), (BytesLogger, "wb")], +) +def test_write_locks_released_on_gc(logger_cls, mode, tmp_path): + """ + WRITE_LOCKS entry is removed automatically when the file object is GC'd. + + Closing the file is not enough — the entry persists until the file object + itself is collected. + """ + gc.collect() + size_before = len(WRITE_LOCKS) + f = (tmp_path / "test.log").open(mode) + logger = logger_cls(f) + assert len(WRITE_LOCKS) == size_before + 1 + + # close() alone does not remove the entry + f.close() + assert len(WRITE_LOCKS) == size_before + 1 + + del logger, f + gc.collect() + + assert len(WRITE_LOCKS) == size_before + + class TestLoggers: """ Tests common to the Print and WriteLoggers. From df8903573c5cbb2e46a98e5a69f7ae004532cb3e Mon Sep 17 00:00:00 2001 From: Hynek Schlawack Date: Sat, 2 May 2026 11:16:34 +0200 Subject: [PATCH 2/3] docs: rewrite changelog from user's perspective --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cc501ec..805bcc8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,8 +29,9 @@ You can find our backwards-compatibility policy [here](https://github.com/hynek/ ### Fixed -- `WRITE_LOCKS` now uses a `weakref.WeakKeyDictionary` instead of a plain `dict`. - File objects used as keys are now released automatically when all application references are dropped, preventing a memory leak in long-running processes that open many log files (e.g., task executors that create a per-task `BytesLogger` or `WriteLogger`). +- `structlog.BytesLogger`, `structlog.PrintLogger`, and `structlog.WriteLogger` now hold *weak* references to the files they use for output. + This prevents their leakage in long-running processes that open many logfiles, such as task executors that create a per-task `BytesLogger` or `WriteLogger`. + [#807](https://github.com/hynek/structlog/pull/807) ### Changed From 88f668d9e834fb9382d3ffedc23d04c67422692b Mon Sep 17 00:00:00 2001 From: Hynek Schlawack Date: Sat, 2 May 2026 11:19:05 +0200 Subject: [PATCH 3/3] tests: adapt style --- tests/test_output.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_output.py b/tests/test_output.py index e2d598e1..460dac41 100644 --- a/tests/test_output.py +++ b/tests/test_output.py @@ -32,17 +32,19 @@ def test_write_locks_released_on_gc(logger_cls, mode, tmp_path): """ WRITE_LOCKS entry is removed automatically when the file object is GC'd. - Closing the file is not enough — the entry persists until the file object + Closing the file is not enough -- the entry persists until the file object itself is collected. """ gc.collect() size_before = len(WRITE_LOCKS) f = (tmp_path / "test.log").open(mode) logger = logger_cls(f) + assert len(WRITE_LOCKS) == size_before + 1 # close() alone does not remove the entry f.close() + assert len(WRITE_LOCKS) == size_before + 1 del logger, f