Allow DBDagBag TTL cache eviction without a size cap

airflow-core/docs/administration-and-deployment/web-stack.rst

@@ -142,8 +142,8 @@ The following configuration options are available in the ``[api]`` section:
 - ``server_type``: ``uvicorn`` (default) or ``gunicorn``
 - ``worker_refresh_interval``: Seconds between worker refresh cycles (0 = disabled, default)
 - ``worker_refresh_batch_size``: Number of workers to refresh per cycle (default: 1)
-- ``dag_cache_size``: Max cached SerializedDAG versions in the API server (default: 64, 0 = unbounded)
-- ``dag_cache_ttl``: TTL in seconds for cached DAGs (default: 3600, 0 = LRU only)
+- ``dag_cache_size``: Max cached SerializedDAG versions in the API server (default: 64, 0 = no size cap)
+- ``dag_cache_ttl``: TTL in seconds for cached DAGs (default: 3600, 0 = LRU only, both 0 = no eviction)
 
 When to Use Gunicorn
 ^^^^^^^^^^^^^^^^^^^^

airflow-core/docs/faq.rst

@@ -725,9 +725,11 @@ in memory. Configure this in the ``[api]`` section:
 .. code-block:: ini
 
     [api]
-    dag_cache_size = 64    ; max cached versions (0 = unbounded, pre-3.2 behavior)
+    dag_cache_size = 64    ; max cached versions (0 = no size cap, TTL still applies)
     dag_cache_ttl = 3600   ; seconds before a cached entry expires (0 = LRU only)
 
+Setting both to 0 disables eviction entirely, matching pre-3.3 behavior.
+
 The cache is keyed by Dag version ID. After a Dag is updated, the API server may serve the
 previous version until the cached entry expires (controlled by ``dag_cache_ttl``).
 

airflow-core/newsfragments/69774.improvement.rst

@@ -0,0 +1 @@
+``[api] dag_cache_ttl`` is now honored when ``[api] dag_cache_size = 0``, giving the API server TTL-based eviction of cached serialized Dags with no size cap. Previously ``dag_cache_size = 0`` disabled eviction entirely and silently ignored ``dag_cache_ttl``. To keep the previous fully-unbounded behavior, set ``dag_cache_ttl = 0`` as well.

airflow-core/src/airflow/api_fastapi/common/dagbag.py

@@ -39,16 +39,12 @@ def create_dag_bag() -> DBDagBag:
     cache_ttl_config = conf.getint("api", "dag_cache_ttl", fallback=3600)
 
     if cache_size < 0:
-        log.warning("dag_cache_size must be >= 0, using unbounded dict")
+        log.warning("dag_cache_size must be >= 0, using no size cap")
         cache_size = 0
     if cache_ttl_config < 0:
         log.warning("dag_cache_ttl must be >= 0, disabling TTL")
         cache_ttl_config = 0
 
-    # Use unbounded dict (no eviction) if cache_size is 0
-    if cache_size <= 0:
-        return DBDagBag(cache_size=0)
-
     # Disable TTL if cache_ttl is 0
     cache_ttl: int | None = cache_ttl_config if cache_ttl_config > 0 else None
 

airflow-core/src/airflow/config_templates/config.yml

@@ -1704,7 +1704,11 @@ api:
     dag_cache_size:
       description: |
         Size of the LRU cache for SerializedDAG objects in the API server.
-        Set to 0 to use an unbounded dict (no eviction, matching pre-3.2 behavior).
+        Set to 0 to remove the size cap. Cached entries are then evicted only by
+        ``dag_cache_ttl``, or never if that is also 0 (matching pre-3.3 behavior).
+        Note that the TTL is then not a ceiling on memory. Entries that keep being
+        requested are refreshed and never expire, so memory usage follows the number
+        of distinct Dag versions in active use.
         The cache is keyed by Dag version ID, so lookups by Dag ID
         (e.g., viewing a Dag's details) always query the database for the latest
         version, but the deserialized result is cached for subsequent
@@ -1717,7 +1721,9 @@ api:
       description: |
         Time-to-live (seconds) for cached SerializedDAG objects in the API server.
         After this time, cached DAGs will be re-fetched from the database on next access.
-        Set to 0 to disable TTL (cache entries will only be evicted by LRU policy).
+        Applies whether or not ``dag_cache_size`` sets a size cap. Set to 0 to disable TTL,
+        leaving eviction to the ``dag_cache_size`` LRU policy, or no eviction at all if
+        ``dag_cache_size`` is also 0.
 
         Note: After a DAG is updated, the API server may serve the previous version
         until the cached entry expires. Lower values reduce staleness but increase

airflow-core/src/airflow/models/dagbag.py

@@ -18,6 +18,7 @@
 from __future__ import annotations
 
 import hashlib
+import math
 import time
 from collections.abc import MutableMapping
 from contextlib import nullcontext
@@ -62,9 +63,9 @@ class DBDagBag:
     """
     Internal class for retrieving dags from the database.
 
-    Optionally supports LRU+TTL caching when cache_size is provided.
-    The scheduler uses this without caching, while the API server can
-    enable caching via configuration.
+    Optionally caches deserialized dags. A size cap enables LRU eviction and a TTL
+    enables age-based eviction, with or without a size cap. The scheduler uses this
+    without caching, while the API server can enable caching via configuration.
 
     :meta private:
     """
@@ -79,21 +80,23 @@ def __init__(
         Initialize DBDagBag.
 
         :param load_op_links: Should the extra operator link be loaded when de-serializing the DAG?
-        :param cache_size: Size of LRU cache. If None or 0, uses unbounded dict (no eviction).
-        :param cache_ttl: Time-to-live for cache entries in seconds. If None or 0, no TTL (LRU only).
+        :param cache_size: Max cached entries; 0 or None means no size cap.
+        :param cache_ttl: Seconds until a cached entry expires. If > 0, entries are evicted by
+            age regardless of ``cache_size`` (with no size cap this gives TTL-only eviction).
         """
         self.load_op_links = load_op_links
         self._dags: MutableMapping[UUID | str, _CacheEntry] = {}
         self._use_cache = False
 
         self._revalidation_interval = conf.getint("core", "min_serialized_dag_update_interval")
 
-        # Initialize bounded cache if cache_size is provided and > 0
-        if cache_size and cache_size > 0:
-            if cache_ttl and cache_ttl > 0:
-                self._dags = TTLCache(maxsize=cache_size, ttl=cache_ttl)
-            else:
-                self._dags = LRUCache(maxsize=cache_size)
+        # Initialize a TTL cache if configured, otherwise a bounded LRU cache.
+        if cache_ttl and cache_ttl > 0:
+            maxsize = cache_size if cache_size and cache_size > 0 else math.inf
+            self._dags = TTLCache(maxsize=maxsize, ttl=cache_ttl)
+            self._use_cache = True
+        elif cache_size and cache_size > 0:
+            self._dags = LRUCache(maxsize=cache_size)
             self._use_cache = True
 
         # Lock required for bounded caches: cachetools caches are NOT thread-safe

airflow-core/tests/unit/api_fastapi/common/test_dagbag.py

@@ -16,6 +16,7 @@
 # under the License.
 from __future__ import annotations
 
+import math
 from unittest import mock
 
 import pytest
@@ -92,8 +93,9 @@ class TestCreateDagBag:
         ("cache_size", "cache_ttl", "expected_use_cache", "expected_dags_type"),
         [
             pytest.param(64, 3600, True, TTLCache, id="default_ttl_cache"),
-            pytest.param(0, 3600, False, dict, id="size_zero_unbounded"),
+            pytest.param(0, 3600, True, TTLCache, id="size_zero_ttl_only"),
             pytest.param(64, 0, True, LRUCache, id="ttl_zero_lru_only"),
+            pytest.param(0, 0, False, dict, id="size_zero_ttl_zero_unbounded"),
         ],
     )
     @mock.patch("airflow.api_fastapi.common.dagbag.conf")
@@ -110,3 +112,14 @@ def test_create_dag_bag_cache_modes(
         dag_bag = create_dag_bag()
         assert dag_bag._use_cache is expected_use_cache
         assert isinstance(dag_bag._dags, expected_dags_type)
+
+    @mock.patch("airflow.api_fastapi.common.dagbag.conf")
+    def test_create_dag_bag_ttl_only_has_no_size_cap(self, mock_conf):
+        from airflow.api_fastapi.common.dagbag import create_dag_bag
+
+        mock_conf.getint.side_effect = lambda section, key, fallback: {
+            "dag_cache_size": 0,
+            "dag_cache_ttl": 3600,
+        }.get(key, fallback)
+
+        assert create_dag_bag()._dags.maxsize == math.inf

airflow-core/tests/unit/models/test_dagbag.py

@@ -16,6 +16,7 @@
 # under the License.
 from __future__ import annotations
 
+import math
 import time
 from concurrent.futures import ThreadPoolExecutor
 from unittest.mock import MagicMock, patch
@@ -259,14 +260,24 @@ def test_lru_cache_enabled_with_cache_size(self):
         assert isinstance(dag_bag._dags, LRUCache)
 
     def test_ttl_cache_enabled_with_cache_size_and_ttl(self):
-        """Test that TTL cache is enabled when both cache_size and cache_ttl are provided."""
+        """Test that a bounded TTL cache is used when both cache_size and cache_ttl are provided."""
         dag_bag = DBDagBag(cache_size=10, cache_ttl=60)
         assert dag_bag._use_cache is True
         assert isinstance(dag_bag._dags, TTLCache)
+        assert dag_bag._dags.maxsize == 10
 
-    def test_zero_cache_size_uses_unbounded_dict(self):
-        """Test that cache_size=0 uses unbounded dict (same as no caching)."""
-        dag_bag = DBDagBag(cache_size=0, cache_ttl=60)
+    @pytest.mark.parametrize("cache_size", [0, None])
+    def test_ttl_only_without_size_cap(self, cache_size):
+        """Test that a positive cache_ttl with no size cap gives a TTL cache with maxsize=inf."""
+        dag_bag = DBDagBag(cache_size=cache_size, cache_ttl=60)
+        assert dag_bag._use_cache is True
+        assert isinstance(dag_bag._dags, TTLCache)
+        assert dag_bag._dags.maxsize == math.inf
+
+    @pytest.mark.parametrize("cache_ttl", [None, 0])
+    def test_zero_cache_size_uses_unbounded_dict(self, cache_ttl):
+        """Test that cache_size=0 without a TTL uses an unbounded dict (same as no caching)."""
+        dag_bag = DBDagBag(cache_size=0, cache_ttl=cache_ttl)
         assert dag_bag._use_cache is False
         assert isinstance(dag_bag._dags, dict)
 
@@ -310,6 +321,21 @@ def test_ttl_cache_expiry(self):
         with time_machine.travel("2025-01-01 00:00:02", tick=False):
             assert dag_bag._dags.get("test_version_id") is None
 
+    def test_ttl_only_evicts_by_ttl_not_size(self):
+        """An unbounded (maxsize=inf) TTL cache keeps every entry until it expires by age."""
+        dag_bag = DBDagBag(cache_size=0, cache_ttl=1)
+        assert dag_bag._dags.maxsize == math.inf
+        dag_bag._dags = TTLCache(maxsize=math.inf, ttl=1, timer=time.time)
+
+        with time_machine.travel("2025-01-01 00:00:00", tick=False):
+            for i in range(500):
+                dag_bag._dags[f"version_{i}"] = MagicMock()
+            assert len(dag_bag._dags) == 500
+
+        with time_machine.travel("2025-01-01 00:00:02", tick=False):
+            assert dag_bag._dags.get("version_0") is None
+            assert len(dag_bag._dags) == 0
+
     def test_lru_eviction(self):
         """Test that LRU eviction works when cache is full."""
         dag_bag = DBDagBag(cache_size=2)