fix(analytics): exclude internal categories by group flag, not by name

FIELDS.md

@@ -67,7 +67,7 @@ Nested inside `Transaction.subtransactions` for split transactions.
 ### Category
 
 **All fields**: `id`, `category_group_id`, `category_group_name`, `name`, `hidden`,
-`original_category_group_id`, `note`, `budgeted`, `activity`, `balance`, `goal_type`,
+`internal`, `original_category_group_id`, `note`, `budgeted`, `activity`, `balance`, `goal_type`,
 `goal_needs_whole_amount`, `goal_day`, `goal_cadence`, `goal_cadence_frequency`,
 `goal_creation_month`, `goal_target`, `goal_target_month`, `goal_target_date`,
 `goal_percentage_complete`, `goal_months_to_budget`, `goal_under_funded`,
@@ -89,10 +89,17 @@ for budgeting analysis.
 
 ### CategoryGroup
 
-**All fields**: `id`, `name`, `hidden`, `deleted`, `categories`
+**All fields**: `id`, `name`, `hidden`, `internal`, `deleted`, `categories`
 
 **Default `exclude_fields`**: `["deleted"]`
 
+`internal` marks a group YNAB maintains itself (`Internal Master Category`,
+`Credit Card Payments`, `Hidden Categories`) rather than one the user created.
+It is kept in responses by default because it is what analytics exclude on. Note
+it does **not** inherit: a Credit Card Payment category is `internal: false`
+inside an `internal: true` group, so the group's flag is the meaningful one.
+`null` means the API did not return the flag.
+
 ### MonthSummary / MonthDetail
 
 **MonthSummary fields**: `month`, `note`, `income`, `budgeted`, `activity`,

src/cache/service.py

@@ -176,7 +176,7 @@ async def api_call():
                 # Store group without nested categories to avoid duplication
                 flat_groups.append(CategoryGroup(
                     id=group.id, name=group.name, hidden=group.hidden,
-                    deleted=group.deleted, categories=[],
+                    internal=group.internal, deleted=group.deleted, categories=[],
                 ))
 
             await self.delta.upsert_entities(plan_id, ENTITY_CATEGORY_GROUP, flat_groups)

src/models/category.py

@@ -23,6 +23,12 @@ class Category(YNABBaseModel):
     category_group_name: str | None = Field(default=None)
     name: str = Field()
     hidden: bool = Field(default=False)
+    # Required on both models in YNAB spec 1.86.0. Nullable here so an absent
+    # flag is distinguishable from a false one: absent must degrade to the legacy
+    # group-name check, false must not. Note the flag does NOT inherit — Credit
+    # Card Payment categories are internal=false inside an internal=true group,
+    # so the *group's* flag is the one that means "not user spending".
+    internal: bool | None = Field(default=None)
     original_category_group_id: str | None = Field(default=None)
     note: str | None = Field(default=None)
     budgeted: Milliunit = Field(default=0)
@@ -50,5 +56,6 @@ class CategoryGroup(YNABBaseModel):
     id: str = Field()
     name: str = Field()
     hidden: bool = Field(default=False)
+    internal: bool | None = Field(default=None)
     deleted: bool = Field(default=False)
     categories: list[Category] = Field(default=[])

src/server/analytics.py

@@ -3,8 +3,17 @@
 
 from src.models.common import milliunits_to_dollars
 from src.server import _shared
+from src.server.internal_categories import InternalCategoryResolver
 
-MONEY_FLOW_EXCLUDE_GROUPS = {"Internal Master Category", "Credit Card Payments"}
+
+async def _internal_resolver(plan_id: str) -> InternalCategoryResolver:
+    """Resolve the category -> group.internal map for a plan.
+
+    One extra delta-synced (cached) call to GET /categories; the month payload
+    does not carry the group's flag.
+    """
+    groups = await _shared.cache.get_categories(plan_id)
+    return InternalCategoryResolver(groups)
 
 
 @_shared.mcp.tool()
@@ -24,14 +33,16 @@ async def get_money_flow(plan_id: str, month: str = "current") -> str:
 
     # Fetch month detail (has categories with activity and income total)
     month_detail = await _shared.cache.get_month(month, plan_id)
+    internal = await _internal_resolver(plan_id)
 
     # Group categories by category_group_name and sum activity
     group_activity: dict[str, int] = {}
     for cat in month_detail.categories:
-        group_name = cat.category_group_name or "Uncategorized"
-        if group_name in MONEY_FLOW_EXCLUDE_GROUPS:
+        if internal.is_internal(cat):
             continue
+        group_name = cat.category_group_name or "Uncategorized"
         group_activity[group_name] = group_activity.get(group_name, 0) + cat.activity
+    internal.log_fallbacks()
 
     # Filter out groups with zero activity
     group_activity = {name: activity for name, activity in group_activity.items() if activity != 0}
@@ -75,14 +86,15 @@ async def get_spending_by_category(plan_id: str, month: str = "current") -> str:
         month = today.replace(day=1).strftime("%Y-%m-%d")
 
     month_detail = await _shared.cache.get_month(month, plan_id)
+    internal = await _internal_resolver(plan_id)
 
     # Collect categories with non-zero activity, excluding internal groups
     categories = []
     total_spent_mu = 0
     for cat in month_detail.categories:
-        group_name = cat.category_group_name or "Uncategorized"
-        if group_name in MONEY_FLOW_EXCLUDE_GROUPS:
+        if internal.is_internal(cat):
             continue
+        group_name = cat.category_group_name or "Uncategorized"
         if cat.activity == 0:
             continue
         spent_mu = abs(cat.activity)
@@ -95,6 +107,8 @@ async def get_spending_by_category(plan_id: str, month: str = "current") -> str:
             "balance_mu": cat.balance,
         })
 
+    internal.log_fallbacks()
+
     # Sort by spending (highest first) and compute percentages
     categories.sort(key=lambda c: c["spent_mu"], reverse=True)
     result_cats = []

src/server/internal_categories.py

@@ -0,0 +1,101 @@
+"""Which categories are YNAB's own bookkeeping rather than user spending.
+
+Analytics must exclude YNAB-internal categories, or credit-funded spending is
+counted twice: a Credit Card Payment category's `activity` is the payment
+movement for spending already counted in the real category.
+
+**The predicate is the category's *group's* `internal` flag.** Measured against a
+live budget: `Internal Master Category`, `Credit Card Payments`, and
+`Hidden Categories` all carry `group.internal = true`, while every Credit Card
+Payment *category* carries `internal = false`. The flag does not inherit. Using
+the category-level flag would readmit every payment category; matching group
+names by string is locale-dependent, breaks on rename, and can be spoofed by a
+user-created group of the same name.
+
+Nothing here knows how many such categories exist in any particular budget.
+"""
+
+import logging
+
+from src.models import Category, CategoryGroup
+
+logger = logging.getLogger(__name__)
+
+# Fallback only, for a category whose group flag cannot be resolved: a historical
+# month can name a category absent from the current category list, and a future
+# API could stop returning `internal`.
+LEGACY_INTERNAL_GROUP_NAMES = frozenset(
+    {"Internal Master Category", "Credit Card Payments"}
+)
+
+
+class InternalCategoryResolver:
+    """Decides, per category, whether it belongs to a YNAB-internal group.
+
+    `group.internal` comes from `GET /plans/{id}/categories`; month analytics read
+    `GET /plans/{id}/months/{month}`, which carries the category-level flag but
+    not the group's. So the map is resolved once from the delta-synced category
+    list and applied to the month's categories.
+
+    Two different things can send a category down the legacy fallback, and they
+    are counted and reported separately because they mean opposite things:
+
+    * **Absent from the current category list** — routine. A historical month
+      names categories deleted since. A 24-month query produces these in bulk.
+    * **Present, but the group carries no `internal` flag** — a schema
+      regression. `internal` is `required` on `CategoryGroup` in spec 1.86.0, so
+      this should never happen, and it is the signal worth acting on.
+
+    Collapsing both into one log event would bury the second under hundreds of
+    the first the moment multi-month analytics exist.
+    """
+
+    def __init__(self, groups: list[CategoryGroup]):
+        self._internal_by_category_id: dict[str, bool | None] = {
+            category.id: group.internal
+            for group in groups
+            for category in group.categories
+        }
+        self.unknown_category_count = 0
+        self.missing_flag_count = 0
+
+    @property
+    def fallback_count(self) -> int:
+        """Total categories that took the legacy path, for either reason."""
+        return self.unknown_category_count + self.missing_flag_count
+
+    def is_internal(self, category: Category) -> bool:
+        """True if `category` belongs to a YNAB-internal group.
+
+        A category whose group flag is unknown is not silently treated as "not
+        internal", which would readmit exactly what this class exists to
+        exclude. It falls back to the legacy group-name check.
+        """
+        if category.id in self._internal_by_category_id:
+            group_internal = self._internal_by_category_id[category.id]
+            if group_internal is not None:
+                return group_internal
+            self.missing_flag_count += 1
+        else:
+            self.unknown_category_count += 1
+
+        return (category.category_group_name or "") in LEGACY_INTERNAL_GROUP_NAMES
+
+    def log_fallbacks(self) -> None:
+        """Report the *mechanism*, once per call. Never category names or amounts."""
+        if self.unknown_category_count:
+            logger.info(
+                "%d categor%s absent from the current category map; "
+                "fell back to legacy group-name exclusion. This is expected for "
+                "months containing categories deleted since",
+                self.unknown_category_count,
+                "y is" if self.unknown_category_count == 1 else "ies are",
+            )
+        if self.missing_flag_count:
+            logger.warning(
+                "CategoryGroup.internal missing for %d resolved categor%s; "
+                "falling back to legacy group-name exclusion. The field is "
+                "required by the YNAB API spec, so this indicates a schema change",
+                self.missing_flag_count,
+                "y" if self.missing_flag_count == 1 else "ies",
+            )