ticket-024: Fallback community detection without networkx and preserve shared LOC

project/ticket-024/README.md

@@ -0,0 +1,24 @@
+# Ticket 024: Fallback community detection without networkx and preserve shared LOC in decision engine
+
+- **ID**: ticket-024
+- **Owner**: agent:gemini
+- **Status**: IN_PROGRESS
+- **Workflow state**: EDIT
+- **Created**: 2026-10-27
+
+SESSION_EXECUTION_AUTHORIZATION: user requested cross-project standardization duplication audit and fix of `semcod/redup` cross-project comparison and decision engine.
+
+## Goal and scope
+
+Provide a pure-Python connected-components fallback in `detect_communities()` when `networkx` is not installed, and fall back to deduplicated matched-block LOC in `recommend()` when `communities` is empty (`--no-community`).
+
+## Acceptance criteria
+
+- [x] AC-01: `detect_communities()` returns connected-component `CodeCommunity` clusters when `networkx` is absent instead of raising `ImportError`.
+- [x] AC-02: `recommend()` computes non-zero `shared_loc` and `overlap_percent` from `comparison.matches` when `communities` is empty.
+- [x] AC-03: Unit tests in `tests/test_compare.py` and `./project/governance-check.sh` pass.
+
+## Tracking boundary
+
+This directory contains the minimal reviewed intent. Optional participant prose
+and raw command logs are not required delivery output.

project/ticket-024/intent.json

@@ -0,0 +1,26 @@
+{
+  "schema": "new-project.intent/v3",
+  "ticket": "ticket-024",
+  "summary": "Fallback community detection without networkx and preserve shared LOC in decision engine",
+  "workstream": "application",
+  "classification": {
+    "kind": "BUG",
+    "priority": "P2",
+    "origin": "requested"
+  },
+  "allowedPaths": [
+    "project/ticket-024/**",
+    "TODO.md",
+    "project/TICKETS.md",
+    "src/redup/core/community.py",
+    "src/redup/core/decision.py",
+    "tests/test_compare.py"
+  ],
+  "forbiddenPaths": [
+    "project/ticket-*/user-*.md"
+  ],
+  "stacks": [],
+  "dependsOn": [],
+  "conflictsWith": [],
+  "integrationTicket": null
+}

src/redup/core/community.py

@@ -1,11 +1,13 @@
 """Community detection — find clusters of mutually similar code across projects.
 
-Uses networkx greedy modularity community detection to identify groups of
-functions that are strong candidates for extraction into a shared library.
+Uses networkx greedy modularity community detection when installed, falling back
+to pure-Python connected-component clustering so cross-project comparison works
+without optional graph dependencies.
 """
 
 from __future__ import annotations
 
+from collections import deque
 from dataclasses import dataclass, field
 
 from redup.core.comparator import CrossProjectComparison
@@ -25,27 +27,20 @@ class CodeCommunity:
 def detect_communities(
     comparison: CrossProjectComparison,
     min_similarity: float = 0.70,
+    use_networkx: bool = True,
 ) -> list[CodeCommunity]:
-    """Build similarity graph and detect communities via networkx.
+    """Build similarity graph and detect communities via networkx or BFS fallback.
 
     Args:
         comparison: Result of ``compare_projects``.
         min_similarity: Minimum edge weight to include.
+        use_networkx: Whether to prefer networkx when installed.
 
     Returns:
         Communities sorted by total LOC (descending).
-
-    Raises:
-        ImportError: If networkx is not installed.
     """
-    try:
-        import networkx as nx
-    except ImportError:
-        raise ImportError(
-            "networkx is required for community detection. Install with: pip install redup[compare]"
-        ) from None
-
-    G = nx.Graph()
+    nodes: dict[str, dict[str, object]] = {}
+    adj: dict[str, dict[str, float]] = {}
 
     for match in comparison.matches:
         if match.similarity < min_similarity:
@@ -54,25 +49,46 @@ def detect_communities(
         node_a = f"{match.project_a}::{match.file_a}::{match.function_a}"
         node_b = f"{match.project_b}::{match.file_b}::{match.function_b}"
 
-        G.add_node(
-            node_a,
-            project=match.project_a,
-            function=match.function_a,
-            loc=match.lines_a[1] - match.lines_a[0] + 1,
+        nodes[node_a] = {
+            "project": match.project_a,
+            "function": match.function_a,
+            "loc": max(0, match.lines_a[1] - match.lines_a[0] + 1),
+        }
+        nodes[node_b] = {
+            "project": match.project_b,
+            "function": match.function_b,
+            "loc": max(0, match.lines_b[1] - match.lines_b[0] + 1),
+        }
+        adj.setdefault(node_a, {})[node_b] = max(
+            adj.get(node_a, {}).get(node_b, 0.0), match.similarity
         )
-        G.add_node(
-            node_b,
-            project=match.project_b,
-            function=match.function_b,
-            loc=match.lines_b[1] - match.lines_b[0] + 1,
+        adj.setdefault(node_b, {})[node_a] = max(
+            adj.get(node_b, {}).get(node_a, 0.0), match.similarity
         )
-        G.add_edge(node_a, node_b, weight=match.similarity)
 
-    if G.number_of_nodes() == 0:
+    if not nodes:
         return []
 
-    # Greedy modularity community detection
-    communities_raw = nx.community.greedy_modularity_communities(G, weight="weight")
+    communities_raw: list[set[str]] | None = None
+    if use_networkx:
+        try:
+            import networkx as nx
+
+            G = nx.Graph()
+            for node_id, meta in nodes.items():
+                G.add_node(node_id, **meta)
+            for u, neighbors in adj.items():
+                for v, weight in neighbors.items():
+                    if u < v:
+                        G.add_edge(u, v, weight=weight)
+            communities_raw = [
+                set(c) for c in nx.community.greedy_modularity_communities(G, weight="weight")
+            ]
+        except ImportError:
+            communities_raw = None
+
+    if communities_raw is None:
+        communities_raw = _connected_components(nodes, adj)
 
     communities: list[CodeCommunity] = []
     for i, member_set in enumerate(communities_raw):
@@ -84,14 +100,16 @@ def detect_communities(
         total_loc = 0
         func_names: list[str] = []
 
-        for node in member_set:
-            data = G.nodes[node]
-            members.append((data["project"], node))
-            total_loc += data.get("loc", 0)
-            func_names.append(data["function"])
+        for node in sorted(member_set):
+            data = nodes[node]
+            members.append((str(data["project"]), node))
+            total_loc += int(data.get("loc", 0))
+            func_names.append(str(data["function"]))
 
-        for u, v in G.subgraph(member_set).edges:
-            similarities.append(G[u][v]["weight"])
+        for u in member_set:
+            for v, weight in adj.get(u, {}).items():
+                if u < v and v in member_set:
+                    similarities.append(weight)
 
         avg_sim = sum(similarities) / len(similarities) if similarities else 0.0
 
@@ -110,6 +128,30 @@ def detect_communities(
     return sorted(communities, key=lambda c: c.total_loc, reverse=True)
 
 
+def _connected_components(
+    nodes: dict[str, dict[str, object]],
+    adj: dict[str, dict[str, float]],
+) -> list[set[str]]:
+    """Extract connected components from an undirected adjacency map via BFS."""
+    visited: set[str] = set()
+    components: list[set[str]] = []
+    for start in sorted(nodes):
+        if start in visited:
+            continue
+        comp: set[str] = set()
+        queue: deque[str] = deque([start])
+        visited.add(start)
+        while queue:
+            curr = queue.popleft()
+            comp.add(curr)
+            for nxt in adj.get(curr, {}):
+                if nxt not in visited:
+                    visited.add(nxt)
+                    queue.append(nxt)
+        components.append(comp)
+    return components
+
+
 def _longest_common_prefix(names: list[str]) -> str:
     """Return the longest common prefix of a list of strings."""
     names = [n for n in names if n]

src/redup/core/decision.py

@@ -32,6 +32,17 @@ class DecisionRecommendation:
     confidence: float = 0.0
 
 
+def _matched_nodes_loc(comparison: CrossProjectComparison) -> int:
+    """Calculate total deduplicated LOC across matched nodes in both projects."""
+    nodes: dict[tuple[str, str, str, tuple[int, int]], int] = {}
+    for match in comparison.matches:
+        key_a = (match.project_a, match.file_a, match.function_a, match.lines_a)
+        key_b = (match.project_b, match.file_b, match.function_b, match.lines_b)
+        nodes[key_a] = max(0, match.lines_a[1] - match.lines_a[0] + 1)
+        nodes[key_b] = max(0, match.lines_b[1] - match.lines_b[0] + 1)
+    return sum(nodes.values())
+
+
 def recommend(
     comparison: CrossProjectComparison,
     communities: list[CodeCommunity],
@@ -55,7 +66,10 @@ def recommend(
     total_loc_b = comparison.stats_b.get("lines", 0)
     total_loc = total_loc_a + total_loc_b
     shared_loc = sum(c.total_loc for c in communities)
+    if shared_loc == 0 and not communities and comparison.matches:
+        shared_loc = _matched_nodes_loc(comparison)
     overlap = shared_loc / total_loc if total_loc > 0 else 0.0
+    cluster_count = len(communities) if communities else len(comparison.matches)
 
     if overlap > 0.60:
         decision = RefactorDecision.MERGE_PROJECTS
@@ -68,13 +82,13 @@ def recommend(
         decision = RefactorDecision.EXTRACT_SHARED_LIB
         rationale = (
             f"{overlap:.0%} overlap ({shared_loc} shared lines, "
-            f"{len(communities)} clusters). Extract to shared library."
+            f"{cluster_count} clusters). Extract to shared library."
         )
         confidence = 0.8
     elif overlap > 0.05:
         decision = RefactorDecision.EXTRACT_SHARED_LIB
         rationale = (
-            f"Small overlap ({overlap:.0%}) but {len(communities)} "
+            f"Small overlap ({overlap:.0%}) but {cluster_count} "
             f"well-defined clusters — worth extracting."
         )
         confidence = 0.6

tests/test_compare.py

@@ -206,23 +206,24 @@ def _make_comparison(self, n_matches: int = 5) -> CrossProjectComparison:
         )
 
     def test_detect_communities_requires_networkx(self):
-        try:
-            import networkx  # noqa: F401
-        except ImportError:
-            pytest.skip("networkx not installed")
-
         from redup.core.community import detect_communities
 
         comparison = self._make_comparison()
         communities = detect_communities(comparison, min_similarity=0.70)
         assert isinstance(communities, list)
+        assert len(communities) == 5
+        assert communities[0].total_loc == 20
 
-    def test_no_matches_yields_no_communities(self):
-        try:
-            import networkx  # noqa: F401
-        except ImportError:
-            pytest.skip("networkx not installed")
+    def test_detect_communities_fallback_without_networkx(self):
+        from redup.core.community import detect_communities
+
+        comparison = self._make_comparison(n_matches=3)
+        communities = detect_communities(comparison, min_similarity=0.70, use_networkx=False)
+        assert len(communities) == 3
+        assert all(c.total_loc == 20 for c in communities)
+        assert all(c.avg_similarity == pytest.approx(0.90) for c in communities)
 
+    def test_no_matches_yields_no_communities(self):
         from redup.core.community import detect_communities
 
         comparison = CrossProjectComparison(
@@ -289,3 +290,29 @@ def test_extract_shared_lib_moderate_overlap(self):
         )
         rec = recommend(comparison, communities=[community])
         assert rec.decision == RefactorDecision.EXTRACT_SHARED_LIB
+
+    def test_recommend_falls_back_to_matches_when_communities_empty(self):
+        comparison = CrossProjectComparison(
+            project_a=Path("/a"),
+            project_b=Path("/b"),
+            matches=[
+                CrossProjectMatch(
+                    project_a="/a",
+                    project_b="/b",
+                    file_a="a.py",
+                    file_b="b.py",
+                    function_a="shared",
+                    function_b="shared",
+                    similarity=1.0,
+                    similarity_type="exact",
+                    lines_a=(1, 150),
+                    lines_b=(1, 150),
+                )
+            ],
+            stats_a={"files": 5, "lines": 500},
+            stats_b={"files": 5, "lines": 500},
+        )
+        rec = recommend(comparison, communities=[])
+        assert rec.shared_loc == 300
+        assert rec.overlap_percent == pytest.approx(0.30)
+        assert rec.decision == RefactorDecision.EXTRACT_SHARED_LIB