From 7cbd5f4e53c1a6e36ba2a17ba445c0dea6bf3d68 Mon Sep 17 00:00:00 2001 From: Tom Softreck Date: Tue, 27 Oct 2026 11:22:40 +0100 Subject: [PATCH] ticket-024: fallback community detection without networkx and preserve shared LOC --- project/ticket-024/README.md | 24 +++++++ project/ticket-024/intent.json | 26 ++++++++ src/redup/core/community.py | 112 ++++++++++++++++++++++----------- src/redup/core/decision.py | 18 +++++- tests/test_compare.py | 47 +++++++++++--- 5 files changed, 180 insertions(+), 47 deletions(-) create mode 100644 project/ticket-024/README.md create mode 100644 project/ticket-024/intent.json diff --git a/project/ticket-024/README.md b/project/ticket-024/README.md new file mode 100644 index 0000000..5182bb9 --- /dev/null +++ b/project/ticket-024/README.md @@ -0,0 +1,24 @@ +# Ticket 024: Fallback community detection without networkx and preserve shared LOC in decision engine + +- **ID**: ticket-024 +- **Owner**: agent:gemini +- **Status**: IN_PROGRESS +- **Workflow state**: EDIT +- **Created**: 2026-10-27 + +SESSION_EXECUTION_AUTHORIZATION: user requested cross-project standardization duplication audit and fix of `semcod/redup` cross-project comparison and decision engine. + +## Goal and scope + +Provide a pure-Python connected-components fallback in `detect_communities()` when `networkx` is not installed, and fall back to deduplicated matched-block LOC in `recommend()` when `communities` is empty (`--no-community`). + +## Acceptance criteria + +- [x] AC-01: `detect_communities()` returns connected-component `CodeCommunity` clusters when `networkx` is absent instead of raising `ImportError`. +- [x] AC-02: `recommend()` computes non-zero `shared_loc` and `overlap_percent` from `comparison.matches` when `communities` is empty. +- [x] AC-03: Unit tests in `tests/test_compare.py` and `./project/governance-check.sh` pass. + +## Tracking boundary + +This directory contains the minimal reviewed intent. Optional participant prose +and raw command logs are not required delivery output. diff --git a/project/ticket-024/intent.json b/project/ticket-024/intent.json new file mode 100644 index 0000000..8579192 --- /dev/null +++ b/project/ticket-024/intent.json @@ -0,0 +1,26 @@ +{ + "schema": "new-project.intent/v3", + "ticket": "ticket-024", + "summary": "Fallback community detection without networkx and preserve shared LOC in decision engine", + "workstream": "application", + "classification": { + "kind": "BUG", + "priority": "P2", + "origin": "requested" + }, + "allowedPaths": [ + "project/ticket-024/**", + "TODO.md", + "project/TICKETS.md", + "src/redup/core/community.py", + "src/redup/core/decision.py", + "tests/test_compare.py" + ], + "forbiddenPaths": [ + "project/ticket-*/user-*.md" + ], + "stacks": [], + "dependsOn": [], + "conflictsWith": [], + "integrationTicket": null +} diff --git a/src/redup/core/community.py b/src/redup/core/community.py index 9081a26..972d3fd 100644 --- a/src/redup/core/community.py +++ b/src/redup/core/community.py @@ -1,11 +1,13 @@ """Community detection — find clusters of mutually similar code across projects. -Uses networkx greedy modularity community detection to identify groups of -functions that are strong candidates for extraction into a shared library. +Uses networkx greedy modularity community detection when installed, falling back +to pure-Python connected-component clustering so cross-project comparison works +without optional graph dependencies. """ from __future__ import annotations +from collections import deque from dataclasses import dataclass, field from redup.core.comparator import CrossProjectComparison @@ -25,27 +27,20 @@ class CodeCommunity: def detect_communities( comparison: CrossProjectComparison, min_similarity: float = 0.70, + use_networkx: bool = True, ) -> list[CodeCommunity]: - """Build similarity graph and detect communities via networkx. + """Build similarity graph and detect communities via networkx or BFS fallback. Args: comparison: Result of ``compare_projects``. min_similarity: Minimum edge weight to include. + use_networkx: Whether to prefer networkx when installed. Returns: Communities sorted by total LOC (descending). - - Raises: - ImportError: If networkx is not installed. """ - try: - import networkx as nx - except ImportError: - raise ImportError( - "networkx is required for community detection. Install with: pip install redup[compare]" - ) from None - - G = nx.Graph() + nodes: dict[str, dict[str, object]] = {} + adj: dict[str, dict[str, float]] = {} for match in comparison.matches: if match.similarity < min_similarity: @@ -54,25 +49,46 @@ def detect_communities( node_a = f"{match.project_a}::{match.file_a}::{match.function_a}" node_b = f"{match.project_b}::{match.file_b}::{match.function_b}" - G.add_node( - node_a, - project=match.project_a, - function=match.function_a, - loc=match.lines_a[1] - match.lines_a[0] + 1, + nodes[node_a] = { + "project": match.project_a, + "function": match.function_a, + "loc": max(0, match.lines_a[1] - match.lines_a[0] + 1), + } + nodes[node_b] = { + "project": match.project_b, + "function": match.function_b, + "loc": max(0, match.lines_b[1] - match.lines_b[0] + 1), + } + adj.setdefault(node_a, {})[node_b] = max( + adj.get(node_a, {}).get(node_b, 0.0), match.similarity ) - G.add_node( - node_b, - project=match.project_b, - function=match.function_b, - loc=match.lines_b[1] - match.lines_b[0] + 1, + adj.setdefault(node_b, {})[node_a] = max( + adj.get(node_b, {}).get(node_a, 0.0), match.similarity ) - G.add_edge(node_a, node_b, weight=match.similarity) - if G.number_of_nodes() == 0: + if not nodes: return [] - # Greedy modularity community detection - communities_raw = nx.community.greedy_modularity_communities(G, weight="weight") + communities_raw: list[set[str]] | None = None + if use_networkx: + try: + import networkx as nx + + G = nx.Graph() + for node_id, meta in nodes.items(): + G.add_node(node_id, **meta) + for u, neighbors in adj.items(): + for v, weight in neighbors.items(): + if u < v: + G.add_edge(u, v, weight=weight) + communities_raw = [ + set(c) for c in nx.community.greedy_modularity_communities(G, weight="weight") + ] + except ImportError: + communities_raw = None + + if communities_raw is None: + communities_raw = _connected_components(nodes, adj) communities: list[CodeCommunity] = [] for i, member_set in enumerate(communities_raw): @@ -84,14 +100,16 @@ def detect_communities( total_loc = 0 func_names: list[str] = [] - for node in member_set: - data = G.nodes[node] - members.append((data["project"], node)) - total_loc += data.get("loc", 0) - func_names.append(data["function"]) + for node in sorted(member_set): + data = nodes[node] + members.append((str(data["project"]), node)) + total_loc += int(data.get("loc", 0)) + func_names.append(str(data["function"])) - for u, v in G.subgraph(member_set).edges: - similarities.append(G[u][v]["weight"]) + for u in member_set: + for v, weight in adj.get(u, {}).items(): + if u < v and v in member_set: + similarities.append(weight) avg_sim = sum(similarities) / len(similarities) if similarities else 0.0 @@ -110,6 +128,30 @@ def detect_communities( return sorted(communities, key=lambda c: c.total_loc, reverse=True) +def _connected_components( + nodes: dict[str, dict[str, object]], + adj: dict[str, dict[str, float]], +) -> list[set[str]]: + """Extract connected components from an undirected adjacency map via BFS.""" + visited: set[str] = set() + components: list[set[str]] = [] + for start in sorted(nodes): + if start in visited: + continue + comp: set[str] = set() + queue: deque[str] = deque([start]) + visited.add(start) + while queue: + curr = queue.popleft() + comp.add(curr) + for nxt in adj.get(curr, {}): + if nxt not in visited: + visited.add(nxt) + queue.append(nxt) + components.append(comp) + return components + + def _longest_common_prefix(names: list[str]) -> str: """Return the longest common prefix of a list of strings.""" names = [n for n in names if n] diff --git a/src/redup/core/decision.py b/src/redup/core/decision.py index 63f21f1..569337b 100644 --- a/src/redup/core/decision.py +++ b/src/redup/core/decision.py @@ -32,6 +32,17 @@ class DecisionRecommendation: confidence: float = 0.0 +def _matched_nodes_loc(comparison: CrossProjectComparison) -> int: + """Calculate total deduplicated LOC across matched nodes in both projects.""" + nodes: dict[tuple[str, str, str, tuple[int, int]], int] = {} + for match in comparison.matches: + key_a = (match.project_a, match.file_a, match.function_a, match.lines_a) + key_b = (match.project_b, match.file_b, match.function_b, match.lines_b) + nodes[key_a] = max(0, match.lines_a[1] - match.lines_a[0] + 1) + nodes[key_b] = max(0, match.lines_b[1] - match.lines_b[0] + 1) + return sum(nodes.values()) + + def recommend( comparison: CrossProjectComparison, communities: list[CodeCommunity], @@ -55,7 +66,10 @@ def recommend( total_loc_b = comparison.stats_b.get("lines", 0) total_loc = total_loc_a + total_loc_b shared_loc = sum(c.total_loc for c in communities) + if shared_loc == 0 and not communities and comparison.matches: + shared_loc = _matched_nodes_loc(comparison) overlap = shared_loc / total_loc if total_loc > 0 else 0.0 + cluster_count = len(communities) if communities else len(comparison.matches) if overlap > 0.60: decision = RefactorDecision.MERGE_PROJECTS @@ -68,13 +82,13 @@ def recommend( decision = RefactorDecision.EXTRACT_SHARED_LIB rationale = ( f"{overlap:.0%} overlap ({shared_loc} shared lines, " - f"{len(communities)} clusters). Extract to shared library." + f"{cluster_count} clusters). Extract to shared library." ) confidence = 0.8 elif overlap > 0.05: decision = RefactorDecision.EXTRACT_SHARED_LIB rationale = ( - f"Small overlap ({overlap:.0%}) but {len(communities)} " + f"Small overlap ({overlap:.0%}) but {cluster_count} " f"well-defined clusters — worth extracting." ) confidence = 0.6 diff --git a/tests/test_compare.py b/tests/test_compare.py index 2c9643d..68e3f88 100644 --- a/tests/test_compare.py +++ b/tests/test_compare.py @@ -206,23 +206,24 @@ def _make_comparison(self, n_matches: int = 5) -> CrossProjectComparison: ) def test_detect_communities_requires_networkx(self): - try: - import networkx # noqa: F401 - except ImportError: - pytest.skip("networkx not installed") - from redup.core.community import detect_communities comparison = self._make_comparison() communities = detect_communities(comparison, min_similarity=0.70) assert isinstance(communities, list) + assert len(communities) == 5 + assert communities[0].total_loc == 20 - def test_no_matches_yields_no_communities(self): - try: - import networkx # noqa: F401 - except ImportError: - pytest.skip("networkx not installed") + def test_detect_communities_fallback_without_networkx(self): + from redup.core.community import detect_communities + + comparison = self._make_comparison(n_matches=3) + communities = detect_communities(comparison, min_similarity=0.70, use_networkx=False) + assert len(communities) == 3 + assert all(c.total_loc == 20 for c in communities) + assert all(c.avg_similarity == pytest.approx(0.90) for c in communities) + def test_no_matches_yields_no_communities(self): from redup.core.community import detect_communities comparison = CrossProjectComparison( @@ -289,3 +290,29 @@ def test_extract_shared_lib_moderate_overlap(self): ) rec = recommend(comparison, communities=[community]) assert rec.decision == RefactorDecision.EXTRACT_SHARED_LIB + + def test_recommend_falls_back_to_matches_when_communities_empty(self): + comparison = CrossProjectComparison( + project_a=Path("/a"), + project_b=Path("/b"), + matches=[ + CrossProjectMatch( + project_a="/a", + project_b="/b", + file_a="a.py", + file_b="b.py", + function_a="shared", + function_b="shared", + similarity=1.0, + similarity_type="exact", + lines_a=(1, 150), + lines_b=(1, 150), + ) + ], + stats_a={"files": 5, "lines": 500}, + stats_b={"files": 5, "lines": 500}, + ) + rec = recommend(comparison, communities=[]) + assert rec.shared_loc == 300 + assert rec.overlap_percent == pytest.approx(0.30) + assert rec.decision == RefactorDecision.EXTRACT_SHARED_LIB