From a6895e414272f7fa5f43a887fc080b8fbaee38cd Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 14:07:00 +0530 Subject: [PATCH 01/19] GML-2211: Add Jira Cloud data source connector MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Jira Cloud REST API client with retry, rate-limit handling, and parallel comment fetching (up to 10 concurrent threads) - Pydantic config models: connection, scope, sync state - ADF → Markdown converter for issue descriptions and comments - Deterministic mapper: JiraIssue, JiraComment, JiraUser, JiraProject vertices + edges; comments chunked directly (bypass ECC Document pipeline) - Log-output filter strips stack traces / log lines from comments before chunking to reduce noise in search results - Sync orchestration: incremental JQL checkpoint, content-hash dedup, batch document loading (every 10 pages), pipelined page fetch, crash-recovery rebuild detection (_has_unprocessed_chunks) - REST endpoints for data source CRUD, sync trigger, schema install - UI: DataSourcesConfig page, Rebuild dialog live sync-status guard - ECC: zero-text guard skips empty chunks before embedding - ECC util: register jira / jira_comment chunker types - Prompt fix: map Jira ticket keys to schema attributes (not vertex IDs) - Embedding store: transient-error retry with exponential back-off - Tests: connector unit tests, cypher generation, structural validation - Design doc: docs/design/jira-data-source-connector.md - Example config: docs/tutorials/configs/graph_configs/data_sources.json - .gitignore: exclude configs/graph_configs/*/data_sources.json (credentials) --- .gitignore | 6 + .../embeddings/tigergraph_embedding_store.py | 114 +- common/llm_services/base_llm.py | 10 +- docs/design/jira-data-source-connector.md | 738 +++++++++ .../configs/graph_configs/data_sources.json | 24 + ecc/app/ecc_util.py | 8 +- ecc/app/graphrag/graph_rag.py | 34 +- ecc/app/graphrag/workers.py | 80 +- ecc/app/main.py | 8 +- ecc/tests/test_jira_comment_chunks.py | 72 + graphrag-ui/src/main.tsx | 5 + .../src/pages/setup/DataSourcesConfig.tsx | 1382 +++++++++++++++++ graphrag-ui/src/pages/setup/KGAdmin.tsx | 3 + graphrag-ui/src/pages/setup/SetupLayout.tsx | 12 +- graphrag-ui/src/types/dataSources.ts | 41 + graphrag/app/connectors/__init__.py | 1 + graphrag/app/connectors/jira/__init__.py | 5 + graphrag/app/connectors/jira/adf.py | 122 ++ graphrag/app/connectors/jira/client.py | 335 ++++ graphrag/app/connectors/jira/config.py | 138 ++ graphrag/app/connectors/jira/mapper.py | 630 ++++++++ graphrag/app/connectors/jira/schema.py | 278 ++++ graphrag/app/connectors/jira/state.py | 150 ++ graphrag/app/connectors/jira/sync.py | 906 +++++++++++ graphrag/app/main.py | 1 + graphrag/app/routers/__init__.py | 1 + graphrag/app/routers/data_sources.py | 578 +++++++ graphrag/app/routers/ui.py | 21 +- graphrag/app/tools/generate_cypher.py | 22 +- graphrag/app/tools/generate_function.py | 6 +- graphrag/app/tools/graphrag_tools.py | 22 +- graphrag/app/tools/validation_utils.py | 18 + graphrag/tests/test_generate_cypher.py | 20 + graphrag/tests/test_jira_connector.py | 1022 ++++++++++++ .../test_structural_function_validation.py | 36 + 35 files changed, 6782 insertions(+), 67 deletions(-) create mode 100644 docs/design/jira-data-source-connector.md create mode 100644 docs/tutorials/configs/graph_configs/data_sources.json create mode 100644 ecc/tests/test_jira_comment_chunks.py create mode 100644 graphrag-ui/src/pages/setup/DataSourcesConfig.tsx create mode 100644 graphrag-ui/src/types/dataSources.ts create mode 100644 graphrag/app/connectors/__init__.py create mode 100644 graphrag/app/connectors/jira/__init__.py create mode 100644 graphrag/app/connectors/jira/adf.py create mode 100644 graphrag/app/connectors/jira/client.py create mode 100644 graphrag/app/connectors/jira/config.py create mode 100644 graphrag/app/connectors/jira/mapper.py create mode 100644 graphrag/app/connectors/jira/schema.py create mode 100644 graphrag/app/connectors/jira/state.py create mode 100644 graphrag/app/connectors/jira/sync.py create mode 100644 graphrag/app/routers/data_sources.py create mode 100644 graphrag/tests/test_generate_cypher.py create mode 100644 graphrag/tests/test_jira_connector.py create mode 100644 graphrag/tests/test_structural_function_validation.py diff --git a/.gitignore b/.gitignore index fbb59583..45aa8905 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,9 @@ __pycache__/ # Release docs /RELEASE_NOTES.md + +# Jira / graph config files contain real credentials — never commit them. +# Use docs/tutorials/configs/graph_configs/data_sources.json as the example. +configs/graph_configs/*/data_sources.json +configs/graph_configs/*/.DS_Store +configs/.DS_Store diff --git a/common/embeddings/tigergraph_embedding_store.py b/common/embeddings/tigergraph_embedding_store.py index 47dbed8e..e21e30a4 100644 --- a/common/embeddings/tigergraph_embedding_store.py +++ b/common/embeddings/tigergraph_embedding_store.py @@ -12,7 +12,9 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +import asyncio import logging +import random import traceback import json from time import sleep, time @@ -287,18 +289,26 @@ def _embed_sync_with_truncation_retry(self, text: str, v_id): truncation gatekeeper for the rationale.""" last_err = None for i, candidate in enumerate(self._truncation_candidates(text)): - try: - return self.embedding_service.embed_query(candidate) - except Exception as e: - last_err = e - if not self._is_embed_overflow(e): - break - LogWriter.warning( - f"Embed for {v_id} overflowed at len={len(candidate)} " - f"(attempt {i + 1}); retrying with shorter prefix" - ) - LogWriter.error(f"Failed to embed {v_id} after truncation: {last_err}") - return None + for retry in range(4): + try: + return self.embedding_service.embed_query(candidate) + except Exception as e: + last_err = e + if self._is_embed_overflow(e): + LogWriter.warning( + f"Embed for {v_id} overflowed at len={len(candidate)} " + f"(attempt {i + 1}); retrying with shorter prefix" + ) + break + if not self._is_transient_embed_error(e) or retry == 3: + return self._log_embed_failure(v_id, last_err) + delay = (2 ** retry) + random.uniform(0, 0.5) + LogWriter.warning( + f"Transient embedding failure for {v_id}; retrying in " + f"{delay:.1f}s (attempt {retry + 2}/4): {e}" + ) + sleep(delay) + return self._log_embed_failure(v_id, last_err) async def _embed_with_truncation_retry(self, text: str, v_id): """Async embed with input-overflow fallback. @@ -309,24 +319,60 @@ async def _embed_with_truncation_retry(self, text: str, v_id): counts vary). Rather than abandoning the whole batch on one bad chunk, truncate the offending text to progressively shorter prefixes until it fits. The persisted chunk text is - unchanged; only the embedding represents the prefix. A chunk - for which no truncation level fits is left without an - embedding — the similarity_search GSQL skips empty vectors. + unchanged; only the embedding represents the prefix. If no + truncation level succeeds, the error is propagated so callers + cannot report a successful build with a missing embedding. """ last_err = None for i, candidate in enumerate(self._truncation_candidates(text)): - try: - return await self.embedding_service.aembed_query(candidate) - except Exception as e: - last_err = e - if not self._is_embed_overflow(e): - break - LogWriter.warning( - f"Embed for {v_id} overflowed at len={len(candidate)} " - f"(attempt {i + 1}); retrying with shorter prefix" - ) - LogWriter.error(f"Failed to embed {v_id} after truncation: {last_err}") - return None + for retry in range(4): + try: + return await self.embedding_service.aembed_query(candidate) + except Exception as e: + last_err = e + if self._is_embed_overflow(e): + LogWriter.warning( + f"Embed for {v_id} overflowed at len={len(candidate)} " + f"(attempt {i + 1}); retrying with shorter prefix" + ) + break + if not self._is_transient_embed_error(e) or retry == 3: + return self._log_embed_failure(v_id, last_err) + delay = (2 ** retry) + random.uniform(0, 0.5) + LogWriter.warning( + f"Transient embedding failure for {v_id}; retrying in " + f"{delay:.1f}s (attempt {retry + 2}/4): {e}" + ) + await asyncio.sleep(delay) + return self._log_embed_failure(v_id, last_err) + + @staticmethod + def _is_transient_embed_error(error: Exception) -> bool: + message = str(error).lower() + return any( + marker in message + for marker in ( + "429", + "500", + "502", + "503", + "504", + "internal", + "rate limit", + "resource exhausted", + "timeout", + "timed out", + "connection reset", + "temporarily unavailable", + ) + ) + + @staticmethod + def _log_embed_failure(v_id, error): + LogWriter.error(f"Failed to embed {v_id} after retries: {error}") + raise RuntimeError( + f"Failed to embed {v_id} after retries" + ) from error def add_embeddings( self, @@ -368,7 +414,9 @@ def add_embeddings( vec_attrs_used.add(vec_attr) embedding = self._embed_sync_with_truncation_retry(text, v_id) if embedding is None: - continue + raise RuntimeError( + f"Embedding provider returned no vector for {v_id}" + ) attr = self.map_attrs([(vec_attr, embedding)]) batch["vertices"][v_type][v_id] = attr @@ -402,6 +450,7 @@ def add_embeddings( f"Ensure these vertex types have the expected vector attribute." ) LogWriter.error(error_message) + raise RuntimeError(error_message) from e async def aadd_embeddings( self, @@ -443,11 +492,9 @@ async def aadd_embeddings( vec_attrs_used.add(vec_attr) embedding = await self._embed_with_truncation_retry(text, v_id) if embedding is None: - # No truncation level worked. Leave this vertex without an - # embedding so similarity_search can skip it (the GSQL - # query filters on v.embedding.size() > 0) — better than - # abandoning the entire batch on one bad chunk. - continue + raise RuntimeError( + f"Embedding provider returned no vector for {v_id}" + ) attr = self.map_attrs([(vec_attr, embedding)]) batch["vertices"][v_type][v_id] = attr @@ -481,6 +528,7 @@ async def aadd_embeddings( f"Ensure these vertex types have the expected vector attribute." ) LogWriter.error(error_message) + raise RuntimeError(error_message) from e def has_embeddings( self, diff --git a/common/llm_services/base_llm.py b/common/llm_services/base_llm.py index fe77ac1f..b5e6499a 100644 --- a/common/llm_services/base_llm.py +++ b/common/llm_services/base_llm.py @@ -681,7 +681,7 @@ def map_question_schema_prompt(self): - If an entity maps to a vertex attribute, consider generating a `WHERE` clause. - For synonyms, output the canonical form from the schema choices. - Generate the **complete** rewritten question. Keep the case of schema elements unchanged. -- Do NOT generate `target_vertex_ids` unless the term `id` is explicitly mentioned in the question. +- Use `target_vertex_ids` only for explicit primary/vertex IDs; map business or external keys to matching schema attributes even when called an "ID". ## Inputs - **Vertices**: {vertices} @@ -712,7 +712,7 @@ def generate_function_prompt(self): - Never pick a function not described in the docstrings below. - If entities map to vertex attributes, consider a `WHERE` clause. - When constructing `WHERE`, quote string attribute values properly. Example: `('Person', where='name="William Torres"')` — applies to every string attribute (name, email, address, etc.). -- Do NOT generate `target_vertex_ids` unless the term `id` is explicitly mentioned in the question. +- Use `getVerticesById` only for actual primary IDs; use `getVertices` with `WHERE` for schema-defined business/external-key attributes. - Pick exactly **one** function to execute. ## Schema @@ -839,7 +839,9 @@ def generate_cypher_prompt(self): - Only use attributes that exist in the schema. Pick the closest matching attribute name when multiple candidates exist. - Prefer attributes over primary IDs when an attribute name is more similar to the keyword in the question. - Keep the query minimal — fewest vertex types, edge types, and attributes possible. -- Do NOT return attributes that aren't explicitly mentioned in the question. If only a vertex is mentioned, return only the vertex. +- Use only schema-defined edge types in relationships, and put vertex labels only in `MATCH`. +- Use primary IDs only for true vertex-ID lookups. Match complete business keys exactly; for a numeric-only portion use delimiter-aware `ENDS WITH` (e.g. ticket `2192` → `"-2192"`). +- For descriptive lookups, return relevant scalar attributes and only explicitly requested relationships; do not invent traversal paths. - Always include the entity from the `WHERE` clause in the final `RETURN`. Use vertex name over ID when available. - Always use **undirected** edge patterns. Ensure edges connect correct vertex types per schema. - Use **double quotes** for strings. @@ -1039,6 +1041,7 @@ def select_retriever_prompt(self): # action, then each next action driven by what the previous result returned. _AGENTIC_AGENT_USER_DEFAULT = """\ - For most questions, make your FIRST action a vector search (graphrag__hybrid_search or graphrag__contextual_search) — it gives the broadest grounding. Skip it only when you are highly confident the question is a pure structured-data request (an exact count, an attribute/id lookup, a relationship traversal, or an aggregation over typed graph data) that a generated graph query fully answers on its own. +- Route business-key lookups (e.g. ticket `ABC-123` or its numeric portion) to graphrag__structural_retrieve and preserve the user's wording. - Let each observation drive the next action: if the passages you got back name specific entities or relationships you still need hard facts about, follow up with a structural query; if a result is thin, empty, or off-target, widen its parameters (top_k, num_hops) or switch method rather than repeating the same call. - Before answering, check that every part of the question is covered with the specific facts and figures it asks for; if a required value, table, or entity is still missing, retrieve again (widen top_k / num_hops or switch method) rather than answering vaguely or partially. - For a specific value, row, total, ranking, or year-over-year comparison, use graphrag__hybrid_search or graphrag__contextual_search with top_k >= 10 (they return atomic table chunks that keep full row/column structure), and quote the exact label, column, year, or unit from the question so the retriever can match it.""" @@ -1089,6 +1092,7 @@ def agentic_agent_prompt(self): # be tuned without touching the role / act model / plan mechanics. _AGENTIC_PLANNER_USER_DEFAULT = """\ - Prioritize including at least one vector search step (graphrag__hybrid_search or graphrag__contextual_search) unless you are highly confident the question is a pure structured-data request — an exact count, an attribute/id lookup, a relationship traversal, or an aggregation over typed graph data — that a generated graph query fully answers on its own. Whenever the answer could plausibly live in document text (what/why/how/describe/summarize, definitions, explanations, figures), include a vector search step. When unsure, include vector search. +- Route business-key lookups (e.g. ticket `ABC-123` or its numeric portion) to graphrag__structural_retrieve and preserve the user's wording. - Use BOTH kinds when a question needs facts from the graph AND supporting text; you may run several of each, in any order. When you use STRUCTURAL, pair it with a vector search step unless the question is a pure structured-data request. - Prefer the smallest plan that will work. Trivial/greeting questions need only the final answer step. - Tabular / numeric questions (a specific value, a row, a column total, a ranking, or a year-over-year comparison from a table or chart): prefer graphrag__contextual_search or graphrag__hybrid_search with top_k>=10 (these return atomic table chunks that preserve full row/column structure); avoid graphrag__similarity_search alone; quote any specific table label, column header, year, or unit from the question (e.g. "ROE 2023"); for "compare X across years/regions/categories" set top_k>=15.""" diff --git a/docs/design/jira-data-source-connector.md b/docs/design/jira-data-source-connector.md new file mode 100644 index 00000000..fe32dac9 --- /dev/null +++ b/docs/design/jira-data-source-connector.md @@ -0,0 +1,738 @@ +# Jira Data Source Connector — Implementation Plan + +Status: Implemented and validated locally; OAuth/ACL security remains planned +Date: 24 September 2026 +Product: TigerGraph GraphRAG +Source: Jira Cloud + +--- + +## 1. Goal + +An operator initializes or selects an existing graph, configures a Jira Cloud +site for that graph, installs the Jira schema, and runs +ingestion. GraphRAG can then answer: + +- Structured questions such as “Which high-priority bugs are still open in + PAY?” and “Who is assigned PAY-123?” +- Narrative questions such as “What did we decide about the checkout + timeout?” + +The connector writes into the graph that GraphRAG already queries. It does not +add a separate answer engine. + +--- + +## 2. Design decisions + +Each Jira issue is written in complementary structural and semantic forms: + +1. POLE+O-aligned domain data: `JiraUser` is a Person subtype; `JiraIssue` and + `JiraProject` are Object subtypes; `JiraComment` is an Event subtype. +2. One issue markdown `Document` containing issue metadata and description. +3. Deterministic `DocumentChunk` records written directly from each Jira + comment, without an intermediate comment `Document`. Short comments produce + one chunk; exceptionally long comments may produce multiple bounded chunks. + Existing hybrid and similarity retrievers read these embeddings without a + connector-specific retrieval path. + +The connector maps Jira fields deterministically. It does not run LLM entity +extraction over Jira issue text. Status, project, assignee, reporter, parent, +and issue links already have authoritative structured values in Jira. + +The entire connector lifecycle lives under **Data sources**: graph selection, +credentials, project scope, connection testing, schema migration, manual sync, +status, and errors. `Ingest to Knowledge Graph` remains the existing +file/cloud-document workflow and does not expose Jira a second time. + +The connector never creates or replaces a graph. The operator first uses the +existing Initialize Knowledge Graph flow. Jira schema installation is an +explicit, additive, idempotent migration against that initialized graph. + +### 2.1 Research basis: Glean + +Glean uses one connector administration surface rather than duplicating Jira +under a generic upload page: + +1. Admin console -> Connectors -> Add connector -> Jira Cloud. +2. The connector’s Setup tab owns authentication and source-specific settings. +3. Manage data owns project/custom-field inclusion rules. +4. The same connector detail shows crawl/index status and supports re-crawl. +5. Glean separates content crawls from identity/permission crawls and uses + webhooks plus incremental/full crawls to keep the index current. +6. Indexed records use stable IDs, object types, searchable bodies, custom + properties, source URLs, and document-level permissions. + +Sources: + +- [Glean: Get started with connectors](https://docs.glean.com/connectors/getting-started) +- [Glean: Jira Cloud connector](https://docs.glean.com/connectors/native/jira/) +- [Glean: Connector settings and visibility](https://docs.glean.com/connectors/connectors-settings-visibility) +- [Glean: Document model](https://developers.glean.com/api-info/indexing/documents/document-model) +- [Glean: Custom properties](https://developers.glean.com/api-info/indexing/datasource/custom-properties) + +Glean configures a datasource schema (`objectDefinitions` and +`propertyDefinitions`) and then indexes stable documents carrying datasource, +object type, body, URL, custom properties, and permissions. GraphRAG requires +one additional connector-owned step: map that normalized Jira model to an +additive TigerGraph schema so structured graph retrieval can traverse issues, +projects, and users. That migration belongs inside the connector detail, +equivalent to Glean’s datasource-schema setup, not on the file-ingest page. + +The implemented connector matches Glean’s connector lifecycle and +normalized content approach, but not its complete security model. Glean +indexes users, groups, memberships, and Jira permissions and enforces those +permissions at query time. GraphRAG currently uses one connector account plus a +project allow-list, so every GraphRAG user with graph access can retrieve the +indexed Jira data. Identity crawls and query-time ACL filtering remain Phase 5 +work; they must not be simulated by adding more Jira business vertices to the +schema. + +Jira access is scoped by the configured service account and project allow-list. +The current GraphRAG retrieval path does not enforce the chatting user’s Jira +permissions. + +--- + +## 3. Existing GraphRAG workflow + +```text +Operator uploads files or selects cloud storage + | + v + Existing ingest API + | + v + Issue Document -> DocumentChunk -> Content + Comment -------> DocumentChunk -> Content + | + v + Existing retrievers + | + v + Answer +``` + +File ingest remains unchanged. Jira is not added as another +`CreateIngestConfig.data_source` because that API creates file loading jobs and +cannot write typed Jira edges such as `JIRA_ASSIGNED_TO`. + +--- + +## 4. Target Jira workflow + +```text +Initialize Knowledge Graph + | + v +Select existing initialized graph + | + v +Data sources -> Add Jira Cloud + | + +-- Configure and test credentials + +-- Select projects and content settings + +-- Inspect live graph schema + | | + | +-- missing/incomplete -> Install Jira schema elements + | +-- conflict ------> Stop with exact conflict + | +-- installed -----> Enable Ingest now + | + v +Ingest now: ingest Jira incrementally + | + +-----------------------------+ + | | + v v +Typed Jira vertices/edges Issue Documents Direct comment chunks + | | + v v + Existing ECC rebuild Existing embedding store + | | + +--------------+--------------+ + | + v + Existing GraphRAG retrievers +``` + +The connector ingestion and GraphRAG build use the existing per-graph lock. +File and cloud ingestion stay independent and continue to use their existing +Ingest to Knowledge Graph flow. + +The sync API loads authoritative Jira vertices, edges, issue documents, and +directly embedded comment chunks, then returns `rebuild_required`. The Data +sources UI calls the existing +`POST /ui/{graph}/rebuild_graph` endpoint when changed documents or deletions +require a GraphRAG rebuild. ECC performs structure-aware chunking and embedding +for issue documents; comment chunks reuse the same embedding store during +connector sync and bypass ECC entity extraction. + +Incremental ingestion orders tickets oldest-to-newest by Jira `updated` time. +After each search page has completed its graph and document writes, the +connector atomically persists that page’s highest `updated` timestamp. After an +unexpected shutdown, the next run resumes from that checkpoint minus a +120-second overlap. Upserts use Jira’s stable numeric issue id, making overlap +duplicates harmless. Newest-to-oldest ordering is intentionally avoided because +it could checkpoint past older tickets that have not yet been committed. + +--- + +## 5. Repository changes + +### 5.1 New backend modules + +| Path | Responsibility | +| --- | --- | +| `graphrag/app/connectors/__init__.py` | Connector package | +| `graphrag/app/connectors/jira/config.py` | Validated Jira source configuration | +| `graphrag/app/connectors/jira/schema.py` | Predefined `SchemaProposal` | +| `graphrag/app/connectors/jira/client.py` | Jira REST API, pagination, retries | +| `graphrag/app/connectors/jira/adf.py` | Atlassian Document Format to markdown | +| `graphrag/app/connectors/jira/mapper.py` | Jira payload to graph records, issue documents, and direct comment chunks | +| `graphrag/app/connectors/jira/state.py` | Atomic graph-scoped config persistence | +| `graphrag/app/connectors/jira/sync.py` | Schema-gated incremental sync | +| `graphrag/app/routers/data_sources.py` | Authenticated data-source API | + +### 5.2 Existing backend integration points + +| Path | Required change | +| --- | --- | +| `graphrag/app/routers/__init__.py` | Export the data-source router | +| `graphrag/app/main.py` | Mount the data-source router | +| `ecc/app/graphrag/graph_rag.py` | Embed residual source-native chunks without duplicate LLM extraction | +| `ecc/app/graphrag/workers.py` | Skip duplicate LLM extraction and link each Jira chunk to its typed issue | +| `common/embeddings/tigergraph_embedding_store.py` | Retry transient embedding-provider failures with bounded backoff | +| `common/llm_services/base_llm.py` | Clarify business-key mapping and concise schema-driven structural lookup | +| `graphrag/app/tools/generate_function.py` | Permit the existing read-only pyTigerGraph function surface | +| `graphrag/app/tools/generate_cypher.py` | Normalize fenced model output safely | +| `graphrag/app/tools/graphrag_tools.py` | Recognize JSON-encoded empty results and improve Cypher retry feedback | +| `graphrag/app/tools/validation_utils.py` | Define the explicit read-only function allow-list | + +### 5.3 Frontend changes + +| Path | Required change | +| --- | --- | +| `graphrag-ui/src/pages/setup/DataSourcesConfig.tsx` | Single graph-scoped connector lifecycle: setup, schema, sync, status | +| `graphrag-ui/src/main.tsx` | Add `/setup/kg-admin/data-sources` | +| `graphrag-ui/src/pages/setup/SetupLayout.tsx` | Add Data sources navigation | + +### 5.4 Existing code to reuse + +| Existing code | Use | +| --- | --- | +| `common/db/schema_utils.py:apply_proposal` | Apply the Jira domain schema additively | +| `upsert_type_metadata` | Give query planning precise Jira type descriptions | +| `emit_structural_link_alters` | Allow `CONTAINS_ENTITY` to target `JiraIssue` | +| `common/chunkers` | Chunk issue markdown and unusually long comments | +| `graphrag/app/supportai/supportai_ingest.py` | Reuse the existing document-loading entry point | +| `common/config.py:get_embedding_store` | Reuse embedding and vector upsert behavior | +| `graphrag/app/tools/graphrag_tools.py` | Existing structured and unstructured answering; no connector-specific retrieval path | +| `common/utils/graph_locks.py` | Prevent sync from racing with ingest/schema operations | +| Existing UI auth and graph selection state | Do not create a second auth or graph-selection mechanism | + +### 5.5 Code that stays unchanged + +- Existing file and cloud storage ingest +- Existing vector and hybrid query implementations +- MCP server configuration +- Unrelated setup pages + +--- + +## 6. Schema + +The Jira schema adds four vertex types and nine edge types. Installation is +idempotent and adds only missing compatible elements. + +A vertex is used only when questions need to traverse from one object to +another. Project, issue, user, and comment are those objects. Status, priority, +labels, components, fix versions, and dates remain issue attributes. Comments +are first-class Event records with separately embedded content. The Jira site +URL remains source configuration. + +### 6.1 Schema picture + +Read each line independently. The left side is the source vertex and the arrow +label is the edge type. + +```text +JiraIssue -- JIRA_BELONGS_TO --> JiraProject + +JiraIssue -- JIRA_ASSIGNED_TO --> JiraUser + +JiraIssue -- JIRA_REPORTED_BY --> JiraUser + +JiraIssue -- JIRA_HAS_PARENT --> JiraIssue + +JiraIssue -- JIRA_LINKS_TO --> JiraIssue + link_type = blocks | is blocked by | duplicates | relates to + +JiraComment -- JIRA_COMMENT_ON --------> JiraIssue + +JiraComment -- JIRA_COMMENTED_BY ------> JiraUser + +JiraComment -- JIRA_COMMENT_REPLIES_TO -> JiraComment + only when supplied explicitly by Jira + +JiraComment -- JIRA_COMMENT_AFTER -----> JiraComment + chronological order on one issue +``` + +Connector-owned edge names use a Jira prefix to avoid collisions with an +existing graph’s domain schema. This is mandatory for `JIRA_HAS_PARENT` and +`JIRA_LINKS_TO` because GraphRAG’s base schema already defines `HAS_PARENT` +and `LINKS_TO` for communities. + +Issue descriptions and comment content reuse GraphRAG’s existing semantic +schema. Comments skip the unnecessary intermediate `Document`: + +```text +issue Document ---- HAS_CHILD ----> issue DocumentChunk + | | + +-- CONTAINS_ENTITY --> JiraIssue +--> JiraIssue + +JiraComment <---- CONTAINS_ENTITY ---- comment DocumentChunk +JiraIssue <---- CONTAINS_ENTITY ---- | + +-- HAS_CONTENT --> Content +``` + +`HAS_CHILD`, `HAS_CONTENT`, and `CONTAINS_ENTITY` already exist. Jira schema +application adds the Jira domain endpoint pairs to `CONTAINS_ENTITY`; this +design change adds no schema type. + +### 6.2 Vertex types + +All ids are prefixed with `jira:{cloudId}:` so different Jira sites cannot +collide. Numeric Jira ids are used because human issue keys can change when an +issue moves project. + +| Vertex | Example id | Attributes | +| --- | --- | --- | +| `JiraProject` | `jira:abc:project:10001` | `project_key STRING`, `name STRING`, `url STRING` | +| `JiraIssue` | `jira:abc:issue:10422` | `issue_key STRING`, `summary STRING`, `issue_type STRING`, `status STRING`, `status_category STRING`, `priority STRING`, `resolution STRING`, `labels STRING`, `components STRING`, `fix_versions STRING`, `created DATETIME`, `updated DATETIME`, `due DATETIME`, `url STRING`, `story_points DOUBLE`, `content_hash STRING` | +| `JiraUser` | `jira:abc:user:557058:account` | `account_id STRING`, `display_name STRING` | +| `JiraComment` | `jira:abc:comment:9001` | `comment_id STRING`, `created DATETIME`, `updated DATETIME`, `visibility STRING`, `is_public BOOL`, `ontology_class STRING`, `content_hash STRING` | + +`labels`, `components`, and `fix_versions` are comma-separated strings. They +do not require separate vertices for the current query set. + +`status_category` contains `new`, `indeterminate`, or `done`, allowing +cross-project open/done filtering while retaining the project-specific status +name in `status`. + +`content_hash` is sync bookkeeping. An unchanged hash skips document +re-chunking and re-embedding. The hash is advanced only after the replacement +document loads successfully, so a failed load is retried on the next sync. + +`story_points` remains empty unless one numeric custom field is configured. + +Email is not stored on `JiraUser`; Jira Cloud can hide it. `accountId` is the +stable user identity. + +### 6.3 Edge types + +| Edge | From | To | Extra attribute | Query purpose | +| --- | --- | --- | --- | --- | +| `JIRA_BELONGS_TO` | `JiraIssue` | `JiraProject` | | Project filtering | +| `JIRA_ASSIGNED_TO` | `JiraIssue` | `JiraUser` | | Current owner | +| `JIRA_REPORTED_BY` | `JiraIssue` | `JiraUser` | | Reporter | +| `JIRA_HAS_PARENT` | `JiraIssue` | `JiraIssue` | | Epic or subtask parent | +| `JIRA_LINKS_TO` | `JiraIssue` | `JiraIssue` | `link_type STRING` | Blocks, duplicates, relates to | +| `JIRA_COMMENT_ON` | `JiraComment` | `JiraIssue` | | Comment’s issue | +| `JIRA_COMMENTED_BY` | `JiraComment` | `JiraUser` | | Comment author | +| `JIRA_COMMENT_REPLIES_TO` | `JiraComment` | `JiraComment` | | Explicit source-provided parent comment | +| `JIRA_COMMENT_AFTER` | `JiraComment` | `JiraComment` | | Chronological order | + +Current-state edges are replaced for each updated issue. This is necessary so +reassignment does not leave an old `JIRA_ASSIGNED_TO` edge. + +Jira’s standard issue comments are normally a flat sequence. Reply edges are +never inferred; `JIRA_COMMENT_REPLIES_TO` is written only when the source +payload explicitly identifies a parent. `JIRA_COMMENT_AFTER` preserves the +available chronological order. + +### 6.4 Type descriptions for query planning + +| Type | Description | +| --- | --- | +| `JiraIssue` | A Jira work item. Filter by issue_key, status, status_category, priority, issue_type, resolution, labels, components, fix_versions, created, updated, and due | +| `JiraProject` | A Jira project. project_key is the short key such as PAY | +| `JiraUser` | An Atlassian account. account_id is identity and display_name is the human name | +| `JiraComment` | A POLE+O Event subtype representing one authored Jira comment with stable identity and timestamps | +| `JIRA_LINKS_TO` | Issue link whose link_type is blocks, is blocked by, duplicates, or relates to | +| `JIRA_HAS_PARENT` | Parent issue, including epic and subtask parent | +| `JIRA_ASSIGNED_TO` | Current assignee | +| `JIRA_REPORTED_BY` | User who reported the issue | +| `JIRA_BELONGS_TO` | Current project | + +These descriptions are stored through `upsert_type_metadata` so the existing +question-to-schema mapper can choose the correct types and attributes. + +### 6.5 Issue document + +Document id: `jira:{cloudId}:issue-doc:{issueId}` + +```text +# PAY-123: Login timeout on checkout +URL: https://acme.atlassian.net/browse/PAY-123 +Project: PAY Payments +Type: Bug +Status: In Progress (indeterminate) +Priority: High +Assignee: Ada Lovelace +Reporter: Grace Hopper +Labels: checkout, auth +Components: API +Fix versions: 2.4 +Updated: 2026-09-20T14:03:00Z + +## Description + + +## Attachments +design.pdf, logs.txt +``` + +### 6.6 Comment event and direct chunk + +Chunk id: +`jira:{cloudId}:comment:{commentId}:chunk:{index}:{contentDigest}` + +```text +# Comment by Ada Lovelace on PAY-123: Login timeout on checkout +Issue: PAY-123 +URL: https://acme.atlassian.net/browse/PAY-123 +Comment ID: 9001 +Author: Ada Lovelace +Created: 2026-09-18T11:02:00Z +Updated: 2026-09-18T11:02:00Z +Visibility: default + +## Comment + +``` + +The connector creates these chunks directly with content type `jira_comment`. +Structure-aware chunking emits one chunk for ordinary short comments and +bounded multiple chunks only for unusually long comments. For long comments, +log-dominated fenced or multiline blocks are replaced by an omission marker; +human explanation surrounding those logs is retained. Detection uses multiple +signals (timestamps, severity levels, stack traces, and structured log fields), +not length alone. + +Every resulting chunk links to both `JiraComment` and `JiraIssue`. The connector +reuses the configured embedding store directly, so no Jira branch is added to +shared ECC processing and no duplicate LLM entity extraction occurs. The +comment content hash advances only after all replacement chunks and embeddings +succeed. Updating one comment therefore re-embeds that comment rather than the +entire issue. Removed comments have their vertex, direct chunks, content, edges, +and embeddings deleted during issue reconciliation. When legacy comment +Documents are detected, the connector performs one resumable full synchronization +to replace them with direct chunks, then returns to incremental checkpoints. + +--- + +## 7. Jira API + +Target Jira Cloud REST API v3 only. + +Current authentication is HTTPS Basic authentication using an Atlassian account +email and API token. + +| Endpoint | Use | +| --- | --- | +| `GET /rest/api/3/myself` | Test connection | +| `GET /_edge/tenant_info` | Resolve `cloudId` | +| `GET /rest/api/3/project/search` | Load all visible projects for the project picker | +| `GET /rest/api/3/project/{projectIdOrKey}` | Load one optional project directly by key | +| `POST /rest/api/3/search/jql` | Initial and incremental issue search | +| `GET /rest/api/3/issue/{id}/comment` | Fetch all comments if the embedded page is truncated | + +Search rules: + +- Use `/rest/api/3/search/jql`, not deprecated `/rest/api/3/search`. +- Use `nextPageToken`, not `startAt`, for issue search. +- Fetch search pages sequentially. +- Use `maxResults = 100`. +- Add `ORDER BY updated ASC, key ASC`. +- Combine the project allow-list with optional created-after, updated-after, + and status-category filters. Additional JQL remains an advanced option. +- On HTTP 429, honor `Retry-After` and retry with bounded backoff. +- Advance the durable checkpoint only after every graph and document write for + a page succeeds; do not advance it after a failed page or failed graph write. +- Reset the incremental checkpoint when scope filters change so a widened + scope performs a full ingestion instead of skipping older matching issues. + +Incremental JQL: + +```text +project in (PAY, PLAT) +AND updated >= "{last successful checkpoint minus 120 seconds}" +ORDER BY updated ASC, key ASC +``` + +Jira search is not a stable snapshot. The overlap and idempotent upsert cover +updates near page boundaries. + +The connector does not persist issue inventory on the local filesystem. +Consequently, incremental sync currently updates and adds records but does not +automatically remove issues deleted in Jira or moved outside the configured +scope. Production deletion reconciliation requires shared ownership state in +TigerGraph or another durable multi-replica state store. + +--- + +## 8. Configuration and UI + +### 8.1 UI locations + +**Data sources** (`/setup/kg-admin/data-sources`) owns the selected graph’s +complete connector lifecycle: + +1. Saved sources with enabled state, last ingestion, and last error. +2. Connection: display name, tenant site URL, Atlassian account email, API + token, and connection status. +3. Enter an optional project key, then test the connection. Connection status + is displayed next to that action. +4. Only after a successful test in the current editing session, load one + requested project or every visible project, then select the projects for the + graph. A direct project-key load replaces stale project results. Changing + connection details invalidates verification and clears loaded results. +5. Configure optional created/updated dates and Jira status categories after + project selection. Additional JQL and comment ingestion remain advanced + scope settings. +6. Preview the approximate Jira ticket count using the unsaved project and + filter scope, then save the scope. +7. Schema: inspect compatibility and install missing compatible elements. +8. Ingest/build actions: Ingest now, last completion, item count, and last error. +9. Connector lifecycle: edit, disable, or delete configuration. + +**Ingest to Knowledge Graph** is unchanged. It remains responsible for local +files, cloud storage, and Amazon BDA. Jira does not appear there. + +The UI requires an API token for every connection test and keeps that token +only in process memory for subsequent connector operations. UI-submitted tokens +are never persisted. If an administrator has already placed `api_token` in the +graph-scoped data-source configuration, saves preserve and may reuse it, but +the API never returns it to the browser. + +### 8.2 Stored configuration + +Path: `configs/graph_configs/{graph}/data_sources.json` + +```json +{ + "sources": [ + { + "id": "jira-acme", + "type": "jira_cloud", + "enabled": true, + "display_name": "Acme Jira", + "connection": { + "site_url": "https://acme.atlassian.net", + "email": "connector@example.com", + "cloud_id": "abc123" + }, + "scope": { + "project_keys": ["PAY", "PLAT"], + "created_after": "2026-01-01", + "updated_after": null, + "status_categories": ["new", "indeterminate"], + "jql_extra": "", + "include_comments": true + }, + "sync": { + "overlap_seconds": 120, + "checkpoint": "2026-09-20T14:03:11.000+0000", + "last_tested_at": "2026-09-20T13:50:00.000+0000", + "last_started_at": null, + "last_completed_at": null, + "last_error": null, + "last_issue_count": 0 + } + } + ] +} +``` + +Sync refuses to run with an empty project list. + +Live TigerGraph metadata is authoritative: every status check validates +required attributes, edge direction, and endpoint pairs. No Jira schema +version number is stored. + +### 8.3 API + +| Method | Path | Behavior | +| --- | --- | --- | +| `GET` | `/ui/{graph}/data-sources` | List graph sources without secrets | +| `PUT` | `/ui/{graph}/data-sources/{id}` | Validate and atomically save | +| `DELETE` | `/ui/{graph}/data-sources/{id}` | Remove configuration only | +| `POST` | `/ui/{graph}/data-sources/{id}/test` | Test a freshly supplied token without persisting it | +| `GET` | `/ui/{graph}/data-sources/{id}/projects` | Return all visible projects, or one project when `project_key` is supplied | +| `POST` | `/ui/{graph}/data-sources/{id}/issues/count` | Preview Jira’s approximate count for an unsaved scope | +| `GET` | `/ui/{graph}/data-sources/jira/schema` | Inspect live Jira schema compatibility | +| `POST` | `/ui/{graph}/data-sources/jira/schema/install` | Add missing compatible Jira schema elements idempotently | +| `POST` | `/ui/{graph}/data-sources/{id}/sync` | Run schema-gated incremental Jira ingestion | +| `GET` | `/ui/{graph}/data-sources/{id}/sync/{run_id}` | Poll the in-memory status of a submitted sync | +| `POST` | `/ui/{graph}/rebuild_graph` | Existing endpoint used by the UI to start the ECC build | + +--- + +## 9. Answer flow after ingestion + +The implementation uses the existing GraphRAG agent and the request’s +TigerGraph connection. It does not yet enforce the chatting user’s Jira +permissions; Jira ACL-safe retrieval is Phase 5. + +| Question | Existing path | Required data | +| --- | --- | --- | +| Which high-priority bugs are still open in PAY? | `structural_retrieve` | JiraIssue attributes and JIRA_BELONGS_TO | +| Who is assigned PAY-123? | `structural_retrieve` | issue_key and JIRA_ASSIGNED_TO | +| What does PAY-123 block? | `structural_retrieve` | JIRA_LINKS_TO.link_type | +| What did we decide about the checkout timeout? | `hybrid_search` | Issue document chunks | +| What status and owner does the issue discussing the timeout have? | hybrid plus structural, then `combine_context` | CONTAINS_ENTITY, status, JIRA_ASSIGNED_TO | +| What themes occur across payments bugs? | `hybrid_search` | Issue documents; community duplication is not part of the connector | + +Jira keys are external identifiers, not TigerGraph primary vertex ids. For an +exact key such as `PAY-123`, the existing structural pipeline maps the value to +`JiraIssue.issue_key` and generates an exact attribute predicate. It must not +pass the key to `getVerticesById`; Jira primary ids are source-qualified, +stable ids based on the Jira cloud and numeric issue id. + +For a numeric-only ticket reference such as `123`, the schema-driven +openCypher fallback uses delimiter-aware suffix matching and returns every +matching business key rather than inventing a project prefix. If more than one +project has that numeric suffix, the answer must expose the ambiguity. + +The structural path first attempts an allowed read-only pyTigerGraph function. +JSON-encoded empty results such as `"[]"` are treated as empty so the existing +openCypher fallback can run. Generated Cypher is retried with execution errors +fed back to the generator. Full-key and numeric-suffix lookups have both been +verified against the development graph. + +Hybrid retrieval is independent of structural ingestion. Issue descriptions +become searchable after ECC builds their document chunks. Comment chunks are +embedded during Jira synchronization and are searchable without passing +through ECC. + +```text +Question + | + v +get_schema + | + +-- count/filter/key/traversal --> existing structural query path + | + +-- description/comments --> existing hybrid/vector retrieval + | + v +combine_context + | + v +Answer with issue key and Jira URL +``` + +--- + +## 10. Implementation phases + +### Phase 1 — Core schema and mapping — Implemented + +- Add validated source models. +- Add the three-vertex, five-edge schema proposal. +- Add ADF-to-markdown conversion. +- Map representative Jira payloads, including unassigned issues, parents, + inward/outward links, hidden user email, missing priority, and custom story + points. + +Done when pure unit tests pass without TigerGraph or Jira. + +### Phase 2 — Jira client and graph sync — Implemented + +- Implement current enhanced-search pagination with `nextPageToken`. +- Implement bounded 429 retry. +- Inspect and validate the live Jira schema before every sync. +- Upsert structured records. +- Replace current-state edges. +- Load issue `Document`/`Content` records for ECC. +- Write bounded comment `DocumentChunk`/`Content` records directly and embed + them with the existing store. +- Skip unchanged documents by `content_hash`. + +Done when a mocked sync proves idempotency and a development graph shows both +the typed issue and linked chunks. + +### Phase 3 — Schema migration, API, and UI — Implemented + +- Add explicit schema status and install endpoints using existing additive + schema utilities. +- Reject conflicts before applying a migration and gate sync on installed + schema status. +- Add authenticated, graph-scoped CRUD/test/project/sync endpoints. +- Add masked secret handling and atomic config writes. +- Keep setup, schema migration, ingestion, build status, and re-ingestion under one + Data sources connector detail. +- Do not add Jira to Ingest to Knowledge Graph. + +### Phase 4 — Exit test — Partially completed locally + +1. Configure a non-admin Jira service account. +2. Select one project. +3. Select the initialized graph under Data sources. +4. Install the Jira schema and confirm a second install is a no-op. +5. Run Ingest now and monitor ingestion/build status on the same page. +6. Ask a structured count question and compare it with Jira. +7. Ask a comment question and confirm the answer names and links the issue. + +Current validation covers schema installation, typed ticket retrieval by full +and numeric-only keys, source document loading, deterministic chunk-to-issue +linkage, direct comment embedding, legacy cleanup, and resumable migration. + +### Phase 5 — OAuth, identity, and permission enforcement — Planned + +- Replace API-token crawling with confidential OAuth 2.0 (3LO) and rotating + refresh tokens. +- Add self-service GraphRAG-user to Atlassian-account linking. +- Crawl accounts, groups, memberships, project permission schemes, project + roles, issue-security schemes, and issue security levels. +- Split or omit role/group-restricted comments; never place them in a document + with only issue-level ACLs. +- Activate permission crawls as validated generations and retain the previous + generation until the replacement is complete. +- Pass the resolved GraphRAG identity into every retrieval path. +- Replace unrestricted Jira structural generation with fixed ACL-aware query + surfaces. Filter semantic candidates before ranking, expansion, and LLM + context construction. +- Deny all Jira results when account linking or permission state is missing, + stale, incomplete, or unsupported. + +--- + +## 11. Out of scope + +- Jira Data Center and Server +- Forge and Connect authentication modes +- Sprints, boards, and backlog rank +- Changelog and worklogs +- Attachment content ingestion +- Shared cross-source Person identity +- LLM or ontology extraction over issue text +- Connector marketplace/plugin loading +- MCP as an ingestion path +- Webhooks and scheduled sync +- Jira Service Management SLA-specific entities + +--- + +## 12. Risks and controls + +| Risk | Control | +| --- | --- | +| Permission leakage | Known limitation: restrict graph access and connector scope; implement OAuth-linked identity, permission snapshots, and fail-closed retrieval in Phase 5 | +| Search pagination drift | Stable id upsert plus 120-second checkpoint overlap | +| Stale assignee or parent | Replace current-state outgoing edges for each updated issue | +| Re-embedding cost | Compare `content_hash` before rewriting chunks | +| Missing Jira embeddings | Embed direct comment chunks transactionally during sync; keep issue-document embedding in the existing ECC stage | +| Schema complexity | Keep only four Jira vertex types | +| Existing feature regressions | Isolated connector/router/page plus narrow, tested fixes to the shared structural-query path | diff --git a/docs/tutorials/configs/graph_configs/data_sources.json b/docs/tutorials/configs/graph_configs/data_sources.json new file mode 100644 index 00000000..6e3249d0 --- /dev/null +++ b/docs/tutorials/configs/graph_configs/data_sources.json @@ -0,0 +1,24 @@ +{ + "sources": [ + { + "id": "my-jira", + "type": "jira_cloud", + "enabled": true, + "display_name": "My Jira Project", + "connection": { + "site_url": "https://your-company.atlassian.net", + "email": "your-email@company.com", + "api_token": "YOUR_JIRA_API_TOKEN" + }, + "scope": { + "project_keys": ["GML", "TSE"], + "created_after": null, + "updated_after": null, + "status_categories": ["new", "indeterminate", "done"], + "jql_extra": "", + "include_comments": true, + "story_points_field": null + } + } + ] +} diff --git a/ecc/app/ecc_util.py b/ecc/app/ecc_util.py index ee96e170..25e9bf64 100644 --- a/ecc/app/ecc_util.py +++ b/ecc/app/ecc_util.py @@ -30,7 +30,13 @@ def get_chunker(chunker_type: str = "", graphname: str = None): chunk_size=chunker_config.get("chunk_size", 0), overlap_size=chunker_config.get("overlap_size", -1), ) - elif chunker_type in ("structured", "markdown", "html"): + elif chunker_type in ( + "structured", + "markdown", + "html", + "jira", + "jira_comment", + ): # Structure-aware chunker for markdown AND HTML: tables/figures/lists/ # code stay atomic (never split mid-row), prose char-splits by size. # Supersedes MarkdownChunker/HTMLChunker, which split structure blindly. diff --git a/ecc/app/graphrag/graph_rag.py b/ecc/app/graphrag/graph_rag.py index 97a0e793..2c57d675 100644 --- a/ecc/app/graphrag/graph_rag.py +++ b/ecc/app/graphrag/graph_rag.py @@ -417,7 +417,9 @@ async def load(conn: AsyncTigerGraphConnection): async def embed( - embed_chan: Channel, embedding_store: EmbeddingStore, graphname: str + embed_chan: Channel, + embedding_store: EmbeddingStore, + graphname: str, ): """ Creates and starts one worker for each embed job @@ -435,7 +437,10 @@ async def embed( v_id = (v_id, index_name) # v_id is a per-vertex identifier derived from user content. logger.debug(f"Embed to {graphname}_{index_name}: {v_id}") - if get_graphrag_config(graphname).get("reuse_embedding", True) and embedding_store.has_embeddings([v_id]): + if get_graphrag_config(graphname).get( + "reuse_embedding", + True, + ) and embedding_store.has_embeddings([v_id]): logger.debug(f"Embeddings for {v_id} already exists, skipping to save cost") n_reused += 1 continue @@ -714,7 +719,11 @@ async def summarize_communities( ) -async def run(graphname: str, conn: AsyncTigerGraphConnection, progress=None): +async def run( + graphname: str, + conn: AsyncTigerGraphConnection, + progress=None, +): """ Set up GraphRAG: - Install necessary queries. @@ -794,7 +803,8 @@ async def new_doc_pipeline(): total_docs = 0 try: count_result = conn.getVertexCount( - "Document", where="epoch_processed=0" + "Document", + where="epoch_processed=0", ) if asyncio.iscoroutine(count_result): count_result = await count_result @@ -828,7 +838,13 @@ async def new_doc_pipeline(): grp.create_task(upsert(upsert_chan)) grp.create_task(load(conn)) - grp.create_task(embed(embed_chan, embedding_store, graphname)) + grp.create_task( + embed( + embed_chan, + embedding_store, + graphname, + ) + ) grp.create_task( extract(extract_chan, upsert_chan, embed_chan, extractor, conn, num_chunk_senders, tracker=doc_tracker) ) @@ -888,7 +904,13 @@ async def new_doc_pipeline(): ) grp.create_task(upsert(upsert_chan)) grp.create_task(load(conn)) - grp.create_task(embed(embed_chan, embedding_store, graphname)) + grp.create_task( + embed( + embed_chan, + embedding_store, + graphname, + ) + ) logger.info("Join comm_process_chan") await comm_process_chan.join() logger.info("Join embed_chan") diff --git a/ecc/app/graphrag/workers.py b/ecc/app/graphrag/workers.py index 518a2d82..eac7ea96 100644 --- a/ecc/app/graphrag/workers.py +++ b/ecc/app/graphrag/workers.py @@ -173,19 +173,24 @@ async def chunk_doc( # send chunks to be upserted (func, args) logger.debug("chunk writes to upsert_chan") - await upsert_chan.put((upsert_chunk, (conn, v_id, chunk_id, chunk, i))) - - # send chunks to have entities extracted - logger.debug("chunk writes to extract_chan") - await extract_chan.put((chunk, chunk_id)) + await upsert_chan.put( + (upsert_chunk, (conn, v_id, chunk_id, chunk, i, chunker_type)) + ) - # When extraction is enabled the extract worker pushes the - # summary-augmented embed message itself (Contextual Retrieval), - # so only embed the raw chunk here when extraction is off. + # Source-native connectors already write authoritative typed + # entities and relationships. Their documents still use this + # production chunk/embed pipeline, but must not create a second, + # LLM-derived copy of the same graph facts. from common.config import entity_extraction_switch - if not entity_extraction_switch: - logger.debug("chunk writes to embed_chan (no extraction)") + skip_extraction = chunker_type in ("jira", "jira_comment") + if entity_extraction_switch and not skip_extraction: + logger.debug("chunk writes to extract_chan") + await extract_chan.put((chunk, chunk_id)) + else: + logger.debug("chunk writes to embed_chan (extraction bypassed)") await embed_chan.put((chunk_id, chunk, "DocumentChunk")) + if tracker is not None: + tracker.chunk_done(chunk_id) return v_id @@ -208,7 +213,14 @@ async def upsert_doc(conn: AsyncTigerGraphConnection, doc_id, ctype, content_tex conn, "Document", doc_id, "HAS_CONTENT", "Content", doc_id ) -async def upsert_chunk(conn: AsyncTigerGraphConnection, doc_id, chunk_id, chunk, idx): +async def upsert_chunk( + conn: AsyncTigerGraphConnection, + doc_id, + chunk_id, + chunk, + idx, + source_type="", +): logger.debug(f"Upserting chunk {chunk_id}") date_added = int(time.time()) # Build the chunk's full vertex + edge bundle and enqueue atomically. @@ -233,6 +245,41 @@ async def upsert_chunk(conn: AsyncTigerGraphConnection, doc_id, chunk_id, chunk, "DocumentChunk", chunk_id, "IS_AFTER", "DocumentChunk", util.process_id(f"{doc_id}_chunk_{idx - 1}"), None, )) + if source_type == "jira" and ":issue-doc:" in doc_id: + # Jira writes authoritative typed issues before ECC runs. Link each + # generated chunk to that stable issue vertex so vector hits can + # traverse into the structured Jira graph without LLM extraction. + issue_id = doc_id.replace(":issue-doc:", ":issue:", 1) + edges.append(( + "DocumentChunk", + chunk_id, + "CONTAINS_ENTITY", + "JiraIssue", + issue_id, + None, + )) + elif source_type == "jira_comment" and ":comment-doc:" in doc_id: + issue_id, comment_id = doc_id.rsplit(":comment-doc:", 1) + cloud_prefix = issue_id.split(":issue:", 1)[0] + comment_vertex_id = f"{cloud_prefix}:comment:{comment_id}" + edges.extend([ + ( + "DocumentChunk", + chunk_id, + "CONTAINS_ENTITY", + "JiraIssue", + issue_id, + None, + ), + ( + "DocumentChunk", + chunk_id, + "CONTAINS_ENTITY", + "JiraComment", + comment_vertex_id, + None, + ), + ]) await util.upsert_group(conn, vertices, edges) @@ -263,6 +310,16 @@ async def embed( async with embed_sem: logger.debug(f"Embedding {v_id}") + # Guard: skip chunks with no text — sending empty content to the + # embedding API returns 500 INTERNAL. Log a warning so the gap + # is visible in the logs without crashing the whole rebuild. + if not content or not content.strip(): + logger.warning( + f"Skipping embed for {v_id}: content is empty. " + "Check the source document for missing text." + ) + return + # if loader is running, wait until it's done if not util.loading_event.is_set(): logger.debug("Embed worker waiting for loading event to finish") @@ -271,6 +328,7 @@ async def embed( await embed_store.aadd_embeddings([(content, [])], [{"vertex_id": v_id}]) except Exception as e: logger.error(f"Failed to add embeddings for {v_id}: {e}") + raise def _is_near_duplicate(new_desc, existing_descs, threshold=0.85): diff --git a/ecc/app/main.py b/ecc/app/main.py index 9624cb36..a1648fbf 100644 --- a/ecc/app/main.py +++ b/ecc/app/main.py @@ -236,7 +236,7 @@ def rebuild_status( "warnings": task_info.get("warnings"), "communities_incomplete": task_info.get("communities_incomplete"), } - + return { "graphname": graphname, "method": ecc_method, @@ -338,7 +338,11 @@ def progress_cb(msg, current=None, total=None, clear_progress=False): ) try: - result = await run_func(graphname, conn, progress=progress_cb) + result = await run_func( + graphname, + conn, + progress=progress_cb, + ) except TypeError: result = await run_func(graphname, conn) completion = {"status": "completed", "completed_at": time.time()} diff --git a/ecc/tests/test_jira_comment_chunks.py b/ecc/tests/test_jira_comment_chunks.py new file mode 100644 index 00000000..9073eb92 --- /dev/null +++ b/ecc/tests/test_jira_comment_chunks.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +import pytest + +from common.embeddings.tigergraph_embedding_store import ( + TigerGraphEmbeddingStore, +) +from graphrag import workers + + +@pytest.mark.asyncio +async def test_jira_comment_chunk_links_comment_and_issue(monkeypatch): + captured: dict = {} + + async def capture_group(conn, vertices, edges): + captured["vertices"] = vertices + captured["edges"] = edges + + monkeypatch.setattr(workers.util, "upsert_group", capture_group) + + await workers.upsert_chunk( + object(), + "jira:cloud-1:issue:10422:comment-doc:9001", + "chunk-1", + "Short Jira comment", + 0, + "jira_comment", + ) + + assert ( + "DocumentChunk", + "chunk-1", + "CONTAINS_ENTITY", + "JiraIssue", + "jira:cloud-1:issue:10422", + None, + ) in captured["edges"] + assert ( + "DocumentChunk", + "chunk-1", + "CONTAINS_ENTITY", + "JiraComment", + "jira:cloud-1:comment:9001", + None, + ) in captured["edges"] + + +def test_exhausted_embedding_retries_raise(): + provider_error = RuntimeError("500 INTERNAL") + + with pytest.raises(RuntimeError, match="Failed to embed chunk-1"): + TigerGraphEmbeddingStore._log_embed_failure( + "chunk-1", + provider_error, + ) + + +@pytest.mark.asyncio +async def test_embedding_worker_propagates_store_failure(): + class FailingStore: + async def aadd_embeddings(self, *args, **kwargs): + raise RuntimeError("embedding provider unavailable") + + workers.util.loading_event.set() + + with pytest.raises(RuntimeError, match="embedding provider unavailable"): + await workers.embed( + object(), + FailingStore(), + ("chunk-1", "DocumentChunk"), + "chunk content", + ) diff --git a/graphrag-ui/src/main.tsx b/graphrag-ui/src/main.tsx index 53239a5c..05191b66 100755 --- a/graphrag-ui/src/main.tsx +++ b/graphrag-ui/src/main.tsx @@ -12,6 +12,7 @@ import LLMConfig from "./pages/setup/LLMConfig.tsx"; import GraphDBConfig from "./pages/setup/GraphDBConfig.tsx"; import GraphRAGConfig from "./pages/setup/GraphRAGConfig.tsx"; import McpServersConfig from "./pages/setup/McpServersConfig.tsx"; +import DataSourcesConfig from "./pages/setup/DataSourcesConfig.tsx"; import CustomizePrompts from "./pages/setup/CustomizePrompts.tsx"; import { ThemeProvider } from "./components/ThemeProvider.tsx"; import { ModeToggle } from "@/components/ModeToggle.tsx"; @@ -79,6 +80,10 @@ const router = createBrowserRouter([ path: "kg-admin/ingest", element: , }, + { + path: "kg-admin/data-sources", + element: , + }, { path: "server-config", element: , diff --git a/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx b/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx new file mode 100644 index 00000000..03d2bd9b --- /dev/null +++ b/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx @@ -0,0 +1,1382 @@ +import React, { useCallback, useEffect, useState } from "react"; +import { + DatabaseZap, + Loader2, + Plus, + Save, + ShieldCheck, + Trash2, +} from "lucide-react"; +import ConfigScopeToggle from "@/components/ConfigScopeToggle"; +import { Button } from "@/components/ui/button"; +import { Input } from "@/components/ui/input"; +import { safeJson } from "@/utils/safeJson"; +import { pauseIdleTimer, pingIdleTimer, resumeIdleTimer } from "@/hooks/useIdleTimeout"; +import type { JiraSchemaStatus, JiraSource } from "@/types/dataSources"; + +const editingSourceKey = (graph: string) => + `graphrag:jira-editing-source:${graph}`; + +const syncFeedbackKey = (graph: string) => + `graphrag:jira-sync-feedback:${graph}`; + +interface JiraProject { + id: string; + key: string; + name: string; +} + +interface ActionFeedback { + type: "success" | "error" | "pending"; + text: string; +} + +const ActionStatus: React.FC<{ feedback: ActionFeedback | null }> = ({ + feedback, +}) => { + if (!feedback) return null; + return ( +
+ {feedback.text} +
+ ); +}; + +const emptySource = (): JiraSource => ({ + id: "", + type: "jira_cloud", + enabled: true, + display_name: "", + connection: { site_url: "", email: "", api_token: "" }, + scope: { + project_keys: [], + created_after: null, + updated_after: null, + status_categories: [], + jql_extra: "", + include_comments: true, + }, + sync: { overlap_seconds: 120, last_issue_count: 0 }, +}); + +const labelClass = "block text-sm font-medium mb-2 text-black dark:text-white"; +const inputClass = "dark:border-[#3D3D3D] dark:bg-background"; +const jiraStatusCategories = [ + { value: "new", label: "To do" }, + { value: "indeterminate", label: "In progress" }, + { value: "done", label: "Done" }, +] as const; + +const errorDetail = (data: any, fallback: string) => { + if (typeof data?.detail === "string") return data.detail; + if (typeof data?.detail?.message === "string") { + const conflicts = Array.isArray(data.detail.conflicts) + ? ` ${data.detail.conflicts.join("; ")}` + : ""; + return `${data.detail.message}${conflicts}`; + } + return fallback; +}; + +const DataSourcesConfig: React.FC = () => { + const [selectedGraph, setSelectedGraph] = useState( + sessionStorage.getItem("selectedGraph") || "" + ); + const [availableGraphs, setAvailableGraphs] = useState([]); + const [sources, setSources] = useState([]); + const [editing, setEditing] = useState(null); + const [connectionVerified, setConnectionVerified] = useState(false); + const [projectSelectionLoaded, setProjectSelectionLoaded] = useState(false); + const [projects, setProjects] = useState([]); + const [directProjectKey, setDirectProjectKey] = useState(""); + const [projectSearch, setProjectSearch] = useState(""); + const [schema, setSchema] = useState(null); + const [loading, setLoading] = useState(false); + const [busy, setBusy] = useState(""); + const [message, setMessage] = useState(""); + const [messageType, setMessageType] = useState<"success" | "error" | "">(""); + const [connectionFeedback, setConnectionFeedback] = + useState(null); + const [scopeFeedback, setScopeFeedback] = + useState(null); + const [saveFeedback, setSaveFeedback] = + useState(null); + const [countFeedback, setCountFeedback] = + useState(null); + const [sourceFeedback, setSourceFeedback] = + useState(null); + const [schemaFeedback, setSchemaFeedback] = + useState(null); + const [syncFeedback, setSyncFeedbackRaw] = + useState(null); + + const setSyncFeedback = ( + feedback: ActionFeedback | null, + graph?: string + ) => { + const key = syncFeedbackKey(graph ?? selectedGraph); + if (feedback) { + sessionStorage.setItem(key, JSON.stringify(feedback)); + } else { + sessionStorage.removeItem(key); + } + setSyncFeedbackRaw(feedback); + }; + + useEffect(() => { + const creds = sessionStorage.getItem("auth"); + if (!creds) return; + fetch("/ui/list_graphs", { headers: { Authorization: creds } }) + .then((response) => (response.ok ? response.json() : null)) + .then((data) => { + const graphs = Array.isArray(data?.graphs) + ? data.graphs + : Array.isArray(data) + ? data + : []; + setAvailableGraphs(graphs); + if (!selectedGraph && graphs.length) setSelectedGraph(graphs[0]); + }) + .catch(() => {}); + }, []); + + useEffect(() => { + const syncSelectedGraph = () => + setSelectedGraph(sessionStorage.getItem("selectedGraph") || ""); + window.addEventListener("graphrag:selectedGraph", syncSelectedGraph); + return () => + window.removeEventListener("graphrag:selectedGraph", syncSelectedGraph); + }, []); + + const loadSources = useCallback(async () => { + if (!selectedGraph) { + setSources([]); + return; + } + setLoading(true); + setMessage(""); + try { + const response = await fetch(`/ui/${selectedGraph}/data-sources`, { + headers: { Authorization: sessionStorage.getItem("auth")! }, + }); + const data = await safeJson(response); + if (!response.ok) throw new Error(data.detail || `HTTP ${response.status}`); + const loadedSources: JiraSource[] = Array.isArray(data.sources) + ? data.sources + : []; + setSources(loadedSources); + setEditing((current) => { + if (current) return current; + const rememberedId = sessionStorage.getItem( + editingSourceKey(selectedGraph) + ); + if (rememberedId === "__closed__") return null; + const remembered = loadedSources.find( + (source) => source.id === rememberedId + ); + return remembered || (loadedSources.length === 1 ? loadedSources[0] : null); + }); + } catch (error: any) { + setMessage(`Failed to load data sources: ${error.message}`); + setMessageType("error"); + } finally { + setLoading(false); + } + }, [selectedGraph]); + + const loadSchema = useCallback(async () => { + if (!selectedGraph) { + setSchema(null); + return false; + } + try { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/jira/schema`, + { headers: { Authorization: sessionStorage.getItem("auth")! } } + ); + const data = await safeJson(response); + if (!response.ok) { + throw new Error( + errorDetail(data, `Failed to inspect schema (${response.status})`) + ); + } + setSchema(data); + return true; + } catch (error: any) { + setSchemaFeedback({ + type: "error", + text: `Failed to inspect Jira schema: ${error.message}`, + }); + return false; + } + }, [selectedGraph]); + + useEffect(() => { + setEditing(null); + setConnectionVerified(false); + setProjectSelectionLoaded(false); + setProjects([]); + setDirectProjectKey(""); + setProjectSearch(""); + setConnectionFeedback(null); + setScopeFeedback(null); + setSaveFeedback(null); + setCountFeedback(null); + setSourceFeedback(null); + setSchemaFeedback(null); + // Restore persisted sync feedback for this graph so navigation doesn't clear it. + const stored = sessionStorage.getItem(syncFeedbackKey(selectedGraph)); + setSyncFeedbackRaw(stored ? (JSON.parse(stored) as ActionFeedback) : null); + loadSources(); + loadSchema(); + }, [loadSchema, loadSources]); + + const patch = (value: Partial) => { + setConnectionFeedback(null); + setScopeFeedback(null); + setSaveFeedback(null); + setCountFeedback(null); + setEditing((current) => (current ? { ...current, ...value } : current)); + }; + + const patchConnectionDetails = (value: Partial) => { + setConnectionVerified(false); + setProjectSelectionLoaded(false); + setProjects([]); + setProjectSearch(""); + patch(value); + }; + + const validateConnection = ( + source: JiraSource, + requireToken = true + ) => { + const missingFields = [ + !source.id.trim() && "source id", + !source.display_name.trim() && "display name", + !source.connection.site_url.trim() && "site URL", + !source.connection.email.trim() && "Atlassian email", + requireToken && !source.connection.api_token.trim() && "API token", + ].filter(Boolean) as string[]; + + if (missingFields.length > 0) { + const fields = + missingFields.length === 1 + ? missingFields[0] + : `${missingFields.slice(0, -1).join(", ")} and ${ + missingFields[missingFields.length - 1] + }`; + setConnectionFeedback({ + type: "error", + text: `${ + fields.charAt(0).toUpperCase() + fields.slice(1) + } ${missingFields.length === 1 ? "is" : "are"} required.`, + }); + return false; + } + return true; + }; + + const persistSource = async (source: JiraSource) => { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(source.id)}`, + { + method: "PUT", + headers: { + Authorization: sessionStorage.getItem("auth")!, + "Content-Type": "application/json", + }, + body: JSON.stringify(source), + } + ); + const data = await safeJson(response); + if (!response.ok) throw new Error(data.detail || `HTTP ${response.status}`); + return data.source as JiraSource; + }; + + const save = async () => { + if ( + !editing || + !selectedGraph || + !connectionVerified || + !validateConnection(editing, false) + ) { + return; + } + if (editing.scope.project_keys.length === 0) { + setSaveFeedback({ + type: "error", + text: "Select at least one Jira project before saving the scope.", + }); + return; + } + setBusy("save"); + setSaveFeedback({ + type: "pending", + text: "Saving Jira project scope…", + }); + try { + const saved = await persistSource(editing); + setEditing(saved); + sessionStorage.setItem(editingSourceKey(selectedGraph), saved.id); + await loadSources(); + setSaveFeedback({ + type: "success", + text: `Project scope saved (${saved.scope.project_keys.length} selected).`, + }); + } catch (error: any) { + setSaveFeedback({ + type: "error", + text: `Failed to save project scope: ${error.message}`, + }); + } finally { + setBusy(""); + } + }; + + const test = async () => { + if (!editing || !selectedGraph) return; + setConnectionVerified(false); + setProjectSelectionLoaded(false); + setProjects([]); + setProjectSearch(""); + setCountFeedback(null); + setSaveFeedback(null); + if (!validateConnection(editing)) return; + setScopeFeedback(null); + setBusy("test"); + setConnectionFeedback({ + type: "pending", + text: "Testing Jira connection…", + }); + try { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(editing.id)}/test`, + { + method: "POST", + headers: { + Authorization: sessionStorage.getItem("auth")!, + "Content-Type": "application/json", + }, + body: JSON.stringify(editing), + } + ); + const data = await safeJson(response); + if (!response.ok) throw new Error(data.detail || `HTTP ${response.status}`); + setConnectionFeedback({ + type: "success", + text: `Connected as ${data.account?.display_name || "Jira user"}.`, + }); + setConnectionVerified(true); + setEditing(data.source); + sessionStorage.setItem(editingSourceKey(selectedGraph), data.source.id); + await loadSources(); + } catch (error: any) { + setConnectionVerified(false); + setConnectionFeedback({ + type: "error", + text: `Connection failed: ${error.message}`, + }); + } finally { + setBusy(""); + } + }; + + const loadProjects = async () => { + if (!editing || !connectionVerified) return; + const projectKey = directProjectKey.trim().toUpperCase(); + setProjectSelectionLoaded(false); + setProjects([]); + setProjectSearch(""); + setCountFeedback(null); + setBusy("projects"); + setScopeFeedback({ + type: "pending", + text: projectKey + ? `Loading Jira project ${projectKey}…` + : "Loading visible Jira projects…", + }); + try { + const query = projectKey + ? `?project_key=${encodeURIComponent(projectKey)}` + : ""; + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(editing.id)}/projects${query}`, + { headers: { Authorization: sessionStorage.getItem("auth")! } } + ); + const data = await safeJson(response); + if (!response.ok) throw new Error(data.detail || `HTTP ${response.status}`); + const visibleProjects = Array.isArray(data.projects) ? data.projects : []; + setProjects(visibleProjects); + setProjectSearch(""); + setProjectSelectionLoaded(true); + if (projectKey && visibleProjects.length === 1) { + setEditing((current) => + current + ? { + ...current, + scope: { + ...current.scope, + project_keys: [visibleProjects[0].key], + }, + } + : current + ); + } + setScopeFeedback({ + type: "success", + text: + visibleProjects.length > 0 + ? projectKey + ? `Loaded and selected ${visibleProjects[0].key} — ${visibleProjects[0].name}.` + : `Loaded ${visibleProjects.length} visible Jira projects.` + : "Connection succeeded, but this account has no visible Jira projects.", + }); + } catch (error: any) { + setProjectSelectionLoaded(false); + setScopeFeedback({ + type: "error", + text: `Failed to load projects: ${error.message}`, + }); + } finally { + setBusy(""); + } + }; + + const previewTicketCount = async () => { + if ( + !editing || + !connectionVerified || + editing.scope.project_keys.length === 0 + ) { + setCountFeedback({ + type: "error", + text: "Select at least one Jira project before previewing tickets.", + }); + return; + } + setBusy("count"); + setCountFeedback({ + type: "pending", + text: "Checking how many Jira tickets match this scope…", + }); + try { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(editing.id)}/issues/count`, + { + method: "POST", + headers: { + Authorization: sessionStorage.getItem("auth")!, + "Content-Type": "application/json", + }, + body: JSON.stringify(editing.scope), + } + ); + const data = await safeJson(response); + if (!response.ok) { + throw new Error(data.detail || `HTTP ${response.status}`); + } + const count = Number(data.count || 0); + setCountFeedback({ + type: "success", + text: `Approximately ${count.toLocaleString()} Jira tickets match this scope.`, + }); + } catch (error: any) { + setCountFeedback({ + type: "error", + text: `Failed to preview ticket count: ${error.message}`, + }); + } finally { + setBusy(""); + } + }; + + const installSchema = async () => { + if (!selectedGraph) return; + setBusy("schema"); + setSchemaFeedback({ + type: "pending", + text: "Installing Jira schema…", + }); + try { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/jira/schema/install`, + { + method: "POST", + headers: { Authorization: sessionStorage.getItem("auth")! }, + } + ); + const data = await safeJson(response); + if (!response.ok) { + throw new Error( + errorDetail(data, `Schema installation failed (${response.status})`) + ); + } + setSchema(data.schema); + setSchemaFeedback({ + type: "success", + text: "Jira schema installed.", + }); + } catch (error: any) { + setSchemaFeedback({ + type: "error", + text: `Schema installation failed: ${error.message}`, + }); + } finally { + setBusy(""); + } + }; + + const pollSync = async (sourceId: string, runId: string) => { + for (;;) { + await new Promise((resolve) => setTimeout(resolve, 3000)); + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(sourceId)}/sync/${runId}`, + { headers: { Authorization: sessionStorage.getItem("auth")! } } + ); + const data = await safeJson(response); + if (!response.ok) { + throw new Error( + errorDetail(data, `Sync status failed (${response.status})`) + ); + } + pingIdleTimer(); + if (data.status === "failed") { + throw new Error(data.error || "Jira synchronization failed"); + } + if (data.status === "completed") return data.result; + } + }; + + const sync = async (source: JiraSource) => { + setBusy(`sync:${source.id}`); + setSources((current) => + current.map((item) => + item.id === source.id + ? { ...item, sync: { ...item.sync, last_error: null } } + : item + ) + ); + setSyncFeedback({ + type: "pending", + text: `Ingesting ${source.display_name} into the graph…`, + }); + pauseIdleTimer(); + try { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(source.id)}/sync`, + { + method: "POST", + headers: { Authorization: sessionStorage.getItem("auth")! }, + } + ); + const data = await safeJson(response); + if (!response.ok) { + throw new Error( + errorDetail(data, `Jira synchronization failed (${response.status})`) + ); + } + const result = await pollSync(source.id, data.run_id); + let buildStarted = false; + const missingChunkEmbeddings = Number( + result.missing_chunk_embeddings || 0 + ); + if (result.rebuild_required) { + setSyncFeedback({ + type: "pending", + text: + missingChunkEmbeddings > 0 && result.documents_loaded === 0 + ? `No Jira changes detected, but ${missingChunkEmbeddings} graph chunks are missing embeddings. Starting a recovery build…` + : "Jira ingestion complete. Starting the GraphRAG build…", + }); + const rebuild = await fetch(`/ui/${selectedGraph}/rebuild_graph`, { + method: "POST", + headers: { Authorization: sessionStorage.getItem("auth")! }, + }); + const rebuildData = await safeJson(rebuild); + if (!rebuild.ok) { + throw new Error( + errorDetail( + rebuildData, + "Jira data was ingested, but the GraphRAG build failed to start." + ) + ); + } + buildStarted = true; + } + await loadSources(); + setSyncFeedback({ + type: "success", + text: + `Jira ingestion complete: ${result.issues_upserted} issues updated, ` + + `${result.issues_deleted || 0} removed, and ` + + `${result.documents_loaded} changed documents loaded.` + + (buildStarted + ? missingChunkEmbeddings > 0 + ? ` A GraphRAG recovery build has started for ${missingChunkEmbeddings} chunks missing embeddings.` + : " The GraphRAG build for chunking and embedding has started." + : " No Jira changes were detected, so no new build was started."), + }); + } catch (error: any) { + setSyncFeedback({ + type: "error", + text: `Jira ingestion failed: ${error.message}`, + }); + } finally { + resumeIdleTimer(); + setBusy(""); + } + }; + + const remove = async (source: JiraSource) => { + if (!window.confirm(`Remove data source "${source.display_name}"?`)) return; + setBusy(`delete:${source.id}`); + setSourceFeedback({ + type: "pending", + text: `Removing ${source.display_name}…`, + }); + try { + const response = await fetch( + `/ui/${selectedGraph}/data-sources/${encodeURIComponent(source.id)}`, + { + method: "DELETE", + headers: { Authorization: sessionStorage.getItem("auth")! }, + } + ); + const data = await safeJson(response); + if (!response.ok) throw new Error(data.detail || `HTTP ${response.status}`); + if (editing?.id === source.id) { + setEditing(null); + sessionStorage.removeItem(editingSourceKey(selectedGraph)); + } + await loadSources(); + setSourceFeedback({ + type: "success", + text: `${source.display_name} was removed.`, + }); + } catch (error: any) { + setSourceFeedback({ + type: "error", + text: `Delete failed: ${error.message}`, + }); + } finally { + setBusy(""); + } + }; + + const normalizedProjectSearch = projectSearch.trim().toLowerCase(); + const filteredProjects = normalizedProjectSearch + ? projects.filter( + (project) => + project.key.toLowerCase().includes(normalizedProjectSearch) || + project.name.toLowerCase().includes(normalizedProjectSearch) + ) + : projects; + + return ( +
+
+
+ +
+
+

+ Data sources +

+

+ Connect, configure, ingest, and monitor external sources for the + selected knowledge graph. +

+
+
+ + {}} + onGraphChange={(graph) => { + setSelectedGraph(graph); + sessionStorage.setItem("selectedGraph", graph); + window.dispatchEvent(new Event("graphrag:selectedGraph")); + }} + /> + + {message && ( +
+ {message} +
+ )} + + {selectedGraph && ( +
+
+
+
+ +
+
+

+ Jira Cloud +

+

+ Connect Jira, select projects, install its graph schema, then + ingest issues and build GraphRAG. +

+
+
+ +
+ +
+ Until per-user Jira ACL enforcement is enabled, graph users can + retrieve every selected-project issue visible to the connector account. +
+ +
+ {sourceFeedback && ( +
+ +
+ )} +
+
+

+ 1. Connection and scope +

+ + {loading ? "Loading…" : `${sources.length} configured source${sources.length === 1 ? "" : "s"}`} + +
+
+ {sources.length === 0 && !loading && ( +

+ No Jira sources configured for this graph. +

+ )} +
+ {sources.map((source) => ( +
+
+
+ {source.display_name} +
+
+ {source.connection.site_url} + {source.scope.project_keys.length > 0 && + ` · ${source.scope.project_keys.join(", ")}`} +
+
+ + {source.sync.last_tested_at + ? "Connection tested" + : "Connection not tested"} + +
+ {source.sync.last_error && ( +
{source.sync.last_error}
+ )} +
+
+ + +
+
+ ))} +
+
+ + {editing && ( +
+

+ Connection details +

+
+
+ + source.id === editing.id)} + onChange={(event) => + patchConnectionDetails({ id: event.target.value }) + } + className={inputClass} + placeholder="jira-acme" + /> +
+
+ + patch({ display_name: event.target.value })} + className={inputClass} + placeholder="Acme Jira" + /> +
+
+ + + patchConnectionDetails({ + connection: { + ...editing.connection, + site_url: event.target.value, + }, + }) + } + className={inputClass} + placeholder="https://your-company.atlassian.net" + /> +

+ Jira tenant base URL—not graphql.atlassian.net or an issue URL. +

+
+
+ + + patchConnectionDetails({ + connection: { + ...editing.connection, + email: event.target.value, + }, + }) + } + className={inputClass} + /> +

+ Email address associated with the API token. +

+
+
+ + + patchConnectionDetails({ + connection: { + ...editing.connection, + api_token: event.target.value, + }, + }) + } + className={inputClass} + placeholder="Required each time you test the connection" + /> +
+
+ + { + setDirectProjectKey(event.target.value); + setProjectSelectionLoaded(false); + setProjects([]); + setProjectSearch(""); + setConnectionFeedback(null); + setScopeFeedback(null); + setSaveFeedback(null); + setCountFeedback(null); + }} + className={inputClass} + placeholder="GML" + /> +

+ Enter a key to load only that Jira project, or leave this + blank to load every project visible to the account. +

+
+
+ +
+ {connectionFeedback && ( +
+ +
+ )} + {connectionVerified && ( + <> +
+

+ Projects to ingest +

+

+ Load the requested Jira project or all projects visible + to this account, then select the projects for this graph. +

+
+
+ +
+ + )} +
+ + {projects.length > 0 && ( +
+
+ {directProjectKey.trim() + ? "Jira project" + : "Visible Jira projects"} +
+ {!directProjectKey.trim() && ( + setProjectSearch(event.target.value)} + className={`${inputClass} mb-2`} + placeholder="Search by project key or name" + aria-label="Search visible Jira projects" + /> + )} +
+ {filteredProjects.map((project) => ( + + ))} + {filteredProjects.length === 0 && ( +

+ No projects match this search. +

+ )} +
+
+ )} + + {connectionVerified && ( + + )} + + {connectionVerified && + projectSelectionLoaded && + projects.length > 0 && ( +
+

+ Ticket filters +

+

+ Optionally limit which tickets are included from the + selected projects. +

+
+
+ + + patch({ + scope: { + ...editing.scope, + created_after: event.target.value || null, + }, + }) + } + className={inputClass} + /> +
+
+ + + patch({ + scope: { + ...editing.scope, + updated_after: event.target.value || null, + }, + }) + } + className={inputClass} + /> +
+
+
Status categories
+
+ {jiraStatusCategories.map((category) => { + const selected = + editing.scope.status_categories || []; + return ( + + ); + })} +
+

+ Leave all categories unselected to include every status. +

+
+ +
+ + Advanced scope options + +
+ + + patch({ + scope: { + ...editing.scope, + jql_extra: event.target.value, + }, + }) + } + className={inputClass} + placeholder="statusCategory != Done" + /> +
+
+
+ +
+ {countFeedback && ( +
+ +
+ )} +
+
+ )} + + {connectionVerified && projectSelectionLoaded && saveFeedback && ( + + )} + +
+ + {connectionVerified && projectSelectionLoaded && ( + + )} +
+
+ )} + + {sources.length > 0 && ( +
+
+
+
+ + 2. Jira graph schema +
+

+ {schema?.status === "installed" + ? "Installed" + : schema?.status === "not_initialized" + ? "Initialize this graph before installing the Jira schema." + : schema?.status === "conflict" + ? "Existing graph schema conflicts with the Jira connector." + : schema?.status === "incomplete" + ? "The Jira schema is incomplete." + : sources.some( + (source) => + source.sync.last_tested_at && + source.scope.project_keys.length > 0 + ) + ? "Ready to install." + : sources.some( + (source) => source.sync.last_tested_at + ) + ? "Select and save at least one project first." + : "Test a saved connection before installing the schema."} +

+
+
+ {schema && + schema.status !== "installed" && + schema.status !== "conflict" && + schema.status !== "not_initialized" && ( + + )} +
+
+ {(schema?.conflicts?.length || 0) > 0 && ( +
    + {schema?.conflicts.map((conflict) => ( +
  • {conflict}
  • + ))} +
+ )} + {schemaFeedback && ( +
+ +
+ )} +
+ )} + + {sources.length > 0 && schema?.status === "installed" && ( +
+

+ 3. Ingest and build +

+

+ Import Jira issues into this graph. The existing GraphRAG loading + pipeline creates chunks and embeddings after structural ingestion. +

+ {syncFeedback && ( +
+ +
+ )} +
+ {sources.map((source) => ( +
+
+
+ {source.display_name} +
+
+ Last ingestion: {source.sync.last_completed_at || "Never"} · + Issues: {source.sync.last_issue_count || 0} +
+ {source.sync.last_error && + busy !== `sync:${source.id}` && ( +
+ {source.sync.last_error} +
+ )} +
+ +
+ ))} +
+
+ )} +
+ )} +
+ ); +}; + +export default DataSourcesConfig; diff --git a/graphrag-ui/src/pages/setup/KGAdmin.tsx b/graphrag-ui/src/pages/setup/KGAdmin.tsx index 2cf23b5a..791fcba3 100644 --- a/graphrag-ui/src/pages/setup/KGAdmin.tsx +++ b/graphrag-ui/src/pages/setup/KGAdmin.tsx @@ -300,6 +300,8 @@ const KGAdmin = () => { setRefreshMessage(""); setPollingActive(false); setRebuildProgress(null); + } else if (refreshGraphName) { + checkRebuildStatus(refreshGraphName); } }; @@ -2734,6 +2736,7 @@ const KGAdmin = () => { setRefreshGraphName(v); sessionStorage.setItem("selectedGraph", v); window.dispatchEvent(new Event("graphrag:selectedGraph")); + checkRebuildStatus(v); }} disabled={isRefreshing || isRebuildRunning || isCheckingStatus} > diff --git a/graphrag-ui/src/pages/setup/SetupLayout.tsx b/graphrag-ui/src/pages/setup/SetupLayout.tsx index 1d2ddde0..426b57cd 100644 --- a/graphrag-ui/src/pages/setup/SetupLayout.tsx +++ b/graphrag-ui/src/pages/setup/SetupLayout.tsx @@ -1,7 +1,7 @@ import React, { useEffect, useState } from "react"; import { Outlet, useNavigate, useLocation } from "react-router-dom"; import { Button } from "@/components/ui/button"; -import { ArrowLeft, Database, Settings, FileText, ChevronRight } from "lucide-react"; +import { ArrowLeft, Database, DatabaseZap, Settings, FileText, ChevronRight } from "lucide-react"; import { cn } from "@/lib/utils"; import { useRoles } from "@/hooks/useRoles"; @@ -41,6 +41,12 @@ const SetupLayout = () => { path: "/setup/kg-admin", subItems: [], }, + { + title: "Data Sources", + icon: DatabaseZap, + path: "/setup/kg-admin/data-sources", + subItems: [], + }, { title: "Server Configuration", icon: Settings, @@ -168,7 +174,9 @@ const SetupLayout = () => { const Icon = item.icon; const hasSubItems = item.subItems.length > 0; const isExpanded = expandedSection === item.path; - const isItemActive = isParentActive(item.path); + const isItemActive = hasSubItems + ? isParentActive(item.path) + : isActive(item.path); return (
diff --git a/graphrag-ui/src/types/dataSources.ts b/graphrag-ui/src/types/dataSources.ts new file mode 100644 index 00000000..eb87568f --- /dev/null +++ b/graphrag-ui/src/types/dataSources.ts @@ -0,0 +1,41 @@ +export interface JiraSource { + id: string; + type: "jira_cloud"; + enabled: boolean; + display_name: string; + connection: { + site_url: string; + email: string; + api_token: string; + cloud_id?: string | null; + }; + scope: { + project_keys: string[]; + created_after?: string | null; + updated_after?: string | null; + status_categories?: Array<"new" | "indeterminate" | "done">; + jql_extra: string; + include_comments: boolean; + story_points_field?: string | null; + }; + sync: { + overlap_seconds: number; + checkpoint?: string | null; + last_tested_at?: string | null; + last_started_at?: string | null; + last_completed_at?: string | null; + last_error?: string | null; + last_issue_count: number; + }; +} + +export interface JiraSchemaStatus { + status: + | "not_initialized" + | "not_installed" + | "incomplete" + | "conflict" + | "installed"; + missing: Record; + conflicts: string[]; +} diff --git a/graphrag/app/connectors/__init__.py b/graphrag/app/connectors/__init__.py new file mode 100644 index 00000000..a1cb1b33 --- /dev/null +++ b/graphrag/app/connectors/__init__.py @@ -0,0 +1 @@ +"""Enterprise data-source connectors.""" diff --git a/graphrag/app/connectors/jira/__init__.py b/graphrag/app/connectors/jira/__init__.py new file mode 100644 index 00000000..5a5ef0bb --- /dev/null +++ b/graphrag/app/connectors/jira/__init__.py @@ -0,0 +1,5 @@ +"""Jira Cloud connector.""" + +from .config import JiraDataSource + +__all__ = ["JiraDataSource"] diff --git a/graphrag/app/connectors/jira/adf.py b/graphrag/app/connectors/jira/adf.py new file mode 100644 index 00000000..5e1ad67d --- /dev/null +++ b/graphrag/app/connectors/jira/adf.py @@ -0,0 +1,122 @@ +"""Convert Atlassian Document Format (ADF) to retrieval-friendly markdown.""" + +from __future__ import annotations + +from typing import Any + + +def _escape_inline(value: str) -> str: + return value.replace("\\", "\\\\").replace("*", "\\*").replace("_", "\\_") + + +def _with_marks(text: str, marks: list[dict[str, Any]]) -> str: + for mark in marks: + mark_type = mark.get("type") + attrs = mark.get("attrs") or {} + if mark_type == "code": + escaped = text.replace("`", "\\`") + text = f"`{escaped}`" + elif mark_type == "strong": + text = f"**{text}**" + elif mark_type == "em": + text = f"*{text}*" + elif mark_type == "strike": + text = f"~~{text}~~" + elif mark_type == "link" and attrs.get("href"): + text = f"[{text}]({attrs['href']})" + return text + + +def _inline(node: dict[str, Any]) -> str: + node_type = node.get("type") + attrs = node.get("attrs") or {} + if node_type == "text": + text = _escape_inline(str(node.get("text") or "")) + return _with_marks(text, node.get("marks") or []) + if node_type == "hardBreak": + return " \n" + if node_type == "mention": + return f"@{attrs.get('text') or attrs.get('displayName') or 'user'}" + if node_type == "emoji": + return str(attrs.get("text") or attrs.get("shortName") or "") + if node_type == "inlineCard": + url = str(attrs.get("url") or "") + return f"[{url}]({url})" if url else "" + return "".join(_inline(child) for child in node.get("content") or []) + + +def _block(node: dict[str, Any], depth: int = 0) -> str: + node_type = node.get("type") + attrs = node.get("attrs") or {} + children = node.get("content") or [] + + if node_type == "doc": + return "\n\n".join( + value for child in children if (value := _block(child, depth)).strip() + ) + if node_type == "paragraph": + return "".join(_inline(child) for child in children).strip() + if node_type == "heading": + level = min(max(int(attrs.get("level") or 1), 1), 6) + return f"{'#' * level} {''.join(_inline(c) for c in children).strip()}" + if node_type == "blockquote": + content = "\n".join(_block(child, depth) for child in children).strip() + return "\n".join(f"> {line}" for line in content.splitlines()) + if node_type == "codeBlock": + language = attrs.get("language") or "" + content = "".join(_inline(child) for child in children) + return f"```{language}\n{content}\n```" + if node_type == "rule": + return "---" + if node_type in ("bulletList", "orderedList"): + ordered = node_type == "orderedList" + start = int(attrs.get("order") or 1) + lines: list[str] = [] + for index, child in enumerate(children): + value = _block(child, depth + 1).strip() + if not value: + continue + prefix = f"{start + index}. " if ordered else "- " + indentation = " " * depth + continuation = "\n".join( + f"{indentation} {line}" for line in value.splitlines()[1:] + ) + first = f"{indentation}{prefix}{value.splitlines()[0]}" + lines.append(f"{first}\n{continuation}".rstrip()) + return "\n".join(lines) + if node_type == "listItem": + return "\n".join( + value for child in children if (value := _block(child, depth)).strip() + ) + if node_type in ("table", "tableRow", "tableCell", "tableHeader"): + # ADF tables can contain arbitrary blocks. Tabs preserve cell + # boundaries for embedding without pretending to provide full GFM. + separator = "\n" if node_type in ("table", "tableRow") else " " + values = [_block(child, depth).strip() for child in children] + values = [value for value in values if value] + if node_type == "tableRow": + separator = " | " + return separator.join(values) + if node_type in ("panel", "expand", "nestedExpand"): + title = str(attrs.get("title") or "").strip() + body = "\n\n".join( + value for child in children if (value := _block(child, depth)).strip() + ) + return f"**{title}**\n\n{body}".strip() if title else body + if node_type == "mediaSingle": + return "\n".join(_block(child, depth) for child in children).strip() + if node_type == "media": + name = attrs.get("alt") or attrs.get("id") or "attachment" + return f"[Attachment: {name}]" + return "".join(_inline(child) for child in children).strip() + + +def adf_to_markdown(value: Any) -> str: + """Return markdown for an ADF document, or a safe string fallback.""" + if value is None: + return "" + if isinstance(value, str): + return value.strip() + if not isinstance(value, dict): + return str(value).strip() + return _block(value).strip() diff --git a/graphrag/app/connectors/jira/client.py b/graphrag/app/connectors/jira/client.py new file mode 100644 index 00000000..8c86b64a --- /dev/null +++ b/graphrag/app/connectors/jira/client.py @@ -0,0 +1,335 @@ +"""Small, testable Jira Cloud REST API client.""" + +from __future__ import annotations + +import concurrent.futures +import time +from collections.abc import Iterator +from datetime import datetime, timedelta, timezone +from email.utils import parsedate_to_datetime +from typing import Any, Callable +from urllib.parse import quote + +import httpx + +from .config import JiraDataSource + + +ISSUE_FIELDS = [ + "summary", + "status", + "issuetype", + "priority", + "resolution", + "project", + "assignee", + "reporter", + "created", + "updated", + "duedate", + "labels", + "components", + "fixVersions", + "parent", + "issuelinks", + "comment", + "description", + "attachment", +] +STATUS_CATEGORY_JQL = { + "new": "To Do", + "indeterminate": "In Progress", + "done": "Done", +} + +# Number of threads used to complete comment pagination within a single page. +# Each thread issues its own HTTP request, so raising this reduces wall-clock +# time proportionally up to the Jira rate-limit ceiling (~100 req/min on free +# plans, higher on paid). 10 is a safe default that won't trigger throttling +# on typical Atlassian Cloud accounts. +COMMENT_FETCH_WORKERS = 10 + + +class JiraAPIError(RuntimeError): + def __init__(self, status_code: int, message: str): + super().__init__(message) + self.status_code = status_code + + +class JiraCloudClient: + def __init__( + self, + source: JiraDataSource, + *, + client: httpx.Client | None = None, + sleep: Callable[[float], None] = time.sleep, + max_attempts: int = 4, + ): + self.source = source + self._sleep = sleep + self._max_attempts = max_attempts + self._owns_client = client is None + self._client = client or httpx.Client( + base_url=source.connection.site_url, + auth=(source.connection.email, source.connection.api_token), + headers={"Accept": "application/json"}, + timeout=httpx.Timeout(30.0, connect=10.0), + ) + + def close(self) -> None: + if self._owns_client: + self._client.close() + + def __enter__(self) -> "JiraCloudClient": + return self + + def __exit__(self, *_args) -> None: + self.close() + + @staticmethod + def _retry_after(response: httpx.Response, attempt: int) -> float: + value = response.headers.get("Retry-After") + if value: + try: + return min(max(float(value), 0.0), 60.0) + except ValueError: + try: + retry_at = parsedate_to_datetime(value) + if retry_at.tzinfo is None: + retry_at = retry_at.replace(tzinfo=timezone.utc) + return min( + max((retry_at - datetime.now(timezone.utc)).total_seconds(), 0), + 60.0, + ) + except (TypeError, ValueError): + pass + return min(2 ** attempt, 30) + + @staticmethod + def _error_message(response: httpx.Response) -> str: + try: + payload = response.json() + except ValueError: + return f"Jira returned HTTP {response.status_code}" + if isinstance(payload, dict): + messages = payload.get("errorMessages") + if isinstance(messages, list) and messages: + return "; ".join(str(message) for message in messages) + errors = payload.get("errors") + if isinstance(errors, dict) and errors: + return "; ".join(f"{key}: {value}" for key, value in errors.items()) + if payload.get("message"): + return str(payload["message"]) + return f"Jira returned HTTP {response.status_code}" + + def _request(self, method: str, path: str, **kwargs) -> Any: + last_error: Exception | None = None + for attempt in range(self._max_attempts): + try: + response = self._client.request(method, path, **kwargs) + except (httpx.TimeoutException, httpx.TransportError) as exc: + last_error = exc + if attempt + 1 == self._max_attempts: + break + self._sleep(min(2 ** attempt, 30)) + continue + + if response.status_code == 429 or response.status_code >= 500: + if attempt + 1 < self._max_attempts: + self._sleep(self._retry_after(response, attempt)) + continue + if response.is_error: + raise JiraAPIError( + response.status_code, + self._error_message(response), + ) + if response.status_code == 204: + return None + try: + return response.json() + except ValueError as exc: + raise JiraAPIError( + response.status_code, + "Jira returned an invalid JSON response", + ) from exc + raise JiraAPIError(503, f"Unable to reach Jira: {last_error}") from last_error + + def myself(self) -> dict[str, Any]: + return self._request("GET", "/rest/api/3/myself") + + def cloud_id(self) -> str: + payload = self._request("GET", "/_edge/tenant_info") + cloud_id = payload.get("cloudId") if isinstance(payload, dict) else None + if not cloud_id: + raise JiraAPIError(502, "Jira did not return a cloudId") + return str(cloud_id) + + def projects(self) -> list[dict[str, Any]]: + start_at = 0 + projects: list[dict[str, Any]] = [] + while True: + payload = self._request( + "GET", + "/rest/api/3/project/search", + params={"startAt": start_at, "maxResults": 100, "orderBy": "key"}, + ) + values = payload.get("values") or [] + projects.extend(values) + if payload.get("isLast", True) or not values: + break + start_at += len(values) + return projects + + def project(self, project_key: str) -> dict[str, Any]: + """Return one project by key without enumerating every visible project.""" + return self._request( + "GET", + f"/rest/api/3/project/{quote(project_key, safe='')}", + ) + + def _jql( + self, + *, + incremental: bool = True, + include_order: bool = True, + ) -> str: + scope = self.source.scope + projects = ", ".join(scope.project_keys) + clauses = [f"project in ({projects})"] + if scope.created_after: + clauses.append(f'created >= "{scope.created_after.isoformat()}"') + if scope.updated_after: + clauses.append(f'updated >= "{scope.updated_after.isoformat()}"') + if scope.status_categories: + categories = ", ".join( + f'"{STATUS_CATEGORY_JQL[category]}"' + for category in scope.status_categories + ) + clauses.append(f"statusCategory in ({categories})") + if scope.jql_extra: + clauses.append(f"({scope.jql_extra})") + if incremental and self.source.sync.checkpoint: + checkpoint = self.source.sync.checkpoint + if checkpoint.tzinfo is None: + checkpoint = checkpoint.replace(tzinfo=timezone.utc) + checkpoint = checkpoint.astimezone(timezone.utc) - timedelta( + seconds=self.source.sync.overlap_seconds + ) + clauses.append(f'updated >= "{checkpoint:%Y-%m-%d %H:%M}"') + jql = " AND ".join(clauses) + return f"{jql} ORDER BY updated ASC, key ASC" if include_order else jql + + def approximate_issue_count(self) -> int: + payload = self._request( + "POST", + "/rest/api/3/search/approximate-count", + json={ + "jql": self._jql( + incremental=False, + include_order=False, + ) + }, + headers={"Content-Type": "application/json"}, + ) + return int(payload.get("count") or 0) + + def _iter_search_pages( + self, + *, + fields: list[str], + incremental: bool, + ) -> Iterator[list[dict[str, Any]]]: + token: str | None = None + jql = self._jql(incremental=incremental) + while True: + body: dict[str, Any] = { + "jql": jql, + "fields": fields, + "fieldsByKeys": False, + "maxResults": 100, + } + if token: + body["nextPageToken"] = token + payload = self._request( + "POST", + "/rest/api/3/search/jql", + json=body, + headers={"Content-Type": "application/json"}, + ) + issues = payload.get("issues") or [] + if issues: + yield issues + token = payload.get("nextPageToken") + if not token: + break + + def _iter_search(self, *, fields: list[str], incremental: bool): + for issues in self._iter_search_pages( + fields=fields, + incremental=incremental, + ): + yield from issues + + def iter_issue_pages(self) -> Iterator[list[dict[str, Any]]]: + """Yield complete issue pages for durable page-level synchronization. + + Comment completion is parallelised across issues within each page so + that the extra Jira API calls needed for issues with >10 comments are + issued concurrently rather than one at a time. ``_complete_comments`` + returns immediately when an issue already carries all its comments, so + it is safe to submit every issue to the pool. + """ + fields = list(ISSUE_FIELDS) + story_points = self.source.scope.story_points_field + if story_points: + fields.append(story_points) + for issues in self._iter_search_pages(fields=fields, incremental=True): + if self.source.scope.include_comments: + # Use up to COMMENT_FETCH_WORKERS threads so that the extra + # comment-pagination API calls for issues with >10 comments run + # in parallel. httpx.Client uses httpcore's connection pool + # which is thread-safe for concurrent requests. + workers = min(len(issues), COMMENT_FETCH_WORKERS) + with concurrent.futures.ThreadPoolExecutor( + max_workers=workers, thread_name_prefix="jira-comment" + ) as pool: + futures = [ + pool.submit(self._complete_comments, issue) + for issue in issues + ] + # Wait for all and propagate any exception immediately. + for fut in concurrent.futures.as_completed(futures): + fut.result() + yield issues + + def iter_issues(self) -> Iterator[dict[str, Any]]: + for issues in self.iter_issue_pages(): + yield from issues + + def _complete_comments(self, issue: dict[str, Any]) -> None: + fields = issue.setdefault("fields", {}) + page = fields.get("comment") or {} + comments = list(page.get("comments") or []) + total = int(page.get("total") or len(comments)) + if len(comments) >= total: + page["comments"] = comments + fields["comment"] = page + return + + issue_id = issue.get("id") or issue.get("key") + start_at = len(comments) + while start_at < total: + payload = self._request( + "GET", + f"/rest/api/3/issue/{issue_id}/comment", + params={"startAt": start_at, "maxResults": 100, "orderBy": "created"}, + ) + batch = payload.get("comments") or [] + if not batch: + break + comments.extend(batch) + start_at += len(batch) + total = int(payload.get("total") or total) + page["comments"] = comments + page["total"] = total + fields["comment"] = page diff --git a/graphrag/app/connectors/jira/config.py b/graphrag/app/connectors/jira/config.py new file mode 100644 index 00000000..1c5dfddd --- /dev/null +++ b/graphrag/app/connectors/jira/config.py @@ -0,0 +1,138 @@ +"""Validated configuration for the Jira Cloud connector.""" + +from __future__ import annotations + +import re +from datetime import date, datetime +from typing import Literal +from urllib.parse import urlparse + +from pydantic import BaseModel, Field, field_validator, model_validator + + +SOURCE_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$") +PROJECT_KEY_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]{0,31}$") + + +class JiraConnection(BaseModel): + site_url: str + email: str = Field(min_length=3) + api_token: str = "" + cloud_id: str | None = None + + @field_validator("site_url") + @classmethod + def validate_site_url(cls, value: str) -> str: + normalized = value.strip().rstrip("/") + parsed = urlparse(normalized) + if parsed.scheme != "https" or not parsed.hostname: + raise ValueError("site_url must be a valid HTTPS URL") + if parsed.path not in ("", "/") or parsed.query or parsed.fragment: + raise ValueError("site_url must not include a path, query, or fragment") + return normalized + + @field_validator("email") + @classmethod + def normalize_email(cls, value: str) -> str: + value = value.strip() + if "@" not in value: + raise ValueError("email must be a valid Atlassian account email") + return value + + +class JiraScope(BaseModel): + project_keys: list[str] = Field(default_factory=list) + created_after: date | None = None + updated_after: date | None = None + status_categories: list[ + Literal["new", "indeterminate", "done"] + ] = Field(default_factory=list) + jql_extra: str = "" + include_comments: bool = True + story_points_field: str | None = None + + @field_validator("project_keys") + @classmethod + def normalize_project_keys(cls, values: list[str]) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for raw in values: + key = raw.strip().upper() + if not PROJECT_KEY_RE.fullmatch(key): + raise ValueError(f"invalid Jira project key: {raw!r}") + if key not in seen: + normalized.append(key) + seen.add(key) + return normalized + + @field_validator("status_categories") + @classmethod + def deduplicate_status_categories( + cls, + values: list[Literal["new", "indeterminate", "done"]], + ) -> list[Literal["new", "indeterminate", "done"]]: + return list(dict.fromkeys(values)) + + @field_validator("jql_extra") + @classmethod + def validate_jql_extra(cls, value: str) -> str: + value = value.strip() + if re.search(r"\border\s+by\b", value, flags=re.IGNORECASE): + raise ValueError("jql_extra must not contain ORDER BY") + return value + + @field_validator("story_points_field") + @classmethod + def validate_story_points_field(cls, value: str | None) -> str | None: + if value is None or not value.strip(): + return None + value = value.strip() + if not re.fullmatch(r"customfield_\d+", value): + raise ValueError("story_points_field must look like customfield_10016") + return value + + +class JiraSyncState(BaseModel): + overlap_seconds: int = Field(default=120, ge=0, le=3600) + checkpoint: datetime | None = None + migrating_legacy_comments: bool = False + last_tested_at: datetime | None = None + last_started_at: datetime | None = None + last_completed_at: datetime | None = None + last_error: str | None = None + last_issue_count: int = Field(default=0, ge=0) + + +class JiraDataSource(BaseModel): + id: str + type: Literal["jira_cloud"] = "jira_cloud" + enabled: bool = True + display_name: str = Field(min_length=1, max_length=100) + connection: JiraConnection + scope: JiraScope + sync: JiraSyncState = Field(default_factory=JiraSyncState) + + @field_validator("id") + @classmethod + def validate_id(cls, value: str) -> str: + value = value.strip() + if not SOURCE_ID_RE.fullmatch(value): + raise ValueError( + "id must be 1-64 characters using letters, numbers, '_' or '-'" + ) + return value + + @field_validator("display_name") + @classmethod + def normalize_display_name(cls, value: str) -> str: + return value.strip() + +class JiraSourceFile(BaseModel): + sources: list[JiraDataSource] = Field(default_factory=list) + + @model_validator(mode="after") + def unique_source_ids(self) -> "JiraSourceFile": + ids = [source.id for source in self.sources] + if len(ids) != len(set(ids)): + raise ValueError("data-source ids must be unique") + return self diff --git a/graphrag/app/connectors/jira/mapper.py b/graphrag/app/connectors/jira/mapper.py new file mode 100644 index 00000000..a597187d --- /dev/null +++ b/graphrag/app/connectors/jira/mapper.py @@ -0,0 +1,630 @@ +"""Deterministic Jira payload to GraphRAG mapping.""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from dateutil import parser as date_parser + +from common.chunkers.structured import StructuredChunker +from common.config import get_graphrag_config + +from .adf import adf_to_markdown +from .config import JiraDataSource +from .schema import ( + JIRA_ASSIGNEE_EDGE, + JIRA_COMMENT_AFTER_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_LINK_EDGE, + JIRA_PARENT_EDGE, + JIRA_PROJECT_EDGE, + JIRA_REPORTER_EDGE, +) + + +@dataclass(frozen=True) +class VertexRecord: + vertex_type: str + vertex_id: str + attributes: dict[str, Any] + + +@dataclass(frozen=True) +class EdgeRecord: + source_type: str + source_id: str + edge_type: str + target_type: str + target_id: str + attributes: dict[str, Any] = field(default_factory=dict) + + +@dataclass +class MappedChunk: + chunk_id: str + index: int + text: str + + +@dataclass +class MappedComment: + comment_id: str + comment_vertex_id: str + issue_vertex_id: str + content_hash: str + chunks: list[MappedChunk] + + +@dataclass +class MappedIssue: + issue_id: str + issue_vertex_id: str + updated: datetime | None + content_hash: str + vertices: list[VertexRecord] + edges: list[EdgeRecord] + document: dict[str, Any] + comments: list[MappedComment] + + +def _datetime(value: Any) -> str | None: + if not value: + return None + try: + return date_parser.parse(str(value)).strftime("%Y-%m-%d %H:%M:%S") + except (TypeError, ValueError, OverflowError): + return None + + +def _parsed_datetime(value: Any) -> datetime | None: + if not value: + return None + try: + return date_parser.parse(str(value)) + except (TypeError, ValueError, OverflowError): + return None + + +def _timestamp(value: Any) -> float: + parsed = _parsed_datetime(value) + return parsed.timestamp() if parsed is not None else 0.0 + + +def _display_name(user: dict[str, Any] | None) -> str: + if not user: + return "" + return str(user.get("displayName") or user.get("accountId") or "").strip() + + +def _comma_names(values: list[Any] | None) -> str: + names: list[str] = [] + for value in values or []: + if isinstance(value, dict): + name = value.get("name") + else: + name = value + if name is not None and str(name).strip(): + names.append(str(name).strip()) + return ", ".join(names) + + +_FENCED_BLOCK_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL) +_LOG_LINE_RE = re.compile( + r"""(?ix) + ^\s*(?: + \[?\d{4}[-/]\d{2}[-/]\d{2}[T\s]\d{2}:\d{2}:\d{2} + |\[?(?:TRACE|DEBUG|INFO|WARN(?:ING)?|ERROR|FATAL|CRITICAL)\]?\b + |(?:Traceback\s+\(most\s+recent\s+call\s+last\)|Caused\s+by:) + |(?:at\s+[\w.$]+\([^)]*(?::\d+)?\)) + |(?:File\s+"[^"]+",\s+line\s+\d+) + |\{.*"(?:timestamp|time|level|severity|logger)"\s*: + ) + """ +) +_LOG_OMISSION = "[Log output omitted from search content.]" + + +def _is_log_heavy(lines: list[str]) -> bool: + non_empty = [line for line in lines if line.strip()] + if len(non_empty) < 5: + return False + matched = sum(bool(_LOG_LINE_RE.search(line)) for line in non_empty) + return matched >= 3 and matched / len(non_empty) >= 0.6 + + +def _filter_long_log_output(text: str, chunk_size: int) -> str: + """Remove log-dominated blocks only when a comment needs chunking.""" + if len(text) <= chunk_size: + return text + + def replace_fence(match: re.Match[str]) -> str: + body = match.group(1) + return _LOG_OMISSION if _is_log_heavy(body.splitlines()) else match.group(0) + + filtered = _FENCED_BLOCK_RE.sub(replace_fence, text) + lines = filtered.splitlines() + output: list[str] = [] + index = 0 + while index < len(lines): + end = index + while end < len(lines) and ( + not lines[end].strip() or _LOG_LINE_RE.search(lines[end]) + ): + end += 1 + block = lines[index:end] + if _is_log_heavy(block): + if not output or output[-1] != _LOG_OMISSION: + output.append(_LOG_OMISSION) + index = end + continue + output.append(lines[index]) + index += 1 + return "\n".join(output).strip() + + +class JiraIssueMapper: + def __init__( + self, + source: JiraDataSource, + graphname: str | None = None, + ): + self.source = source + chunker_config = get_graphrag_config(graphname).get( + "chunker_config", + {}, + ) + self.comment_chunker = StructuredChunker( + chunk_size=chunker_config.get("chunk_size", 0), + overlap_size=chunker_config.get("overlap_size", -1), + ) + + def _id(self, object_type: str, object_id: Any) -> str: + # cloud_id intentionally excluded — vertex IDs use only the portable + # object_type + object_id so they are stable across graph recreations + # and don't leak internal tenant identifiers. + return f"jira:{object_type}:{object_id}".lower() + + def _issue_id(self, issue_key: str) -> str: + # Use the human-readable ticket key so GenerateFunction can construct + # the vertex ID directly from what the user says (e.g. "GML-2191" + # → "jira:gml-2191:issue"). cloud_id is NOT included — ticket keys + # are unique within a graph's connected Jira project scope. + return f"jira:{issue_key}:issue".lower() + + def _user_vertex( + self, user: dict[str, Any] | None + ) -> VertexRecord | None: + if not user or not user.get("accountId"): + return None + account_id = str(user["accountId"]) + return VertexRecord( + "JiraUser", + self._id("user", account_id), + { + "account_id": account_id, + "display_name": _display_name(user), + }, + ) + + def map(self, issue: dict[str, Any]) -> MappedIssue: + issue_id = str(issue["id"]) + issue_key = str(issue.get("key") or issue_id) + fields = issue.get("fields") or {} + project = fields.get("project") or {} + project_id = str(project.get("id") or project.get("key") or "unknown") + issue_vertex_id = self._issue_id(issue_key) + project_vertex_id = self._id("project", project_id) + site_url = self.source.connection.site_url + issue_url = f"{site_url}/browse/{issue_key}" + + description = adf_to_markdown(fields.get("description")) + document_text = self._document_text( + issue_key=issue_key, + issue_url=issue_url, + fields=fields, + project=project, + description=description, + ) + content_hash = hashlib.sha256(document_text.encode("utf-8")).hexdigest() + + status = fields.get("status") or {} + status_category = status.get("statusCategory") or {} + priority = fields.get("priority") or {} + resolution = fields.get("resolution") or {} + issue_type = fields.get("issuetype") or {} + story_points_field = self.source.scope.story_points_field + story_points = fields.get(story_points_field) if story_points_field else None + + issue_attrs = { + "issue_key": issue_key, + "summary": str(fields.get("summary") or ""), + "issue_type": str(issue_type.get("name") or ""), + "status": str(status.get("name") or ""), + "status_category": str(status_category.get("key") or ""), + "priority": str(priority.get("name") or ""), + "resolution": str(resolution.get("name") or ""), + "labels": _comma_names(fields.get("labels")), + "components": _comma_names(fields.get("components")), + "fix_versions": _comma_names(fields.get("fixVersions")), + "created": _datetime(fields.get("created")), + "updated": _datetime(fields.get("updated")), + "due": _datetime(fields.get("duedate")), + "url": issue_url, + "content_hash": content_hash, + } + if story_points is not None: + try: + issue_attrs["story_points"] = float(story_points) + except (TypeError, ValueError): + pass + issue_attrs = { + key: value for key, value in issue_attrs.items() if value not in (None, "") + } + + vertices: dict[tuple[str, str], VertexRecord] = {} + + def add_vertex(vertex: VertexRecord | None) -> None: + if not vertex: + return + key = (vertex.vertex_type, vertex.vertex_id) + existing = vertices.get(key) + if existing and existing.attributes.keys() - vertex.attributes.keys(): + return + vertices[key] = vertex + + add_vertex( + VertexRecord( + "JiraProject", + project_vertex_id, + { + "project_key": str(project.get("key") or ""), + "name": str(project.get("name") or ""), + "url": ( + f"{site_url}/jira/software/projects/{project.get('key')}" + if project.get("key") + else site_url + ), + }, + ) + ) + add_vertex(VertexRecord("JiraIssue", issue_vertex_id, issue_attrs)) + + edges = [ + EdgeRecord( + "JiraIssue", + issue_vertex_id, + JIRA_PROJECT_EDGE, + "JiraProject", + project_vertex_id, + ) + ] + + for field_name, edge_type in ( + ("assignee", JIRA_ASSIGNEE_EDGE), + ("reporter", JIRA_REPORTER_EDGE), + ): + user_vertex = self._user_vertex(fields.get(field_name)) + add_vertex(user_vertex) + if user_vertex: + edges.append( + EdgeRecord( + "JiraIssue", + issue_vertex_id, + edge_type, + "JiraUser", + user_vertex.vertex_id, + ) + ) + + mapped_comments: list[MappedComment] = [] + previous_comment_vertex_id: str | None = None + comments = ( + (fields.get("comment") or {}).get("comments") or [] + if self.source.scope.include_comments + else [] + ) + comments = sorted( + comments, + key=lambda comment: ( + _timestamp(comment.get("created")), + str(comment.get("id") or ""), + ), + ) + for comment in comments: + raw_comment_id = comment.get("id") + if raw_comment_id is None: + continue + comment_id = str(raw_comment_id) + comment_vertex_id = self._id("comment", comment_id) + author_vertex = self._user_vertex(comment.get("author")) + add_vertex(author_vertex) + visibility = comment.get("visibility") + if not isinstance(visibility, dict): + visibility = {} + visibility_text = ":".join( + str(value) + for value in ( + visibility.get("type"), + visibility.get("value"), + ) + if value + ) + comment_body = adf_to_markdown(comment.get("body")) or "(Empty comment)" + comment_body = _filter_long_log_output( + comment_body, + self.comment_chunker.chunk_size, + ) + comment_text = self._comment_document_text( + issue_key=issue_key, + issue_url=issue_url, + issue_summary=str(fields.get("summary") or ""), + comment=comment, + visibility=visibility_text, + body=comment_body, + ) + chunks = [str(chunk).strip() for chunk in self.comment_chunker.chunk(comment_text)] + chunks = [chunk for chunk in chunks if chunk] + comment_hash = hashlib.sha256( + ( + "direct-comment-chunks\0" + + "\0".join(chunks) + ).encode("utf-8") + ).hexdigest() + comment_attrs = { + "comment_id": comment_id, + "created": _datetime(comment.get("created")), + "updated": _datetime(comment.get("updated")), + "visibility": visibility_text, + "is_public": comment.get("jsdPublic"), + "ontology_class": "Event", + "content_hash": comment_hash, + } + add_vertex( + VertexRecord( + "JiraComment", + comment_vertex_id, + { + key: value + for key, value in comment_attrs.items() + if value not in (None, "") + }, + ) + ) + edges.append( + EdgeRecord( + "JiraComment", + comment_vertex_id, + JIRA_COMMENT_ISSUE_EDGE, + "JiraIssue", + issue_vertex_id, + ) + ) + if author_vertex: + edges.append( + EdgeRecord( + "JiraComment", + comment_vertex_id, + JIRA_COMMENT_AUTHOR_EDGE, + "JiraUser", + author_vertex.vertex_id, + ) + ) + parent_comment = comment.get("parent") + if not isinstance(parent_comment, dict): + parent_comment = {} + parent_comment_id = comment.get("parentId") or parent_comment.get("id") + if parent_comment_id: + parent_comment_vertex_id = self._id( + "comment", + parent_comment_id, + ) + add_vertex( + VertexRecord( + "JiraComment", + parent_comment_vertex_id, + { + "comment_id": str(parent_comment_id), + "ontology_class": "Event", + }, + ) + ) + edges.append( + EdgeRecord( + "JiraComment", + comment_vertex_id, + JIRA_COMMENT_REPLY_EDGE, + "JiraComment", + parent_comment_vertex_id, + ) + ) + if previous_comment_vertex_id: + edges.append( + EdgeRecord( + "JiraComment", + comment_vertex_id, + JIRA_COMMENT_AFTER_EDGE, + "JiraComment", + previous_comment_vertex_id, + ) + ) + mapped_comments.append( + MappedComment( + comment_id=comment_id, + comment_vertex_id=comment_vertex_id, + issue_vertex_id=issue_vertex_id, + content_hash=comment_hash, + chunks=[ + MappedChunk( + chunk_id=( + f"{comment_vertex_id}:chunk:{index}:" + f"{hashlib.sha256(text.encode('utf-8')).hexdigest()[:12]}" + ), + index=index, + text=text, + ) + for index, text in enumerate(chunks) + ], + ) + ) + previous_comment_vertex_id = comment_vertex_id + + parent = fields.get("parent") or {} + if parent.get("id"): + parent_key = str(parent.get("key") or parent["id"]) + parent_id = self._issue_id(parent_key) + parent_fields = parent.get("fields") or {} + add_vertex( + VertexRecord( + "JiraIssue", + parent_id, + { + "issue_key": str(parent.get("key") or ""), + "summary": str(parent_fields.get("summary") or ""), + "url": ( + f"{site_url}/browse/{parent.get('key')}" + if parent.get("key") + else site_url + ), + }, + ) + ) + edges.append( + EdgeRecord( + "JiraIssue", + issue_vertex_id, + JIRA_PARENT_EDGE, + "JiraIssue", + parent_id, + ) + ) + + for link in fields.get("issuelinks") or []: + link_type = link.get("type") or {} + target = link.get("outwardIssue") + relation = link_type.get("outward") + if not target: + target = link.get("inwardIssue") + relation = link_type.get("inward") + if not target or not target.get("id"): + continue + target_key = str(target.get("key") or target["id"]) + target_id = self._issue_id(target_key) + target_fields = target.get("fields") or {} + add_vertex( + VertexRecord( + "JiraIssue", + target_id, + { + "issue_key": target_key, + "summary": str(target_fields.get("summary") or ""), + "url": f"{site_url}/browse/{target_key}" if target_key else site_url, + }, + ) + ) + edges.append( + EdgeRecord( + "JiraIssue", + issue_vertex_id, + JIRA_LINK_EDGE, + "JiraIssue", + target_id, + {"link_type": str(relation or link_type.get("name") or "relates to")}, + ) + ) + + return MappedIssue( + issue_id=issue_id, + issue_vertex_id=issue_vertex_id, + updated=_parsed_datetime(fields.get("updated")), + content_hash=content_hash, + vertices=list(vertices.values()), + edges=edges, + document={ + "doc_id": self._id("issue-doc", issue_id), + "doc_type": "jira", + "content": document_text, + "position": 0, + }, + comments=mapped_comments, + ) + + def _comment_document_text( + self, + *, + issue_key: str, + issue_url: str, + issue_summary: str, + comment: dict[str, Any], + visibility: str, + body: str, + ) -> str: + author = _display_name(comment.get("author")) or "Unknown user" + return "\n".join( + [ + f"# Comment by {author} on {issue_key}: {issue_summary}", + f"Issue: {issue_key}", + f"URL: {issue_url}", + f"Comment ID: {comment.get('id') or ''}", + f"Author: {author}", + f"Created: {comment.get('created') or ''}", + f"Updated: {comment.get('updated') or ''}", + f"Visibility: {visibility or 'default'}", + "", + "## Comment", + body, + ] + ).strip() + "\n" + + def _document_text( + self, + *, + issue_key: str, + issue_url: str, + fields: dict[str, Any], + project: dict[str, Any], + description: str, + ) -> str: + status = fields.get("status") or {} + status_category = status.get("statusCategory") or {} + issue_type = fields.get("issuetype") or {} + priority = fields.get("priority") or {} + lines = [ + f"# {issue_key}: {fields.get('summary') or ''}", + f"URL: {issue_url}", + f"Project: {project.get('key') or ''} {project.get('name') or ''}".rstrip(), + f"Type: {issue_type.get('name') or ''}", + ( + f"Status: {status.get('name') or ''} " + f"({status_category.get('key') or ''})" + ).rstrip(), + f"Priority: {priority.get('name') or ''}", + f"Assignee: {_display_name(fields.get('assignee')) or 'Unassigned'}", + f"Reporter: {_display_name(fields.get('reporter'))}", + f"Labels: {_comma_names(fields.get('labels'))}", + f"Components: {_comma_names(fields.get('components'))}", + f"Fix versions: {_comma_names(fields.get('fixVersions'))}", + f"Updated: {fields.get('updated') or ''}", + "", + "## Description", + description or "(No description)", + ] + attachments = fields.get("attachment") or [] + if attachments: + lines.extend( + [ + "", + "## Attachments", + ", ".join( + str(item.get("filename")) + for item in attachments + if item.get("filename") + ), + ] + ) + return "\n".join(lines).strip() + "\n" diff --git a/graphrag/app/connectors/jira/schema.py b/graphrag/app/connectors/jira/schema.py new file mode 100644 index 00000000..85b0d8ef --- /dev/null +++ b/graphrag/app/connectors/jira/schema.py @@ -0,0 +1,278 @@ +"""Predefined Jira domain schema.""" + +from __future__ import annotations + +from typing import Any + +from common.db.schema_utils import SchemaProposal, read_existing_schema + + +JIRA_PROJECT_EDGE = "JIRA_BELONGS_TO" +JIRA_ASSIGNEE_EDGE = "JIRA_ASSIGNED_TO" +JIRA_REPORTER_EDGE = "JIRA_REPORTED_BY" +JIRA_PARENT_EDGE = "JIRA_HAS_PARENT" +JIRA_LINK_EDGE = "JIRA_LINKS_TO" +JIRA_COMMENT_ISSUE_EDGE = "JIRA_COMMENT_ON" +JIRA_COMMENT_AUTHOR_EDGE = "JIRA_COMMENTED_BY" +JIRA_COMMENT_REPLY_EDGE = "JIRA_COMMENT_REPLIES_TO" +JIRA_COMMENT_AFTER_EDGE = "JIRA_COMMENT_AFTER" + + +def _attribute_types(metadata: dict[str, Any]) -> dict[str, str]: + attributes: dict[str, str] = {} + primary_id = (metadata.get("PrimaryId") or {}).get("AttributeName") + for attribute in metadata.get("Attributes") or []: + name = attribute.get("AttributeName") + if not name or name == primary_id: + continue + attribute_type = ( + (attribute.get("AttributeType") or {}).get("Name") or "STRING" + ) + attributes[str(name).casefold()] = str(attribute_type).upper() + return attributes + + +def _edge_pairs(metadata: dict[str, Any]) -> set[tuple[str, str]]: + pairs: set[tuple[str, str]] = set() + source = metadata.get("FromVertexTypeName") + target = metadata.get("ToVertexTypeName") + if source and target and source != "*" and target != "*": + pairs.add((str(source).casefold(), str(target).casefold())) + for pair in metadata.get("EdgePairs") or []: + source = pair.get("From") + target = pair.get("To") + if source and target: + pairs.add((str(source).casefold(), str(target).casefold())) + return pairs + + +def jira_schema_proposal() -> SchemaProposal: + """Return the bounded Jira schema consumed by GraphRAG.""" + proposal = SchemaProposal(domain_label="Jira Cloud") + proposal.add_vertex( + "JiraProject", + ( + "A Jira project, modeled as a POLE+O Object subtype and work " + "container. project_key is the short key, such as PAY." + ), + [ + ("project_key", "STRING"), + ("name", "STRING"), + ("url", "STRING"), + ], + ) + proposal.add_vertex( + "JiraIssue", + ( + "A Jira work item modeled as a POLE+O Object subtype. Filter using " + "issue_key, status, status_category (new, indeterminate, or done), " + "priority, issue_type, resolution, labels, components, " + "fix_versions, created, updated, or due." + ), + [ + ("issue_key", "STRING"), + ("summary", "STRING"), + ("issue_type", "STRING"), + ("status", "STRING"), + ("status_category", "STRING"), + ("priority", "STRING"), + ("resolution", "STRING"), + ("labels", "STRING"), + ("components", "STRING"), + ("fix_versions", "STRING"), + ("created", "DATETIME"), + ("updated", "DATETIME"), + ("due", "DATETIME"), + ("url", "STRING"), + ("story_points", "DOUBLE"), + ("content_hash", "STRING"), + ], + ) + proposal.add_vertex( + "JiraUser", + ( + "An Atlassian account modeled as a POLE+O Person subtype. " + "account_id is the stable identity and display_name is the " + "human-readable name." + ), + [ + ("account_id", "STRING"), + ("display_name", "STRING"), + ], + ) + proposal.add_vertex( + "JiraComment", + ( + "A Jira comment modeled as a POLE+O Event subtype. Its content is " + "embedded through deterministic comment-specific document chunks; " + "use graph edges for issue, author, ordering, and explicit replies." + ), + [ + ("comment_id", "STRING"), + ("created", "DATETIME"), + ("updated", "DATETIME"), + ("visibility", "STRING"), + ("is_public", "BOOL"), + ("ontology_class", "STRING"), + ("content_hash", "STRING"), + ], + ) + + proposal.add_edge_pair( + JIRA_PROJECT_EDGE, + "JiraIssue", + "JiraProject", + "The Jira issue's current project.", + ) + proposal.add_edge_pair( + JIRA_ASSIGNEE_EDGE, + "JiraIssue", + "JiraUser", + "The Jira issue's current assignee.", + ) + proposal.add_edge_pair( + JIRA_REPORTER_EDGE, + "JiraIssue", + "JiraUser", + "The Atlassian account that reported the Jira issue.", + ) + proposal.add_edge_pair( + JIRA_PARENT_EDGE, + "JiraIssue", + "JiraIssue", + "The parent Jira issue, including epic and subtask parents.", + ) + proposal.add_edge_pair( + JIRA_LINK_EDGE, + "JiraIssue", + "JiraIssue", + ( + "A directed Jira issue link. link_type is the phrase from the " + "source issue toward the target, such as blocks or is blocked by." + ), + [("link_type", "STRING")], + ) + proposal.add_edge_pair( + JIRA_COMMENT_ISSUE_EDGE, + "JiraComment", + "JiraIssue", + "The Jira issue on which the comment was posted.", + ) + proposal.add_edge_pair( + JIRA_COMMENT_AUTHOR_EDGE, + "JiraComment", + "JiraUser", + "The Atlassian account that authored the comment.", + ) + proposal.add_edge_pair( + JIRA_COMMENT_REPLY_EDGE, + "JiraComment", + "JiraComment", + "An explicit source-provided parent comment; never inferred.", + ) + proposal.add_edge_pair( + JIRA_COMMENT_AFTER_EDGE, + "JiraComment", + "JiraComment", + "Chronological order between adjacent comments on one issue.", + ) + return proposal + + +def jira_schema_status(conn) -> dict[str, Any]: + """Inspect whether the one current Jira schema is fully installed.""" + proposal = jira_schema_proposal() + existing = read_existing_schema(conn) + conflicts: list[str] = [] + missing_vertices: list[str] = [] + missing_edges: list[str] = [] + missing_pairs: list[str] = [] + + required_core = ("Document", "DocumentChunk", "Content") + missing_core = [name for name in required_core if not existing.has_vertex(name)] + required_core_edges = ("CONTAINS_ENTITY", "HAS_CHILD", "HAS_CONTENT") + missing_core_edges = [ + name for name in required_core_edges if not existing.has_edge(name) + ] + if missing_core or missing_core_edges: + return { + "status": "not_initialized", + "missing": { + "core_vertices": missing_core, + "core_edges": missing_core_edges, + }, + "conflicts": [], + } + + for vertex in proposal.vertices: + if not existing.has_vertex(vertex.name): + missing_vertices.append(vertex.name) + continue + metadata = conn.getVertexType(vertex.name) or {} + actual = _attribute_types(metadata) + for attribute in vertex.attributes: + actual_type = actual.get(attribute.name.casefold()) + if actual_type is None: + conflicts.append( + f"{vertex.name}.{attribute.name} is missing" + ) + elif actual_type != attribute.type.upper(): + conflicts.append( + f"{vertex.name}.{attribute.name} must be " + f"{attribute.type.upper()}, found {actual_type}" + ) + + for edge in proposal.edges: + if not existing.has_edge(edge.name): + missing_edges.append(edge.name) + continue + metadata = conn.getEdgeType(edge.name) or {} + if bool(metadata.get("IsDirected")) != edge.directed: + expected = "directed" if edge.directed else "undirected" + conflicts.append(f"{edge.name} must be {expected}") + actual_attributes = _attribute_types(metadata) + for attribute in edge.attributes: + actual_type = actual_attributes.get(attribute.name.casefold()) + if actual_type is None: + conflicts.append(f"{edge.name}.{attribute.name} is missing") + elif actual_type != attribute.type.upper(): + conflicts.append( + f"{edge.name}.{attribute.name} must be " + f"{attribute.type.upper()}, found {actual_type}" + ) + actual_pairs = _edge_pairs(metadata) + for source, target in edge.pairs: + if (source.casefold(), target.casefold()) not in actual_pairs: + missing_pairs.append(f"{edge.name}: {source} -> {target}") + + required_links = ( + ("CONTAINS_ENTITY", "Document", "JiraIssue"), + ("CONTAINS_ENTITY", "DocumentChunk", "JiraIssue"), + ("CONTAINS_ENTITY", "Document", "JiraComment"), + ("CONTAINS_ENTITY", "DocumentChunk", "JiraComment"), + ) + for edge, source, target in required_links: + if not existing.has_edge_pair(edge, source, target): + missing_pairs.append(f"{edge}: {source} -> {target}") + + missing = { + "vertices": missing_vertices, + "edges": missing_edges, + "pairs": missing_pairs, + } + if conflicts: + status = "conflict" + elif not any(missing.values()): + status = "installed" + elif any( + existing.has_vertex(vertex.name) for vertex in proposal.vertices + ): + status = "incomplete" + else: + status = "not_installed" + + return { + "status": status, + "missing": missing, + "conflicts": conflicts, + } diff --git a/graphrag/app/connectors/jira/state.py b/graphrag/app/connectors/jira/state.py new file mode 100644 index 00000000..e99e1a4e --- /dev/null +++ b/graphrag/app/connectors/jira/state.py @@ -0,0 +1,150 @@ +"""Atomic persistence and secret redaction for Jira data sources.""" + +from __future__ import annotations + +import json +import os +from pathlib import Path +from urllib.parse import urlparse + +from common.config import _config_file_lock, validate_graphname + +from .config import JiraDataSource, JiraSourceFile, JiraSyncState + + +MASKED_SECRET = "********" +NON_TENANT_ATLASSIAN_HOSTS = { + "api.atlassian.com", + "atlassian.net", + "graphql.atlassian.net", + "id.atlassian.com", +} + + +class JiraSourceStore: + def __init__(self, config_root: str = "configs/graph_configs"): + self.config_root = Path(config_root) + + def path(self, graphname: str) -> Path: + validate_graphname(graphname) + return self.config_root / graphname / "data_sources.json" + + def load(self, graphname: str) -> JiraSourceFile: + path = self.path(graphname) + if not path.exists(): + return JiraSourceFile() + with _config_file_lock: + with path.open("r", encoding="utf-8") as stream: + payload = json.load(stream) + return JiraSourceFile.model_validate(payload) + + def save(self, graphname: str, config: JiraSourceFile) -> None: + path = self.path(graphname) + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + payload = config.model_dump(mode="json") + for source in payload["sources"]: + if not source["connection"].get("api_token"): + source["connection"].pop("api_token", None) + with _config_file_lock: + try: + with tmp.open("w", encoding="utf-8") as stream: + json.dump(payload, stream, indent=2) + stream.flush() + os.fsync(stream.fileno()) + os.replace(tmp, path) + finally: + if tmp.exists(): + tmp.unlink() + + def list(self, graphname: str, *, redact: bool = True) -> list[dict]: + sources = self.load(graphname).sources + return [self.redact(source) if redact else source.model_dump(mode="json") for source in sources] + + def get(self, graphname: str, source_id: str) -> JiraDataSource: + for source in self.load(graphname).sources: + if source.id == source_id: + return source + raise KeyError(source_id) + + def upsert(self, graphname: str, submitted: JiraDataSource) -> JiraDataSource: + hostname = urlparse(submitted.connection.site_url).hostname + if hostname and hostname.lower() in NON_TENANT_ATLASSIAN_HOSTS: + raise ValueError( + "site_url must be your Jira tenant URL, for example " + "https://your-company.atlassian.net" + ) + config = self.load(graphname) + sources = list(config.sources) + existing = next((s for s in sources if s.id == submitted.id), None) + overlap_seconds = submitted.sync.overlap_seconds + submitted.connection.api_token = ( + existing.connection.api_token + if existing is not None + and existing.connection.api_token != MASKED_SECRET + else "" + ) + if existing is not None: + credentials_changed = ( + submitted.connection.email != existing.connection.email + ) + if submitted.connection.site_url != existing.connection.site_url: + unused_draft = ( + existing.sync.last_tested_at is None + and existing.sync.checkpoint is None + and existing.sync.last_issue_count == 0 + ) + if not unused_draft: + raise ValueError( + "site_url cannot be changed after a source has been used; " + "create a new source so existing Jira data can be reconciled" + ) + credentials_changed = True + else: + pass + submitted.sync = existing.sync.model_copy(deep=True) + submitted.sync.overlap_seconds = overlap_seconds + if credentials_changed: + submitted.sync.last_tested_at = None + if submitted.scope != existing.scope: + submitted.sync.checkpoint = None + else: + submitted.sync = JiraSyncState(overlap_seconds=overlap_seconds) + sources = [source for source in sources if source.id != submitted.id] + sources.append(submitted) + sources.sort(key=lambda source: source.id) + config.sources = sources + self.save(graphname, config) + return submitted + + def delete(self, graphname: str, source_id: str) -> None: + config = self.load(graphname) + remaining = [source for source in config.sources if source.id != source_id] + if len(remaining) == len(config.sources): + raise KeyError(source_id) + config.sources = remaining + self.save(graphname, config) + + def update_runtime_state( + self, + graphname: str, + source: JiraDataSource, + ) -> None: + """Persist connector-owned sync fields without losing edits.""" + config = self.load(graphname) + sources = list(config.sources) + for index, stored in enumerate(sources): + if stored.id != source.id: + continue + stored.sync = source.sync + sources[index] = stored + config.sources = sources + self.save(graphname, config) + return + raise KeyError(source.id) + + @staticmethod + def redact(source: JiraDataSource) -> dict: + payload = source.model_dump(mode="json") + payload["connection"]["api_token"] = "" + return payload diff --git a/graphrag/app/connectors/jira/sync.py b/graphrag/app/connectors/jira/sync.py new file mode 100644 index 00000000..b73be29e --- /dev/null +++ b/graphrag/app/connectors/jira/sync.py @@ -0,0 +1,906 @@ +"""Jira Cloud synchronization into the existing GraphRAG pipeline.""" + +from __future__ import annotations + +import asyncio +import json +import logging +import queue +import tempfile +import threading +import time +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterator + +from common.config import get_embedding_store +from common.db.health import embedding_coverage +from common.py_schemas import LoadingInfo +from supportai import supportai + +from .client import JiraCloudClient +from .config import JiraDataSource +from .mapper import ( + EdgeRecord, + JiraIssueMapper, + MappedComment, + MappedIssue, + VertexRecord, +) +from .schema import ( + JIRA_ASSIGNEE_EDGE, + JIRA_COMMENT_AFTER_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_LINK_EDGE, + JIRA_PARENT_EDGE, + JIRA_PROJECT_EDGE, + JIRA_REPORTER_EDGE, + jira_schema_status, +) +from .state import JiraSourceStore + +logger = logging.getLogger(__name__) + +CURRENT_STATE_EDGES = ( + JIRA_PROJECT_EDGE, + JIRA_ASSIGNEE_EDGE, + JIRA_REPORTER_EDGE, + JIRA_PARENT_EDGE, + JIRA_LINK_EDGE, +) +COMMENT_STATE_EDGES = ( + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_COMMENT_AFTER_EDGE, +) +UPSERT_BATCH_SIZE = 500 +EMBEDDING_BATCH_SIZE = 32 +# Flush accumulated documents and comment-chunks to TigerGraph every this +# many Jira pages. A higher value means fewer supportai.ingest() round-trips +# (the dominant overhead for large initial loads) at the cost of a larger +# in-memory accumulation. 10 pages × 100 issues = 1 000 issues per flush. +DOC_LOAD_BATCH_PAGES = 10 + + +def _attrs(attributes: dict[str, Any]) -> dict[str, dict[str, Any]]: + return { + key: {"value": value} + for key, value in attributes.items() + if value is not None + } + + +def _upsert_payload( + vertices: list[VertexRecord], + edges: list[EdgeRecord], +) -> dict[str, Any]: + payload: dict[str, Any] = { + "vertices": defaultdict(dict), + "edges": defaultdict( + lambda: defaultdict( + lambda: defaultdict(lambda: defaultdict(dict)) + ) + ), + } + for vertex in vertices: + payload["vertices"][vertex.vertex_type][vertex.vertex_id] = _attrs( + vertex.attributes + ) + for edge in edges: + payload["edges"][edge.source_type][edge.source_id][edge.edge_type][ + edge.target_type + ][edge.target_id] = _attrs(edge.attributes) + return payload + + +class JiraSyncService: + def __init__( + self, + graphname: str, + source: JiraDataSource, + conn, + *, + store: JiraSourceStore | None = None, + client: JiraCloudClient | None = None, + ): + self.graphname = graphname + self.source = source + self.conn = conn + self.store = store or JiraSourceStore() + self.client = client or JiraCloudClient(source) + self._owns_client = client is None + + def run(self) -> dict[str, Any]: + if not self.source.enabled: + raise ValueError("Cannot sync a disabled data source") + + started = datetime.now(timezone.utc) + self.source.sync.last_started_at = started + self.source.sync.last_error = None + self.store.update_runtime_state(self.graphname, self.source) + + try: + schema_result = jira_schema_status(self.conn) + if schema_result["status"] != "installed": + raise RuntimeError( + "Jira schema is not installed for this graph. " + "Install it from Data Sources before synchronization." + ) + self._prepare_legacy_comment_migration() + + mapper = JiraIssueMapper( + self.source, + graphname=self.graphname, + ) + existing_hashes = self._all_existing_hashes() + # If the graph has no JiraIssue vertices but a checkpoint exists, + # the graph was likely cleared or recreated. Reset the checkpoint + # so the next iteration performs a full sync instead of a no-op. + if not existing_hashes and self.source.sync.checkpoint is not None: + logger.warning( + "Graph appears empty but checkpoint is set — resetting " + "checkpoint for graph=%s source=%s to trigger a full sync", + self.graphname, + self.source.id, + ) + self.source.sync.checkpoint = None + self.store.update_runtime_state(self.graphname, self.source) + existing_comment_hashes = self._all_existing_comment_hashes() + issues_upserted = 0 + documents_loaded = 0 + comments_deleted = 0 + + # Pending batches — accumulated across DOC_LOAD_BATCH_PAGES pages + # before being flushed to TigerGraph in one supportai.ingest call. + pending_docs: list[MappedIssue] = [] + pending_comments: list[MappedComment] = [] + pages_since_flush = 0 + # Track the highest "updated" timestamp seen so far. We only + # advance the checkpoint AFTER a successful batch flush so that a + # crash before the flush leaves the checkpoint behind the lost + # pages, ensuring they are re-fetched on the next sync run. + pending_checkpoint = self.source.sync.checkpoint + + for issues in self._iter_pages_pipelined(): + mapped = [mapper.map(issue) for issue in issues] + changed_docs, changed_comms, deleted_count = ( + self._upsert_issue_records( + mapped, + existing_hashes, + existing_comment_hashes, + ) + ) + pending_docs.extend(changed_docs) + pending_comments.extend(changed_comms) + documents_loaded += len(changed_docs) + comments_deleted += deleted_count + issues_upserted += len(mapped) + pages_since_flush += 1 + + page_max = max( + (item.updated for item in mapped if item.updated is not None), + default=None, + ) + if page_max is not None and ( + pending_checkpoint is None or page_max > pending_checkpoint + ): + pending_checkpoint = page_max + + if pages_since_flush >= DOC_LOAD_BATCH_PAGES: + self._finalize_batch( + pending_docs, + pending_comments, + existing_hashes, + existing_comment_hashes, + ) + pending_docs = [] + pending_comments = [] + pages_since_flush = 0 + if pending_checkpoint != self.source.sync.checkpoint: + self.source.sync.checkpoint = pending_checkpoint + self.store.update_runtime_state( + self.graphname, self.source + ) + + # Final flush for any pages that did not fill a complete batch. + if pending_docs or pending_comments: + self._finalize_batch( + pending_docs, + pending_comments, + existing_hashes, + existing_comment_hashes, + ) + if pending_checkpoint != self.source.sync.checkpoint: + self.source.sync.checkpoint = pending_checkpoint + self.store.update_runtime_state(self.graphname, self.source) + + chunk_coverage = embedding_coverage( + self.conn, + "DocumentChunk", + ) + logger.info( + "embedding_coverage graph=%s: %s", + self.graphname, + chunk_coverage, + ) + missing_chunk_embeddings = ( + int(chunk_coverage["missing"]) + if chunk_coverage is not None + else 0 + ) + + # Recovery check: if this sync loaded nothing new but there are + # DocumentChunks still at epoch_processed=0, a previous ECC rebuild + # must have failed partway through. Trigger a fresh rebuild so + # those chunks get embedded without the user having to intervene. + if not documents_loaded and not comments_deleted and not missing_chunk_embeddings: + if self._has_unprocessed_chunks(): + logger.info( + "Detected unprocessed DocumentChunks from a prior failed " + "ECC rebuild for graph=%s source=%s — triggering recovery rebuild", + self.graphname, + self.source.id, + ) + missing_chunk_embeddings = 1 + self.source.sync.last_completed_at = datetime.now(timezone.utc) + self.source.sync.last_issue_count = issues_upserted + self.source.sync.last_error = None + self.source.sync.migrating_legacy_comments = False + self.store.update_runtime_state(self.graphname, self.source) + return { + "status": "completed", + "issues_upserted": issues_upserted, + "issues_deleted": 0, + "comments_deleted": comments_deleted, + "documents_loaded": documents_loaded, + "checkpoint": ( + self.source.sync.checkpoint.isoformat() + if self.source.sync.checkpoint is not None + else None + ), + "schema": schema_result["status"], + "missing_chunk_embeddings": missing_chunk_embeddings, + "rebuild_required": bool( + documents_loaded + or comments_deleted + or missing_chunk_embeddings + ), + } + except Exception as exc: + logger.exception( + "Jira sync failed for graph=%s source=%s", + self.graphname, + self.source.id, + ) + self.source.sync.last_error = str(exc)[:1000] + self.store.update_runtime_state(self.graphname, self.source) + raise + finally: + if self._owns_client: + self.client.close() + + def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: + # pyTigerGraph's getVerticesById raises error 601 as soon as any + # requested STRING ID does not exist, which is the normal state during + # an initial or incremental ingestion. Read the lightweight hash + # projection once and filter it in memory. + vertices = self.conn.getVertices( + vertex_type, + select="content_hash", + ) or [] + return { + str(vertex.get("v_id")): str( + (vertex.get("attributes") or {}).get("content_hash") or "" + ) + for vertex in vertices + } + + def _all_existing_hashes(self) -> dict[str, str]: + return self._all_content_hashes("JiraIssue") + + def _all_existing_comment_hashes(self) -> dict[str, str]: + return self._all_content_hashes("JiraComment") + + def _has_unprocessed_chunks(self) -> bool: + """True when at least one DocumentChunk has epoch_processed=0. + + Called only when this sync loaded no new content (documents_loaded=0, + comments_deleted=0). In that case any epoch_processed=0 chunk is a + survivor from a prior ECC rebuild that failed before it could embed + everything. We signal rebuild_required so the next ECC run can + complete the job without the user having to force a re-ingest. + """ + try: + hits = self.conn.getVertices( + "DocumentChunk", + where="epoch_processed=0", + select="epoch_processed", + limit=1, + ) or [] + logger.info( + "_has_unprocessed_chunks graph=%s: found %d chunk(s) with epoch_processed=0", + self.graphname, + len(hits), + ) + return len(hits) > 0 + except Exception as exc: + logger.warning("_has_unprocessed_chunks check failed: %s", exc) + return False + + def _prepare_legacy_comment_migration(self) -> None: + if self.source.sync.migrating_legacy_comments: + return + documents = self.conn.getVertices("Document", select="id") or [] + has_legacy_comments = any( + ":comment-doc:" in str(document.get("v_id") or "") + for document in documents + ) + if not has_legacy_comments: + return + self.source.sync.migrating_legacy_comments = True + self.source.sync.checkpoint = None + self.store.update_runtime_state(self.graphname, self.source) + + def _existing_hashes(self, issue_ids: list[str]) -> dict[str, str]: + if not issue_ids: + return {} + requested = set(issue_ids) + return { + issue_id: content_hash + for issue_id, content_hash in self._all_existing_hashes().items() + if issue_id in requested + } + + def _iter_pages_pipelined(self) -> Iterator[list[dict[str, Any]]]: + """Yield Jira issue pages pre-fetched by a background thread. + + The background thread requests the next page from the Jira API while + the main thread is writing the current page to TigerGraph, overlapping + network I/O with graph I/O. A queue capacity of 2 bounds memory: at + most two extra pages are held in RAM at any moment. + """ + page_queue: queue.Queue[tuple[str, Any]] = queue.Queue(maxsize=2) + error_holder: list[Exception] = [] + + def _fetch() -> None: + try: + for page in self.client.iter_issue_pages(): + page_queue.put(("page", page)) + except Exception as exc: # noqa: BLE001 + error_holder.append(exc) + finally: + page_queue.put(("done", None)) + + thread = threading.Thread( + target=_fetch, daemon=True, name="jira-page-fetcher" + ) + thread.start() + try: + while True: + kind, value = page_queue.get() + if kind == "done": + if error_holder: + raise error_holder[0] + break + yield value + finally: + thread.join(timeout=60) + + def _upsert_issue_records( + self, + mapped: list[MappedIssue], + existing_hashes: dict[str, str], + existing_comment_hashes: dict[str, str], + ) -> tuple[list[MappedIssue], list[MappedComment], int]: + """Write graph vertices/edges for one page; return items to load later. + + Document loading and comment-chunk embedding are deliberately NOT done + here. The caller accumulates the returned lists across DOC_LOAD_BATCH_PAGES + pages and flushes them together via _finalize_batch(), reducing the + number of supportai.ingest() round-trips from one-per-page to + one-per-batch. + + Returns (changed_documents, changed_comments, comments_deleted). + """ + changed_documents = self._changed_documents(mapped, existing_hashes) + mapped_comments = [ + comment + for item in mapped + for comment in item.comments + ] + changed_comments = self._changed_comments( + mapped_comments, + existing_comment_hashes, + ) + comments_deleted = 0 + for item in mapped: + comments_deleted += self._reconcile_issue_comments( + item, + existing_comment_hashes, + ) + + all_vertices: dict[tuple[str, str], VertexRecord] = {} + all_edges: list[EdgeRecord] = [] + for item in mapped: + for vertex in item.vertices: + key = (vertex.vertex_type, vertex.vertex_id) + previous = all_vertices.get(key) + if ( + previous + and previous.attributes.keys() - vertex.attributes.keys() + ): + # Keep richer attributes if a linked-issue placeholder + # arrives after a full issue record in the same page. + continue + all_vertices[key] = vertex + all_edges.extend(item.edges) + if item.issue_vertex_id in existing_hashes: + self._delete_current_edges(item.issue_vertex_id) + for comment in item.comments: + if comment.comment_vertex_id in existing_comment_hashes: + self._delete_current_comment_edges( + comment.comment_vertex_id + ) + + # Do not advance a changed item's content hash until its document load + # succeeds. A failed batch is therefore safe to replay after restart. + def strip_content_hash(vertex_type: str, vertex_id: str) -> None: + key = (vertex_type, vertex_id) + vertex = all_vertices.get(key) + if vertex is not None: + attributes = dict(vertex.attributes) + attributes.pop("content_hash", None) + all_vertices[key] = VertexRecord( + vertex.vertex_type, + vertex.vertex_id, + attributes, + ) + + for item in changed_documents: + strip_content_hash("JiraIssue", item.issue_vertex_id) + for comment in changed_comments: + strip_content_hash("JiraComment", comment.comment_vertex_id) + + self._upsert_records(list(all_vertices.values()), all_edges) + return changed_documents, changed_comments, comments_deleted + + def _finalize_batch( + self, + changed_documents: list[MappedIssue], + changed_comments: list[MappedComment], + existing_hashes: dict[str, str], + existing_comment_hashes: dict[str, str], + ) -> None: + """Load documents and embed comment-chunks for a batch of pages. + + Called after every DOC_LOAD_BATCH_PAGES pages (and once at the end of + the sync). By batching, we replace N-per-page supportai.ingest calls + with one call per batch, cutting fixed HTTP/job-submission overhead by + DOC_LOAD_BATCH_PAGES × for a large initial load. + """ + if changed_documents: + self._load_documents(changed_documents) + for item in changed_documents: + document_id = item.document["doc_id"].lower() + self.conn.upsertEdge( + "Document", + document_id, + "CONTAINS_ENTITY", + "JiraIssue", + item.issue_vertex_id, + ) + self.conn.upsertVertex( + "JiraIssue", + item.issue_vertex_id, + attributes={"content_hash": item.content_hash}, + ) + existing_hashes[item.issue_vertex_id] = item.content_hash + if changed_comments: + self._upsert_comment_chunks(changed_comments) + self._embed_comment_chunks(changed_comments) + for comment in changed_comments: + self.conn.upsertVertex( + "JiraComment", + comment.comment_vertex_id, + attributes={"content_hash": comment.content_hash}, + ) + existing_comment_hashes[ + comment.comment_vertex_id + ] = comment.content_hash + + def _upsert_issue_page( + self, + mapped: list[MappedIssue], + existing_hashes: dict[str, str], + existing_comment_hashes: dict[str, str], + ) -> tuple[int, int]: + changed_documents = self._changed_documents(mapped, existing_hashes) + mapped_comments = [ + comment + for item in mapped + for comment in item.comments + ] + changed_comments = self._changed_comments( + mapped_comments, + existing_comment_hashes, + ) + comments_deleted = 0 + for item in mapped: + comments_deleted += self._reconcile_issue_comments( + item, + existing_comment_hashes, + ) + + all_vertices: dict[tuple[str, str], VertexRecord] = {} + all_edges: list[EdgeRecord] = [] + for item in mapped: + for vertex in item.vertices: + key = (vertex.vertex_type, vertex.vertex_id) + previous = all_vertices.get(key) + if ( + previous + and previous.attributes.keys() - vertex.attributes.keys() + ): + # Keep richer attributes if a linked-issue placeholder + # arrives after a full issue record in the same page. + continue + all_vertices[key] = vertex + all_edges.extend(item.edges) + if item.issue_vertex_id in existing_hashes: + self._delete_current_edges(item.issue_vertex_id) + for comment in item.comments: + if comment.comment_vertex_id in existing_comment_hashes: + self._delete_current_comment_edges( + comment.comment_vertex_id + ) + + # Do not advance a changed issue's content hash until its document load + # succeeds. A failed page is therefore safe to replay after restart. + def strip_content_hash(vertex_type: str, vertex_id: str) -> None: + key = (vertex_type, vertex_id) + vertex = all_vertices.get(key) + if vertex is not None: + attributes = dict(vertex.attributes) + attributes.pop("content_hash", None) + all_vertices[key] = VertexRecord( + vertex.vertex_type, + vertex.vertex_id, + attributes, + ) + + for item in changed_documents: + strip_content_hash("JiraIssue", item.issue_vertex_id) + for comment in changed_comments: + strip_content_hash("JiraComment", comment.comment_vertex_id) + + self._upsert_records(list(all_vertices.values()), all_edges) + + documents_to_load: list[MappedIssue | MappedComment] = [ + *changed_documents, + *changed_comments, + ] + if documents_to_load: + if changed_documents: + self._load_documents(changed_documents) + if changed_comments: + self._upsert_comment_chunks(changed_comments) + self._embed_comment_chunks(changed_comments) + for item in changed_documents: + document_id = item.document["doc_id"].lower() + self.conn.upsertEdge( + "Document", + document_id, + "CONTAINS_ENTITY", + "JiraIssue", + item.issue_vertex_id, + ) + self.conn.upsertVertex( + "JiraIssue", + item.issue_vertex_id, + attributes={"content_hash": item.content_hash}, + ) + existing_hashes[item.issue_vertex_id] = item.content_hash + for comment in changed_comments: + self.conn.upsertVertex( + "JiraComment", + comment.comment_vertex_id, + attributes={"content_hash": comment.content_hash}, + ) + existing_comment_hashes[ + comment.comment_vertex_id + ] = comment.content_hash + + return len(documents_to_load), comments_deleted + + def _upsert_records( + self, + vertices: list[VertexRecord], + edges: list[EdgeRecord], + ) -> None: + # Vertices must exist before their edge batches are applied. Keeping + # payloads bounded avoids REST request-size failures on large projects. + for records, are_edges in ((vertices, False), (edges, True)): + for start in range(0, len(records), UPSERT_BATCH_SIZE): + batch = records[start : start + UPSERT_BATCH_SIZE] + payload = _upsert_payload( + [] if are_edges else batch, + batch if are_edges else [], + ) + result = self.conn.upsertData(json.dumps(payload)) + if isinstance(result, dict) and ( + result.get("skipped_vertices") or result.get("skipped_edges") + ): + raise RuntimeError( + "TigerGraph rejected part of the Jira graph upsert" + ) + + def _changed_documents( + self, + mapped: list[MappedIssue], + existing: dict[str, str], + ) -> list[MappedIssue]: + changed: list[MappedIssue] = [] + for item in mapped: + if existing.get(item.issue_vertex_id) == item.content_hash: + continue + if item.issue_vertex_id in existing: + self._delete_document_chunks(item.document["doc_id"].lower()) + changed.append(item) + return changed + + def _changed_comments( + self, + comments: list[MappedComment], + existing: dict[str, str], + ) -> list[MappedComment]: + changed: list[MappedComment] = [] + for comment in comments: + if existing.get(comment.comment_vertex_id) == comment.content_hash: + continue + if comment.comment_vertex_id in existing: + self._delete_comment_content( + comment.issue_vertex_id, + comment.comment_vertex_id, + ) + changed.append(comment) + return changed + + def _issue_comment_ids(self, issue_vertex_id: str) -> set[str]: + try: + edges = self.conn.getEdges( + "JiraIssue", + issue_vertex_id, + f"reverse_{JIRA_COMMENT_ISSUE_EDGE}", + ) or [] + except Exception as exc: + if "is not a valid vertex id" in str(exc): + return set() + raise + return { + str(edge.get("to_id")) + for edge in edges + if edge.get("to_id") is not None + } + + def _reconcile_issue_comments( + self, + issue: MappedIssue, + existing_hashes: dict[str, str], + ) -> int: + current_ids = { + comment.comment_vertex_id for comment in issue.comments + } + stale_ids = self._issue_comment_ids(issue.issue_vertex_id) - current_ids + for comment_vertex_id in stale_ids: + self._delete_comment( + issue.issue_vertex_id, + comment_vertex_id, + ) + existing_hashes.pop(comment_vertex_id, None) + return len(stale_ids) + + def _delete_current_edges(self, issue_vertex_id: str) -> None: + for edge_type in CURRENT_STATE_EDGES: + self.conn.delEdges("JiraIssue", issue_vertex_id, edge_type) + + def _delete_current_comment_edges( + self, + comment_vertex_id: str, + ) -> None: + for edge_type in COMMENT_STATE_EDGES: + self.conn.delEdges("JiraComment", comment_vertex_id, edge_type) + + def _delete_comment( + self, + issue_vertex_id: str, + comment_vertex_id: str, + ) -> None: + self._delete_comment_content(issue_vertex_id, comment_vertex_id) + self.conn.delVerticesById("JiraComment", [comment_vertex_id]) + + def _comment_chunk_ids(self, comment_vertex_id: str) -> list[str]: + try: + edges = self.conn.getEdges( + "JiraComment", + comment_vertex_id, + "reverse_CONTAINS_ENTITY", + ) or [] + except Exception as exc: + if "is not a valid vertex id" in str(exc): + return [] + raise + return [ + str(edge["to_id"]) + for edge in edges + if edge.get("to_type") == "DocumentChunk" + and edge.get("to_id") is not None + ] + + def _delete_comment_content( + self, + issue_vertex_id: str, + comment_vertex_id: str, + ) -> None: + direct_chunk_ids = self._comment_chunk_ids(comment_vertex_id) + if direct_chunk_ids: + get_embedding_store( + graphname=self.graphname + ).remove_embeddings(ids=direct_chunk_ids) + self.conn.delVerticesById("DocumentChunk", direct_chunk_ids) + self.conn.delVerticesById("Content", direct_chunk_ids) + + # Remove records created by the former Document -> ECC chunking path. + comment_id = comment_vertex_id.rsplit(":comment:", 1)[-1] + legacy_document_id = f"{issue_vertex_id}:comment-doc:{comment_id}" + self._delete_document_chunks(legacy_document_id) + self.conn.delVerticesById("Document", [legacy_document_id]) + self.conn.delVerticesById("Content", [legacy_document_id]) + + def _delete_document_chunks(self, document_id: str) -> None: + try: + edges = self.conn.getEdges("Document", document_id, "HAS_CHILD") or [] + except Exception as exc: + if "is not a valid vertex id" in str(exc): + return + raise + chunk_ids = [ + str(edge.get("to_id")) + for edge in edges + if edge.get("to_id") is not None + ] + if not chunk_ids: + return + get_embedding_store(graphname=self.graphname).remove_embeddings(ids=chunk_ids) + self.conn.delVerticesById("DocumentChunk", chunk_ids) + self.conn.delVerticesById("Content", chunk_ids) + + def _load_documents( + self, + mapped: list[MappedIssue], + ) -> None: + with tempfile.TemporaryDirectory( + prefix=f"jira-{self.graphname}-{self.source.id}-" + ) as directory: + path = Path(directory) / "documents.jsonl" + with path.open("w", encoding="utf-8") as stream: + for item in mapped: + stream.write(json.dumps(item.document, ensure_ascii=False) + "\n") + result = supportai.ingest( + self.graphname, + LoadingInfo( + load_job_id="load_documents_content_json", + data_source_id={ + "data_source": "server", + "data_source_id": "DocumentContent", + "data_path": directory, + }, + file_path="documents.jsonl", + ), + self.conn, + ) + failed_files = result.get("failed_files") if isinstance(result, dict) else None + if failed_files: + raise RuntimeError( + "Jira document loading failed; retry the sync to resume" + ) + + def _upsert_comment_chunks( + self, + comments: list[MappedComment], + ) -> None: + epoch_added = int(time.time()) + vertices: list[VertexRecord] = [] + edges: list[EdgeRecord] = [] + for comment in comments: + for chunk in comment.chunks: + vertices.extend( + [ + VertexRecord( + "DocumentChunk", + chunk.chunk_id, + { + "idx": chunk.index, + "epoch_added": epoch_added, + "epoch_processing": 0, + "epoch_processed": 0, + }, + ), + VertexRecord( + "Content", + chunk.chunk_id, + { + "ctype": "jira_comment", + "text": chunk.text, + "epoch_added": epoch_added, + }, + ), + ] + ) + edges.extend( + [ + EdgeRecord( + "DocumentChunk", + chunk.chunk_id, + "HAS_CONTENT", + "Content", + chunk.chunk_id, + ), + EdgeRecord( + "DocumentChunk", + chunk.chunk_id, + "CONTAINS_ENTITY", + "JiraComment", + comment.comment_vertex_id, + ), + EdgeRecord( + "DocumentChunk", + chunk.chunk_id, + "CONTAINS_ENTITY", + "JiraIssue", + comment.issue_vertex_id, + ), + ] + ) + self._upsert_records(vertices, edges) + + def _embed_comment_chunks( + self, + comments: list[MappedComment], + ) -> None: + chunks = [ + chunk + for comment in comments + for chunk in comment.chunks + ] + if not chunks: + return + store = get_embedding_store(graphname=self.graphname) + + async def embed_batches() -> None: + for start in range(0, len(chunks), EMBEDDING_BATCH_SIZE): + batch = chunks[start : start + EMBEDDING_BATCH_SIZE] + await store.aadd_embeddings( + [(chunk.text, []) for chunk in batch], + [ + { + "vertex_id": ( + chunk.chunk_id, + "DocumentChunk", + ) + } + for chunk in batch + ], + ) + + asyncio.run(embed_batches()) + processed_at = int(time.time()) + for chunk in chunks: + self.conn.upsertVertex( + "DocumentChunk", + chunk.chunk_id, + attributes={"epoch_processed": processed_at}, + ) diff --git a/graphrag/app/main.py b/graphrag/app/main.py index cfa79aaa..30c01eb4 100644 --- a/graphrag/app/main.py +++ b/graphrag/app/main.py @@ -99,6 +99,7 @@ async def lifespan(app: FastAPI): app.include_router(routers.queryai_router, prefix=PATH_PREFIX) app.include_router(routers.ui_router, prefix=PATH_PREFIX) app.include_router(routers.mcp_servers_router, prefix=PATH_PREFIX) +app.include_router(routers.data_sources_router, prefix=PATH_PREFIX) excluded_metrics_paths = ("/docs", "/openapi.json", "/metrics") diff --git a/graphrag/app/routers/__init__.py b/graphrag/app/routers/__init__.py index 054cc955..d2785067 100644 --- a/graphrag/app/routers/__init__.py +++ b/graphrag/app/routers/__init__.py @@ -4,3 +4,4 @@ from .supportai import router as supportai_router from .ui import router as ui_router from .mcp_servers import router as mcp_servers_router +from .data_sources import router as data_sources_router diff --git a/graphrag/app/routers/data_sources.py b/graphrag/app/routers/data_sources.py new file mode 100644 index 00000000..8be0fa64 --- /dev/null +++ b/graphrag/app/routers/data_sources.py @@ -0,0 +1,578 @@ +"""Graph-scoped enterprise data-source management.""" + +from __future__ import annotations + +import logging +import threading +import uuid +from datetime import datetime, timezone +from typing import Annotated, Any + +from fastapi import APIRouter, BackgroundTasks, Depends, HTTPException, Request +from fastapi.security import HTTPBasicCredentials + +from common.config import validate_graphname +from common.db.schema_utils import apply_proposal +from common.utils.graph_locks import ( + acquire_graph_lock, + get_current_operation, + release_graph_lock, +) +from connectors.jira.client import JiraAPIError, JiraCloudClient +from connectors.jira.config import PROJECT_KEY_RE, JiraDataSource, JiraScope +from connectors.jira.schema import ( + jira_schema_proposal, + jira_schema_status, +) +from connectors.jira.state import JiraSourceStore +from connectors.jira.sync import JiraSyncService + +logger = logging.getLogger(__name__) +router = APIRouter(tags=["Data Sources"]) +route_prefix = "/ui" +store = JiraSourceStore() + +_sync_state: dict[str, dict[str, Any]] = {} +_sync_state_lock = threading.Lock() +_jira_tokens: dict[tuple[str, str], str] = {} +_jira_tokens_lock = threading.Lock() + + +def _ui_basic_auth(): + from routers.ui import ui_basic_auth + + return ui_basic_auth + + +def _require_access( + graphname: str, + auth: tuple[list[str], HTTPBasicCredentials], +) -> HTTPBasicCredentials: + validate_graphname(graphname) + graphs, credentials = auth + if graphname not in graphs: + raise HTTPException(status_code=403, detail="Graph access is required.") + from routers.ui import _require_roles + + _require_roles(credentials, {"superuser", "globaldesigner"}) + return credentials + + +def _source_or_404(graphname: str, source_id: str) -> JiraDataSource: + try: + return store.get(graphname, source_id) + except KeyError: + raise HTTPException(status_code=404, detail="Data source not found.") + + +def _source_with_credentials( + graphname: str, + source_id: str, +) -> JiraDataSource: + source = _source_or_404(graphname, source_id) + with _jira_tokens_lock: + token = _jira_tokens.get((graphname, source_id)) + token = token or source.connection.api_token + if not token: + raise HTTPException( + status_code=409, + detail=( + "Jira API token is not available. Test the connection again " + "and provide the token." + ), + ) + resolved = source.model_copy(deep=True) + resolved.connection.api_token = token + return resolved + + +def _jira_error(exc: JiraAPIError) -> HTTPException: + if exc.status_code == 404: + detail = ( + "Jira REST API was not found at the configured Site URL. " + "Use the Jira tenant base URL, for example " + "https://your-company.atlassian.net." + ) + elif exc.status_code == 401: + detail = ( + "Jira rejected the credentials. Use the Atlassian account email " + "that owns the API token and a valid Jira API token." + ) + elif exc.status_code == 403: + detail = ( + "Jira accepted the credentials but denied access. Verify that the " + "account can access Jira and browse the required projects." + ) + else: + detail = str(exc) + status_code = 400 if 400 <= exc.status_code < 500 else 502 + return HTTPException(status_code=status_code, detail=detail) + + +def _connection(request: Request, graphname: str): + authorization = request.headers.get("Authorization") + if not authorization: + raise HTTPException(status_code=401, detail="Missing Authorization header.") + from routers.ui import ws_basic_auth + + _, conn = ws_basic_auth(authorization, graphname) + return conn + + +def _set_sync_state(sync_run_id: str, **updates: Any) -> None: + with _sync_state_lock: + current = dict(_sync_state.get(sync_run_id) or {}) + current.update(updates) + _sync_state[sync_run_id] = current + + +@router.get(f"{route_prefix}/{{graphname}}/data-sources") +def list_data_sources( + graphname: str, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + return {"sources": store.list(graphname)} + + +@router.put(f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}") +def save_data_source( + graphname: str, + source_id: str, + source: JiraDataSource, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + if source.id != source_id: + raise HTTPException( + status_code=400, + detail="Path source id must match the request body.", + ) + try: + saved = store.upsert(graphname, source) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + return {"source": store.redact(saved)} + + +@router.delete(f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}") +def delete_data_source( + graphname: str, + source_id: str, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + try: + store.delete(graphname, source_id) + except KeyError: + raise HTTPException(status_code=404, detail="Data source not found.") + with _jira_tokens_lock: + _jira_tokens.pop((graphname, source_id), None) + return {"status": "deleted"} + + +@router.post(f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}/test") +def test_data_source( + graphname: str, + source_id: str, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], + candidate: JiraDataSource | None = None, +): + _require_access(graphname, auth) + if candidate is None or not candidate.connection.api_token.strip(): + raise HTTPException( + status_code=400, + detail="Enter the Jira API token before testing the connection.", + ) + if candidate.id != source_id: + raise HTTPException( + status_code=400, + detail="Path source id must match the request body.", + ) + token = candidate.connection.api_token.strip() + try: + with JiraCloudClient(candidate) as client: + account = client.myself() + cloud_id = candidate.connection.cloud_id or client.cloud_id() + except JiraAPIError as exc: + raise _jira_error(exc) + try: + source = store.upsert(graphname, candidate) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) + with _jira_tokens_lock: + _jira_tokens[(graphname, source_id)] = token + source.connection.cloud_id = cloud_id + source.sync.last_tested_at = datetime.now(timezone.utc) + store.update_runtime_state(graphname, source) + return { + "status": "connected", + "account": { + "account_id": account.get("accountId"), + "display_name": account.get("displayName"), + }, + "cloud_id": cloud_id, + "tested_at": source.sync.last_tested_at.isoformat(), + "source": store.redact(source), + } + + +@router.get(f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}/projects") +def list_jira_projects( + graphname: str, + source_id: str, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], + project_key: str | None = None, +): + _require_access(graphname, auth) + source = _source_with_credentials(graphname, source_id) + if source.sync.last_tested_at is None: + raise HTTPException( + status_code=409, + detail="Test this Jira connection before loading projects.", + ) + normalized_project_key = project_key.strip().upper() if project_key else None + if normalized_project_key and not PROJECT_KEY_RE.fullmatch( + normalized_project_key + ): + raise HTTPException( + status_code=400, + detail=( + "Project key must start with a letter and contain only letters, " + "numbers, or underscores." + ), + ) + try: + with JiraCloudClient(source) as client: + projects = ( + [client.project(normalized_project_key)] + if normalized_project_key + else client.projects() + ) + except JiraAPIError as exc: + if normalized_project_key and exc.status_code == 404: + raise HTTPException( + status_code=400, + detail=( + f"Jira project {normalized_project_key} was not found or " + "is not visible to this account." + ), + ) + raise _jira_error(exc) + return { + "projects": [ + { + "id": project.get("id"), + "key": project.get("key"), + "name": project.get("name"), + } + for project in projects + ] + } + + +@router.post( + f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}/issues/count" +) +def preview_jira_issue_count( + graphname: str, + source_id: str, + scope: JiraScope, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + """Estimate how many Jira issues match the unsaved project scope.""" + _require_access(graphname, auth) + if not scope.project_keys: + raise HTTPException( + status_code=400, + detail="Select at least one Jira project before previewing tickets.", + ) + source = _source_with_credentials(graphname, source_id) + if source.sync.last_tested_at is None: + raise HTTPException( + status_code=409, + detail="Test this Jira connection before previewing tickets.", + ) + preview_source = source.model_copy(deep=True) + preview_source.scope = scope + try: + with JiraCloudClient(preview_source) as client: + count = client.approximate_issue_count() + except JiraAPIError as exc: + raise _jira_error(exc) + return {"count": count, "approximate": True} + + +@router.get(f"{route_prefix}/{{graphname}}/data-sources/jira/schema") +def get_jira_schema_status( + graphname: str, + request: Request, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + conn = _connection(request, graphname) + return jira_schema_status(conn) + + +@router.post(f"{route_prefix}/{{graphname}}/data-sources/jira/schema/install") +def install_jira_schema( + graphname: str, + request: Request, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + sources = store.load(graphname).sources + if not sources: + raise HTTPException( + status_code=409, + detail="Save a Jira connection before installing its schema.", + ) + if not any(source.sync.last_tested_at for source in sources): + raise HTTPException( + status_code=409, + detail="Test a Jira connection before installing its schema.", + ) + if not any( + source.sync.last_tested_at and source.scope.project_keys + for source in sources + ): + raise HTTPException( + status_code=409, + detail="Select and save at least one Jira project before installing its schema.", + ) + operation = "install_jira_schema" + if not acquire_graph_lock(graphname, operation): + current = get_current_operation(graphname) or "another operation" + raise HTTPException( + status_code=409, + detail=f"Graph '{graphname}' is busy with '{current}'.", + ) + try: + conn = _connection(request, graphname) + before = jira_schema_status(conn) + if before["status"] == "not_initialized": + raise HTTPException( + status_code=409, + detail=( + "Initialize the knowledge graph before installing " + "the Jira connector schema." + ), + ) + if before["status"] == "conflict": + raise HTTPException( + status_code=409, + detail={ + "message": "Existing graph schema conflicts with Jira schema.", + "conflicts": before["conflicts"], + }, + ) + + result = apply_proposal(conn, graphname, jira_schema_proposal()) + if result.get("status") == "error": + raise HTTPException( + status_code=500, + detail=result.get("error") or "Jira schema installation failed.", + ) + retrievers = result.get("retrievers") or {} + if retrievers.get("status") == "error": + raise HTTPException( + status_code=500, + detail=( + "Jira schema was applied, but retriever installation failed: " + f"{retrievers.get('error', 'unknown error')}" + ), + ) + + # The connection used to apply the SCHEMA_CHANGE JOB can retain the + # pre-migration schema snapshot. Verify through a fresh connection so + # a successful migration is not reported as a false 500. + verification_conn = _connection(request, graphname) + after = jira_schema_status(verification_conn) + if after["status"] != "installed": + logger.error( + "Jira schema verification failed for graph %s: %s", + graphname, + after, + ) + raise HTTPException( + status_code=500, + detail={ + "message": ( + "Jira schema installation completed, but verification " + "did not find a compatible schema." + ), + "missing": after.get("missing"), + "conflicts": after.get("conflicts"), + }, + ) + return { + "status": "installed", + "schema": after, + "migration": { + "status": result.get("status"), + "statements": result.get("statements") or [], + "summary": result.get("summary") or {}, + "retrievers": retrievers, + }, + } + finally: + release_graph_lock(graphname, operation) + + +@router.post(f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}/sync") +def start_sync( + graphname: str, + source_id: str, + request: Request, + background_tasks: BackgroundTasks, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + source = _source_with_credentials(graphname, source_id) + if not source.enabled: + raise HTTPException(status_code=409, detail="Jira data source is disabled.") + if source.sync.last_tested_at is None: + raise HTTPException( + status_code=409, + detail="Test this Jira connection before synchronization.", + ) + if not source.scope.project_keys: + raise HTTPException( + status_code=409, + detail="Select at least one Jira project before synchronization.", + ) + authorization = request.headers.get("Authorization") + if not authorization: + raise HTTPException(status_code=401, detail="Missing Authorization header.") + conn = _connection(request, graphname) + schema = jira_schema_status(conn) + if schema["status"] != "installed": + raise HTTPException( + status_code=409, + detail=( + "Jira schema is not installed for this graph. " + "Install it from Data Sources before synchronization." + ), + ) + + operation = f"jira_sync:{source_id}" + if not acquire_graph_lock(graphname, operation): + current = get_current_operation(graphname) or "another operation" + raise HTTPException( + status_code=409, + detail=f"Graph '{graphname}' is busy with '{current}'.", + ) + + run_id = uuid.uuid4().hex + try: + _set_sync_state( + run_id, + run_id=run_id, + graphname=graphname, + source_id=source_id, + status="queued", + started_at=datetime.now(timezone.utc).isoformat(), + ) + except Exception: + release_graph_lock(graphname, operation) + raise + + def run() -> None: + result: dict[str, Any] | None = None + error: Exception | None = None + try: + _set_sync_state(run_id, status="running") + from routers.ui import ws_basic_auth + + _, conn = ws_basic_auth(authorization, graphname) + source = _source_with_credentials(graphname, source_id) + result = JiraSyncService( + graphname, + source, + conn, + store=store, + ).run() + except Exception as exc: + error = exc + logger.exception( + "Jira data-source sync failed graph=%s source=%s", + graphname, + source_id, + ) + finally: + # Release before publishing a terminal state. The UI starts the + # GraphRAG rebuild as soon as it observes "completed". + release_graph_lock(graphname, operation) + + if error is not None: + _set_sync_state( + run_id, + status="failed", + completed_at=datetime.now(timezone.utc).isoformat(), + error=str(error)[:1000], + ) + else: + _set_sync_state( + run_id, + status="completed", + completed_at=datetime.now(timezone.utc).isoformat(), + result=result, + ) + + try: + background_tasks.add_task(run) + except Exception: + release_graph_lock(graphname, operation) + raise + return {"status": "submitted", "run_id": run_id} + + +@router.get( + f"{route_prefix}/{{graphname}}/data-sources/{{source_id}}/sync/{{run_id}}" +) +def get_sync_status( + graphname: str, + source_id: str, + run_id: str, + auth: Annotated[ + tuple[list[str], HTTPBasicCredentials], + Depends(_ui_basic_auth()), + ], +): + _require_access(graphname, auth) + with _sync_state_lock: + state = dict(_sync_state.get(run_id) or {}) + if ( + not state + or state.get("graphname") != graphname + or state.get("source_id") != source_id + ): + raise HTTPException(status_code=404, detail="Sync run not found.") + return state diff --git a/graphrag/app/routers/ui.py b/graphrag/app/routers/ui.py index a482e90f..71602acf 100644 --- a/graphrag/app/routers/ui.py +++ b/graphrag/app/routers/ui.py @@ -2400,12 +2400,18 @@ def get_rebuild_status( """ Check if a GraphRAG rebuild is currently in progress for the specified graph. Returns the current status without triggering a new rebuild. + Also surfaces any blocking graph-level operation (e.g. jira_sync) so the + UI can show a meaningful message before the user attempts a rebuild. Uses HTTP Basic Authentication to get credentials. """ # Extract credentials from the dependency creds = creds[1] auth_header = _ecc_auth_header(creds) + # Check the graph-level lock first — a jira_sync or other operation may be + # in flight and is about to trigger a rebuild even if ECC hasn't started yet. + current_op = get_current_operation(graphname) + try: ecc_status_url = ( graphrag_config.get("ecc", "http://graphrag-ecc:8001") @@ -2420,13 +2426,21 @@ def get_rebuild_status( ) if response.status_code == 200: - return response.json() + payload = response.json() + # Merge graph-lock info so the UI can detect a pending rebuild + # triggered by a data-source sync that hasn't handed off to ECC yet. + payload["current_operation"] = current_op + if current_op and not payload.get("is_running"): + payload["is_running"] = True + payload["status"] = "pending" + return payload else: LogWriter.warning(f"ECC status check returned {response.status_code}") return { "graphname": graphname, - "is_running": False, - "status": "unknown", + "is_running": bool(current_op), + "status": "pending" if current_op else "unknown", + "current_operation": current_op, "error": f"ECC service returned status {response.status_code}" } except httpx.TimeoutException as e: @@ -2439,6 +2453,7 @@ def get_rebuild_status( "graphname": graphname, "is_running": True, "status": cached.get("status", "unknown"), + "current_operation": current_op, "error": "ECC is busy processing, status check timed out. Rebuild likely still in progress." } except Exception as e: diff --git a/graphrag/app/tools/generate_cypher.py b/graphrag/app/tools/generate_cypher.py index d9199de4..a8d2d362 100644 --- a/graphrag/app/tools/generate_cypher.py +++ b/graphrag/app/tools/generate_cypher.py @@ -13,6 +13,7 @@ # along with this program. If not, see . import logging +import re from typing import Iterable from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import PromptTemplate @@ -27,6 +28,22 @@ logger = logging.getLogger(__name__) +def _clean_cypher_output(value: str) -> str: + """Remove an optional Markdown language fence without altering the query.""" + text = value.strip() + fenced = re.fullmatch( + r"```(?:open)?cypher\s*\n?(.*?)\n?```", + text, + flags=re.IGNORECASE | re.DOTALL, + ) + if fenced: + return fenced.group(1).strip() + lines = text.splitlines() + if lines and lines[0].strip().casefold() in {"cypher", "opencypher"}: + return "\n".join(lines[1:]).strip() + return text + + class GenerateCypher(BaseTool): """GenerateCypher Tool. Tool to generate and execute the appropriate Cypher query for the question. @@ -94,11 +111,12 @@ def generate_cypher(self, question: str, history: Iterable[str]) -> str: schema = self._generate_schema_rep() logger.debug_pii("Prompt to LLM:\n" + PROMPT.invoke({"question": question, "schema": schema, "history": history}).to_string()) - out = self.llm.invoke_with_parser( + raw = self.llm.invoke_with_parser( PROMPT, StrOutputParser(), {"question": question, "schema": schema, "history": history}, caller_name="generate_cypher", - ).strip("```cypher").strip("```").strip() + ) + out = _clean_cypher_output(raw) # Validate the LLM output looks like a Cypher query out_upper = out.upper() diff --git a/graphrag/app/tools/generate_function.py b/graphrag/app/tools/generate_function.py index 073cfe38..d6ec2c6d 100644 --- a/graphrag/app/tools/generate_function.py +++ b/graphrag/app/tools/generate_function.py @@ -33,6 +33,7 @@ InvalidFunctionCallException, MapQuestionToSchemaException, NoDocumentsFoundException, + SAFE_READ_FUNCTIONS, validate_function_call, validate_schema, ) @@ -171,12 +172,15 @@ def _run( # Prioritize pyTigerGraph docs over custom docs docs = pytg_docs + custom_docs - valid_function_calls = [ + registered_function_calls = [ x["function_header"] for x in self.embedding_store.list_registered_documents( output_fields=["function_header"] ) ] + valid_function_calls = list( + set(registered_function_calls) | SAFE_READ_FUNCTIONS + ) if len(docs) == 0: LogWriter.warning(f"request_id={req_id_cv.get()} WARN no documents found") diff --git a/graphrag/app/tools/graphrag_tools.py b/graphrag/app/tools/graphrag_tools.py index ca4b9f93..5d388b5b 100644 --- a/graphrag/app/tools/graphrag_tools.py +++ b/graphrag/app/tools/graphrag_tools.py @@ -28,6 +28,7 @@ from __future__ import annotations +import json import logging from dataclasses import dataclass, field from typing import Any, Callable, Dict, Optional @@ -85,7 +86,14 @@ def _empty(summary: str) -> dict: def _result_is_empty(result: Any) -> bool: if result is None: return True - if isinstance(result, (list, dict, str)) and len(result) == 0: + if isinstance(result, str): + if not result.strip(): + return True + try: + return _result_is_empty(json.loads(result)) + except (TypeError, ValueError): + return False + if isinstance(result, (list, dict)) and len(result) == 0: return True return False @@ -167,7 +175,6 @@ def structural_retrieve(ctx: GraphRAGToolContext, question: str) -> dict: def _cypher_retrieve(ctx: GraphRAGToolContext, question: str) -> dict: - import json ctx.emit("Generating a graph query") gen_history: list = [] for i in range(3): @@ -176,13 +183,22 @@ def _cypher_retrieve(ctx: GraphRAGToolContext, question: str) -> dict: except ValueError as exc: gen_history.append(f"{i}: Error: {exc}\n") continue - response = ctx.conn.gsql(cypher) + try: + response = ctx.conn.gsql(cypher) + except Exception as exc: + logger.warning("openCypher execution failed: %s", exc) + gen_history.append(f"{i}: {cypher}\n\tExecution error: {exc}\n") + continue json_str = "\n".join(response.split("\n")[1:]) try: parsed = json.loads(json_str) except Exception: gen_history.append(f"{i}: {cypher}\n\tError: {json_str}\n") continue + if parsed.get("error"): + message = parsed.get("message") or parsed + gen_history.append(f"{i}: {cypher}\n\tError: {message}\n") + continue rows = parsed.get("results", [{}]) first = rows[0] if rows else None if not _result_is_empty(first): diff --git a/graphrag/app/tools/validation_utils.py b/graphrag/app/tools/validation_utils.py index 6b18b351..416a4569 100644 --- a/graphrag/app/tools/validation_utils.py +++ b/graphrag/app/tools/validation_utils.py @@ -23,6 +23,24 @@ class InvalidFunctionCallException(Exception): pass +# Built-in pyTigerGraph reads remain available when an embedding-store backend +# cannot enumerate its registered function documents. Keep this list read-only: +# generated calls execute with the request's TigerGraph connection. +SAFE_READ_FUNCTIONS = frozenset({ + "getEdgeCount", + "getEdgeType", + "getEdgeTypes", + "getEdges", + "getSchema", + "getVertexCount", + "getVertexStats", + "getVertexType", + "getVertexTypes", + "getVertices", + "getVerticesById", +}) + + def validate_schema(conn, v_types, e_types, v_attrs, e_attrs): LogWriter.info(f"request_id={req_id_cv.get()} ENTRY validate_schema()") if v_types: diff --git a/graphrag/tests/test_generate_cypher.py b/graphrag/tests/test_generate_cypher.py new file mode 100644 index 00000000..7e6f7a00 --- /dev/null +++ b/graphrag/tests/test_generate_cypher.py @@ -0,0 +1,20 @@ +"""Regression tests for generated openCypher normalization.""" + +from tools.generate_cypher import _clean_cypher_output + + +def test_removes_cypher_markdown_fence(): + assert _clean_cypher_output("```cypher\nMATCH (n) RETURN n\n```") == ( + "MATCH (n) RETURN n" + ) + + +def test_removes_opencypher_markdown_fence(): + assert _clean_cypher_output("```opencypher\nMATCH (n) RETURN n\n```") == ( + "MATCH (n) RETURN n" + ) + + +def test_does_not_strip_query_characters(): + query = 'MATCH (issue:JiraIssue) WHERE issue.issue_key = "GML-2191" RETURN issue' + assert _clean_cypher_output(query) == query diff --git a/graphrag/tests/test_jira_connector.py b/graphrag/tests/test_jira_connector.py new file mode 100644 index 00000000..018cf804 --- /dev/null +++ b/graphrag/tests/test_jira_connector.py @@ -0,0 +1,1022 @@ +from __future__ import annotations + +import json +from datetime import date, datetime, timezone + +import httpx +import pytest + +from connectors.jira.adf import adf_to_markdown +from connectors.jira.client import JiraCloudClient +from connectors.jira.config import JiraDataSource +from connectors.jira.mapper import JiraIssueMapper +from connectors.jira.schema import ( + JIRA_ASSIGNEE_EDGE, + JIRA_COMMENT_AFTER_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_LINK_EDGE, + JIRA_PARENT_EDGE, + JIRA_PROJECT_EDGE, + JIRA_REPORTER_EDGE, + jira_schema_proposal, + jira_schema_status, +) +from connectors.jira.state import JiraSourceStore +from connectors.jira.sync import JiraSyncService + + +def test_sync_state_records_public_run_id_without_argument_collision(): + from routers import data_sources + + run_id = "test-run-id" + try: + data_sources._set_sync_state( + run_id, + run_id=run_id, + status="queued", + ) + assert data_sources._sync_state[run_id] == { + "run_id": run_id, + "status": "queued", + } + finally: + with data_sources._sync_state_lock: + data_sources._sync_state.pop(run_id, None) + + +def source(**overrides) -> JiraDataSource: + payload = { + "id": "jira-acme", + "display_name": "Acme Jira", + "connection": { + "site_url": "https://acme.atlassian.net/", + "email": "svc@example.com", + "api_token": "secret", + }, + "scope": { + "project_keys": ["pay"], + "include_comments": True, + }, + } + payload.update(overrides) + return JiraDataSource.model_validate(payload) + + +def test_source_config_normalizes_url_and_projects(): + config = source() + assert config.connection.site_url == "https://acme.atlassian.net" + assert config.scope.project_keys == ["PAY"] + + +def test_store_rejects_atlassian_service_url(tmp_path): + store = JiraSourceStore(str(tmp_path)) + with pytest.raises(ValueError, match="Jira tenant URL"): + store.upsert( + "TestGraph", + source( + connection={ + "site_url": "https://graphql.atlassian.net", + "email": "svc@example.com", + "api_token": "secret", + } + ), + ) + + +def test_source_config_allows_connection_draft_without_scope(): + config = source(scope={"project_keys": []}) + assert config.scope.project_keys == [] + + +def test_adf_to_markdown_preserves_structure_and_links(): + adf = { + "type": "doc", + "version": 1, + "content": [ + { + "type": "heading", + "attrs": {"level": 2}, + "content": [{"type": "text", "text": "Decision"}], + }, + { + "type": "paragraph", + "content": [ + {"type": "text", "text": "Use "}, + { + "type": "text", + "text": "GraphRAG", + "marks": [ + {"type": "strong"}, + { + "type": "link", + "attrs": {"href": "https://example.com"}, + }, + ], + }, + ], + }, + ], + } + markdown = adf_to_markdown(adf) + assert "## Decision" in markdown + assert "[**GraphRAG**](https://example.com)" in markdown + + +def test_enhanced_search_uses_next_page_token_and_completes_comments(): + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + if request.url.path.endswith("/search/jql"): + body = json.loads(request.content) + if body.get("nextPageToken") == "page-2": + return httpx.Response(200, json={"issues": [{"id": "2", "fields": {}}]}) + return httpx.Response( + 200, + json={ + "issues": [ + { + "id": "1", + "fields": { + "comment": { + "total": 2, + "comments": [{"id": "10"}], + } + }, + } + ], + "nextPageToken": "page-2", + }, + ) + if request.url.path.endswith("/issue/1/comment"): + return httpx.Response( + 200, + json={"total": 2, "comments": [{"id": "11"}]}, + ) + raise AssertionError(f"unexpected request: {request.url}") + + http_client = httpx.Client( + base_url="https://acme.atlassian.net", + transport=httpx.MockTransport(handler), + ) + issues = list(JiraCloudClient(source(), client=http_client).iter_issues()) + assert [issue["id"] for issue in issues] == ["1", "2"] + assert len(issues[0]["fields"]["comment"]["comments"]) == 2 + search_bodies = [ + json.loads(request.content) + for request in requests + if request.url.path.endswith("/search/jql") + ] + assert search_bodies[1]["nextPageToken"] == "page-2" + assert "/rest/api/3/search/jql" in str(requests[0].url) + + +def test_project_loads_one_project_by_key(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + assert request.url.path == "/rest/api/3/project/PAY" + return httpx.Response( + 200, + json={"id": "10001", "key": "PAY", "name": "Payments"}, + ) + + http_client = httpx.Client( + base_url="https://acme.atlassian.net", + transport=httpx.MockTransport(handler), + ) + project = JiraCloudClient(source(), client=http_client).project("PAY") + assert project == {"id": "10001", "key": "PAY", "name": "Payments"} + + +def test_project_endpoint_loads_only_requested_project(monkeypatch): + from routers import data_sources + + configured = source( + sync={"last_tested_at": datetime.now(timezone.utc)} + ) + + class Client: + def __init__(self, jira_source): + assert jira_source == configured + + def __enter__(self): + return self + + def __exit__(self, *args): + return None + + def project(self, project_key): + assert project_key == "PAY" + return {"id": "10001", "key": "PAY", "name": "Payments"} + + monkeypatch.setattr(data_sources, "_require_access", lambda *args: None) + monkeypatch.setattr( + data_sources, "_source_or_404", lambda *args: configured + ) + monkeypatch.setattr(data_sources, "JiraCloudClient", Client) + + result = data_sources.list_jira_projects( + "TestGraph", + "jira-acme", + auth=(["TestGraph"], object()), + project_key=" pay ", + ) + assert result == { + "projects": [{"id": "10001", "key": "PAY", "name": "Payments"}] + } + + +def test_search_applies_structured_scope_filters(): + request_bodies: list[dict] = [] + + def handler(request: httpx.Request) -> httpx.Response: + request_bodies.append(json.loads(request.content)) + return httpx.Response(200, json={"issues": []}) + + configured = source( + scope={ + "project_keys": ["PAY"], + "created_after": "2026-01-01", + "updated_after": "2026-06-01", + "status_categories": ["new", "indeterminate"], + "include_comments": True, + } + ) + http_client = httpx.Client( + base_url="https://acme.atlassian.net", + transport=httpx.MockTransport(handler), + ) + assert list( + JiraCloudClient(configured, client=http_client).iter_issues() + ) == [] + assert request_bodies[0]["jql"] == ( + 'project in (PAY) AND created >= "2026-01-01" ' + 'AND updated >= "2026-06-01" ' + 'AND statusCategory in ("To Do", "In Progress") ' + "ORDER BY updated ASC, key ASC" + ) + + +def test_approximate_count_uses_scope_without_ordering(): + def handler(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/rest/api/3/search/approximate-count" + body = json.loads(request.content) + assert body["jql"] == ( + 'project in (PAY) AND created >= "2026-01-01" ' + 'AND statusCategory in ("To Do", "In Progress")' + ) + assert "ORDER BY" not in body["jql"] + return httpx.Response(200, json={"count": 42}) + + configured = source( + scope={ + "project_keys": ["PAY"], + "created_after": "2026-01-01", + "status_categories": ["new", "indeterminate"], + } + ) + http_client = httpx.Client( + base_url="https://acme.atlassian.net", + transport=httpx.MockTransport(handler), + ) + assert ( + JiraCloudClient( + configured, + client=http_client, + ).approximate_issue_count() + == 42 + ) + + +def test_existing_hash_lookup_treats_new_issue_ids_as_missing(tmp_path): + new_id = "jira:cloud-1:issue:100" + existing_id = "jira:cloud-1:issue:existing" + + class Connection: + def getVertices(self, vertex_type, select=""): + assert vertex_type == "JiraIssue" + assert select == "content_hash" + return [ + { + "v_id": existing_id, + "attributes": {"content_hash": "existing-hash"}, + } + ] + + service = JiraSyncService( + "TestGraph", + source(), + Connection(), + store=JiraSourceStore(str(tmp_path)), + client=object(), + ) + assert service._existing_hashes([new_id, existing_id]) == { + existing_id: "existing-hash" + } + + +def test_comment_reconciliation_deletes_only_removed_comments( + tmp_path, + monkeypatch, +): + issue = { + "id": "10422", + "key": "PAY-123", + "fields": { + "summary": "Login timeout", + "project": {"id": "10001", "key": "PAY", "name": "Payments"}, + "comment": { + "comments": [ + { + "id": "9001", + "author": { + "accountId": "ada", + "displayName": "Ada", + }, + "created": "2026-09-18T11:02:00.000+0000", + "body": "Keep this comment.", + } + ] + }, + }, + } + mapped = JiraIssueMapper(source(), "cloud-1").map(issue) + current_id = "jira:cloud-1:comment:9001" + stale_id = "jira:cloud-1:comment:9002" + + class Connection: + def getEdges(self, vertex_type, vertex_id, edge_type): + assert vertex_type == "JiraIssue" + assert vertex_id == mapped.issue_vertex_id + assert edge_type == f"reverse_{JIRA_COMMENT_ISSUE_EDGE}" + return [{"to_id": current_id}, {"to_id": stale_id}] + + service = JiraSyncService( + "TestGraph", + source(), + Connection(), + store=JiraSourceStore(str(tmp_path)), + client=object(), + ) + deleted: list[tuple[str, str]] = [] + monkeypatch.setattr( + service, + "_delete_comment", + lambda issue_id, comment_id: deleted.append( + (issue_id, comment_id) + ), + ) + hashes = {current_id: "current", stale_id: "stale"} + + assert service._reconcile_issue_comments(mapped, hashes) == 1 + assert deleted == [(mapped.issue_vertex_id, stale_id)] + assert hashes == {current_id: "current"} + + +def test_sync_persists_checkpoint_after_each_committed_page( + tmp_path, + monkeypatch, +): + from connectors.jira import sync as sync_module + + store = JiraSourceStore(str(tmp_path)) + configured = store.upsert("TestGraph", source()) + issue = { + "id": "10422", + "key": "PAY-123", + "fields": { + "summary": "Login timeout", + "project": {"id": "10001", "key": "PAY", "name": "Payments"}, + "updated": "2026-09-20T14:03:00.000+0000", + }, + } + + class Client: + def cloud_id(self): + return "cloud-1" + + def iter_issue_pages(self): + yield [issue] + assert store.get( + "TestGraph", + "jira-acme", + ).sync.checkpoint == datetime( + 2026, + 9, + 20, + 14, + 3, + tzinfo=timezone.utc, + ) + raise RuntimeError("simulated shutdown") + + class Connection: + def getVertices(self, vertex_type, select=""): + assert vertex_type in {"Document", "JiraIssue", "JiraComment"} + assert select == ( + "id" if vertex_type == "Document" else "content_hash" + ) + return [] + + monkeypatch.setattr( + sync_module, + "jira_schema_status", + lambda *args: {"status": "installed"}, + ) + service = JiraSyncService( + "TestGraph", + configured, + Connection(), + store=store, + client=Client(), + ) + monkeypatch.setattr(service, "_upsert_issue_page", lambda *args: (0, 0)) + + with pytest.raises(RuntimeError, match="simulated shutdown"): + service.run() + + stored = store.get("TestGraph", "jira-acme") + assert stored.sync.checkpoint == datetime( + 2026, + 9, + 20, + 14, + 3, + tzinfo=timezone.utc, + ) + assert stored.sync.last_completed_at is None + assert stored.sync.last_error == "simulated shutdown" + + +def test_sync_requires_rebuild_when_existing_chunks_lack_embeddings( + tmp_path, + monkeypatch, +): + from connectors.jira import sync as sync_module + + store = JiraSourceStore(str(tmp_path)) + configured = store.upsert("TestGraph", source()) + + class Client: + def cloud_id(self): + return "cloud-1" + + def iter_issue_pages(self): + return iter(()) + + class Connection: + def getVertices(self, vertex_type, select=""): + return [] + + monkeypatch.setattr( + sync_module, + "jira_schema_status", + lambda *args: {"status": "installed"}, + ) + monkeypatch.setattr( + sync_module, + "embedding_coverage", + lambda *args: {"total": 12, "missing": 3}, + ) + + result = JiraSyncService( + "TestGraph", + configured, + Connection(), + store=store, + client=Client(), + ).run() + + assert result["documents_loaded"] == 0 + assert result["missing_chunk_embeddings"] == 3 + assert result["rebuild_required"] is True + + +def test_mapper_writes_small_schema_and_searchable_document(): + issue = { + "id": "10422", + "key": "PAY-123", + "fields": { + "summary": "Login timeout", + "project": {"id": "10001", "key": "PAY", "name": "Payments"}, + "issuetype": {"name": "Bug"}, + "status": { + "name": "In Progress", + "statusCategory": {"key": "indeterminate"}, + }, + "priority": {"name": "High"}, + "assignee": {"accountId": "ada", "displayName": "Ada"}, + "reporter": {"accountId": "grace", "displayName": "Grace"}, + "created": "2026-09-01T10:00:00.000+0000", + "updated": "2026-09-20T14:03:00.000+0000", + "description": { + "type": "doc", + "content": [ + { + "type": "paragraph", + "content": [{"type": "text", "text": "Timeout at checkout"}], + } + ], + }, + "comment": { + "comments": [ + { + "id": "9001", + "author": { + "accountId": "ada", + "displayName": "Ada", + }, + "created": "2026-09-18T11:02:00.000+0000", + "body": "Increase the gateway timeout.", + }, + { + "id": "9002", + "parentId": "9001", + "author": { + "accountId": "grace", + "displayName": "Grace", + }, + "created": "2026-09-18T12:02:00.000+0000", + "body": "The timeout was increased.", + } + ] + }, + "issuelinks": [], + }, + } + mapped = JiraIssueMapper(source(), "cloud-1").map(issue) + assert {vertex.vertex_type for vertex in mapped.vertices} == { + "JiraComment", + "JiraIssue", + "JiraProject", + "JiraUser", + } + assert {edge.edge_type for edge in mapped.edges} == { + JIRA_PROJECT_EDGE, + JIRA_ASSIGNEE_EDGE, + JIRA_REPORTER_EDGE, + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_COMMENT_AFTER_EDGE, + } + assert mapped.document["doc_type"] == "jira" + assert "PAY-123" in mapped.document["content"] + assert "Increase the gateway timeout." not in mapped.document["content"] + assert len(mapped.comments) == 2 + assert len(mapped.comments[0].chunks) == 1 + assert mapped.comments[0].chunks[0].chunk_id.startswith( + "jira:cloud-1:comment:9001:chunk:0:" + ) + assert "Increase the gateway timeout." in mapped.comments[0].chunks[0].text + + +def test_long_jira_comment_drops_log_heavy_blocks_before_chunking(): + log_lines = "\n".join( + f"2026-09-28 12:00:{index:02d} ERROR request failed" + for index in range(80) + ) + issue = { + "id": "10422", + "key": "PAY-123", + "fields": { + "summary": "Login timeout", + "project": {"id": "10001", "key": "PAY", "name": "Payments"}, + "comment": { + "comments": [ + { + "id": "9001", + "author": { + "accountId": "ada", + "displayName": "Ada", + }, + "body": ( + "The gateway failed during checkout.\n\n" + f"```\n{log_lines}\n```\n\n" + "Please inspect the timeout configuration." + ), + } + ] + }, + }, + } + + mapped = JiraIssueMapper(source(), "cloud-1").map(issue) + content = "\n".join( + chunk.text for chunk in mapped.comments[0].chunks + ) + assert "The gateway failed during checkout." in content + assert "Please inspect the timeout configuration." in content + assert "[Log output omitted from search content.]" in content + assert "ERROR request failed" not in content + + +def test_comment_chunks_are_upserted_directly_without_document(tmp_path, monkeypatch): + issue = { + "id": "10422", + "key": "PAY-123", + "fields": { + "summary": "Login timeout", + "project": {"id": "10001", "key": "PAY", "name": "Payments"}, + "comment": { + "comments": [ + { + "id": "9001", + "author": { + "accountId": "ada", + "displayName": "Ada", + }, + "body": "Increase the gateway timeout.", + } + ] + }, + }, + } + comment = JiraIssueMapper(source(), "cloud-1").map(issue).comments[0] + service = JiraSyncService( + "TestGraph", + source(), + object(), + store=JiraSourceStore(str(tmp_path)), + client=object(), + ) + captured: dict = {} + monkeypatch.setattr( + service, + "_upsert_records", + lambda vertices, edges: captured.update( + vertices=vertices, + edges=edges, + ), + ) + + service._upsert_comment_chunks([comment]) + + assert {vertex.vertex_type for vertex in captured["vertices"]} == { + "DocumentChunk", + "Content", + } + assert "Document" not in { + vertex.vertex_type for vertex in captured["vertices"] + } + chunk_id = comment.chunks[0].chunk_id + assert { + (edge.source_type, edge.edge_type, edge.target_type) + for edge in captured["edges"] + } == { + ("DocumentChunk", "HAS_CONTENT", "Content"), + ("DocumentChunk", "CONTAINS_ENTITY", "JiraComment"), + ("DocumentChunk", "CONTAINS_ENTITY", "JiraIssue"), + } + chunk_vertex = next( + vertex + for vertex in captured["vertices"] + if vertex.vertex_type == "DocumentChunk" + ) + assert chunk_vertex.vertex_id == chunk_id + assert chunk_vertex.attributes["epoch_processed"] == 0 + + +def test_comment_chunks_are_embedded_by_existing_store(tmp_path, monkeypatch): + issue = { + "id": "10422", + "key": "PAY-123", + "fields": { + "summary": "Login timeout", + "project": {"id": "10001", "key": "PAY", "name": "Payments"}, + "comment": { + "comments": [ + { + "id": "9001", + "body": "Increase the gateway timeout.", + } + ] + }, + }, + } + comment = JiraIssueMapper(source(), "cloud-1").map(issue).comments[0] + embedded: list[tuple[list, list]] = [] + processed: list[tuple[str, str, dict]] = [] + + class Store: + async def aadd_embeddings(self, embeddings, metadatas): + embedded.append((embeddings, metadatas)) + + class Connection: + def upsertVertex(self, vertex_type, vertex_id, attributes): + processed.append((vertex_type, vertex_id, attributes)) + + monkeypatch.setattr( + "connectors.jira.sync.get_embedding_store", + lambda **kwargs: Store(), + ) + service = JiraSyncService( + "TestGraph", + source(), + Connection(), + store=JiraSourceStore(str(tmp_path)), + client=object(), + ) + + service._embed_comment_chunks([comment]) + + assert embedded == [ + ( + [(comment.chunks[0].text, [])], + [ + { + "vertex_id": ( + comment.chunks[0].chunk_id, + "DocumentChunk", + ) + } + ], + ) + ] + assert processed[0][:2] == ( + "DocumentChunk", + comment.chunks[0].chunk_id, + ) + assert processed[0][2]["epoch_processed"] > 0 + + +def test_comment_cleanup_removes_direct_and_legacy_content(tmp_path, monkeypatch): + direct_chunk_id = "jira:cloud-1:comment:9001:chunk:0:abc" + + class Connection: + def getEdges(self, vertex_type, vertex_id, edge_type): + if vertex_type == "JiraComment": + return [ + { + "to_type": "DocumentChunk", + "to_id": direct_chunk_id, + } + ] + if vertex_type == "Document": + return [{"to_id": "legacy-chunk"}] + raise AssertionError((vertex_type, vertex_id, edge_type)) + + def delVerticesById(self, vertex_type, vertex_ids): + deleted.append((vertex_type, tuple(vertex_ids))) + + deleted: list[tuple[str, tuple[str, ...]]] = [] + service = JiraSyncService( + "TestGraph", + source(), + Connection(), + store=JiraSourceStore(str(tmp_path)), + client=object(), + ) + monkeypatch.setattr( + "connectors.jira.sync.get_embedding_store", + lambda **kwargs: type( + "Store", + (), + {"remove_embeddings": lambda self, ids: None}, + )(), + ) + + service._delete_comment_content( + "jira:cloud-1:issue:10422", + "jira:cloud-1:comment:9001", + ) + + assert ("DocumentChunk", (direct_chunk_id,)) in deleted + assert ("Content", (direct_chunk_id,)) in deleted + assert ("DocumentChunk", ("legacy-chunk",)) in deleted + assert ( + "Document", + ("jira:cloud-1:issue:10422:comment-doc:9001",), + ) in deleted + + +def test_legacy_comment_migration_resets_checkpoint_only_once(tmp_path): + store = JiraSourceStore(str(tmp_path)) + configured = store.upsert("TestGraph", source()) + configured.sync.checkpoint = datetime(2026, 9, 24, tzinfo=timezone.utc) + store.update_runtime_state("TestGraph", configured) + + class Connection: + def getVertices(self, vertex_type, select=""): + assert vertex_type == "Document" + assert select == "id" + return [ + { + "v_id": ( + "jira:cloud-1:issue:10422:" + "comment-doc:9001" + ) + } + ] + + service = JiraSyncService( + "TestGraph", + store.get("TestGraph", "jira-acme"), + Connection(), + store=store, + client=object(), + ) + service._prepare_legacy_comment_migration() + + migrated = store.get("TestGraph", "jira-acme") + assert migrated.sync.checkpoint is None + assert migrated.sync.migrating_legacy_comments is True + + migrated.sync.checkpoint = datetime(2026, 9, 25, tzinfo=timezone.utc) + store.update_runtime_state("TestGraph", migrated) + resumed = JiraSyncService( + "TestGraph", + store.get("TestGraph", "jira-acme"), + Connection(), + store=store, + client=object(), + ) + resumed._prepare_legacy_comment_migration() + assert store.get( + "TestGraph", + "jira-acme", + ).sync.checkpoint == datetime(2026, 9, 25, tzinfo=timezone.utc) + + +def test_schema_is_bounded(): + proposal = jira_schema_proposal() + assert {vertex.name for vertex in proposal.vertices} == { + "JiraComment", + "JiraIssue", + "JiraProject", + "JiraUser", + } + assert {edge.name for edge in proposal.edges} == { + JIRA_PROJECT_EDGE, + JIRA_ASSIGNEE_EDGE, + JIRA_REPORTER_EDGE, + JIRA_PARENT_EDGE, + JIRA_LINK_EDGE, + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_COMMENT_AFTER_EDGE, + } + + +def test_store_only_preserves_manually_configured_token(tmp_path): + store = JiraSourceStore(str(tmp_path)) + original = source() + tested_at = datetime(2026, 9, 24, tzinfo=timezone.utc) + store.upsert("TestGraph", original) + config_path = tmp_path / "TestGraph" / "data_sources.json" + payload = json.loads(config_path.read_text()) + assert "api_token" not in payload["sources"][0]["connection"] + + payload["sources"][0]["connection"]["api_token"] = "configured-secret" + config_path.write_text(json.dumps(payload)) + runtime_source = store.get("TestGraph", "jira-acme") + runtime_source.sync.last_tested_at = tested_at + store.update_runtime_state("TestGraph", runtime_source) + redacted = store.list("TestGraph")[0] + assert redacted["connection"]["api_token"] == "" + + submitted = JiraDataSource.model_validate(redacted) + submitted.display_name = "Renamed" + store.upsert("TestGraph", submitted) + assert ( + store.get("TestGraph", "jira-acme").connection.api_token + == "configured-secret" + ) + assert store.get("TestGraph", "jira-acme").sync.last_tested_at == tested_at + + changed_credentials = store.get("TestGraph", "jira-acme") + changed_credentials.connection.api_token = "new-secret" + store.upsert("TestGraph", changed_credentials) + assert ( + store.get("TestGraph", "jira-acme").connection.api_token + == "configured-secret" + ) + + submitted.display_name = "Renamed again" + store.upsert("TestGraph", submitted) + +def test_store_resets_checkpoint_when_ingestion_filters_change(tmp_path): + store = JiraSourceStore(str(tmp_path)) + store.upsert("TestGraph", source()) + configured = store.get("TestGraph", "jira-acme") + configured.sync.checkpoint = datetime(2026, 9, 24, tzinfo=timezone.utc) + store.update_runtime_state("TestGraph", configured) + + changed = store.get("TestGraph", "jira-acme") + changed.scope.created_after = date(2026, 1, 1) + store.upsert("TestGraph", changed) + + assert store.get("TestGraph", "jira-acme").sync.checkpoint is None + + +class SchemaConnection: + def __init__(self): + proposal = jira_schema_proposal() + self.vertices = { + "Document": {}, + "DocumentChunk": {}, + "Content": {}, + **{ + vertex.name: { + "PrimaryId": {"AttributeName": "id"}, + "Attributes": [ + { + "AttributeName": attribute.name, + "AttributeType": {"Name": attribute.type}, + } + for attribute in vertex.attributes + ], + } + for vertex in proposal.vertices + }, + } + self.edges = { + edge.name: { + "FromVertexTypeName": edge.pairs[0][0], + "ToVertexTypeName": edge.pairs[0][1], + "IsDirected": edge.directed, + "Attributes": [ + { + "AttributeName": attribute.name, + "AttributeType": {"Name": attribute.type}, + } + for attribute in edge.attributes + ], + } + for edge in proposal.edges + } + self.edges["CONTAINS_ENTITY"] = { + "FromVertexTypeName": "*", + "ToVertexTypeName": "*", + "IsDirected": True, + "EdgePairs": [ + {"From": "Document", "To": "JiraIssue"}, + {"From": "DocumentChunk", "To": "JiraIssue"}, + {"From": "Document", "To": "JiraComment"}, + {"From": "DocumentChunk", "To": "JiraComment"}, + ], + } + self.edges["HAS_CHILD"] = { + "FromVertexTypeName": "Document", + "ToVertexTypeName": "DocumentChunk", + "IsDirected": True, + } + self.edges["HAS_CONTENT"] = { + "FromVertexTypeName": "*", + "ToVertexTypeName": "*", + "IsDirected": True, + "EdgePairs": [ + {"From": "Document", "To": "Content"}, + {"From": "DocumentChunk", "To": "Content"}, + ], + } + + def getVertexTypes(self): + return list(self.vertices) + + def getVertexType(self, name): + return self.vertices[name] + + def getEdgeTypes(self): + return list(self.edges) + + def getEdgeType(self, name): + return self.edges[name] + + +def test_schema_status_detects_installed_and_conflicting_schema(): + conn = SchemaConnection() + assert jira_schema_status(conn)["status"] == "installed" + + issue_attributes = conn.vertices["JiraIssue"]["Attributes"] + next( + attribute + for attribute in issue_attributes + if attribute["AttributeName"] == "story_points" + )["AttributeType"]["Name"] = "STRING" + result = jira_schema_status(conn) + assert result["status"] == "conflict" + assert "JiraIssue.story_points must be DOUBLE" in result["conflicts"][0] + + +def test_schema_status_marks_partial_graph_incomplete(): + conn = SchemaConnection() + conn.vertices.pop("JiraComment") + for edge_type in ( + JIRA_COMMENT_ISSUE_EDGE, + JIRA_COMMENT_AUTHOR_EDGE, + JIRA_COMMENT_REPLY_EDGE, + JIRA_COMMENT_AFTER_EDGE, + ): + conn.edges.pop(edge_type) + conn.edges["CONTAINS_ENTITY"]["EdgePairs"] = [ + pair + for pair in conn.edges["CONTAINS_ENTITY"]["EdgePairs"] + if pair["To"] != "JiraComment" + ] + + result = jira_schema_status(conn) + assert result["status"] == "incomplete" diff --git a/graphrag/tests/test_structural_function_validation.py b/graphrag/tests/test_structural_function_validation.py new file mode 100644 index 00000000..495c2c7c --- /dev/null +++ b/graphrag/tests/test_structural_function_validation.py @@ -0,0 +1,36 @@ +"""Regression tests for the built-in structural read-function surface.""" + +import pytest + +from tools.graphrag_tools import _result_is_empty +from tools.validation_utils import ( + InvalidFunctionCallException, + SAFE_READ_FUNCTIONS, + validate_function_call, +) + + +class _Connection: + graphname = "JiraTest" + + def getEndpoints(self, dynamic=True): + return {} + + +def test_get_vertices_is_available_without_registered_function_documents(): + call = 'getVertices("JiraIssue", where=\'issue_key == "GML-2186"\')' + + assert validate_function_call(_Connection(), call, SAFE_READ_FUNCTIONS) == call + + +def test_mutating_function_is_not_in_builtin_read_surface(): + with pytest.raises(InvalidFunctionCallException): + validate_function_call( + _Connection(), + 'upsertVertex("JiraIssue", "x", {})', + SAFE_READ_FUNCTIONS, + ) + + +def test_json_encoded_empty_result_triggers_structural_fallback(): + assert _result_is_empty("[]") From 4cd4c806b78ab3641de1148f2820905627c2f361 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 14:08:04 +0530 Subject: [PATCH 02/19] GML-2211: Move example data_sources.json to docs/tutorials/configs/ --- docs/design/jira-data-source-connector.md | 738 ------------------ .../{graph_configs => }/data_sources.json | 0 2 files changed, 738 deletions(-) delete mode 100644 docs/design/jira-data-source-connector.md rename docs/tutorials/configs/{graph_configs => }/data_sources.json (100%) diff --git a/docs/design/jira-data-source-connector.md b/docs/design/jira-data-source-connector.md deleted file mode 100644 index fe32dac9..00000000 --- a/docs/design/jira-data-source-connector.md +++ /dev/null @@ -1,738 +0,0 @@ -# Jira Data Source Connector — Implementation Plan - -Status: Implemented and validated locally; OAuth/ACL security remains planned -Date: 24 September 2026 -Product: TigerGraph GraphRAG -Source: Jira Cloud - ---- - -## 1. Goal - -An operator initializes or selects an existing graph, configures a Jira Cloud -site for that graph, installs the Jira schema, and runs -ingestion. GraphRAG can then answer: - -- Structured questions such as “Which high-priority bugs are still open in - PAY?” and “Who is assigned PAY-123?” -- Narrative questions such as “What did we decide about the checkout - timeout?” - -The connector writes into the graph that GraphRAG already queries. It does not -add a separate answer engine. - ---- - -## 2. Design decisions - -Each Jira issue is written in complementary structural and semantic forms: - -1. POLE+O-aligned domain data: `JiraUser` is a Person subtype; `JiraIssue` and - `JiraProject` are Object subtypes; `JiraComment` is an Event subtype. -2. One issue markdown `Document` containing issue metadata and description. -3. Deterministic `DocumentChunk` records written directly from each Jira - comment, without an intermediate comment `Document`. Short comments produce - one chunk; exceptionally long comments may produce multiple bounded chunks. - Existing hybrid and similarity retrievers read these embeddings without a - connector-specific retrieval path. - -The connector maps Jira fields deterministically. It does not run LLM entity -extraction over Jira issue text. Status, project, assignee, reporter, parent, -and issue links already have authoritative structured values in Jira. - -The entire connector lifecycle lives under **Data sources**: graph selection, -credentials, project scope, connection testing, schema migration, manual sync, -status, and errors. `Ingest to Knowledge Graph` remains the existing -file/cloud-document workflow and does not expose Jira a second time. - -The connector never creates or replaces a graph. The operator first uses the -existing Initialize Knowledge Graph flow. Jira schema installation is an -explicit, additive, idempotent migration against that initialized graph. - -### 2.1 Research basis: Glean - -Glean uses one connector administration surface rather than duplicating Jira -under a generic upload page: - -1. Admin console -> Connectors -> Add connector -> Jira Cloud. -2. The connector’s Setup tab owns authentication and source-specific settings. -3. Manage data owns project/custom-field inclusion rules. -4. The same connector detail shows crawl/index status and supports re-crawl. -5. Glean separates content crawls from identity/permission crawls and uses - webhooks plus incremental/full crawls to keep the index current. -6. Indexed records use stable IDs, object types, searchable bodies, custom - properties, source URLs, and document-level permissions. - -Sources: - -- [Glean: Get started with connectors](https://docs.glean.com/connectors/getting-started) -- [Glean: Jira Cloud connector](https://docs.glean.com/connectors/native/jira/) -- [Glean: Connector settings and visibility](https://docs.glean.com/connectors/connectors-settings-visibility) -- [Glean: Document model](https://developers.glean.com/api-info/indexing/documents/document-model) -- [Glean: Custom properties](https://developers.glean.com/api-info/indexing/datasource/custom-properties) - -Glean configures a datasource schema (`objectDefinitions` and -`propertyDefinitions`) and then indexes stable documents carrying datasource, -object type, body, URL, custom properties, and permissions. GraphRAG requires -one additional connector-owned step: map that normalized Jira model to an -additive TigerGraph schema so structured graph retrieval can traverse issues, -projects, and users. That migration belongs inside the connector detail, -equivalent to Glean’s datasource-schema setup, not on the file-ingest page. - -The implemented connector matches Glean’s connector lifecycle and -normalized content approach, but not its complete security model. Glean -indexes users, groups, memberships, and Jira permissions and enforces those -permissions at query time. GraphRAG currently uses one connector account plus a -project allow-list, so every GraphRAG user with graph access can retrieve the -indexed Jira data. Identity crawls and query-time ACL filtering remain Phase 5 -work; they must not be simulated by adding more Jira business vertices to the -schema. - -Jira access is scoped by the configured service account and project allow-list. -The current GraphRAG retrieval path does not enforce the chatting user’s Jira -permissions. - ---- - -## 3. Existing GraphRAG workflow - -```text -Operator uploads files or selects cloud storage - | - v - Existing ingest API - | - v - Issue Document -> DocumentChunk -> Content - Comment -------> DocumentChunk -> Content - | - v - Existing retrievers - | - v - Answer -``` - -File ingest remains unchanged. Jira is not added as another -`CreateIngestConfig.data_source` because that API creates file loading jobs and -cannot write typed Jira edges such as `JIRA_ASSIGNED_TO`. - ---- - -## 4. Target Jira workflow - -```text -Initialize Knowledge Graph - | - v -Select existing initialized graph - | - v -Data sources -> Add Jira Cloud - | - +-- Configure and test credentials - +-- Select projects and content settings - +-- Inspect live graph schema - | | - | +-- missing/incomplete -> Install Jira schema elements - | +-- conflict ------> Stop with exact conflict - | +-- installed -----> Enable Ingest now - | - v -Ingest now: ingest Jira incrementally - | - +-----------------------------+ - | | - v v -Typed Jira vertices/edges Issue Documents Direct comment chunks - | | - v v - Existing ECC rebuild Existing embedding store - | | - +--------------+--------------+ - | - v - Existing GraphRAG retrievers -``` - -The connector ingestion and GraphRAG build use the existing per-graph lock. -File and cloud ingestion stay independent and continue to use their existing -Ingest to Knowledge Graph flow. - -The sync API loads authoritative Jira vertices, edges, issue documents, and -directly embedded comment chunks, then returns `rebuild_required`. The Data -sources UI calls the existing -`POST /ui/{graph}/rebuild_graph` endpoint when changed documents or deletions -require a GraphRAG rebuild. ECC performs structure-aware chunking and embedding -for issue documents; comment chunks reuse the same embedding store during -connector sync and bypass ECC entity extraction. - -Incremental ingestion orders tickets oldest-to-newest by Jira `updated` time. -After each search page has completed its graph and document writes, the -connector atomically persists that page’s highest `updated` timestamp. After an -unexpected shutdown, the next run resumes from that checkpoint minus a -120-second overlap. Upserts use Jira’s stable numeric issue id, making overlap -duplicates harmless. Newest-to-oldest ordering is intentionally avoided because -it could checkpoint past older tickets that have not yet been committed. - ---- - -## 5. Repository changes - -### 5.1 New backend modules - -| Path | Responsibility | -| --- | --- | -| `graphrag/app/connectors/__init__.py` | Connector package | -| `graphrag/app/connectors/jira/config.py` | Validated Jira source configuration | -| `graphrag/app/connectors/jira/schema.py` | Predefined `SchemaProposal` | -| `graphrag/app/connectors/jira/client.py` | Jira REST API, pagination, retries | -| `graphrag/app/connectors/jira/adf.py` | Atlassian Document Format to markdown | -| `graphrag/app/connectors/jira/mapper.py` | Jira payload to graph records, issue documents, and direct comment chunks | -| `graphrag/app/connectors/jira/state.py` | Atomic graph-scoped config persistence | -| `graphrag/app/connectors/jira/sync.py` | Schema-gated incremental sync | -| `graphrag/app/routers/data_sources.py` | Authenticated data-source API | - -### 5.2 Existing backend integration points - -| Path | Required change | -| --- | --- | -| `graphrag/app/routers/__init__.py` | Export the data-source router | -| `graphrag/app/main.py` | Mount the data-source router | -| `ecc/app/graphrag/graph_rag.py` | Embed residual source-native chunks without duplicate LLM extraction | -| `ecc/app/graphrag/workers.py` | Skip duplicate LLM extraction and link each Jira chunk to its typed issue | -| `common/embeddings/tigergraph_embedding_store.py` | Retry transient embedding-provider failures with bounded backoff | -| `common/llm_services/base_llm.py` | Clarify business-key mapping and concise schema-driven structural lookup | -| `graphrag/app/tools/generate_function.py` | Permit the existing read-only pyTigerGraph function surface | -| `graphrag/app/tools/generate_cypher.py` | Normalize fenced model output safely | -| `graphrag/app/tools/graphrag_tools.py` | Recognize JSON-encoded empty results and improve Cypher retry feedback | -| `graphrag/app/tools/validation_utils.py` | Define the explicit read-only function allow-list | - -### 5.3 Frontend changes - -| Path | Required change | -| --- | --- | -| `graphrag-ui/src/pages/setup/DataSourcesConfig.tsx` | Single graph-scoped connector lifecycle: setup, schema, sync, status | -| `graphrag-ui/src/main.tsx` | Add `/setup/kg-admin/data-sources` | -| `graphrag-ui/src/pages/setup/SetupLayout.tsx` | Add Data sources navigation | - -### 5.4 Existing code to reuse - -| Existing code | Use | -| --- | --- | -| `common/db/schema_utils.py:apply_proposal` | Apply the Jira domain schema additively | -| `upsert_type_metadata` | Give query planning precise Jira type descriptions | -| `emit_structural_link_alters` | Allow `CONTAINS_ENTITY` to target `JiraIssue` | -| `common/chunkers` | Chunk issue markdown and unusually long comments | -| `graphrag/app/supportai/supportai_ingest.py` | Reuse the existing document-loading entry point | -| `common/config.py:get_embedding_store` | Reuse embedding and vector upsert behavior | -| `graphrag/app/tools/graphrag_tools.py` | Existing structured and unstructured answering; no connector-specific retrieval path | -| `common/utils/graph_locks.py` | Prevent sync from racing with ingest/schema operations | -| Existing UI auth and graph selection state | Do not create a second auth or graph-selection mechanism | - -### 5.5 Code that stays unchanged - -- Existing file and cloud storage ingest -- Existing vector and hybrid query implementations -- MCP server configuration -- Unrelated setup pages - ---- - -## 6. Schema - -The Jira schema adds four vertex types and nine edge types. Installation is -idempotent and adds only missing compatible elements. - -A vertex is used only when questions need to traverse from one object to -another. Project, issue, user, and comment are those objects. Status, priority, -labels, components, fix versions, and dates remain issue attributes. Comments -are first-class Event records with separately embedded content. The Jira site -URL remains source configuration. - -### 6.1 Schema picture - -Read each line independently. The left side is the source vertex and the arrow -label is the edge type. - -```text -JiraIssue -- JIRA_BELONGS_TO --> JiraProject - -JiraIssue -- JIRA_ASSIGNED_TO --> JiraUser - -JiraIssue -- JIRA_REPORTED_BY --> JiraUser - -JiraIssue -- JIRA_HAS_PARENT --> JiraIssue - -JiraIssue -- JIRA_LINKS_TO --> JiraIssue - link_type = blocks | is blocked by | duplicates | relates to - -JiraComment -- JIRA_COMMENT_ON --------> JiraIssue - -JiraComment -- JIRA_COMMENTED_BY ------> JiraUser - -JiraComment -- JIRA_COMMENT_REPLIES_TO -> JiraComment - only when supplied explicitly by Jira - -JiraComment -- JIRA_COMMENT_AFTER -----> JiraComment - chronological order on one issue -``` - -Connector-owned edge names use a Jira prefix to avoid collisions with an -existing graph’s domain schema. This is mandatory for `JIRA_HAS_PARENT` and -`JIRA_LINKS_TO` because GraphRAG’s base schema already defines `HAS_PARENT` -and `LINKS_TO` for communities. - -Issue descriptions and comment content reuse GraphRAG’s existing semantic -schema. Comments skip the unnecessary intermediate `Document`: - -```text -issue Document ---- HAS_CHILD ----> issue DocumentChunk - | | - +-- CONTAINS_ENTITY --> JiraIssue +--> JiraIssue - -JiraComment <---- CONTAINS_ENTITY ---- comment DocumentChunk -JiraIssue <---- CONTAINS_ENTITY ---- | - +-- HAS_CONTENT --> Content -``` - -`HAS_CHILD`, `HAS_CONTENT`, and `CONTAINS_ENTITY` already exist. Jira schema -application adds the Jira domain endpoint pairs to `CONTAINS_ENTITY`; this -design change adds no schema type. - -### 6.2 Vertex types - -All ids are prefixed with `jira:{cloudId}:` so different Jira sites cannot -collide. Numeric Jira ids are used because human issue keys can change when an -issue moves project. - -| Vertex | Example id | Attributes | -| --- | --- | --- | -| `JiraProject` | `jira:abc:project:10001` | `project_key STRING`, `name STRING`, `url STRING` | -| `JiraIssue` | `jira:abc:issue:10422` | `issue_key STRING`, `summary STRING`, `issue_type STRING`, `status STRING`, `status_category STRING`, `priority STRING`, `resolution STRING`, `labels STRING`, `components STRING`, `fix_versions STRING`, `created DATETIME`, `updated DATETIME`, `due DATETIME`, `url STRING`, `story_points DOUBLE`, `content_hash STRING` | -| `JiraUser` | `jira:abc:user:557058:account` | `account_id STRING`, `display_name STRING` | -| `JiraComment` | `jira:abc:comment:9001` | `comment_id STRING`, `created DATETIME`, `updated DATETIME`, `visibility STRING`, `is_public BOOL`, `ontology_class STRING`, `content_hash STRING` | - -`labels`, `components`, and `fix_versions` are comma-separated strings. They -do not require separate vertices for the current query set. - -`status_category` contains `new`, `indeterminate`, or `done`, allowing -cross-project open/done filtering while retaining the project-specific status -name in `status`. - -`content_hash` is sync bookkeeping. An unchanged hash skips document -re-chunking and re-embedding. The hash is advanced only after the replacement -document loads successfully, so a failed load is retried on the next sync. - -`story_points` remains empty unless one numeric custom field is configured. - -Email is not stored on `JiraUser`; Jira Cloud can hide it. `accountId` is the -stable user identity. - -### 6.3 Edge types - -| Edge | From | To | Extra attribute | Query purpose | -| --- | --- | --- | --- | --- | -| `JIRA_BELONGS_TO` | `JiraIssue` | `JiraProject` | | Project filtering | -| `JIRA_ASSIGNED_TO` | `JiraIssue` | `JiraUser` | | Current owner | -| `JIRA_REPORTED_BY` | `JiraIssue` | `JiraUser` | | Reporter | -| `JIRA_HAS_PARENT` | `JiraIssue` | `JiraIssue` | | Epic or subtask parent | -| `JIRA_LINKS_TO` | `JiraIssue` | `JiraIssue` | `link_type STRING` | Blocks, duplicates, relates to | -| `JIRA_COMMENT_ON` | `JiraComment` | `JiraIssue` | | Comment’s issue | -| `JIRA_COMMENTED_BY` | `JiraComment` | `JiraUser` | | Comment author | -| `JIRA_COMMENT_REPLIES_TO` | `JiraComment` | `JiraComment` | | Explicit source-provided parent comment | -| `JIRA_COMMENT_AFTER` | `JiraComment` | `JiraComment` | | Chronological order | - -Current-state edges are replaced for each updated issue. This is necessary so -reassignment does not leave an old `JIRA_ASSIGNED_TO` edge. - -Jira’s standard issue comments are normally a flat sequence. Reply edges are -never inferred; `JIRA_COMMENT_REPLIES_TO` is written only when the source -payload explicitly identifies a parent. `JIRA_COMMENT_AFTER` preserves the -available chronological order. - -### 6.4 Type descriptions for query planning - -| Type | Description | -| --- | --- | -| `JiraIssue` | A Jira work item. Filter by issue_key, status, status_category, priority, issue_type, resolution, labels, components, fix_versions, created, updated, and due | -| `JiraProject` | A Jira project. project_key is the short key such as PAY | -| `JiraUser` | An Atlassian account. account_id is identity and display_name is the human name | -| `JiraComment` | A POLE+O Event subtype representing one authored Jira comment with stable identity and timestamps | -| `JIRA_LINKS_TO` | Issue link whose link_type is blocks, is blocked by, duplicates, or relates to | -| `JIRA_HAS_PARENT` | Parent issue, including epic and subtask parent | -| `JIRA_ASSIGNED_TO` | Current assignee | -| `JIRA_REPORTED_BY` | User who reported the issue | -| `JIRA_BELONGS_TO` | Current project | - -These descriptions are stored through `upsert_type_metadata` so the existing -question-to-schema mapper can choose the correct types and attributes. - -### 6.5 Issue document - -Document id: `jira:{cloudId}:issue-doc:{issueId}` - -```text -# PAY-123: Login timeout on checkout -URL: https://acme.atlassian.net/browse/PAY-123 -Project: PAY Payments -Type: Bug -Status: In Progress (indeterminate) -Priority: High -Assignee: Ada Lovelace -Reporter: Grace Hopper -Labels: checkout, auth -Components: API -Fix versions: 2.4 -Updated: 2026-09-20T14:03:00Z - -## Description - - -## Attachments -design.pdf, logs.txt -``` - -### 6.6 Comment event and direct chunk - -Chunk id: -`jira:{cloudId}:comment:{commentId}:chunk:{index}:{contentDigest}` - -```text -# Comment by Ada Lovelace on PAY-123: Login timeout on checkout -Issue: PAY-123 -URL: https://acme.atlassian.net/browse/PAY-123 -Comment ID: 9001 -Author: Ada Lovelace -Created: 2026-09-18T11:02:00Z -Updated: 2026-09-18T11:02:00Z -Visibility: default - -## Comment - -``` - -The connector creates these chunks directly with content type `jira_comment`. -Structure-aware chunking emits one chunk for ordinary short comments and -bounded multiple chunks only for unusually long comments. For long comments, -log-dominated fenced or multiline blocks are replaced by an omission marker; -human explanation surrounding those logs is retained. Detection uses multiple -signals (timestamps, severity levels, stack traces, and structured log fields), -not length alone. - -Every resulting chunk links to both `JiraComment` and `JiraIssue`. The connector -reuses the configured embedding store directly, so no Jira branch is added to -shared ECC processing and no duplicate LLM entity extraction occurs. The -comment content hash advances only after all replacement chunks and embeddings -succeed. Updating one comment therefore re-embeds that comment rather than the -entire issue. Removed comments have their vertex, direct chunks, content, edges, -and embeddings deleted during issue reconciliation. When legacy comment -Documents are detected, the connector performs one resumable full synchronization -to replace them with direct chunks, then returns to incremental checkpoints. - ---- - -## 7. Jira API - -Target Jira Cloud REST API v3 only. - -Current authentication is HTTPS Basic authentication using an Atlassian account -email and API token. - -| Endpoint | Use | -| --- | --- | -| `GET /rest/api/3/myself` | Test connection | -| `GET /_edge/tenant_info` | Resolve `cloudId` | -| `GET /rest/api/3/project/search` | Load all visible projects for the project picker | -| `GET /rest/api/3/project/{projectIdOrKey}` | Load one optional project directly by key | -| `POST /rest/api/3/search/jql` | Initial and incremental issue search | -| `GET /rest/api/3/issue/{id}/comment` | Fetch all comments if the embedded page is truncated | - -Search rules: - -- Use `/rest/api/3/search/jql`, not deprecated `/rest/api/3/search`. -- Use `nextPageToken`, not `startAt`, for issue search. -- Fetch search pages sequentially. -- Use `maxResults = 100`. -- Add `ORDER BY updated ASC, key ASC`. -- Combine the project allow-list with optional created-after, updated-after, - and status-category filters. Additional JQL remains an advanced option. -- On HTTP 429, honor `Retry-After` and retry with bounded backoff. -- Advance the durable checkpoint only after every graph and document write for - a page succeeds; do not advance it after a failed page or failed graph write. -- Reset the incremental checkpoint when scope filters change so a widened - scope performs a full ingestion instead of skipping older matching issues. - -Incremental JQL: - -```text -project in (PAY, PLAT) -AND updated >= "{last successful checkpoint minus 120 seconds}" -ORDER BY updated ASC, key ASC -``` - -Jira search is not a stable snapshot. The overlap and idempotent upsert cover -updates near page boundaries. - -The connector does not persist issue inventory on the local filesystem. -Consequently, incremental sync currently updates and adds records but does not -automatically remove issues deleted in Jira or moved outside the configured -scope. Production deletion reconciliation requires shared ownership state in -TigerGraph or another durable multi-replica state store. - ---- - -## 8. Configuration and UI - -### 8.1 UI locations - -**Data sources** (`/setup/kg-admin/data-sources`) owns the selected graph’s -complete connector lifecycle: - -1. Saved sources with enabled state, last ingestion, and last error. -2. Connection: display name, tenant site URL, Atlassian account email, API - token, and connection status. -3. Enter an optional project key, then test the connection. Connection status - is displayed next to that action. -4. Only after a successful test in the current editing session, load one - requested project or every visible project, then select the projects for the - graph. A direct project-key load replaces stale project results. Changing - connection details invalidates verification and clears loaded results. -5. Configure optional created/updated dates and Jira status categories after - project selection. Additional JQL and comment ingestion remain advanced - scope settings. -6. Preview the approximate Jira ticket count using the unsaved project and - filter scope, then save the scope. -7. Schema: inspect compatibility and install missing compatible elements. -8. Ingest/build actions: Ingest now, last completion, item count, and last error. -9. Connector lifecycle: edit, disable, or delete configuration. - -**Ingest to Knowledge Graph** is unchanged. It remains responsible for local -files, cloud storage, and Amazon BDA. Jira does not appear there. - -The UI requires an API token for every connection test and keeps that token -only in process memory for subsequent connector operations. UI-submitted tokens -are never persisted. If an administrator has already placed `api_token` in the -graph-scoped data-source configuration, saves preserve and may reuse it, but -the API never returns it to the browser. - -### 8.2 Stored configuration - -Path: `configs/graph_configs/{graph}/data_sources.json` - -```json -{ - "sources": [ - { - "id": "jira-acme", - "type": "jira_cloud", - "enabled": true, - "display_name": "Acme Jira", - "connection": { - "site_url": "https://acme.atlassian.net", - "email": "connector@example.com", - "cloud_id": "abc123" - }, - "scope": { - "project_keys": ["PAY", "PLAT"], - "created_after": "2026-01-01", - "updated_after": null, - "status_categories": ["new", "indeterminate"], - "jql_extra": "", - "include_comments": true - }, - "sync": { - "overlap_seconds": 120, - "checkpoint": "2026-09-20T14:03:11.000+0000", - "last_tested_at": "2026-09-20T13:50:00.000+0000", - "last_started_at": null, - "last_completed_at": null, - "last_error": null, - "last_issue_count": 0 - } - } - ] -} -``` - -Sync refuses to run with an empty project list. - -Live TigerGraph metadata is authoritative: every status check validates -required attributes, edge direction, and endpoint pairs. No Jira schema -version number is stored. - -### 8.3 API - -| Method | Path | Behavior | -| --- | --- | --- | -| `GET` | `/ui/{graph}/data-sources` | List graph sources without secrets | -| `PUT` | `/ui/{graph}/data-sources/{id}` | Validate and atomically save | -| `DELETE` | `/ui/{graph}/data-sources/{id}` | Remove configuration only | -| `POST` | `/ui/{graph}/data-sources/{id}/test` | Test a freshly supplied token without persisting it | -| `GET` | `/ui/{graph}/data-sources/{id}/projects` | Return all visible projects, or one project when `project_key` is supplied | -| `POST` | `/ui/{graph}/data-sources/{id}/issues/count` | Preview Jira’s approximate count for an unsaved scope | -| `GET` | `/ui/{graph}/data-sources/jira/schema` | Inspect live Jira schema compatibility | -| `POST` | `/ui/{graph}/data-sources/jira/schema/install` | Add missing compatible Jira schema elements idempotently | -| `POST` | `/ui/{graph}/data-sources/{id}/sync` | Run schema-gated incremental Jira ingestion | -| `GET` | `/ui/{graph}/data-sources/{id}/sync/{run_id}` | Poll the in-memory status of a submitted sync | -| `POST` | `/ui/{graph}/rebuild_graph` | Existing endpoint used by the UI to start the ECC build | - ---- - -## 9. Answer flow after ingestion - -The implementation uses the existing GraphRAG agent and the request’s -TigerGraph connection. It does not yet enforce the chatting user’s Jira -permissions; Jira ACL-safe retrieval is Phase 5. - -| Question | Existing path | Required data | -| --- | --- | --- | -| Which high-priority bugs are still open in PAY? | `structural_retrieve` | JiraIssue attributes and JIRA_BELONGS_TO | -| Who is assigned PAY-123? | `structural_retrieve` | issue_key and JIRA_ASSIGNED_TO | -| What does PAY-123 block? | `structural_retrieve` | JIRA_LINKS_TO.link_type | -| What did we decide about the checkout timeout? | `hybrid_search` | Issue document chunks | -| What status and owner does the issue discussing the timeout have? | hybrid plus structural, then `combine_context` | CONTAINS_ENTITY, status, JIRA_ASSIGNED_TO | -| What themes occur across payments bugs? | `hybrid_search` | Issue documents; community duplication is not part of the connector | - -Jira keys are external identifiers, not TigerGraph primary vertex ids. For an -exact key such as `PAY-123`, the existing structural pipeline maps the value to -`JiraIssue.issue_key` and generates an exact attribute predicate. It must not -pass the key to `getVerticesById`; Jira primary ids are source-qualified, -stable ids based on the Jira cloud and numeric issue id. - -For a numeric-only ticket reference such as `123`, the schema-driven -openCypher fallback uses delimiter-aware suffix matching and returns every -matching business key rather than inventing a project prefix. If more than one -project has that numeric suffix, the answer must expose the ambiguity. - -The structural path first attempts an allowed read-only pyTigerGraph function. -JSON-encoded empty results such as `"[]"` are treated as empty so the existing -openCypher fallback can run. Generated Cypher is retried with execution errors -fed back to the generator. Full-key and numeric-suffix lookups have both been -verified against the development graph. - -Hybrid retrieval is independent of structural ingestion. Issue descriptions -become searchable after ECC builds their document chunks. Comment chunks are -embedded during Jira synchronization and are searchable without passing -through ECC. - -```text -Question - | - v -get_schema - | - +-- count/filter/key/traversal --> existing structural query path - | - +-- description/comments --> existing hybrid/vector retrieval - | - v -combine_context - | - v -Answer with issue key and Jira URL -``` - ---- - -## 10. Implementation phases - -### Phase 1 — Core schema and mapping — Implemented - -- Add validated source models. -- Add the three-vertex, five-edge schema proposal. -- Add ADF-to-markdown conversion. -- Map representative Jira payloads, including unassigned issues, parents, - inward/outward links, hidden user email, missing priority, and custom story - points. - -Done when pure unit tests pass without TigerGraph or Jira. - -### Phase 2 — Jira client and graph sync — Implemented - -- Implement current enhanced-search pagination with `nextPageToken`. -- Implement bounded 429 retry. -- Inspect and validate the live Jira schema before every sync. -- Upsert structured records. -- Replace current-state edges. -- Load issue `Document`/`Content` records for ECC. -- Write bounded comment `DocumentChunk`/`Content` records directly and embed - them with the existing store. -- Skip unchanged documents by `content_hash`. - -Done when a mocked sync proves idempotency and a development graph shows both -the typed issue and linked chunks. - -### Phase 3 — Schema migration, API, and UI — Implemented - -- Add explicit schema status and install endpoints using existing additive - schema utilities. -- Reject conflicts before applying a migration and gate sync on installed - schema status. -- Add authenticated, graph-scoped CRUD/test/project/sync endpoints. -- Add masked secret handling and atomic config writes. -- Keep setup, schema migration, ingestion, build status, and re-ingestion under one - Data sources connector detail. -- Do not add Jira to Ingest to Knowledge Graph. - -### Phase 4 — Exit test — Partially completed locally - -1. Configure a non-admin Jira service account. -2. Select one project. -3. Select the initialized graph under Data sources. -4. Install the Jira schema and confirm a second install is a no-op. -5. Run Ingest now and monitor ingestion/build status on the same page. -6. Ask a structured count question and compare it with Jira. -7. Ask a comment question and confirm the answer names and links the issue. - -Current validation covers schema installation, typed ticket retrieval by full -and numeric-only keys, source document loading, deterministic chunk-to-issue -linkage, direct comment embedding, legacy cleanup, and resumable migration. - -### Phase 5 — OAuth, identity, and permission enforcement — Planned - -- Replace API-token crawling with confidential OAuth 2.0 (3LO) and rotating - refresh tokens. -- Add self-service GraphRAG-user to Atlassian-account linking. -- Crawl accounts, groups, memberships, project permission schemes, project - roles, issue-security schemes, and issue security levels. -- Split or omit role/group-restricted comments; never place them in a document - with only issue-level ACLs. -- Activate permission crawls as validated generations and retain the previous - generation until the replacement is complete. -- Pass the resolved GraphRAG identity into every retrieval path. -- Replace unrestricted Jira structural generation with fixed ACL-aware query - surfaces. Filter semantic candidates before ranking, expansion, and LLM - context construction. -- Deny all Jira results when account linking or permission state is missing, - stale, incomplete, or unsupported. - ---- - -## 11. Out of scope - -- Jira Data Center and Server -- Forge and Connect authentication modes -- Sprints, boards, and backlog rank -- Changelog and worklogs -- Attachment content ingestion -- Shared cross-source Person identity -- LLM or ontology extraction over issue text -- Connector marketplace/plugin loading -- MCP as an ingestion path -- Webhooks and scheduled sync -- Jira Service Management SLA-specific entities - ---- - -## 12. Risks and controls - -| Risk | Control | -| --- | --- | -| Permission leakage | Known limitation: restrict graph access and connector scope; implement OAuth-linked identity, permission snapshots, and fail-closed retrieval in Phase 5 | -| Search pagination drift | Stable id upsert plus 120-second checkpoint overlap | -| Stale assignee or parent | Replace current-state outgoing edges for each updated issue | -| Re-embedding cost | Compare `content_hash` before rewriting chunks | -| Missing Jira embeddings | Embed direct comment chunks transactionally during sync; keep issue-document embedding in the existing ECC stage | -| Schema complexity | Keep only four Jira vertex types | -| Existing feature regressions | Isolated connector/router/page plus narrow, tested fixes to the shared structural-query path | diff --git a/docs/tutorials/configs/graph_configs/data_sources.json b/docs/tutorials/configs/data_sources.json similarity index 100% rename from docs/tutorials/configs/graph_configs/data_sources.json rename to docs/tutorials/configs/data_sources.json From 9d99bb4b69a9addb61642a3c55ae0723b9deb1b9 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 14:46:04 +0530 Subject: [PATCH 03/19] GML-2211: Revert tigergraph_embedding_store.py to main (out of scope) --- .../embeddings/tigergraph_embedding_store.py | 114 +++++------------- 1 file changed, 33 insertions(+), 81 deletions(-) diff --git a/common/embeddings/tigergraph_embedding_store.py b/common/embeddings/tigergraph_embedding_store.py index e21e30a4..47dbed8e 100644 --- a/common/embeddings/tigergraph_embedding_store.py +++ b/common/embeddings/tigergraph_embedding_store.py @@ -12,9 +12,7 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . -import asyncio import logging -import random import traceback import json from time import sleep, time @@ -289,26 +287,18 @@ def _embed_sync_with_truncation_retry(self, text: str, v_id): truncation gatekeeper for the rationale.""" last_err = None for i, candidate in enumerate(self._truncation_candidates(text)): - for retry in range(4): - try: - return self.embedding_service.embed_query(candidate) - except Exception as e: - last_err = e - if self._is_embed_overflow(e): - LogWriter.warning( - f"Embed for {v_id} overflowed at len={len(candidate)} " - f"(attempt {i + 1}); retrying with shorter prefix" - ) - break - if not self._is_transient_embed_error(e) or retry == 3: - return self._log_embed_failure(v_id, last_err) - delay = (2 ** retry) + random.uniform(0, 0.5) - LogWriter.warning( - f"Transient embedding failure for {v_id}; retrying in " - f"{delay:.1f}s (attempt {retry + 2}/4): {e}" - ) - sleep(delay) - return self._log_embed_failure(v_id, last_err) + try: + return self.embedding_service.embed_query(candidate) + except Exception as e: + last_err = e + if not self._is_embed_overflow(e): + break + LogWriter.warning( + f"Embed for {v_id} overflowed at len={len(candidate)} " + f"(attempt {i + 1}); retrying with shorter prefix" + ) + LogWriter.error(f"Failed to embed {v_id} after truncation: {last_err}") + return None async def _embed_with_truncation_retry(self, text: str, v_id): """Async embed with input-overflow fallback. @@ -319,60 +309,24 @@ async def _embed_with_truncation_retry(self, text: str, v_id): counts vary). Rather than abandoning the whole batch on one bad chunk, truncate the offending text to progressively shorter prefixes until it fits. The persisted chunk text is - unchanged; only the embedding represents the prefix. If no - truncation level succeeds, the error is propagated so callers - cannot report a successful build with a missing embedding. + unchanged; only the embedding represents the prefix. A chunk + for which no truncation level fits is left without an + embedding — the similarity_search GSQL skips empty vectors. """ last_err = None for i, candidate in enumerate(self._truncation_candidates(text)): - for retry in range(4): - try: - return await self.embedding_service.aembed_query(candidate) - except Exception as e: - last_err = e - if self._is_embed_overflow(e): - LogWriter.warning( - f"Embed for {v_id} overflowed at len={len(candidate)} " - f"(attempt {i + 1}); retrying with shorter prefix" - ) - break - if not self._is_transient_embed_error(e) or retry == 3: - return self._log_embed_failure(v_id, last_err) - delay = (2 ** retry) + random.uniform(0, 0.5) - LogWriter.warning( - f"Transient embedding failure for {v_id}; retrying in " - f"{delay:.1f}s (attempt {retry + 2}/4): {e}" - ) - await asyncio.sleep(delay) - return self._log_embed_failure(v_id, last_err) - - @staticmethod - def _is_transient_embed_error(error: Exception) -> bool: - message = str(error).lower() - return any( - marker in message - for marker in ( - "429", - "500", - "502", - "503", - "504", - "internal", - "rate limit", - "resource exhausted", - "timeout", - "timed out", - "connection reset", - "temporarily unavailable", - ) - ) - - @staticmethod - def _log_embed_failure(v_id, error): - LogWriter.error(f"Failed to embed {v_id} after retries: {error}") - raise RuntimeError( - f"Failed to embed {v_id} after retries" - ) from error + try: + return await self.embedding_service.aembed_query(candidate) + except Exception as e: + last_err = e + if not self._is_embed_overflow(e): + break + LogWriter.warning( + f"Embed for {v_id} overflowed at len={len(candidate)} " + f"(attempt {i + 1}); retrying with shorter prefix" + ) + LogWriter.error(f"Failed to embed {v_id} after truncation: {last_err}") + return None def add_embeddings( self, @@ -414,9 +368,7 @@ def add_embeddings( vec_attrs_used.add(vec_attr) embedding = self._embed_sync_with_truncation_retry(text, v_id) if embedding is None: - raise RuntimeError( - f"Embedding provider returned no vector for {v_id}" - ) + continue attr = self.map_attrs([(vec_attr, embedding)]) batch["vertices"][v_type][v_id] = attr @@ -450,7 +402,6 @@ def add_embeddings( f"Ensure these vertex types have the expected vector attribute." ) LogWriter.error(error_message) - raise RuntimeError(error_message) from e async def aadd_embeddings( self, @@ -492,9 +443,11 @@ async def aadd_embeddings( vec_attrs_used.add(vec_attr) embedding = await self._embed_with_truncation_retry(text, v_id) if embedding is None: - raise RuntimeError( - f"Embedding provider returned no vector for {v_id}" - ) + # No truncation level worked. Leave this vertex without an + # embedding so similarity_search can skip it (the GSQL + # query filters on v.embedding.size() > 0) — better than + # abandoning the entire batch on one bad chunk. + continue attr = self.map_attrs([(vec_attr, embedding)]) batch["vertices"][v_type][v_id] = attr @@ -528,7 +481,6 @@ async def aadd_embeddings( f"Ensure these vertex types have the expected vector attribute." ) LogWriter.error(error_message) - raise RuntimeError(error_message) from e def has_embeddings( self, From 0d5699d17ef4ac7129bc130c1b63cdaa17426599 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 15:02:01 +0530 Subject: [PATCH 04/19] =?UTF-8?q?GML-2211:=20Fix=20base=5Fllm=20prompt=20?= =?UTF-8?q?=E2=80=94=20revert=20out-of-scope=20changes,=20add=20Jira=20tic?= =?UTF-8?q?ket=20query=20behavior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- common/llm_services/base_llm.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/common/llm_services/base_llm.py b/common/llm_services/base_llm.py index b5e6499a..8b2a8976 100644 --- a/common/llm_services/base_llm.py +++ b/common/llm_services/base_llm.py @@ -681,7 +681,7 @@ def map_question_schema_prompt(self): - If an entity maps to a vertex attribute, consider generating a `WHERE` clause. - For synonyms, output the canonical form from the schema choices. - Generate the **complete** rewritten question. Keep the case of schema elements unchanged. -- Use `target_vertex_ids` only for explicit primary/vertex IDs; map business or external keys to matching schema attributes even when called an "ID". +- Do NOT generate `target_vertex_ids` unless the term `id` is explicitly mentioned in the question. ## Inputs - **Vertices**: {vertices} @@ -712,7 +712,7 @@ def generate_function_prompt(self): - Never pick a function not described in the docstrings below. - If entities map to vertex attributes, consider a `WHERE` clause. - When constructing `WHERE`, quote string attribute values properly. Example: `('Person', where='name="William Torres"')` — applies to every string attribute (name, email, address, etc.). -- Use `getVerticesById` only for actual primary IDs; use `getVertices` with `WHERE` for schema-defined business/external-key attributes. +- Do NOT generate `target_vertex_ids` unless the term `id` is explicitly mentioned in the question. - Pick exactly **one** function to execute. ## Schema @@ -839,9 +839,8 @@ def generate_cypher_prompt(self): - Only use attributes that exist in the schema. Pick the closest matching attribute name when multiple candidates exist. - Prefer attributes over primary IDs when an attribute name is more similar to the keyword in the question. - Keep the query minimal — fewest vertex types, edge types, and attributes possible. -- Use only schema-defined edge types in relationships, and put vertex labels only in `MATCH`. -- Use primary IDs only for true vertex-ID lookups. Match complete business keys exactly; for a numeric-only portion use delimiter-aware `ENDS WITH` (e.g. ticket `2192` → `"-2192"`). -- For descriptive lookups, return relevant scalar attributes and only explicitly requested relationships; do not invent traversal paths. +- Do NOT return attributes that aren't explicitly mentioned in the question. If only a vertex is mentioned, return only the vertex. +- For Jira business-key lookups where only the numeric portion is given (e.g. `2192`), use `ENDS WITH "-2192"` on the issue_key attribute rather than an exact match. - Always include the entity from the `WHERE` clause in the final `RETURN`. Use vertex name over ID when available. - Always use **undirected** edge patterns. Ensure edges connect correct vertex types per schema. - Use **double quotes** for strings. @@ -1041,6 +1040,8 @@ def select_retriever_prompt(self): # action, then each next action driven by what the previous result returned. _AGENTIC_AGENT_USER_DEFAULT = """\ - For most questions, make your FIRST action a vector search (graphrag__hybrid_search or graphrag__contextual_search) — it gives the broadest grounding. Skip it only when you are highly confident the question is a pure structured-data request (an exact count, an attribute/id lookup, a relationship traversal, or an aggregation over typed graph data) that a generated graph query fully answers on its own. +- If the user refers to a ticket by number only with no project prefix (e.g. "ticket 20012" or "issue 20012"), do NOT guess the project — ask a clarifying question first, e.g. "Are you looking for GML-20012 or TSE-20012?" before making any retrieval. +- If the user provides a fully qualified Jira ticket ID (e.g. `GML-20012`), ALWAYS use BOTH graphrag__structural_retrieve (to fetch structured fields: status, assignee, priority, reporter, fix version, and recent comments) AND graphrag__hybrid_search or graphrag__contextual_search for broader context; combine both into one comprehensive answer that includes recent activity and comments — do not return a purely structural answer. - Route business-key lookups (e.g. ticket `ABC-123` or its numeric portion) to graphrag__structural_retrieve and preserve the user's wording. - Let each observation drive the next action: if the passages you got back name specific entities or relationships you still need hard facts about, follow up with a structural query; if a result is thin, empty, or off-target, widen its parameters (top_k, num_hops) or switch method rather than repeating the same call. - Before answering, check that every part of the question is covered with the specific facts and figures it asks for; if a required value, table, or entity is still missing, retrieve again (widen top_k / num_hops or switch method) rather than answering vaguely or partially. @@ -1092,6 +1093,8 @@ def agentic_agent_prompt(self): # be tuned without touching the role / act model / plan mechanics. _AGENTIC_PLANNER_USER_DEFAULT = """\ - Prioritize including at least one vector search step (graphrag__hybrid_search or graphrag__contextual_search) unless you are highly confident the question is a pure structured-data request — an exact count, an attribute/id lookup, a relationship traversal, or an aggregation over typed graph data — that a generated graph query fully answers on its own. Whenever the answer could plausibly live in document text (what/why/how/describe/summarize, definitions, explanations, figures), include a vector search step. When unsure, include vector search. +- If the user refers to a ticket by number only with no project prefix (e.g. "ticket 20012"), plan a single final "answer" step that asks for clarification (e.g. "Are you looking for GML-20012 or TSE-20012?") — do NOT plan any retrieval steps until the project is known. +- If the user provides a fully qualified Jira ticket ID (e.g. `GML-20012`), plan BOTH a graphrag__structural_retrieve step (for structured fields: status, assignee, priority, reporter, fix version, and recent comments) AND a graphrag__hybrid_search or graphrag__contextual_search step for broader context; the final answer step must depend on both and produce a comprehensive response including recent activity. - Route business-key lookups (e.g. ticket `ABC-123` or its numeric portion) to graphrag__structural_retrieve and preserve the user's wording. - Use BOTH kinds when a question needs facts from the graph AND supporting text; you may run several of each, in any order. When you use STRUCTURAL, pair it with a vector search step unless the question is a pure structured-data request. - Prefer the smallest plan that will work. Trivial/greeting questions need only the final answer step. From 9b0eb435b2534374367689b3738eacb734816c94 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 15:16:54 +0530 Subject: [PATCH 05/19] GML-2211: Remove Jira-specific agent prompt rules from base_llm defaults (configurable via UI) --- common/llm_services/base_llm.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/common/llm_services/base_llm.py b/common/llm_services/base_llm.py index 8b2a8976..c982c053 100644 --- a/common/llm_services/base_llm.py +++ b/common/llm_services/base_llm.py @@ -1040,9 +1040,6 @@ def select_retriever_prompt(self): # action, then each next action driven by what the previous result returned. _AGENTIC_AGENT_USER_DEFAULT = """\ - For most questions, make your FIRST action a vector search (graphrag__hybrid_search or graphrag__contextual_search) — it gives the broadest grounding. Skip it only when you are highly confident the question is a pure structured-data request (an exact count, an attribute/id lookup, a relationship traversal, or an aggregation over typed graph data) that a generated graph query fully answers on its own. -- If the user refers to a ticket by number only with no project prefix (e.g. "ticket 20012" or "issue 20012"), do NOT guess the project — ask a clarifying question first, e.g. "Are you looking for GML-20012 or TSE-20012?" before making any retrieval. -- If the user provides a fully qualified Jira ticket ID (e.g. `GML-20012`), ALWAYS use BOTH graphrag__structural_retrieve (to fetch structured fields: status, assignee, priority, reporter, fix version, and recent comments) AND graphrag__hybrid_search or graphrag__contextual_search for broader context; combine both into one comprehensive answer that includes recent activity and comments — do not return a purely structural answer. -- Route business-key lookups (e.g. ticket `ABC-123` or its numeric portion) to graphrag__structural_retrieve and preserve the user's wording. - Let each observation drive the next action: if the passages you got back name specific entities or relationships you still need hard facts about, follow up with a structural query; if a result is thin, empty, or off-target, widen its parameters (top_k, num_hops) or switch method rather than repeating the same call. - Before answering, check that every part of the question is covered with the specific facts and figures it asks for; if a required value, table, or entity is still missing, retrieve again (widen top_k / num_hops or switch method) rather than answering vaguely or partially. - For a specific value, row, total, ranking, or year-over-year comparison, use graphrag__hybrid_search or graphrag__contextual_search with top_k >= 10 (they return atomic table chunks that keep full row/column structure), and quote the exact label, column, year, or unit from the question so the retriever can match it.""" @@ -1093,9 +1090,6 @@ def agentic_agent_prompt(self): # be tuned without touching the role / act model / plan mechanics. _AGENTIC_PLANNER_USER_DEFAULT = """\ - Prioritize including at least one vector search step (graphrag__hybrid_search or graphrag__contextual_search) unless you are highly confident the question is a pure structured-data request — an exact count, an attribute/id lookup, a relationship traversal, or an aggregation over typed graph data — that a generated graph query fully answers on its own. Whenever the answer could plausibly live in document text (what/why/how/describe/summarize, definitions, explanations, figures), include a vector search step. When unsure, include vector search. -- If the user refers to a ticket by number only with no project prefix (e.g. "ticket 20012"), plan a single final "answer" step that asks for clarification (e.g. "Are you looking for GML-20012 or TSE-20012?") — do NOT plan any retrieval steps until the project is known. -- If the user provides a fully qualified Jira ticket ID (e.g. `GML-20012`), plan BOTH a graphrag__structural_retrieve step (for structured fields: status, assignee, priority, reporter, fix version, and recent comments) AND a graphrag__hybrid_search or graphrag__contextual_search step for broader context; the final answer step must depend on both and produce a comprehensive response including recent activity. -- Route business-key lookups (e.g. ticket `ABC-123` or its numeric portion) to graphrag__structural_retrieve and preserve the user's wording. - Use BOTH kinds when a question needs facts from the graph AND supporting text; you may run several of each, in any order. When you use STRUCTURAL, pair it with a vector search step unless the question is a pure structured-data request. - Prefer the smallest plan that will work. Trivial/greeting questions need only the final answer step. - Tabular / numeric questions (a specific value, a row, a column total, a ranking, or a year-over-year comparison from a table or chart): prefer graphrag__contextual_search or graphrag__hybrid_search with top_k>=10 (these return atomic table chunks that preserve full row/column structure); avoid graphrag__similarity_search alone; quote any specific table label, column header, year, or unit from the question (e.g. "ROE 2023"); for "compare X across years/regions/categories" set top_k>=15.""" From 76334bc477512e81ce06dab66547586f87d6f51a Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 15:23:58 +0530 Subject: [PATCH 06/19] GML-2211: Add Jira-specific agent prompt overrides for JiraTest graph config From a1646cafdfb1382e4b8bf182e40bc42329113eff Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 17:17:01 +0530 Subject: [PATCH 07/19] =?UTF-8?q?GML-2211:=20Fix=20REST-4000=20=E2=80=94?= =?UTF-8?q?=20paginate=20getVertices=20hash=20fetch=20to=20avoid=204MB=20T?= =?UTF-8?q?G=20limit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- graphrag/app/connectors/jira/sync.py | 32 +++++++++++++++++++--------- 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/graphrag/app/connectors/jira/sync.py b/graphrag/app/connectors/jira/sync.py index b73be29e..a0b0479b 100644 --- a/graphrag/app/connectors/jira/sync.py +++ b/graphrag/app/connectors/jira/sync.py @@ -288,16 +288,28 @@ def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: # requested STRING ID does not exist, which is the normal state during # an initial or incremental ingestion. Read the lightweight hash # projection once and filter it in memory. - vertices = self.conn.getVertices( - vertex_type, - select="content_hash", - ) or [] - return { - str(vertex.get("v_id")): str( - (vertex.get("attributes") or {}).get("content_hash") or "" - ) - for vertex in vertices - } + # + # Paginate to stay under TigerGraph's 4MB REST response limit — a + # single getVertices call over a large vertex set (e.g. JiraComment) + # can exceed 4MB and raise REST-4000. + _PAGE = 10_000 + result: dict[str, str] = {} + offset = 0 + while True: + page = self.conn.getVertices( + vertex_type, + select="content_hash", + limit=_PAGE, + offset=offset, + ) or [] + for vertex in page: + result[str(vertex.get("v_id"))] = str( + (vertex.get("attributes") or {}).get("content_hash") or "" + ) + if len(page) < _PAGE: + break + offset += _PAGE + return result def _all_existing_hashes(self) -> dict[str, str]: return self._all_content_hashes("JiraIssue") From 7dd9dfe16010370971061577bc661463cd5db596 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 17:35:23 +0530 Subject: [PATCH 08/19] GML-2211: Skip bot/empty comments at ingest; paginate hash fetch to avoid REST-4000 --- graphrag/app/connectors/jira/mapper.py | 17 +++++++++++++++-- graphrag/app/connectors/jira/sync.py | 6 +++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/graphrag/app/connectors/jira/mapper.py b/graphrag/app/connectors/jira/mapper.py index a597187d..7efb43d6 100644 --- a/graphrag/app/connectors/jira/mapper.py +++ b/graphrag/app/connectors/jira/mapper.py @@ -340,9 +340,23 @@ def add_vertex(vertex: VertexRecord | None) -> None: raw_comment_id = comment.get("id") if raw_comment_id is None: continue + + # Skip bot/automation comments — accountType "app" means a Jira + # automation rule, CI integration, or service-account bot. These + # produce high-volume noise (build status, deploy notifications, + # auto-transitions) with no useful search content. + author = comment.get("author") or {} + if author.get("accountType") == "app": + continue + + # Skip empty comments — nothing meaningful to store or search. + comment_body = adf_to_markdown(comment.get("body")) + if not comment_body or not comment_body.strip(): + continue + comment_id = str(raw_comment_id) comment_vertex_id = self._id("comment", comment_id) - author_vertex = self._user_vertex(comment.get("author")) + author_vertex = self._user_vertex(author) add_vertex(author_vertex) visibility = comment.get("visibility") if not isinstance(visibility, dict): @@ -355,7 +369,6 @@ def add_vertex(vertex: VertexRecord | None) -> None: ) if value ) - comment_body = adf_to_markdown(comment.get("body")) or "(Empty comment)" comment_body = _filter_long_log_output( comment_body, self.comment_chunker.chunk_size, diff --git a/graphrag/app/connectors/jira/sync.py b/graphrag/app/connectors/jira/sync.py index a0b0479b..55d61027 100644 --- a/graphrag/app/connectors/jira/sync.py +++ b/graphrag/app/connectors/jira/sync.py @@ -289,9 +289,9 @@ def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: # an initial or incremental ingestion. Read the lightweight hash # projection once and filter it in memory. # - # Paginate to stay under TigerGraph's 4MB REST response limit — a - # single getVertices call over a large vertex set (e.g. JiraComment) - # can exceed 4MB and raise REST-4000. + # Paginate in 10 k-vertex pages to stay under TigerGraph's 4 MB REST + # response limit (REST-4000). An unbounded getVertices call will fail + # on any graph with a large number of vertices of that type. _PAGE = 10_000 result: dict[str, str] = {} offset = 0 From ab9fab842466fdf1aaa32db944c1a1a36b1fcbb9 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Wed, 30 Sep 2026 17:50:11 +0530 Subject: [PATCH 09/19] =?UTF-8?q?GML-2211:=20Fix=20pagination=20=E2=80=94?= =?UTF-8?q?=20use=20GSQL=20runInterpretedQuery=20with=20LIMIT/OFFSET=20ins?= =?UTF-8?q?tead=20of=20unsupported=20getVertices=20offset=20param?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- graphrag/app/connectors/jira/sync.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/graphrag/app/connectors/jira/sync.py b/graphrag/app/connectors/jira/sync.py index 55d61027..197c50db 100644 --- a/graphrag/app/connectors/jira/sync.py +++ b/graphrag/app/connectors/jira/sync.py @@ -289,21 +289,26 @@ def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: # an initial or incremental ingestion. Read the lightweight hash # projection once and filter it in memory. # - # Paginate in 10 k-vertex pages to stay under TigerGraph's 4 MB REST - # response limit (REST-4000). An unbounded getVertices call will fail - # on any graph with a large number of vertices of that type. + # getVertices() in this version of pyTigerGraph does not support an + # offset parameter, and an unbounded call hits TigerGraph's 4 MB REST + # limit (REST-4000) on large vertex sets. Use a GSQL interpreted query + # with LIMIT/OFFSET for proper pagination without that constraint. _PAGE = 10_000 result: dict[str, str] = {} offset = 0 while True: - page = self.conn.getVertices( - vertex_type, - select="content_hash", - limit=_PAGE, - offset=offset, - ) or [] + query = ( + f"INTERPRET QUERY() FOR GRAPH {self.graphname} {{\n" + f" verts = {{{vertex_type}.*}};\n" + f" res = SELECT v FROM verts:v\n" + f" LIMIT {_PAGE} OFFSET {offset};\n" + f" PRINT res[res.content_hash];\n" + f"}}" + ) + response = self.conn.runInterpretedQuery(query) or [] + page = response[0].get("res", []) if response else [] for vertex in page: - result[str(vertex.get("v_id"))] = str( + result[str(vertex.get("v_id", ""))] = str( (vertex.get("attributes") or {}).get("content_hash") or "" ) if len(page) < _PAGE: From 39a5fcf39ea78cbd8c5f5f347b93c2a4553e60f1 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 17:57:43 +0530 Subject: [PATCH 10/19] feat(jira): Gemini Embedding v2, live ECC progress, fix CONTAINS_ENTITY edges MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch embedding model to gemini-embedding-2 (fixes 500/503 transient errors) - Bump default_concurrency to 5 (embed_sem=10 concurrent embeddings) - workers.py: look up Document's CONTAINS_ENTITY edge to get correct JiraIssue vertex ID for chunk edges (fixes jira:issue:207120 → jira:gml-2191:issue) - DataSourcesConfig.tsx: poll rebuild_status live during ECC build, show 'Processing documents (X/Y — Z%)' instead of static 'build started' - client.py: bulk-fetch changelogs for all issues in one POST per page - mapper.py: include status/changelog history in issue document text - sync.py: reconcile comment chunks, recovery build on missing embeddings --- ecc/app/graphrag/workers.py | 27 ++- .../src/pages/setup/DataSourcesConfig.tsx | 35 +++- graphrag/app/connectors/jira/client.py | 127 +++++++++++-- graphrag/app/connectors/jira/mapper.py | 44 +++++ graphrag/app/connectors/jira/sync.py | 169 ++++++++++++++++-- 5 files changed, 373 insertions(+), 29 deletions(-) diff --git a/ecc/app/graphrag/workers.py b/ecc/app/graphrag/workers.py index eac7ea96..560389e5 100644 --- a/ecc/app/graphrag/workers.py +++ b/ecc/app/graphrag/workers.py @@ -154,6 +154,25 @@ async def chunk_doc( v_id = doc["v_id"].lower() + # For Jira issue documents, look up the authoritative JiraIssue vertex ID + # from the Document's CONTAINS_ENTITY edge (set by the connector mapper). + # workers.py cannot reliably derive the key-based vertex ID from the + # numeric issue doc_id alone (e.g. "jira:issue-doc:207120" → wrong + # "jira:issue:207120", correct is "jira:gml-2191:issue"). + jira_issue_vertex_id: str | None = None + if chunker_type == "jira" and ":issue-doc:" in v_id: + try: + edges = await conn.getEdges( + "Document", v_id, "CONTAINS_ENTITY", "JiraIssue" + ) + if edges: + jira_issue_vertex_id = edges[0]["to_id"] + except Exception as exc: + logger.warning( + f"Could not look up JiraIssue vertex for {v_id}: {exc}; " + "falling back to derived ID" + ) + # Use get_chunker for all types (including images) # For images, get_chunker returns SingleChunker which preserves markdown image references chunker = ecc_util.get_chunker(chunker_type, graphname=conn.graphname) @@ -174,7 +193,7 @@ async def chunk_doc( # send chunks to be upserted (func, args) logger.debug("chunk writes to upsert_chan") await upsert_chan.put( - (upsert_chunk, (conn, v_id, chunk_id, chunk, i, chunker_type)) + (upsert_chunk, (conn, v_id, chunk_id, chunk, i, chunker_type, jira_issue_vertex_id)) ) # Source-native connectors already write authoritative typed @@ -220,6 +239,7 @@ async def upsert_chunk( chunk, idx, source_type="", + jira_issue_vertex_id: "str | None" = None, ): logger.debug(f"Upserting chunk {chunk_id}") date_added = int(time.time()) @@ -249,7 +269,10 @@ async def upsert_chunk( # Jira writes authoritative typed issues before ECC runs. Link each # generated chunk to that stable issue vertex so vector hits can # traverse into the structured Jira graph without LLM extraction. - issue_id = doc_id.replace(":issue-doc:", ":issue:", 1) + # Prefer the caller-supplied vertex ID (looked up from the Document's + # CONTAINS_ENTITY edge in chunk_doc); fall back to the derived form + # only when the lookup failed. + issue_id = jira_issue_vertex_id or doc_id.replace(":issue-doc:", ":issue:", 1) edges.append(( "DocumentChunk", chunk_id, diff --git a/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx b/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx index 03d2bd9b..641a7e89 100644 --- a/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx +++ b/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx @@ -612,6 +612,37 @@ const DataSourcesConfig: React.FC = () => { ); } buildStarted = true; + + // Poll rebuild_status and show live ECC progress on the Data Sources + // page, identical to the progress bar shown on the KGAdmin page. + const creds = sessionStorage.getItem("auth")!; + const baseMsg = + `Jira ingestion complete: ${result.issues_upserted} issues updated, ` + + `${result.issues_deleted || 0} removed, and ` + + `${result.documents_loaded} changed documents loaded. `; + let pollDone = false; + while (!pollDone) { + await new Promise((r) => setTimeout(r, 3000)); + try { + const statusResp = await fetch( + `/ui/${selectedGraph}/rebuild_status`, + { headers: { Authorization: creds } } + ); + if (!statusResp.ok) break; + const statusData = await statusResp.json(); + if (statusData.is_running) { + const stage = statusData.stage ? ` — ${statusData.stage}` : " — Building…"; + setSyncFeedback({ + type: "pending", + text: baseMsg + `GraphRAG build in progress${stage}`, + }); + } else { + pollDone = true; + } + } catch { + break; + } + } } await loadSources(); setSyncFeedback({ @@ -622,8 +653,8 @@ const DataSourcesConfig: React.FC = () => { `${result.documents_loaded} changed documents loaded.` + (buildStarted ? missingChunkEmbeddings > 0 - ? ` A GraphRAG recovery build has started for ${missingChunkEmbeddings} chunks missing embeddings.` - : " The GraphRAG build for chunking and embedding has started." + ? ` GraphRAG recovery build complete.` + : " GraphRAG build for chunking and embedding complete." : " No Jira changes were detected, so no new build was started."), }); } catch (error: any) { diff --git a/graphrag/app/connectors/jira/client.py b/graphrag/app/connectors/jira/client.py index 8c86b64a..7cae672a 100644 --- a/graphrag/app/connectors/jira/client.py +++ b/graphrag/app/connectors/jira/client.py @@ -3,6 +3,7 @@ from __future__ import annotations import concurrent.futures +import logging import time from collections.abc import Iterator from datetime import datetime, timedelta, timezone @@ -14,6 +15,8 @@ from .config import JiraDataSource +logger = logging.getLogger(__name__) + ISSUE_FIELDS = [ "summary", @@ -273,11 +276,9 @@ def _iter_search(self, *, fields: list[str], incremental: bool): def iter_issue_pages(self) -> Iterator[list[dict[str, Any]]]: """Yield complete issue pages for durable page-level synchronization. - Comment completion is parallelised across issues within each page so - that the extra Jira API calls needed for issues with >10 comments are - issued concurrently rather than one at a time. ``_complete_comments`` - returns immediately when an issue already carries all its comments, so - it is safe to submit every issue to the pool. + Comment completion and changelog fetching are parallelised / batched + across issues within each page so that extra Jira API calls run + concurrently rather than one at a time. """ fields = list(ISSUE_FIELDS) story_points = self.source.scope.story_points_field @@ -285,10 +286,6 @@ def iter_issue_pages(self) -> Iterator[list[dict[str, Any]]]: fields.append(story_points) for issues in self._iter_search_pages(fields=fields, incremental=True): if self.source.scope.include_comments: - # Use up to COMMENT_FETCH_WORKERS threads so that the extra - # comment-pagination API calls for issues with >10 comments run - # in parallel. httpx.Client uses httpcore's connection pool - # which is thread-safe for concurrent requests. workers = min(len(issues), COMMENT_FETCH_WORKERS) with concurrent.futures.ThreadPoolExecutor( max_workers=workers, thread_name_prefix="jira-comment" @@ -297,9 +294,58 @@ def iter_issue_pages(self) -> Iterator[list[dict[str, Any]]]: pool.submit(self._complete_comments, issue) for issue in issues ] - # Wait for all and propagate any exception immediately. for fut in concurrent.futures.as_completed(futures): fut.result() + # Bulk-fetch changelogs for all issues in this page in one POST. + self._bulk_fetch_changelogs(issues) + yield issues + + def iter_issues_by_keys(self, keys: set[str]) -> Iterator[list[dict[str, Any]]]: + """Fetch specific Jira issues by key without a full incremental scan. + + Used for recovery: when a previous sync wrote structural vertices but + failed before embedding, this fetches only the failed issues by key + (JQL ``key in (...)``), avoiding a full re-scan of all Jira issues. + Yields pages of up to 100 issues with comments completed, identical in + structure to ``iter_issue_pages()``. + """ + if not keys: + return + fields = list(ISSUE_FIELDS) + story_points = self.source.scope.story_points_field + if story_points: + fields.append(story_points) + keys_sorted = sorted(keys) + for i in range(0, len(keys_sorted), 100): + batch = keys_sorted[i : i + 100] + jql = "key in (" + ", ".join(batch) + ") ORDER BY updated ASC, key ASC" + payload = self._request( + "POST", + "/rest/api/3/search/jql", + json={ + "jql": jql, + "fields": fields, + "fieldsByKeys": False, + "maxResults": 100, + }, + headers={"Content-Type": "application/json"}, + ) + issues = payload.get("issues") or [] + if not issues: + continue + if self.source.scope.include_comments: + workers = min(len(issues), COMMENT_FETCH_WORKERS) + with concurrent.futures.ThreadPoolExecutor( + max_workers=workers, thread_name_prefix="jira-recovery" + ) as pool: + futures = [ + pool.submit(self._complete_comments, issue) + for issue in issues + ] + for fut in concurrent.futures.as_completed(futures): + fut.result() + # Bulk-fetch changelogs for all issues in this page in one POST. + self._bulk_fetch_changelogs(issues) yield issues def iter_issues(self) -> Iterator[dict[str, Any]]: @@ -333,3 +379,64 @@ def _complete_comments(self, issue: dict[str, Any]) -> None: page["comments"] = comments page["total"] = total fields["comment"] = page + + def _bulk_fetch_changelogs(self, issues: list[dict[str, Any]]) -> None: + """Bulk-fetch changelogs for up to 1,000 issues in a single POST. + + Uses POST /rest/api/3/changelog/bulkfetch which returns changelogs for + all requested issues in one round-trip instead of N individual GETs. + Paginates via nextPageToken if the response is truncated. + Results are attached at issue["changelog"]["histories"] keyed by + Jira issue ID so the mapper can access them as issue.get("changelog"). + On any error the method returns silently — no issue is blocked. + """ + if not issues: + return + + # Build a mapping from Jira numeric id → issue dict for fast lookup. + id_to_issue: dict[str, dict[str, Any]] = { + str(issue["id"]): issue for issue in issues if issue.get("id") + } + keys = [issue["key"] for issue in issues if issue.get("key")] + if not keys: + return + + # Initialise empty changelog on every issue so mapper never sees None. + for issue in issues: + issue.setdefault("changelog", {"histories": []}) + + try: + token: str | None = None + while True: + body: dict[str, Any] = { + "issueIdsOrKeys": keys, + "maxResults": 100, + } + if token: + body["nextPageToken"] = token + payload = self._request( + "POST", + "/rest/api/3/changelog/bulkfetch", + json=body, + headers={"Content-Type": "application/json"}, + ) + for entry in payload.get("issueChangeLogs") or []: + issue_id = str(entry.get("issueId") or "") + issue = id_to_issue.get(issue_id) + if issue is None: + continue + histories = issue["changelog"].setdefault("histories", []) + for h in entry.get("changeHistories") or []: + # Normalise ms-epoch timestamp → ISO string for mapper. + created = h.get("created") + if isinstance(created, (int, float)): + created = datetime.fromtimestamp( + created / 1000, tz=timezone.utc + ).strftime("%Y-%m-%dT%H:%M:%S+00:00") + h = dict(h, created=created) + histories.append(h) + token = payload.get("nextPageToken") + if not token: + break + except Exception as exc: + logger.warning("bulk changelog fetch failed: %s — skipping", exc) diff --git a/graphrag/app/connectors/jira/mapper.py b/graphrag/app/connectors/jira/mapper.py index 7efb43d6..60192ed4 100644 --- a/graphrag/app/connectors/jira/mapper.py +++ b/graphrag/app/connectors/jira/mapper.py @@ -230,6 +230,7 @@ def map(self, issue: dict[str, Any]) -> MappedIssue: fields=fields, project=project, description=description, + changelog=issue.get("changelog") or {}, ) content_hash = hashlib.sha256(document_text.encode("utf-8")).hexdigest() @@ -602,6 +603,7 @@ def _document_text( fields: dict[str, Any], project: dict[str, Any], description: str, + changelog: dict[str, Any] | None = None, ) -> str: status = fields.get("status") or {} status_category = status.get("statusCategory") or {} @@ -640,4 +642,46 @@ def _document_text( ), ] ) + + # Status / assignee / priority change history from the Jira changelog. + # Sorted oldest-first so the timeline reads naturally. + history_lines = self._changelog_lines(changelog or {}) + if history_lines: + lines.append("") + lines.append("## Status History") + lines.extend(history_lines) + return "\n".join(lines).strip() + "\n" + + # Fields we care about in the changelog — skip noise (attachment changes, + # rank updates, etc.). + _CHANGELOG_FIELDS = {"status", "assignee", "priority", "resolution"} + + def _changelog_lines(self, changelog: dict[str, Any]) -> list[str]: + """Return formatted history lines for tracked field changes.""" + histories = changelog.get("histories") or [] + # Sort oldest-first for a readable timeline + histories = sorted(histories, key=lambda h: h.get("created") or "") + lines = [] + for history in histories: + author = _display_name(history.get("author")) or "Unknown" + created = _datetime(history.get("created")) or history.get("created") or "" + items = history.get("items") or [] + for item in items: + field = str(item.get("field") or "").lower() + if field not in self._CHANGELOG_FIELDS: + continue + from_val = str(item.get("fromString") or "").strip() + to_val = str(item.get("toString") or "").strip() + if not to_val: + continue + if from_val: + lines.append( + f"- {created}: {author} changed {field} from" + f" \"{from_val}\" to \"{to_val}\"" + ) + else: + lines.append( + f"- {created}: {author} set {field} to \"{to_val}\"" + ) + return lines diff --git a/graphrag/app/connectors/jira/sync.py b/graphrag/app/connectors/jira/sync.py index 197c50db..1f8bfb5b 100644 --- a/graphrag/app/connectors/jira/sync.py +++ b/graphrag/app/connectors/jira/sync.py @@ -165,6 +165,53 @@ def run(self) -> dict[str, Any]: # pages, ensuring they are re-fetched on the next sync run. pending_checkpoint = self.source.sync.checkpoint + # ── Recovery phase ──────────────────────────────────────────────── + # If a previous sync wrote structural vertices but failed before + # embedding, those vertices have empty content_hash. Re-fetch only + # those specific issues from Jira (targeted key lookup) rather than + # re-scanning all issues from the checkpoint. + recovery_keys = self._find_recovery_issue_keys() + if recovery_keys: + recovery_docs: list[MappedIssue] = [] + recovery_comments: list[MappedComment] = [] + for issues in self.client.iter_issues_by_keys(recovery_keys): + mapped = [mapper.map(issue) for issue in issues] + changed_docs, changed_comms, deleted = self._upsert_issue_records( + mapped, existing_hashes, existing_comment_hashes + ) + recovery_docs.extend(changed_docs) + recovery_comments.extend(changed_comms) + documents_loaded += len(changed_docs) + comments_deleted += deleted + issues_upserted += len(mapped) + for item in mapped: + if item.updated and ( + pending_checkpoint is None + or item.updated > pending_checkpoint + ): + pending_checkpoint = item.updated + + if recovery_docs or recovery_comments: + self._finalize_batch( + recovery_docs, + recovery_comments, + existing_hashes, + existing_comment_hashes, + ) + # Advance checkpoint past the recovered items so the normal + # incremental JQL scan starts from after this recovery batch, + # not from the old pre-failure checkpoint. + if pending_checkpoint != self.source.sync.checkpoint: + self.source.sync.checkpoint = pending_checkpoint + self.store.update_runtime_state(self.graphname, self.source) + logger.info( + "Recovery complete: %d doc(s), %d comment(s) re-embedded " + "for graph=%s source=%s", + len(recovery_docs), len(recovery_comments), + self.graphname, self.source.id, + ) + # ── End recovery phase ──────────────────────────────────────────── + for issues in self._iter_pages_pipelined(): mapped = [mapper.map(issue) for issue in issues] changed_docs, changed_comms, deleted_count = ( @@ -283,6 +330,83 @@ def run(self) -> dict[str, Any]: if self._owns_client: self.client.close() + def _find_recovery_issue_keys(self) -> set[str]: + """Return issue keys whose embeddings are missing from a previous failed sync. + + When _finalize_batch() fails, strip_content_hash() has already cleared + content_hash on the affected JiraIssue and JiraComment vertices. Those + empty-hash vertices are the exact set that needs re-processing — no full + Jira re-scan required. + + Uses REST++ getVertices/getEdges (not GSQL interpreted queries) so it + works with the app's connection pool authentication. + """ + keys: set[str] = set() + + # 1. JiraIssue vertices with empty content_hash — key is in the v_id. + try: + issue_verts = self.conn.getVertices( + "JiraIssue", + where='content_hash=""', + select="content_hash", + limit=5000, + ) or [] + for v in issue_verts: + v_id = str(v.get("v_id", "")) + # v_id format: "jira::issue" + key = v_id.removeprefix("jira:").removesuffix(":issue").upper() + if key: + keys.add(key) + except Exception as exc: + logger.warning( + "Recovery: could not query empty-hash JiraIssue vertices " + "graph=%s: %s", self.graphname, exc, + ) + + # 2. JiraComment vertices with empty content_hash → walk JIRA_COMMENT_ON + # edge to get the parent JiraIssue key. + try: + comment_verts = self.conn.getVertices( + "JiraComment", + where='content_hash=""', + select="content_hash", + limit=5000, + ) or [] + for cv in comment_verts: + comment_v_id = str(cv.get("v_id", "")) + try: + edges = self.conn.getEdges( + "JiraComment", + comment_v_id, + JIRA_COMMENT_ISSUE_EDGE, + ) or [] + for edge in edges: + parent_v_id = str(edge.get("to_id", "")) + key = ( + parent_v_id + .removeprefix("jira:") + .removesuffix(":issue") + .upper() + ) + if key: + keys.add(key) + except Exception: + pass # skip individual comment if edge lookup fails + except Exception as exc: + logger.warning( + "Recovery: could not query empty-hash JiraComment vertices " + "graph=%s: %s", self.graphname, exc, + ) + + if keys: + logger.info( + "Recovery: found %d issue(s) with missing embeddings in " + "graph=%s source=%s — re-fetching from Jira instead of " + "full scan", + len(keys), self.graphname, self.source.id, + ) + return keys + def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: # pyTigerGraph's getVerticesById raises error 601 as soon as any # requested STRING ID does not exist, which is the normal state during @@ -301,6 +425,7 @@ def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: f"INTERPRET QUERY() FOR GRAPH {self.graphname} {{\n" f" verts = {{{vertex_type}.*}};\n" f" res = SELECT v FROM verts:v\n" + f" ORDER BY v.content_hash ASC\n" f" LIMIT {_PAGE} OFFSET {offset};\n" f" PRINT res[res.content_hash];\n" f"}}" @@ -308,8 +433,10 @@ def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: response = self.conn.runInterpretedQuery(query) or [] page = response[0].get("res", []) if response else [] for vertex in page: + # TG prefixes the attribute with the result-set alias: + # "res.content_hash" rather than plain "content_hash". result[str(vertex.get("v_id", ""))] = str( - (vertex.get("attributes") or {}).get("content_hash") or "" + (vertex.get("attributes") or {}).get("res.content_hash") or "" ) if len(page) < _PAGE: break @@ -898,20 +1025,32 @@ def _embed_comment_chunks( store = get_embedding_store(graphname=self.graphname) async def embed_batches() -> None: - for start in range(0, len(chunks), EMBEDDING_BATCH_SIZE): - batch = chunks[start : start + EMBEDDING_BATCH_SIZE] - await store.aadd_embeddings( - [(chunk.text, []) for chunk in batch], - [ - { - "vertex_id": ( - chunk.chunk_id, - "DocumentChunk", - ) - } - for chunk in batch - ], - ) + # Cap at 5 concurrent aadd_embeddings calls. Each call makes ~32 + # sequential Gemini requests; 20 concurrent was causing traffic + # spikes that trigger Gemini 500 INTERNAL (server overload). + # 5 concurrent × ~32 requests = ~160 in-flight, safe for the API. + sem = asyncio.Semaphore(5) + + async def _run_batch(batch: list) -> None: + async with sem: + await store.aadd_embeddings( + [(chunk.text, []) for chunk in batch], + [ + { + "vertex_id": ( + chunk.chunk_id, + "DocumentChunk", + ) + } + for chunk in batch + ], + ) + + tasks = [ + _run_batch(chunks[s : s + EMBEDDING_BATCH_SIZE]) + for s in range(0, len(chunks), EMBEDDING_BATCH_SIZE) + ] + await asyncio.gather(*tasks) asyncio.run(embed_batches()) processed_at = int(time.time()) From 2f9bb93d07bee4688bd7ec4942f56ee047cb0bc1 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 19:55:31 +0530 Subject: [PATCH 11/19] =?UTF-8?q?fix:=20revert=20graph=5Frag.py=20formatti?= =?UTF-8?q?ng=20noise=20=E2=80=94=20restore=20to=20main?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecc/app/graphrag/graph_rag.py | 34 ++++++---------------------------- 1 file changed, 6 insertions(+), 28 deletions(-) diff --git a/ecc/app/graphrag/graph_rag.py b/ecc/app/graphrag/graph_rag.py index 2c57d675..97a0e793 100644 --- a/ecc/app/graphrag/graph_rag.py +++ b/ecc/app/graphrag/graph_rag.py @@ -417,9 +417,7 @@ async def load(conn: AsyncTigerGraphConnection): async def embed( - embed_chan: Channel, - embedding_store: EmbeddingStore, - graphname: str, + embed_chan: Channel, embedding_store: EmbeddingStore, graphname: str ): """ Creates and starts one worker for each embed job @@ -437,10 +435,7 @@ async def embed( v_id = (v_id, index_name) # v_id is a per-vertex identifier derived from user content. logger.debug(f"Embed to {graphname}_{index_name}: {v_id}") - if get_graphrag_config(graphname).get( - "reuse_embedding", - True, - ) and embedding_store.has_embeddings([v_id]): + if get_graphrag_config(graphname).get("reuse_embedding", True) and embedding_store.has_embeddings([v_id]): logger.debug(f"Embeddings for {v_id} already exists, skipping to save cost") n_reused += 1 continue @@ -719,11 +714,7 @@ async def summarize_communities( ) -async def run( - graphname: str, - conn: AsyncTigerGraphConnection, - progress=None, -): +async def run(graphname: str, conn: AsyncTigerGraphConnection, progress=None): """ Set up GraphRAG: - Install necessary queries. @@ -803,8 +794,7 @@ async def new_doc_pipeline(): total_docs = 0 try: count_result = conn.getVertexCount( - "Document", - where="epoch_processed=0", + "Document", where="epoch_processed=0" ) if asyncio.iscoroutine(count_result): count_result = await count_result @@ -838,13 +828,7 @@ async def new_doc_pipeline(): grp.create_task(upsert(upsert_chan)) grp.create_task(load(conn)) - grp.create_task( - embed( - embed_chan, - embedding_store, - graphname, - ) - ) + grp.create_task(embed(embed_chan, embedding_store, graphname)) grp.create_task( extract(extract_chan, upsert_chan, embed_chan, extractor, conn, num_chunk_senders, tracker=doc_tracker) ) @@ -904,13 +888,7 @@ async def new_doc_pipeline(): ) grp.create_task(upsert(upsert_chan)) grp.create_task(load(conn)) - grp.create_task( - embed( - embed_chan, - embedding_store, - graphname, - ) - ) + grp.create_task(embed(embed_chan, embedding_store, graphname)) logger.info("Join comm_process_chan") await comm_process_chan.join() logger.info("Join embed_chan") From 2f63dc896ee09c4210f1b5fa142c0819d2b7b8bf Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 20:03:11 +0530 Subject: [PATCH 12/19] =?UTF-8?q?fix:=20remove=20formatting=20noise=20?= =?UTF-8?q?=E2=80=94=20restore=20main.py=20to=20main,=20trim=20verbose=20c?= =?UTF-8?q?omments=20in=20workers.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecc/app/graphrag/workers.py | 24 ++++++------------------ ecc/app/main.py | 8 ++------ 2 files changed, 8 insertions(+), 24 deletions(-) diff --git a/ecc/app/graphrag/workers.py b/ecc/app/graphrag/workers.py index 560389e5..1c39e13c 100644 --- a/ecc/app/graphrag/workers.py +++ b/ecc/app/graphrag/workers.py @@ -154,11 +154,8 @@ async def chunk_doc( v_id = doc["v_id"].lower() - # For Jira issue documents, look up the authoritative JiraIssue vertex ID - # from the Document's CONTAINS_ENTITY edge (set by the connector mapper). - # workers.py cannot reliably derive the key-based vertex ID from the - # numeric issue doc_id alone (e.g. "jira:issue-doc:207120" → wrong - # "jira:issue:207120", correct is "jira:gml-2191:issue"). + # Look up the authoritative JiraIssue vertex ID from the Document's + # CONTAINS_ENTITY edge — derived IDs are unreliable for key-based vertices. jira_issue_vertex_id: str | None = None if chunker_type == "jira" and ":issue-doc:" in v_id: try: @@ -196,10 +193,8 @@ async def chunk_doc( (upsert_chunk, (conn, v_id, chunk_id, chunk, i, chunker_type, jira_issue_vertex_id)) ) - # Source-native connectors already write authoritative typed - # entities and relationships. Their documents still use this - # production chunk/embed pipeline, but must not create a second, - # LLM-derived copy of the same graph facts. + # Skip LLM entity extraction for source-native connectors (jira) + # that already write authoritative typed entities. from common.config import entity_extraction_switch skip_extraction = chunker_type in ("jira", "jira_comment") if entity_extraction_switch and not skip_extraction: @@ -266,12 +261,7 @@ async def upsert_chunk( "DocumentChunk", util.process_id(f"{doc_id}_chunk_{idx - 1}"), None, )) if source_type == "jira" and ":issue-doc:" in doc_id: - # Jira writes authoritative typed issues before ECC runs. Link each - # generated chunk to that stable issue vertex so vector hits can - # traverse into the structured Jira graph without LLM extraction. - # Prefer the caller-supplied vertex ID (looked up from the Document's - # CONTAINS_ENTITY edge in chunk_doc); fall back to the derived form - # only when the lookup failed. + # Link chunk to the authoritative JiraIssue vertex for graph traversal. issue_id = jira_issue_vertex_id or doc_id.replace(":issue-doc:", ":issue:", 1) edges.append(( "DocumentChunk", @@ -333,9 +323,7 @@ async def embed( async with embed_sem: logger.debug(f"Embedding {v_id}") - # Guard: skip chunks with no text — sending empty content to the - # embedding API returns 500 INTERNAL. Log a warning so the gap - # is visible in the logs without crashing the whole rebuild. + # Skip empty chunks — embedding API returns 500 on empty content. if not content or not content.strip(): logger.warning( f"Skipping embed for {v_id}: content is empty. " diff --git a/ecc/app/main.py b/ecc/app/main.py index a1648fbf..9624cb36 100644 --- a/ecc/app/main.py +++ b/ecc/app/main.py @@ -236,7 +236,7 @@ def rebuild_status( "warnings": task_info.get("warnings"), "communities_incomplete": task_info.get("communities_incomplete"), } - + return { "graphname": graphname, "method": ecc_method, @@ -338,11 +338,7 @@ def progress_cb(msg, current=None, total=None, clear_progress=False): ) try: - result = await run_func( - graphname, - conn, - progress=progress_cb, - ) + result = await run_func(graphname, conn, progress=progress_cb) except TypeError: result = await run_func(graphname, conn) completion = {"status": "completed", "completed_at": time.time()} From 46307e2fe43ae78baabc7c0b4be2c937940be1f7 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 20:08:27 +0530 Subject: [PATCH 13/19] =?UTF-8?q?fix:=20restore=20original=20comments=20in?= =?UTF-8?q?=20workers.py=20=E2=80=94=20keep=20only=20real=20code=20changes?= =?UTF-8?q?=20in=20diff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecc/app/graphrag/workers.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ecc/app/graphrag/workers.py b/ecc/app/graphrag/workers.py index 1c39e13c..ced3a1f7 100644 --- a/ecc/app/graphrag/workers.py +++ b/ecc/app/graphrag/workers.py @@ -193,15 +193,18 @@ async def chunk_doc( (upsert_chunk, (conn, v_id, chunk_id, chunk, i, chunker_type, jira_issue_vertex_id)) ) - # Skip LLM entity extraction for source-native connectors (jira) - # that already write authoritative typed entities. - from common.config import entity_extraction_switch + # send chunks to have entities extracted skip_extraction = chunker_type in ("jira", "jira_comment") - if entity_extraction_switch and not skip_extraction: + if not skip_extraction: logger.debug("chunk writes to extract_chan") await extract_chan.put((chunk, chunk_id)) - else: - logger.debug("chunk writes to embed_chan (extraction bypassed)") + + # When extraction is enabled the extract worker pushes the + # summary-augmented embed message itself (Contextual Retrieval), + # so only embed the raw chunk here when extraction is off. + from common.config import entity_extraction_switch + if not entity_extraction_switch or skip_extraction: + logger.debug("chunk writes to embed_chan (no extraction)") await embed_chan.put((chunk_id, chunk, "DocumentChunk")) if tracker is not None: tracker.chunk_done(chunk_id) From e2b5c48391bbddef58adc8ae4794cd962dbb5d94 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 20:20:31 +0530 Subject: [PATCH 14/19] =?UTF-8?q?revert:=20remove=20SAFE=5FREAD=5FFUNCTION?= =?UTF-8?q?S=20=E2=80=94=20unrelated=20to=20Jira,=20affects=20all=20graphs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- graphrag/app/tools/generate_function.py | 6 +----- graphrag/app/tools/validation_utils.py | 18 ------------------ 2 files changed, 1 insertion(+), 23 deletions(-) diff --git a/graphrag/app/tools/generate_function.py b/graphrag/app/tools/generate_function.py index d6ec2c6d..073cfe38 100644 --- a/graphrag/app/tools/generate_function.py +++ b/graphrag/app/tools/generate_function.py @@ -33,7 +33,6 @@ InvalidFunctionCallException, MapQuestionToSchemaException, NoDocumentsFoundException, - SAFE_READ_FUNCTIONS, validate_function_call, validate_schema, ) @@ -172,15 +171,12 @@ def _run( # Prioritize pyTigerGraph docs over custom docs docs = pytg_docs + custom_docs - registered_function_calls = [ + valid_function_calls = [ x["function_header"] for x in self.embedding_store.list_registered_documents( output_fields=["function_header"] ) ] - valid_function_calls = list( - set(registered_function_calls) | SAFE_READ_FUNCTIONS - ) if len(docs) == 0: LogWriter.warning(f"request_id={req_id_cv.get()} WARN no documents found") diff --git a/graphrag/app/tools/validation_utils.py b/graphrag/app/tools/validation_utils.py index 416a4569..6b18b351 100644 --- a/graphrag/app/tools/validation_utils.py +++ b/graphrag/app/tools/validation_utils.py @@ -23,24 +23,6 @@ class InvalidFunctionCallException(Exception): pass -# Built-in pyTigerGraph reads remain available when an embedding-store backend -# cannot enumerate its registered function documents. Keep this list read-only: -# generated calls execute with the request's TigerGraph connection. -SAFE_READ_FUNCTIONS = frozenset({ - "getEdgeCount", - "getEdgeType", - "getEdgeTypes", - "getEdges", - "getSchema", - "getVertexCount", - "getVertexStats", - "getVertexType", - "getVertexTypes", - "getVertices", - "getVerticesById", -}) - - def validate_schema(conn, v_types, e_types, v_attrs, e_attrs): LogWriter.info(f"request_id={req_id_cv.get()} ENTRY validate_schema()") if v_types: From 6628602b3462cfecb6cacbc35688e381578e1170 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 20:23:37 +0530 Subject: [PATCH 15/19] fix(generate_cypher): handle plain backtick fence; add test coverage for all LLM output formats --- graphrag/app/tools/generate_cypher.py | 2 +- graphrag/tests/test_generate_cypher.py | 17 +++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/graphrag/app/tools/generate_cypher.py b/graphrag/app/tools/generate_cypher.py index a8d2d362..5ac3857e 100644 --- a/graphrag/app/tools/generate_cypher.py +++ b/graphrag/app/tools/generate_cypher.py @@ -32,7 +32,7 @@ def _clean_cypher_output(value: str) -> str: """Remove an optional Markdown language fence without altering the query.""" text = value.strip() fenced = re.fullmatch( - r"```(?:open)?cypher\s*\n?(.*?)\n?```", + r"```(?:(?:open)?cypher)?\s*\n?(.*?)\n?```", text, flags=re.IGNORECASE | re.DOTALL, ) diff --git a/graphrag/tests/test_generate_cypher.py b/graphrag/tests/test_generate_cypher.py index 7e6f7a00..b8aa2f25 100644 --- a/graphrag/tests/test_generate_cypher.py +++ b/graphrag/tests/test_generate_cypher.py @@ -18,3 +18,20 @@ def test_removes_opencypher_markdown_fence(): def test_does_not_strip_query_characters(): query = 'MATCH (issue:JiraIssue) WHERE issue.issue_key = "GML-2191" RETURN issue' assert _clean_cypher_output(query) == query + + +def test_removes_plain_markdown_fence(): + assert _clean_cypher_output("```\nMATCH (n) RETURN n\n```") == ( + "MATCH (n) RETURN n" + ) + + +def test_removes_uppercase_fence(): + assert _clean_cypher_output("```CYPHER\nMATCH (n) RETURN n\n```") == ( + "MATCH (n) RETURN n" + ) + + +def test_plain_query_unchanged(): + query = "MATCH (n) RETURN n" + assert _clean_cypher_output(query) == query From aed7662516445dafb44e2181e61d5e01242c7f3f Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 1 Oct 2026 20:29:11 +0530 Subject: [PATCH 16/19] =?UTF-8?q?revert:=20restore=20.gitignore=20to=20mai?= =?UTF-8?q?n=20=E2=80=94=20no=20unauthorized=20changes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.gitignore b/.gitignore index 45aa8905..fbb59583 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,3 @@ __pycache__/ # Release docs /RELEASE_NOTES.md - -# Jira / graph config files contain real credentials — never commit them. -# Use docs/tutorials/configs/graph_configs/data_sources.json as the example. -configs/graph_configs/*/data_sources.json -configs/graph_configs/*/.DS_Store -configs/.DS_Store From ad31744c77f9157cb2c1b65f56744f5b32515492 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Tue, 6 Oct 2026 21:40:18 +0530 Subject: [PATCH 17/19] Restore shared retrieval and ECC code to main. The Jira connector should not depend on changes in the shared search path. --- common/llm_services/base_llm.py | 1 - ecc/app/ecc_util.py | 8 +- ecc/app/graphrag/workers.py | 82 ++----------------- ecc/tests/test_jira_comment_chunks.py | 72 ---------------- graphrag/app/routers/ui.py | 21 +---- graphrag/app/tools/generate_cypher.py | 22 +---- graphrag/app/tools/graphrag_tools.py | 22 +---- graphrag/tests/test_generate_cypher.py | 37 --------- .../test_structural_function_validation.py | 36 -------- 9 files changed, 14 insertions(+), 287 deletions(-) delete mode 100644 ecc/tests/test_jira_comment_chunks.py delete mode 100644 graphrag/tests/test_generate_cypher.py delete mode 100644 graphrag/tests/test_structural_function_validation.py diff --git a/common/llm_services/base_llm.py b/common/llm_services/base_llm.py index c982c053..fe77ac1f 100644 --- a/common/llm_services/base_llm.py +++ b/common/llm_services/base_llm.py @@ -840,7 +840,6 @@ def generate_cypher_prompt(self): - Prefer attributes over primary IDs when an attribute name is more similar to the keyword in the question. - Keep the query minimal — fewest vertex types, edge types, and attributes possible. - Do NOT return attributes that aren't explicitly mentioned in the question. If only a vertex is mentioned, return only the vertex. -- For Jira business-key lookups where only the numeric portion is given (e.g. `2192`), use `ENDS WITH "-2192"` on the issue_key attribute rather than an exact match. - Always include the entity from the `WHERE` clause in the final `RETURN`. Use vertex name over ID when available. - Always use **undirected** edge patterns. Ensure edges connect correct vertex types per schema. - Use **double quotes** for strings. diff --git a/ecc/app/ecc_util.py b/ecc/app/ecc_util.py index 25e9bf64..ee96e170 100644 --- a/ecc/app/ecc_util.py +++ b/ecc/app/ecc_util.py @@ -30,13 +30,7 @@ def get_chunker(chunker_type: str = "", graphname: str = None): chunk_size=chunker_config.get("chunk_size", 0), overlap_size=chunker_config.get("overlap_size", -1), ) - elif chunker_type in ( - "structured", - "markdown", - "html", - "jira", - "jira_comment", - ): + elif chunker_type in ("structured", "markdown", "html"): # Structure-aware chunker for markdown AND HTML: tables/figures/lists/ # code stay atomic (never split mid-row), prose char-splits by size. # Supersedes MarkdownChunker/HTMLChunker, which split structure blindly. diff --git a/ecc/app/graphrag/workers.py b/ecc/app/graphrag/workers.py index ced3a1f7..518a2d82 100644 --- a/ecc/app/graphrag/workers.py +++ b/ecc/app/graphrag/workers.py @@ -154,22 +154,6 @@ async def chunk_doc( v_id = doc["v_id"].lower() - # Look up the authoritative JiraIssue vertex ID from the Document's - # CONTAINS_ENTITY edge — derived IDs are unreliable for key-based vertices. - jira_issue_vertex_id: str | None = None - if chunker_type == "jira" and ":issue-doc:" in v_id: - try: - edges = await conn.getEdges( - "Document", v_id, "CONTAINS_ENTITY", "JiraIssue" - ) - if edges: - jira_issue_vertex_id = edges[0]["to_id"] - except Exception as exc: - logger.warning( - f"Could not look up JiraIssue vertex for {v_id}: {exc}; " - "falling back to derived ID" - ) - # Use get_chunker for all types (including images) # For images, get_chunker returns SingleChunker which preserves markdown image references chunker = ecc_util.get_chunker(chunker_type, graphname=conn.graphname) @@ -189,25 +173,19 @@ async def chunk_doc( # send chunks to be upserted (func, args) logger.debug("chunk writes to upsert_chan") - await upsert_chan.put( - (upsert_chunk, (conn, v_id, chunk_id, chunk, i, chunker_type, jira_issue_vertex_id)) - ) + await upsert_chan.put((upsert_chunk, (conn, v_id, chunk_id, chunk, i))) # send chunks to have entities extracted - skip_extraction = chunker_type in ("jira", "jira_comment") - if not skip_extraction: - logger.debug("chunk writes to extract_chan") - await extract_chan.put((chunk, chunk_id)) + logger.debug("chunk writes to extract_chan") + await extract_chan.put((chunk, chunk_id)) # When extraction is enabled the extract worker pushes the # summary-augmented embed message itself (Contextual Retrieval), # so only embed the raw chunk here when extraction is off. from common.config import entity_extraction_switch - if not entity_extraction_switch or skip_extraction: + if not entity_extraction_switch: logger.debug("chunk writes to embed_chan (no extraction)") await embed_chan.put((chunk_id, chunk, "DocumentChunk")) - if tracker is not None: - tracker.chunk_done(chunk_id) return v_id @@ -230,15 +208,7 @@ async def upsert_doc(conn: AsyncTigerGraphConnection, doc_id, ctype, content_tex conn, "Document", doc_id, "HAS_CONTENT", "Content", doc_id ) -async def upsert_chunk( - conn: AsyncTigerGraphConnection, - doc_id, - chunk_id, - chunk, - idx, - source_type="", - jira_issue_vertex_id: "str | None" = None, -): +async def upsert_chunk(conn: AsyncTigerGraphConnection, doc_id, chunk_id, chunk, idx): logger.debug(f"Upserting chunk {chunk_id}") date_added = int(time.time()) # Build the chunk's full vertex + edge bundle and enqueue atomically. @@ -263,39 +233,6 @@ async def upsert_chunk( "DocumentChunk", chunk_id, "IS_AFTER", "DocumentChunk", util.process_id(f"{doc_id}_chunk_{idx - 1}"), None, )) - if source_type == "jira" and ":issue-doc:" in doc_id: - # Link chunk to the authoritative JiraIssue vertex for graph traversal. - issue_id = jira_issue_vertex_id or doc_id.replace(":issue-doc:", ":issue:", 1) - edges.append(( - "DocumentChunk", - chunk_id, - "CONTAINS_ENTITY", - "JiraIssue", - issue_id, - None, - )) - elif source_type == "jira_comment" and ":comment-doc:" in doc_id: - issue_id, comment_id = doc_id.rsplit(":comment-doc:", 1) - cloud_prefix = issue_id.split(":issue:", 1)[0] - comment_vertex_id = f"{cloud_prefix}:comment:{comment_id}" - edges.extend([ - ( - "DocumentChunk", - chunk_id, - "CONTAINS_ENTITY", - "JiraIssue", - issue_id, - None, - ), - ( - "DocumentChunk", - chunk_id, - "CONTAINS_ENTITY", - "JiraComment", - comment_vertex_id, - None, - ), - ]) await util.upsert_group(conn, vertices, edges) @@ -326,14 +263,6 @@ async def embed( async with embed_sem: logger.debug(f"Embedding {v_id}") - # Skip empty chunks — embedding API returns 500 on empty content. - if not content or not content.strip(): - logger.warning( - f"Skipping embed for {v_id}: content is empty. " - "Check the source document for missing text." - ) - return - # if loader is running, wait until it's done if not util.loading_event.is_set(): logger.debug("Embed worker waiting for loading event to finish") @@ -342,7 +271,6 @@ async def embed( await embed_store.aadd_embeddings([(content, [])], [{"vertex_id": v_id}]) except Exception as e: logger.error(f"Failed to add embeddings for {v_id}: {e}") - raise def _is_near_duplicate(new_desc, existing_descs, threshold=0.85): diff --git a/ecc/tests/test_jira_comment_chunks.py b/ecc/tests/test_jira_comment_chunks.py deleted file mode 100644 index 9073eb92..00000000 --- a/ecc/tests/test_jira_comment_chunks.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations - -import pytest - -from common.embeddings.tigergraph_embedding_store import ( - TigerGraphEmbeddingStore, -) -from graphrag import workers - - -@pytest.mark.asyncio -async def test_jira_comment_chunk_links_comment_and_issue(monkeypatch): - captured: dict = {} - - async def capture_group(conn, vertices, edges): - captured["vertices"] = vertices - captured["edges"] = edges - - monkeypatch.setattr(workers.util, "upsert_group", capture_group) - - await workers.upsert_chunk( - object(), - "jira:cloud-1:issue:10422:comment-doc:9001", - "chunk-1", - "Short Jira comment", - 0, - "jira_comment", - ) - - assert ( - "DocumentChunk", - "chunk-1", - "CONTAINS_ENTITY", - "JiraIssue", - "jira:cloud-1:issue:10422", - None, - ) in captured["edges"] - assert ( - "DocumentChunk", - "chunk-1", - "CONTAINS_ENTITY", - "JiraComment", - "jira:cloud-1:comment:9001", - None, - ) in captured["edges"] - - -def test_exhausted_embedding_retries_raise(): - provider_error = RuntimeError("500 INTERNAL") - - with pytest.raises(RuntimeError, match="Failed to embed chunk-1"): - TigerGraphEmbeddingStore._log_embed_failure( - "chunk-1", - provider_error, - ) - - -@pytest.mark.asyncio -async def test_embedding_worker_propagates_store_failure(): - class FailingStore: - async def aadd_embeddings(self, *args, **kwargs): - raise RuntimeError("embedding provider unavailable") - - workers.util.loading_event.set() - - with pytest.raises(RuntimeError, match="embedding provider unavailable"): - await workers.embed( - object(), - FailingStore(), - ("chunk-1", "DocumentChunk"), - "chunk content", - ) diff --git a/graphrag/app/routers/ui.py b/graphrag/app/routers/ui.py index 71602acf..a482e90f 100644 --- a/graphrag/app/routers/ui.py +++ b/graphrag/app/routers/ui.py @@ -2400,18 +2400,12 @@ def get_rebuild_status( """ Check if a GraphRAG rebuild is currently in progress for the specified graph. Returns the current status without triggering a new rebuild. - Also surfaces any blocking graph-level operation (e.g. jira_sync) so the - UI can show a meaningful message before the user attempts a rebuild. Uses HTTP Basic Authentication to get credentials. """ # Extract credentials from the dependency creds = creds[1] auth_header = _ecc_auth_header(creds) - # Check the graph-level lock first — a jira_sync or other operation may be - # in flight and is about to trigger a rebuild even if ECC hasn't started yet. - current_op = get_current_operation(graphname) - try: ecc_status_url = ( graphrag_config.get("ecc", "http://graphrag-ecc:8001") @@ -2426,21 +2420,13 @@ def get_rebuild_status( ) if response.status_code == 200: - payload = response.json() - # Merge graph-lock info so the UI can detect a pending rebuild - # triggered by a data-source sync that hasn't handed off to ECC yet. - payload["current_operation"] = current_op - if current_op and not payload.get("is_running"): - payload["is_running"] = True - payload["status"] = "pending" - return payload + return response.json() else: LogWriter.warning(f"ECC status check returned {response.status_code}") return { "graphname": graphname, - "is_running": bool(current_op), - "status": "pending" if current_op else "unknown", - "current_operation": current_op, + "is_running": False, + "status": "unknown", "error": f"ECC service returned status {response.status_code}" } except httpx.TimeoutException as e: @@ -2453,7 +2439,6 @@ def get_rebuild_status( "graphname": graphname, "is_running": True, "status": cached.get("status", "unknown"), - "current_operation": current_op, "error": "ECC is busy processing, status check timed out. Rebuild likely still in progress." } except Exception as e: diff --git a/graphrag/app/tools/generate_cypher.py b/graphrag/app/tools/generate_cypher.py index 5ac3857e..d9199de4 100644 --- a/graphrag/app/tools/generate_cypher.py +++ b/graphrag/app/tools/generate_cypher.py @@ -13,7 +13,6 @@ # along with this program. If not, see . import logging -import re from typing import Iterable from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import PromptTemplate @@ -28,22 +27,6 @@ logger = logging.getLogger(__name__) -def _clean_cypher_output(value: str) -> str: - """Remove an optional Markdown language fence without altering the query.""" - text = value.strip() - fenced = re.fullmatch( - r"```(?:(?:open)?cypher)?\s*\n?(.*?)\n?```", - text, - flags=re.IGNORECASE | re.DOTALL, - ) - if fenced: - return fenced.group(1).strip() - lines = text.splitlines() - if lines and lines[0].strip().casefold() in {"cypher", "opencypher"}: - return "\n".join(lines[1:]).strip() - return text - - class GenerateCypher(BaseTool): """GenerateCypher Tool. Tool to generate and execute the appropriate Cypher query for the question. @@ -111,12 +94,11 @@ def generate_cypher(self, question: str, history: Iterable[str]) -> str: schema = self._generate_schema_rep() logger.debug_pii("Prompt to LLM:\n" + PROMPT.invoke({"question": question, "schema": schema, "history": history}).to_string()) - raw = self.llm.invoke_with_parser( + out = self.llm.invoke_with_parser( PROMPT, StrOutputParser(), {"question": question, "schema": schema, "history": history}, caller_name="generate_cypher", - ) - out = _clean_cypher_output(raw) + ).strip("```cypher").strip("```").strip() # Validate the LLM output looks like a Cypher query out_upper = out.upper() diff --git a/graphrag/app/tools/graphrag_tools.py b/graphrag/app/tools/graphrag_tools.py index 5d388b5b..ca4b9f93 100644 --- a/graphrag/app/tools/graphrag_tools.py +++ b/graphrag/app/tools/graphrag_tools.py @@ -28,7 +28,6 @@ from __future__ import annotations -import json import logging from dataclasses import dataclass, field from typing import Any, Callable, Dict, Optional @@ -86,14 +85,7 @@ def _empty(summary: str) -> dict: def _result_is_empty(result: Any) -> bool: if result is None: return True - if isinstance(result, str): - if not result.strip(): - return True - try: - return _result_is_empty(json.loads(result)) - except (TypeError, ValueError): - return False - if isinstance(result, (list, dict)) and len(result) == 0: + if isinstance(result, (list, dict, str)) and len(result) == 0: return True return False @@ -175,6 +167,7 @@ def structural_retrieve(ctx: GraphRAGToolContext, question: str) -> dict: def _cypher_retrieve(ctx: GraphRAGToolContext, question: str) -> dict: + import json ctx.emit("Generating a graph query") gen_history: list = [] for i in range(3): @@ -183,22 +176,13 @@ def _cypher_retrieve(ctx: GraphRAGToolContext, question: str) -> dict: except ValueError as exc: gen_history.append(f"{i}: Error: {exc}\n") continue - try: - response = ctx.conn.gsql(cypher) - except Exception as exc: - logger.warning("openCypher execution failed: %s", exc) - gen_history.append(f"{i}: {cypher}\n\tExecution error: {exc}\n") - continue + response = ctx.conn.gsql(cypher) json_str = "\n".join(response.split("\n")[1:]) try: parsed = json.loads(json_str) except Exception: gen_history.append(f"{i}: {cypher}\n\tError: {json_str}\n") continue - if parsed.get("error"): - message = parsed.get("message") or parsed - gen_history.append(f"{i}: {cypher}\n\tError: {message}\n") - continue rows = parsed.get("results", [{}]) first = rows[0] if rows else None if not _result_is_empty(first): diff --git a/graphrag/tests/test_generate_cypher.py b/graphrag/tests/test_generate_cypher.py deleted file mode 100644 index b8aa2f25..00000000 --- a/graphrag/tests/test_generate_cypher.py +++ /dev/null @@ -1,37 +0,0 @@ -"""Regression tests for generated openCypher normalization.""" - -from tools.generate_cypher import _clean_cypher_output - - -def test_removes_cypher_markdown_fence(): - assert _clean_cypher_output("```cypher\nMATCH (n) RETURN n\n```") == ( - "MATCH (n) RETURN n" - ) - - -def test_removes_opencypher_markdown_fence(): - assert _clean_cypher_output("```opencypher\nMATCH (n) RETURN n\n```") == ( - "MATCH (n) RETURN n" - ) - - -def test_does_not_strip_query_characters(): - query = 'MATCH (issue:JiraIssue) WHERE issue.issue_key = "GML-2191" RETURN issue' - assert _clean_cypher_output(query) == query - - -def test_removes_plain_markdown_fence(): - assert _clean_cypher_output("```\nMATCH (n) RETURN n\n```") == ( - "MATCH (n) RETURN n" - ) - - -def test_removes_uppercase_fence(): - assert _clean_cypher_output("```CYPHER\nMATCH (n) RETURN n\n```") == ( - "MATCH (n) RETURN n" - ) - - -def test_plain_query_unchanged(): - query = "MATCH (n) RETURN n" - assert _clean_cypher_output(query) == query diff --git a/graphrag/tests/test_structural_function_validation.py b/graphrag/tests/test_structural_function_validation.py deleted file mode 100644 index 495c2c7c..00000000 --- a/graphrag/tests/test_structural_function_validation.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Regression tests for the built-in structural read-function surface.""" - -import pytest - -from tools.graphrag_tools import _result_is_empty -from tools.validation_utils import ( - InvalidFunctionCallException, - SAFE_READ_FUNCTIONS, - validate_function_call, -) - - -class _Connection: - graphname = "JiraTest" - - def getEndpoints(self, dynamic=True): - return {} - - -def test_get_vertices_is_available_without_registered_function_documents(): - call = 'getVertices("JiraIssue", where=\'issue_key == "GML-2186"\')' - - assert validate_function_call(_Connection(), call, SAFE_READ_FUNCTIONS) == call - - -def test_mutating_function_is_not_in_builtin_read_surface(): - with pytest.raises(InvalidFunctionCallException): - validate_function_call( - _Connection(), - 'upsertVertex("JiraIssue", "x", {})', - SAFE_READ_FUNCTIONS, - ) - - -def test_json_encoded_empty_result_triggers_structural_fallback(): - assert _result_is_empty("[]") From 23d0354f49f4d23b4eb42613d7332cc419a9a5a9 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Tue, 6 Oct 2026 21:40:18 +0530 Subject: [PATCH 18/19] Store each Jira ticket as separate facts and keep a new project on the full scan. Recovery no longer treats link stubs as failed embeddings or moves the checkpoint, and the data-sources page shows prompts to append. --- .../src/pages/setup/ConnectorPromptGuide.tsx | 223 +++++++++ .../src/pages/setup/DataSourcesConfig.tsx | 2 + graphrag/app/connectors/jira/mapper.py | 331 +++++++++---- graphrag/app/connectors/jira/schema.py | 43 +- graphrag/app/connectors/jira/sync.py | 463 +++++++++++------- graphrag/tests/test_jira_connector.py | 368 ++++++++++++-- 6 files changed, 1106 insertions(+), 324 deletions(-) create mode 100644 graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx diff --git a/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx b/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx new file mode 100644 index 00000000..24d6c9f1 --- /dev/null +++ b/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx @@ -0,0 +1,223 @@ +import React, { useState } from "react"; +import { Link } from "react-router-dom"; +import { BookOpen, Check, ChevronDown, Copy } from "lucide-react"; +import { Button } from "@/components/ui/button"; +import { cn } from "@/lib/utils"; + +interface ExamplePrompt { + id: string; + name: string; + where: string; + text: string; +} + +interface ConnectorGuide { + title: string; + summary: string; + points: string[]; + examples: ExamplePrompt[]; +} + +const JIRA_PLANNER_EXAMPLE = `## Jira issues +Each Jira issue is stored in two places. They answer different parts of the same question, so a question about one issue needs both. + +- Structural (graphrag__structural_retrieve): the current record. JiraIssue attributes are issue_key, summary, issue_type, status, status_category, priority, resolution, labels, components, fix_versions, created, updated, due, url, story_points. Assignee and reporter are JiraUser vertices reached by JIRA_ASSIGNED_TO and JIRA_REPORTED_BY. Project, parent, and issue links are edges. Comment text is JiraComment.body. Status, assignee, priority, and resolution changes are JiraChange events (field, from_value, to_value, created) reached by JIRA_HAS_CHANGE. The person who made a change is JIRA_CHANGE_BY. +- Unstructured (graphrag__hybrid_search): short document chunks for that issue. Each chunk starts with the issue key. They hold the description, comments, and one chunk per change: who changed status, assignee, priority, or resolution, when, and from which value to which. + +Decide which issue the user means, using this message and ## Conversation: +- A project key plus a number (any casing, such as gml-2191) identifies one issue. +- Only a number (ticket 2191, summary of 2191) does not. Retrieve every issue_key that ends with that number so the answer can ask which project they mean. Do not plan a full write-up of one issue until a single key is known. +- A follow-up that does not repeat the key refers to the issue already discussed. Carry that key into every retrieval step. + +Once one issue is identified, plan both retrievals. Put the full issue key and the aspect the user asked about into both tool questions. Set hybrid top_k to at least 8. +- A broad question (what is it, summarize, tell me about) needs the current fields plus the description, recent comments, and notable changes. +- History, timeline, progress, or who did what needs the JiraChange events and the comment text, together with the current status, assignee, and resolution. +- Release, fix version, due date, or when it will be fixed needs fix_versions, due, status, and resolution, plus any release or fix mentioned in the description or comments. + +For the structural step, say which match to use: an exact case-insensitive match on issue_key when the full key is known, or issue_key ending with the number when only a number is known. Name the attributes and neighbour names to return. Returning the vertex alone is not enough. + +A count, list, or filter across many issues stays a structural query. Add hybrid search only when that question also needs description, comment, or change-log text. + +The final answer step depends on every retrieval step.`; + +const JIRA_AGENT_EXAMPLE = `## Jira issues +Each issue has a current record in the graph and short document chunks. For one issue, call both graphrag__structural_retrieve and graphrag__hybrid_search before you answer. Use the conversation to resolve which issue is meant when this message does not repeat the key. + +- Structural returns current fields: issue_key, summary, type, status, priority, resolution, labels, components, fix_versions, created, updated, due, url, story_points, plus assignee and reporter names. Comment text is JiraComment.body. Status, assignee, priority, and resolution changes are JiraChange events. Ask for an exact case-insensitive issue_key match when the full key is known. When the user gave only a number, ask for every issue_key that ends with that number and return the keys and summaries. Always ask for the attributes by name. A bare vertex id has no fields. +- Hybrid search (top_k at least 8) returns chunks that start with the issue key: the description, comments, and the change log. Include the issue key and the aspect the user asked about in the search text. + +If several project keys share that number, ask which one they mean and list the full keys. Do not mix those issues into one answer. + +Answer only what was asked, using both sources. A broad question gets the current fields plus the description and recent activity. A history question gets the change log in time order and the comments, with the current status beside it. A release or "when will this be fixed" question uses fix_versions, due, status, and resolution, plus any release mentioned in the text. If that fact is in neither source, say it is not available.`; + +const JIRA_RESPONSE_EXAMPLE = `## Jira issues +Treat structured rows and document passages as one issue, not as competing answers. + +- Structured rows are the current record: issue key, summary, status, priority, resolution, fix version, due date, url, assignee, reporter, labels, components, and links. Comment text is the comment body. Each status, assignee, priority, or resolution change is its own event: field, previous value, new value, who, and when. +- Document passages are the description, comments, and the change log. Use a passage only when it names the same issue key. + +If the user named only a number and the context has more than one project key for it, ask which issue they mean and list the full keys. Do not include details from those issues. If only one key matches, confirm that key and then answer. + +If the user message is only a full issue key, they are selecting that issue. Answer with its current fields and a short account of the description and recent activity. + +Otherwise answer the question they asked. +- A broad question (what is it, summarize, tell me about) gets the title and url, the current fields, then the description and the recent comments or changes. +- History, timeline, progress, or who did what gets the change log in time order, then the comments. State the current status from the structured row. +- Release, fix version, or when it will be fixed uses fix version, due date, status, and resolution, plus any release or fix stated in the description or comments. If none of those say when or in which release, say that is not in the available data. An empty fix version is not a release. A due date of 1970-01-01 means no due date was set. + +Prefer a structured value when the same field also appears in a passage. Do not invent a person, date, status, or release.`; + +const GUIDES: Record = { + jira_cloud: { + title: "Jira Cloud", + summary: + "Keep the prompt that is already on this graph. Add a short Jira section at the end so one ticket is answered from the current record and from the description, comments, and change history.", + points: [ + "A project key plus a number, in any casing, is one ticket. Copy that key into both the structural question and the hybrid question.", + "A number alone is not one ticket. Retrieve every issue key that ends with that number. If more than one project matches, ask which key they mean and list the full keys.", + "A follow-up that does not repeat the key is about the ticket already in the conversation. Carry that key into the next retrieval.", + "Structural search returns the current fields, the people, and the JiraChange events. Hybrid search returns the description, comment text, and one chunk per change. Use both for a question about one ticket.", + "Do not put a real ticket key in the prompt. The model will treat that key as the ticket to retrieve. If fix version or due date is empty, say it is not available.", + ], + examples: [ + { + id: "agentic_planner", + name: "Agentic Planner", + where: "Customize Prompts → Agentic Planner, on this graph", + text: JIRA_PLANNER_EXAMPLE, + }, + { + id: "agentic_agent", + name: "React Agent", + where: "Customize Prompts → React Agent, on this graph", + text: JIRA_AGENT_EXAMPLE, + }, + { + id: "chatbot_response", + name: "Chatbot Responses", + where: "Customize Prompts → Chatbot Responses, on this graph", + text: JIRA_RESPONSE_EXAMPLE, + }, + ], + }, +}; + +const ConnectorPromptGuide: React.FC<{ connectorType: string }> = ({ + connectorType, +}) => { + const guide = GUIDES[connectorType]; + const [openId, setOpenId] = useState(null); + const [copiedId, setCopiedId] = useState(null); + + if (!guide) return null; + + const copyExample = async (example: ExamplePrompt) => { + try { + await navigator.clipboard.writeText(example.text); + setCopiedId(example.id); + window.setTimeout(() => { + setCopiedId((current) => (current === example.id ? null : current)); + }, 2000); + } catch { + setCopiedId(null); + } + }; + + return ( +
+
+ +
+

+ 4. Prompt guide +

+

+ How to write the {guide.title} prompt for this graph. +

+
+
+ +
+

+ How to build the prompt +

+

{guide.summary}

+
    + {guide.points.map((point) => ( +
  • {point}
  • + ))} +
+
+ +
+ These examples are an addition, not a replacement. Open{" "} + + Customize Prompts + + , select this graph, and open the prompt named on the example. Leave + the instructions already in that prompt, scroll to the end, and paste + the example there. Then save. Do not delete the existing prompt text. +
+ +
+

+ Example prompts +

+ {guide.examples.map((example) => { + const open = openId === example.id; + return ( +
+
+ + +
+ {open && ( +
+                  {example.text}
+                
+ )} +
+ ); + })} +
+
+ ); +}; + +export default ConnectorPromptGuide; diff --git a/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx b/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx index 641a7e89..1593e7c1 100644 --- a/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx +++ b/graphrag-ui/src/pages/setup/DataSourcesConfig.tsx @@ -13,6 +13,7 @@ import { Input } from "@/components/ui/input"; import { safeJson } from "@/utils/safeJson"; import { pauseIdleTimer, pingIdleTimer, resumeIdleTimer } from "@/hooks/useIdleTimeout"; import type { JiraSchemaStatus, JiraSource } from "@/types/dataSources"; +import ConnectorPromptGuide from "@/pages/setup/ConnectorPromptGuide"; const editingSourceKey = (graph: string) => `graphrag:jira-editing-source:${graph}`; @@ -1402,6 +1403,7 @@ const DataSourcesConfig: React.FC = () => {
))} + )} diff --git a/graphrag/app/connectors/jira/mapper.py b/graphrag/app/connectors/jira/mapper.py index 60192ed4..184f2614 100644 --- a/graphrag/app/connectors/jira/mapper.py +++ b/graphrag/app/connectors/jira/mapper.py @@ -17,6 +17,8 @@ from .config import JiraDataSource from .schema import ( JIRA_ASSIGNEE_EDGE, + JIRA_CHANGE_AUTHOR_EDGE, + JIRA_CHANGE_EDGE, JIRA_COMMENT_AFTER_EDGE, JIRA_COMMENT_AUTHOR_EDGE, JIRA_COMMENT_ISSUE_EDGE, @@ -61,6 +63,17 @@ class MappedComment: chunks: list[MappedChunk] +@dataclass +class MappedFact: + """One short searchable fact for an issue. + + entities are extra CONTAINS_ENTITY targets besides the issue itself. + """ + + chunk: MappedChunk + entities: tuple[tuple[str, str], ...] = () + + @dataclass class MappedIssue: issue_id: str @@ -69,7 +82,9 @@ class MappedIssue: content_hash: str vertices: list[VertexRecord] edges: list[EdgeRecord] - document: dict[str, Any] + facts: list[MappedFact] + legacy_document_id: str + change_vertex_ids: list[str] comments: list[MappedComment] @@ -224,15 +239,6 @@ def map(self, issue: dict[str, Any]) -> MappedIssue: issue_url = f"{site_url}/browse/{issue_key}" description = adf_to_markdown(fields.get("description")) - document_text = self._document_text( - issue_key=issue_key, - issue_url=issue_url, - fields=fields, - project=project, - description=description, - changelog=issue.get("changelog") or {}, - ) - content_hash = hashlib.sha256(document_text.encode("utf-8")).hexdigest() status = fields.get("status") or {} status_category = status.get("statusCategory") or {} @@ -257,7 +263,6 @@ def map(self, issue: dict[str, Any]) -> MappedIssue: "updated": _datetime(fields.get("updated")), "due": _datetime(fields.get("duedate")), "url": issue_url, - "content_hash": content_hash, } if story_points is not None: try: @@ -376,13 +381,13 @@ def add_vertex(vertex: VertexRecord | None) -> None: ) comment_text = self._comment_document_text( issue_key=issue_key, - issue_url=issue_url, - issue_summary=str(fields.get("summary") or ""), comment=comment, - visibility=visibility_text, body=comment_body, ) - chunks = [str(chunk).strip() for chunk in self.comment_chunker.chunk(comment_text)] + chunks = [ + self._lead_with_issue(issue_key, str(chunk)) + for chunk in self.comment_chunker.chunk(comment_text) + ] chunks = [chunk for chunk in chunks if chunk] comment_hash = hashlib.sha256( ( @@ -392,6 +397,7 @@ def add_vertex(vertex: VertexRecord | None) -> None: ).hexdigest() comment_attrs = { "comment_id": comment_id, + "body": comment_body, "created": _datetime(comment.get("created")), "updated": _datetime(comment.get("updated")), "visibility": visibility_text, @@ -552,6 +558,42 @@ def add_vertex(vertex: VertexRecord | None) -> None: ) ) + change_vertices, change_edges, change_parts = self._map_changes( + issue_id=issue_id, + issue_key=issue_key, + changelog=issue.get("changelog") or {}, + ) + for vertex in change_vertices: + add_vertex(vertex) + edges.extend(change_edges) + facts = self._facts( + issue_vertex_id, + [ + ( + self._record_text( + issue_key=issue_key, + issue_url=issue_url, + fields=fields, + project=project, + ), + (), + ), + *self._description_parts(issue_key, description), + *change_parts, + ], + ) + content_hash = hashlib.sha256( + "\0".join(fact.chunk.text for fact in facts).encode("utf-8") + ).hexdigest() + issue_vertex = vertices[("JiraIssue", issue_vertex_id)] + issue_attributes = dict(issue_vertex.attributes) + issue_attributes["content_hash"] = content_hash + vertices[("JiraIssue", issue_vertex_id)] = VertexRecord( + "JiraIssue", + issue_vertex_id, + issue_attributes, + ) + return MappedIssue( issue_id=issue_id, issue_vertex_id=issue_vertex_id, @@ -559,12 +601,9 @@ def add_vertex(vertex: VertexRecord | None) -> None: content_hash=content_hash, vertices=list(vertices.values()), edges=edges, - document={ - "doc_id": self._id("issue-doc", issue_id), - "doc_type": "jira", - "content": document_text, - "position": 0, - }, + facts=facts, + legacy_document_id=self._id("issue-doc", issue_id), + change_vertex_ids=[vertex.vertex_id for vertex in change_vertices], comments=mapped_comments, ) @@ -572,102 +611,113 @@ def _comment_document_text( self, *, issue_key: str, - issue_url: str, - issue_summary: str, comment: dict[str, Any], - visibility: str, body: str, ) -> str: author = _display_name(comment.get("author")) or "Unknown user" - return "\n".join( - [ - f"# Comment by {author} on {issue_key}: {issue_summary}", - f"Issue: {issue_key}", - f"URL: {issue_url}", - f"Comment ID: {comment.get('id') or ''}", - f"Author: {author}", - f"Created: {comment.get('created') or ''}", - f"Updated: {comment.get('updated') or ''}", - f"Visibility: {visibility or 'default'}", - "", - "## Comment", - body, - ] - ).strip() + "\n" + created = _datetime(comment.get("created")) or "" + when = f" at {created}" if created else "" + return ( + f"Issue: {issue_key}\n" + f"Comment by {author}{when}:\n" + f"{body.strip()}" + ) - def _document_text( + def _lead_with_issue(self, issue_key: str, text: str) -> str: + cleaned = text.strip() + prefix = f"Issue: {issue_key}" + if not cleaned or cleaned.startswith(prefix): + return cleaned + return f"{prefix}\n{cleaned}" + + def _record_text( self, *, issue_key: str, issue_url: str, fields: dict[str, Any], project: dict[str, Any], - description: str, - changelog: dict[str, Any] | None = None, ) -> str: status = fields.get("status") or {} status_category = status.get("statusCategory") or {} issue_type = fields.get("issuetype") or {} priority = fields.get("priority") or {} - lines = [ - f"# {issue_key}: {fields.get('summary') or ''}", - f"URL: {issue_url}", - f"Project: {project.get('key') or ''} {project.get('name') or ''}".rstrip(), - f"Type: {issue_type.get('name') or ''}", - ( - f"Status: {status.get('name') or ''} " - f"({status_category.get('key') or ''})" - ).rstrip(), - f"Priority: {priority.get('name') or ''}", - f"Assignee: {_display_name(fields.get('assignee')) or 'Unassigned'}", - f"Reporter: {_display_name(fields.get('reporter'))}", - f"Labels: {_comma_names(fields.get('labels'))}", - f"Components: {_comma_names(fields.get('components'))}", - f"Fix versions: {_comma_names(fields.get('fixVersions'))}", - f"Updated: {fields.get('updated') or ''}", - "", - "## Description", - description or "(No description)", - ] - attachments = fields.get("attachment") or [] - if attachments: - lines.extend( - [ - "", - "## Attachments", - ", ".join( - str(item.get("filename")) - for item in attachments - if item.get("filename") - ), - ] - ) - - # Status / assignee / priority change history from the Jira changelog. - # Sorted oldest-first so the timeline reads naturally. - history_lines = self._changelog_lines(changelog or {}) - if history_lines: - lines.append("") - lines.append("## Status History") - lines.extend(history_lines) + resolution = fields.get("resolution") or {} + project_line = ( + f"{project.get('key') or ''} {project.get('name') or ''}".strip() + ) + status_name = str(status.get("name") or "") + category = str(status_category.get("key") or "") + status_line = ( + f"{status_name} ({category})".strip() + if category + else status_name + ) + lines = [f"Issue: {issue_key}"] + for label, value in ( + ("Summary", fields.get("summary")), + ("URL", issue_url), + ("Project", project_line), + ("Type", issue_type.get("name")), + ("Status", status_line), + ("Priority", priority.get("name")), + ("Resolution", resolution.get("name")), + ("Assignee", _display_name(fields.get("assignee")) or "Unassigned"), + ("Reporter", _display_name(fields.get("reporter"))), + ("Labels", _comma_names(fields.get("labels"))), + ("Components", _comma_names(fields.get("components"))), + ("Fix versions", _comma_names(fields.get("fixVersions"))), + ("Due", _datetime(fields.get("duedate"))), + ("Updated", _datetime(fields.get("updated"))), + ): + text = str(value or "").strip() + if text: + lines.append(f"{label}: {text}") + return "\n".join(lines) - return "\n".join(lines).strip() + "\n" + def _description_parts( + self, + issue_key: str, + description: str, + ) -> list[tuple[str, tuple[tuple[str, str], ...]]]: + if not description or not description.strip(): + return [] + source_text = f"Issue: {issue_key}\nDescription:\n{description.strip()}" + return [ + (self._lead_with_issue(issue_key, str(piece)), ()) + for piece in self.comment_chunker.chunk(source_text) + if str(piece).strip() + ] - # Fields we care about in the changelog — skip noise (attachment changes, - # rank updates, etc.). _CHANGELOG_FIELDS = {"status", "assignee", "priority", "resolution"} - def _changelog_lines(self, changelog: dict[str, Any]) -> list[str]: - """Return formatted history lines for tracked field changes.""" - histories = changelog.get("histories") or [] - # Sort oldest-first for a readable timeline - histories = sorted(histories, key=lambda h: h.get("created") or "") - lines = [] + def _map_changes( + self, + *, + issue_id: str, + issue_key: str, + changelog: dict[str, Any], + ) -> tuple[ + list[VertexRecord], + list[EdgeRecord], + list[tuple[str, tuple[tuple[str, str], ...]]], + ]: + """One JiraChange event and one fact per tracked changelog item.""" + histories = sorted( + changelog.get("histories") or [], + key=lambda history: history.get("created") or "", + ) + vertices: list[VertexRecord] = [] + edges: list[EdgeRecord] = [] + parts: list[tuple[str, tuple[tuple[str, str], ...]]] = [] + change_index = 0 + issue_vertex_id = self._issue_id(issue_key) for history in histories: - author = _display_name(history.get("author")) or "Unknown" - created = _datetime(history.get("created")) or history.get("created") or "" - items = history.get("items") or [] - for item in items: + author = history.get("author") or {} + author_name = _display_name(author) or "Unknown" + created = _datetime(history.get("created")) + history_id = str(history.get("id") or "") + for item in history.get("items") or []: field = str(item.get("field") or "").lower() if field not in self._CHANGELOG_FIELDS: continue @@ -675,13 +725,90 @@ def _changelog_lines(self, changelog: dict[str, Any]) -> list[str]: to_val = str(item.get("toString") or "").strip() if not to_val: continue + change_index += 1 + change_key = ( + f"{issue_id}:{history_id or change_index}:" + f"{field}:{change_index}" + ) + change_vertex_id = self._id("change", change_key) + attributes = { + "change_id": change_key, + "field": field, + "from_value": from_val, + "to_value": to_val, + "created": created, + "ontology_class": "Event", + } + vertices.append( + VertexRecord( + "JiraChange", + change_vertex_id, + { + key: value + for key, value in attributes.items() + if value not in (None, "") + }, + ) + ) + edges.append( + EdgeRecord( + "JiraIssue", + issue_vertex_id, + JIRA_CHANGE_EDGE, + "JiraChange", + change_vertex_id, + ) + ) + author_vertex = self._user_vertex(author) + if author_vertex: + vertices.append(author_vertex) + edges.append( + EdgeRecord( + "JiraChange", + change_vertex_id, + JIRA_CHANGE_AUTHOR_EDGE, + "JiraUser", + author_vertex.vertex_id, + ) + ) + when = f"{created}: " if created else "" if from_val: - lines.append( - f"- {created}: {author} changed {field} from" - f" \"{from_val}\" to \"{to_val}\"" + sentence = ( + f"{when}{author_name} changed {field} from " + f"\"{from_val}\" to \"{to_val}\"" ) else: - lines.append( - f"- {created}: {author} set {field} to \"{to_val}\"" + sentence = ( + f"{when}{author_name} set {field} to \"{to_val}\"" ) - return lines + parts.append( + ( + f"Issue: {issue_key}\n{sentence}", + (("JiraChange", change_vertex_id),), + ) + ) + return vertices, edges, parts + + def _facts( + self, + issue_vertex_id: str, + parts: list[tuple[str, tuple[tuple[str, str], ...]]], + ) -> list[MappedFact]: + facts: list[MappedFact] = [] + for text, entities in parts: + cleaned = text.strip() + if not cleaned: + continue + digest = hashlib.sha256(cleaned.encode("utf-8")).hexdigest()[:12] + index = len(facts) + facts.append( + MappedFact( + chunk=MappedChunk( + chunk_id=f"{issue_vertex_id}:fact:{index}:{digest}", + index=index, + text=cleaned + "\n", + ), + entities=entities, + ) + ) + return facts diff --git a/graphrag/app/connectors/jira/schema.py b/graphrag/app/connectors/jira/schema.py index 85b0d8ef..007b9344 100644 --- a/graphrag/app/connectors/jira/schema.py +++ b/graphrag/app/connectors/jira/schema.py @@ -16,6 +16,8 @@ JIRA_COMMENT_AUTHOR_EDGE = "JIRA_COMMENTED_BY" JIRA_COMMENT_REPLY_EDGE = "JIRA_COMMENT_REPLIES_TO" JIRA_COMMENT_AFTER_EDGE = "JIRA_COMMENT_AFTER" +JIRA_CHANGE_EDGE = "JIRA_HAS_CHANGE" +JIRA_CHANGE_AUTHOR_EDGE = "JIRA_CHANGE_BY" def _attribute_types(metadata: dict[str, Any]) -> dict[str, str]: @@ -67,7 +69,8 @@ def jira_schema_proposal() -> SchemaProposal: "A Jira work item modeled as a POLE+O Object subtype. Filter using " "issue_key, status, status_category (new, indeterminate, or done), " "priority, issue_type, resolution, labels, components, " - "fix_versions, created, updated, or due." + "fix_versions, created, updated, or due. Status, assignee, " + "priority, and resolution changes are JiraChange events." ), [ ("issue_key", "STRING"), @@ -103,12 +106,13 @@ def jira_schema_proposal() -> SchemaProposal: proposal.add_vertex( "JiraComment", ( - "A Jira comment modeled as a POLE+O Event subtype. Its content is " - "embedded through deterministic comment-specific document chunks; " - "use graph edges for issue, author, ordering, and explicit replies." + "A Jira comment modeled as a POLE+O Event subtype. body is the " + "comment text. Searchable text is also stored as document chunks. " + "Use graph edges for issue, author, ordering, and explicit replies." ), [ ("comment_id", "STRING"), + ("body", "STRING"), ("created", "DATETIME"), ("updated", "DATETIME"), ("visibility", "STRING"), @@ -117,6 +121,23 @@ def jira_schema_proposal() -> SchemaProposal: ("content_hash", "STRING"), ], ) + proposal.add_vertex( + "JiraChange", + ( + "One status, assignee, priority, or resolution change, modeled as " + "a POLE+O Event subtype. field is the changed field, from_value " + "and to_value are the previous and new values, and created is " + "when the change happened. The author is JIRA_CHANGE_BY." + ), + [ + ("change_id", "STRING"), + ("field", "STRING"), + ("from_value", "STRING"), + ("to_value", "STRING"), + ("created", "DATETIME"), + ("ontology_class", "STRING"), + ], + ) proposal.add_edge_pair( JIRA_PROJECT_EDGE, @@ -176,6 +197,18 @@ def jira_schema_proposal() -> SchemaProposal: "JiraComment", "Chronological order between adjacent comments on one issue.", ) + proposal.add_edge_pair( + JIRA_CHANGE_EDGE, + "JiraIssue", + "JiraChange", + "A status, assignee, priority, or resolution change on this issue.", + ) + proposal.add_edge_pair( + JIRA_CHANGE_AUTHOR_EDGE, + "JiraChange", + "JiraUser", + "The Atlassian account that made the change.", + ) return proposal @@ -250,6 +283,8 @@ def jira_schema_status(conn) -> dict[str, Any]: ("CONTAINS_ENTITY", "DocumentChunk", "JiraIssue"), ("CONTAINS_ENTITY", "Document", "JiraComment"), ("CONTAINS_ENTITY", "DocumentChunk", "JiraComment"), + ("CONTAINS_ENTITY", "Document", "JiraChange"), + ("CONTAINS_ENTITY", "DocumentChunk", "JiraChange"), ) for edge, source, target in required_links: if not existing.has_edge_pair(edge, source, target): diff --git a/graphrag/app/connectors/jira/sync.py b/graphrag/app/connectors/jira/sync.py index 1f8bfb5b..e26b3004 100644 --- a/graphrag/app/connectors/jira/sync.py +++ b/graphrag/app/connectors/jira/sync.py @@ -6,30 +6,29 @@ import json import logging import queue -import tempfile import threading import time from collections import defaultdict from datetime import datetime, timezone -from pathlib import Path from typing import Any, Iterator from common.config import get_embedding_store from common.db.health import embedding_coverage -from common.py_schemas import LoadingInfo -from supportai import supportai from .client import JiraCloudClient from .config import JiraDataSource from .mapper import ( EdgeRecord, JiraIssueMapper, + MappedChunk, MappedComment, MappedIssue, VertexRecord, ) from .schema import ( JIRA_ASSIGNEE_EDGE, + JIRA_CHANGE_AUTHOR_EDGE, + JIRA_CHANGE_EDGE, JIRA_COMMENT_AFTER_EDGE, JIRA_COMMENT_AUTHOR_EDGE, JIRA_COMMENT_ISSUE_EDGE, @@ -50,6 +49,7 @@ JIRA_REPORTER_EDGE, JIRA_PARENT_EDGE, JIRA_LINK_EDGE, + JIRA_CHANGE_EDGE, ) COMMENT_STATE_EDGES = ( JIRA_COMMENT_ISSUE_EDGE, @@ -166,10 +166,9 @@ def run(self) -> dict[str, Any]: pending_checkpoint = self.source.sync.checkpoint # ── Recovery phase ──────────────────────────────────────────────── - # If a previous sync wrote structural vertices but failed before - # embedding, those vertices have empty content_hash. Re-fetch only - # those specific issues from Jira (targeted key lookup) rather than - # re-scanning all issues from the checkpoint. + # Re-embed ingested issues whose content hash was cleared before + # embedding finished. This does not change the checkpoint and does + # not replace the project scan below. recovery_keys = self._find_recovery_issue_keys() if recovery_keys: recovery_docs: list[MappedIssue] = [] @@ -184,12 +183,6 @@ def run(self) -> dict[str, Any]: documents_loaded += len(changed_docs) comments_deleted += deleted issues_upserted += len(mapped) - for item in mapped: - if item.updated and ( - pending_checkpoint is None - or item.updated > pending_checkpoint - ): - pending_checkpoint = item.updated if recovery_docs or recovery_comments: self._finalize_batch( @@ -198,15 +191,14 @@ def run(self) -> dict[str, Any]: existing_hashes, existing_comment_hashes, ) - # Advance checkpoint past the recovered items so the normal - # incremental JQL scan starts from after this recovery batch, - # not from the old pre-failure checkpoint. - if pending_checkpoint != self.source.sync.checkpoint: - self.source.sync.checkpoint = pending_checkpoint - self.store.update_runtime_state(self.graphname, self.source) + # Do not move the checkpoint here. A cleared checkpoint means a + # new project must be scanned from the start. Link stubs are + # not failed embeddings, and their update times must not become + # the checkpoint for that scan. logger.info( "Recovery complete: %d doc(s), %d comment(s) re-embedded " - "for graph=%s source=%s", + "for graph=%s source=%s. The project scan continues from " + "the saved checkpoint.", len(recovery_docs), len(recovery_comments), self.graphname, self.source.id, ) @@ -331,30 +323,26 @@ def run(self) -> dict[str, Any]: self.client.close() def _find_recovery_issue_keys(self) -> set[str]: - """Return issue keys whose embeddings are missing from a previous failed sync. + """Return issue keys whose own ingest was written but not embedded. - When _finalize_batch() fails, strip_content_hash() has already cleared - content_hash on the affected JiraIssue and JiraComment vertices. Those - empty-hash vertices are the exact set that needs re-processing — no full - Jira re-scan required. - - Uses REST++ getVertices/getEdges (not GSQL interpreted queries) so it - works with the app's connection pool authentication. + Link and parent placeholders also have an empty content_hash. They + were never ingested, so they are not failed embeddings. Only a full + issue (status or updated set) or a real comment (body or created set) + is recovered. This does not replace the project scan. """ keys: set[str] = set() - # 1. JiraIssue vertices with empty content_hash — key is in the v_id. try: issue_verts = self.conn.getVertices( "JiraIssue", where='content_hash=""', - select="content_hash", + select="content_hash,status,updated", limit=5000, ) or [] - for v in issue_verts: - v_id = str(v.get("v_id", "")) - # v_id format: "jira::issue" - key = v_id.removeprefix("jira:").removesuffix(":issue").upper() + for vertex in issue_verts: + if not self._issue_embed_incomplete(vertex): + continue + key = self._issue_key_from_vertex_id(str(vertex.get("v_id", ""))) if key: keys.add(key) except Exception as exc: @@ -363,35 +351,29 @@ def _find_recovery_issue_keys(self) -> set[str]: "graph=%s: %s", self.graphname, exc, ) - # 2. JiraComment vertices with empty content_hash → walk JIRA_COMMENT_ON - # edge to get the parent JiraIssue key. try: comment_verts = self.conn.getVertices( "JiraComment", where='content_hash=""', - select="content_hash", + select="content_hash,body,created", limit=5000, ) or [] - for cv in comment_verts: - comment_v_id = str(cv.get("v_id", "")) + for vertex in comment_verts: + if not self._comment_embed_incomplete(vertex): + continue + comment_v_id = str(vertex.get("v_id", "")) try: edges = self.conn.getEdges( "JiraComment", comment_v_id, JIRA_COMMENT_ISSUE_EDGE, ) or [] - for edge in edges: - parent_v_id = str(edge.get("to_id", "")) - key = ( - parent_v_id - .removeprefix("jira:") - .removesuffix(":issue") - .upper() - ) - if key: - keys.add(key) except Exception: - pass # skip individual comment if edge lookup fails + continue + for edge in edges: + key = self._issue_key_from_vertex_id(str(edge.get("to_id", ""))) + if key: + keys.add(key) except Exception as exc: logger.warning( "Recovery: could not query empty-hash JiraComment vertices " @@ -400,13 +382,51 @@ def _find_recovery_issue_keys(self) -> set[str]: if keys: logger.info( - "Recovery: found %d issue(s) with missing embeddings in " - "graph=%s source=%s — re-fetching from Jira instead of " - "full scan", + "Recovery: found %d ingested issue(s) with missing embeddings " + "in graph=%s source=%s. Re-embedding them without moving the " + "project checkpoint.", len(keys), self.graphname, self.source.id, ) return keys + @staticmethod + def _vertex_attr(vertex: dict, name: str) -> str: + attributes = vertex.get("attributes") or {} + value = attributes.get(name) + if isinstance(value, dict): + value = value.get("value") + return str(value or "").strip() + + @classmethod + def _issue_embed_incomplete(cls, vertex: dict) -> bool: + """True for an ingested issue whose content hash was not saved. + + A link placeholder has no status and no real updated time. + """ + status = cls._vertex_attr(vertex, "status") + updated = cls._vertex_attr(vertex, "updated") + return bool(status) or cls._is_real_timestamp(updated) + + @classmethod + def _comment_embed_incomplete(cls, vertex: dict) -> bool: + """True for a stored comment whose content hash was not saved. + + A parent-comment placeholder has no body and no created time. + """ + body = cls._vertex_attr(vertex, "body") + created = cls._vertex_attr(vertex, "created") + return bool(body) or cls._is_real_timestamp(created) + + @staticmethod + def _is_real_timestamp(value: str) -> bool: + return bool(value) and not value.startswith("1970-01-01") + + @staticmethod + def _issue_key_from_vertex_id(vertex_id: str) -> str: + if not vertex_id.startswith("jira:") or not vertex_id.endswith(":issue"): + return "" + return vertex_id.removeprefix("jira:").removesuffix(":issue").upper() + def _all_content_hashes(self, vertex_type: str) -> dict[str, str]: # pyTigerGraph's getVerticesById raises error 601 as soon as any # requested STRING ID does not exist, which is the normal state during @@ -540,13 +560,10 @@ def _upsert_issue_records( existing_hashes: dict[str, str], existing_comment_hashes: dict[str, str], ) -> tuple[list[MappedIssue], list[MappedComment], int]: - """Write graph vertices/edges for one page; return items to load later. + """Write graph vertices/edges for one page; return items to embed later. - Document loading and comment-chunk embedding are deliberately NOT done - here. The caller accumulates the returned lists across DOC_LOAD_BATCH_PAGES - pages and flushes them together via _finalize_batch(), reducing the - number of supportai.ingest() round-trips from one-per-page to - one-per-batch. + Fact chunks and comment chunks are written by the caller in batches + via _finalize_batch(). Returns (changed_documents, changed_comments, comments_deleted). """ @@ -566,6 +583,7 @@ def _upsert_issue_records( item, existing_comment_hashes, ) + self._reconcile_issue_changes(item) all_vertices: dict[tuple[str, str], VertexRecord] = {} all_edges: list[EdgeRecord] = [] @@ -584,6 +602,8 @@ def _upsert_issue_records( all_edges.extend(item.edges) if item.issue_vertex_id in existing_hashes: self._delete_current_edges(item.issue_vertex_id) + for change_vertex_id in item.change_vertex_ids: + self._delete_change_author_edges(change_vertex_id) for comment in item.comments: if comment.comment_vertex_id in existing_comment_hashes: self._delete_current_comment_edges( @@ -619,33 +639,23 @@ def _finalize_batch( existing_hashes: dict[str, str], existing_comment_hashes: dict[str, str], ) -> None: - """Load documents and embed comment-chunks for a batch of pages. + """Write issue fact chunks and embed comment chunks for a batch. Called after every DOC_LOAD_BATCH_PAGES pages (and once at the end of - the sync). By batching, we replace N-per-page supportai.ingest calls - with one call per batch, cutting fixed HTTP/job-submission overhead by - DOC_LOAD_BATCH_PAGES × for a large initial load. + the sync). Facts are written by the connector, one short chunk per + record, description piece, and change, instead of one document blob. """ if changed_documents: - self._load_documents(changed_documents) - for item in changed_documents: - document_id = item.document["doc_id"].lower() - self.conn.upsertEdge( - "Document", - document_id, - "CONTAINS_ENTITY", - "JiraIssue", - item.issue_vertex_id, - ) - self.conn.upsertVertex( - "JiraIssue", - item.issue_vertex_id, - attributes={"content_hash": item.content_hash}, - ) - existing_hashes[item.issue_vertex_id] = item.content_hash + self._write_issue_facts(changed_documents, existing_hashes) if changed_comments: self._upsert_comment_chunks(changed_comments) - self._embed_comment_chunks(changed_comments) + self._embed_chunks( + [ + chunk + for comment in changed_comments + for chunk in comment.chunks + ] + ) for comment in changed_comments: self.conn.upsertVertex( "JiraComment", @@ -678,6 +688,7 @@ def _upsert_issue_page( item, existing_comment_hashes, ) + self._reconcile_issue_changes(item) all_vertices: dict[tuple[str, str], VertexRecord] = {} all_edges: list[EdgeRecord] = [] @@ -696,6 +707,8 @@ def _upsert_issue_page( all_edges.extend(item.edges) if item.issue_vertex_id in existing_hashes: self._delete_current_edges(item.issue_vertex_id) + for change_vertex_id in item.change_vertex_ids: + self._delete_change_author_edges(change_vertex_id) for comment in item.comments: if comment.comment_vertex_id in existing_comment_hashes: self._delete_current_comment_edges( @@ -729,25 +742,16 @@ def strip_content_hash(vertex_type: str, vertex_id: str) -> None: ] if documents_to_load: if changed_documents: - self._load_documents(changed_documents) + self._write_issue_facts(changed_documents, existing_hashes) if changed_comments: self._upsert_comment_chunks(changed_comments) - self._embed_comment_chunks(changed_comments) - for item in changed_documents: - document_id = item.document["doc_id"].lower() - self.conn.upsertEdge( - "Document", - document_id, - "CONTAINS_ENTITY", - "JiraIssue", - item.issue_vertex_id, - ) - self.conn.upsertVertex( - "JiraIssue", - item.issue_vertex_id, - attributes={"content_hash": item.content_hash}, + self._embed_chunks( + [ + chunk + for comment in changed_comments + for chunk in comment.chunks + ] ) - existing_hashes[item.issue_vertex_id] = item.content_hash for comment in changed_comments: self.conn.upsertVertex( "JiraComment", @@ -792,7 +796,7 @@ def _changed_documents( if existing.get(item.issue_vertex_id) == item.content_hash: continue if item.issue_vertex_id in existing: - self._delete_document_chunks(item.document["doc_id"].lower()) + self._delete_issue_search_content(item) changed.append(item) return changed @@ -847,6 +851,42 @@ def _reconcile_issue_comments( existing_hashes.pop(comment_vertex_id, None) return len(stale_ids) + def _reconcile_issue_changes(self, issue: MappedIssue) -> None: + current_ids = set(issue.change_vertex_ids) + for change_vertex_id in ( + self._issue_change_ids(issue.issue_vertex_id) - current_ids + ): + self.conn.delVerticesById("JiraChange", [change_vertex_id]) + + def _issue_change_ids(self, issue_vertex_id: str) -> set[str]: + try: + edges = self.conn.getEdges( + "JiraIssue", + issue_vertex_id, + JIRA_CHANGE_EDGE, + ) or [] + except Exception as exc: + if "is not a valid vertex id" in str(exc): + return set() + raise + return { + str(edge.get("to_id")) + for edge in edges + if edge.get("to_id") is not None + } + + def _delete_change_author_edges(self, change_vertex_id: str) -> None: + try: + self.conn.delEdges( + "JiraChange", + change_vertex_id, + JIRA_CHANGE_AUTHOR_EDGE, + ) + except Exception as exc: + if "is not a valid vertex id" in str(exc): + return + raise + def _delete_current_edges(self, issue_vertex_id: str) -> None: for edge_type in CURRENT_STATE_EDGES: self.conn.delEdges("JiraIssue", issue_vertex_id, edge_type) @@ -922,35 +962,149 @@ def _delete_document_chunks(self, document_id: str) -> None: self.conn.delVerticesById("DocumentChunk", chunk_ids) self.conn.delVerticesById("Content", chunk_ids) - def _load_documents( + def _write_issue_facts( self, - mapped: list[MappedIssue], + issues: list[MappedIssue], + existing_hashes: dict[str, str], ) -> None: - with tempfile.TemporaryDirectory( - prefix=f"jira-{self.graphname}-{self.source.id}-" - ) as directory: - path = Path(directory) / "documents.jsonl" - with path.open("w", encoding="utf-8") as stream: - for item in mapped: - stream.write(json.dumps(item.document, ensure_ascii=False) + "\n") - result = supportai.ingest( - self.graphname, - LoadingInfo( - load_job_id="load_documents_content_json", - data_source_id={ - "data_source": "server", - "data_source_id": "DocumentContent", - "data_path": directory, - }, - file_path="documents.jsonl", - ), - self.conn, + self._upsert_issue_facts(issues) + self._embed_chunks( + [fact.chunk for issue in issues for fact in issue.facts] + ) + for issue in issues: + self.conn.upsertVertex( + "JiraIssue", + issue.issue_vertex_id, + attributes={"content_hash": issue.content_hash}, ) - failed_files = result.get("failed_files") if isinstance(result, dict) else None - if failed_files: - raise RuntimeError( - "Jira document loading failed; retry the sync to resume" + existing_hashes[issue.issue_vertex_id] = issue.content_hash + + def _delete_issue_search_content(self, issue: MappedIssue) -> None: + chunk_ids = self._issue_fact_chunk_ids(issue.issue_vertex_id) + if chunk_ids: + get_embedding_store( + graphname=self.graphname + ).remove_embeddings(ids=chunk_ids) + self.conn.delVerticesById("DocumentChunk", chunk_ids) + self.conn.delVerticesById("Content", chunk_ids) + document_id = issue.legacy_document_id.lower() + self._delete_document_chunks(document_id) + self.conn.delVerticesById("Document", [document_id]) + self.conn.delVerticesById("Content", [document_id]) + + def _issue_fact_chunk_ids(self, issue_vertex_id: str) -> list[str]: + try: + edges = self.conn.getEdges( + "JiraIssue", + issue_vertex_id, + "reverse_CONTAINS_ENTITY", + ) or [] + except Exception as exc: + if "is not a valid vertex id" in str(exc): + return [] + raise + return [ + str(edge["to_id"]) + for edge in edges + if edge.get("to_type") == "DocumentChunk" + and edge.get("to_id") is not None + and ":fact:" in str(edge["to_id"]) + ] + + def _upsert_issue_facts(self, issues: list[MappedIssue]) -> None: + epoch_added = int(time.time()) + vertices: list[VertexRecord] = [] + edges: list[EdgeRecord] = [] + for issue in issues: + previous_chunk_id: str | None = None + for fact in issue.facts: + chunk = fact.chunk + vertices.extend( + self._chunk_vertices(chunk, epoch_added, "jira_issue") + ) + edges.extend( + self._chunk_edges( + chunk.chunk_id, + issue.issue_vertex_id, + fact.entities, + previous_chunk_id, + ) + ) + previous_chunk_id = chunk.chunk_id + self._upsert_records(vertices, edges) + + def _chunk_vertices( + self, + chunk: MappedChunk, + epoch_added: int, + ctype: str, + ) -> list[VertexRecord]: + return [ + VertexRecord( + "DocumentChunk", + chunk.chunk_id, + { + "idx": chunk.index, + "epoch_added": epoch_added, + "epoch_processing": 0, + "epoch_processed": 0, + }, + ), + VertexRecord( + "Content", + chunk.chunk_id, + { + "ctype": ctype, + "text": chunk.text, + "epoch_added": epoch_added, + }, + ), + ] + + def _chunk_edges( + self, + chunk_id: str, + issue_vertex_id: str, + entities: tuple[tuple[str, str], ...], + previous_chunk_id: str | None, + ) -> list[EdgeRecord]: + edges = [ + EdgeRecord( + "DocumentChunk", + chunk_id, + "HAS_CONTENT", + "Content", + chunk_id, + ), + EdgeRecord( + "DocumentChunk", + chunk_id, + "CONTAINS_ENTITY", + "JiraIssue", + issue_vertex_id, + ), + ] + for vertex_type, vertex_id in entities: + edges.append( + EdgeRecord( + "DocumentChunk", + chunk_id, + "CONTAINS_ENTITY", + vertex_type, + vertex_id, ) + ) + if previous_chunk_id: + edges.append( + EdgeRecord( + "DocumentChunk", + chunk_id, + "IS_AFTER", + "DocumentChunk", + previous_chunk_id, + ) + ) + return edges def _upsert_comment_chunks( self, @@ -960,66 +1114,23 @@ def _upsert_comment_chunks( vertices: list[VertexRecord] = [] edges: list[EdgeRecord] = [] for comment in comments: + previous_chunk_id: str | None = None for chunk in comment.chunks: vertices.extend( - [ - VertexRecord( - "DocumentChunk", - chunk.chunk_id, - { - "idx": chunk.index, - "epoch_added": epoch_added, - "epoch_processing": 0, - "epoch_processed": 0, - }, - ), - VertexRecord( - "Content", - chunk.chunk_id, - { - "ctype": "jira_comment", - "text": chunk.text, - "epoch_added": epoch_added, - }, - ), - ] + self._chunk_vertices(chunk, epoch_added, "jira_comment") ) edges.extend( - [ - EdgeRecord( - "DocumentChunk", - chunk.chunk_id, - "HAS_CONTENT", - "Content", - chunk.chunk_id, - ), - EdgeRecord( - "DocumentChunk", - chunk.chunk_id, - "CONTAINS_ENTITY", - "JiraComment", - comment.comment_vertex_id, - ), - EdgeRecord( - "DocumentChunk", - chunk.chunk_id, - "CONTAINS_ENTITY", - "JiraIssue", - comment.issue_vertex_id, - ), - ] + self._chunk_edges( + chunk.chunk_id, + comment.issue_vertex_id, + (("JiraComment", comment.comment_vertex_id),), + previous_chunk_id, + ) ) + previous_chunk_id = chunk.chunk_id self._upsert_records(vertices, edges) - def _embed_comment_chunks( - self, - comments: list[MappedComment], - ) -> None: - chunks = [ - chunk - for comment in comments - for chunk in comment.chunks - ] + def _embed_chunks(self, chunks: list[MappedChunk]) -> None: if not chunks: return store = get_embedding_store(graphname=self.graphname) diff --git a/graphrag/tests/test_jira_connector.py b/graphrag/tests/test_jira_connector.py index 018cf804..674cf19f 100644 --- a/graphrag/tests/test_jira_connector.py +++ b/graphrag/tests/test_jira_connector.py @@ -12,6 +12,8 @@ from connectors.jira.mapper import JiraIssueMapper from connectors.jira.schema import ( JIRA_ASSIGNEE_EDGE, + JIRA_CHANGE_AUTHOR_EDGE, + JIRA_CHANGE_EDGE, JIRA_COMMENT_AFTER_EDGE, JIRA_COMMENT_AUTHOR_EDGE, JIRA_COMMENT_ISSUE_EDGE, @@ -295,13 +297,18 @@ def test_existing_hash_lookup_treats_new_issue_ids_as_missing(tmp_path): existing_id = "jira:cloud-1:issue:existing" class Connection: - def getVertices(self, vertex_type, select=""): - assert vertex_type == "JiraIssue" - assert select == "content_hash" + def runInterpretedQuery(self, query): + assert "JiraIssue" in query return [ { - "v_id": existing_id, - "attributes": {"content_hash": "existing-hash"}, + "res": [ + { + "v_id": existing_id, + "attributes": { + "res.content_hash": "existing-hash", + }, + } + ] } ] @@ -342,9 +349,9 @@ def test_comment_reconciliation_deletes_only_removed_comments( }, }, } - mapped = JiraIssueMapper(source(), "cloud-1").map(issue) - current_id = "jira:cloud-1:comment:9001" - stale_id = "jira:cloud-1:comment:9002" + mapped = JiraIssueMapper(source()).map(issue) + current_id = mapped.comments[0].comment_vertex_id + stale_id = "jira:comment:9002" class Connection: def getEdges(self, vertex_type, vertex_id, edge_type): @@ -375,7 +382,7 @@ def getEdges(self, vertex_type, vertex_id, edge_type): assert hashes == {current_id: "current"} -def test_sync_persists_checkpoint_after_each_committed_page( +def test_sync_keeps_checkpoint_when_a_page_batch_is_interrupted( tmp_path, monkeypatch, ): @@ -399,27 +406,18 @@ def cloud_id(self): def iter_issue_pages(self): yield [issue] - assert store.get( - "TestGraph", - "jira-acme", - ).sync.checkpoint == datetime( - 2026, - 9, - 20, - 14, - 3, - tzinfo=timezone.utc, - ) raise RuntimeError("simulated shutdown") class Connection: def getVertices(self, vertex_type, select=""): - assert vertex_type in {"Document", "JiraIssue", "JiraComment"} - assert select == ( - "id" if vertex_type == "Document" else "content_hash" - ) + assert vertex_type == "Document" + assert select == "id" return [] + def runInterpretedQuery(self, query): + assert "JiraIssue" in query or "JiraComment" in query + return [{"res": []}] + monkeypatch.setattr( sync_module, "jira_schema_status", @@ -432,24 +430,141 @@ def getVertices(self, vertex_type, select=""): store=store, client=Client(), ) - monkeypatch.setattr(service, "_upsert_issue_page", lambda *args: (0, 0)) + monkeypatch.setattr( + service, + "_upsert_issue_records", + lambda *args: ([], [], 0), + ) with pytest.raises(RuntimeError, match="simulated shutdown"): service.run() stored = store.get("TestGraph", "jira-acme") - assert stored.sync.checkpoint == datetime( - 2026, - 9, - 20, - 14, - 3, - tzinfo=timezone.utc, - ) + assert stored.sync.checkpoint is None assert stored.sync.last_completed_at is None assert stored.sync.last_error == "simulated shutdown" +def test_recovery_ignores_link_stubs_and_does_not_move_checkpoint( + tmp_path, + monkeypatch, +): + from connectors.jira import sync as sync_module + + store = JiraSourceStore(str(tmp_path)) + configured = store.upsert("TestGraph", source()) + configured.sync.checkpoint = None + store.update_runtime_state("TestGraph", configured) + recovered: dict = {} + + class Client: + def iter_issues_by_keys(self, keys): + recovered["keys"] = set(keys) + yield [ + { + "id": "1", + "key": "OLD-1", + "fields": { + "summary": "Already started", + "project": {"id": "1", "key": "PAY", "name": "Payments"}, + "status": {"name": "Done"}, + "updated": "2099-01-01T00:00:00.000+0000", + }, + } + ] + + def iter_issue_pages(self): + yield [ + { + "id": "2", + "key": "GLE-10", + "fields": { + "summary": "New project ticket", + "project": {"id": "2", "key": "GLE", "name": "GLE"}, + "updated": "2024-01-01T00:00:00.000+0000", + }, + } + ] + + class Connection: + def getVertices(self, vertex_type, select="", where="", limit=0): + if vertex_type == "Document": + return [] + if vertex_type == "JiraIssue": + return [ + { + "v_id": "jira:gle-stub:issue", + "attributes": { + "content_hash": "", + "status": "", + "updated": "1970-01-01 00:00:00", + }, + }, + { + "v_id": "jira:old-1:issue", + "attributes": { + "content_hash": "", + "status": "Done", + "updated": "2099-01-01 00:00:00", + }, + }, + ] + if vertex_type == "JiraComment": + return [ + { + "v_id": "jira:comment:parent", + "attributes": {"content_hash": "", "body": "", "created": ""}, + }, + { + "v_id": "jira:comment:real", + "attributes": { + "content_hash": "", + "body": "A real comment that was not embedded.", + "created": "2024-02-01 00:00:00", + }, + }, + ] + return [] + + def getEdges(self, vertex_type, vertex_id, edge_type): + if vertex_id == "jira:comment:real": + return [{"to_id": "jira:pay-9:issue"}] + return [] + + def runInterpretedQuery(self, query): + return [{"res": []}] + + monkeypatch.setattr( + sync_module, + "jira_schema_status", + lambda *args: {"status": "installed"}, + ) + monkeypatch.setattr( + sync_module, + "embedding_coverage", + lambda *args: {"total": 1, "missing": 0}, + ) + service = JiraSyncService( + "TestGraph", + store.get("TestGraph", "jira-acme"), + Connection(), + store=store, + client=Client(), + ) + monkeypatch.setattr( + service, + "_upsert_issue_records", + lambda *args: ([], [], 0), + ) + + result = service.run() + + assert recovered["keys"] == {"OLD-1", "PAY-9"} + assert result["status"] == "completed" + stored = store.get("TestGraph", "jira-acme") + assert stored.sync.checkpoint == datetime(2024, 1, 1, tzinfo=timezone.utc) + + def test_sync_requires_rebuild_when_existing_chunks_lack_embeddings( tmp_path, monkeypatch, @@ -470,6 +585,9 @@ class Connection: def getVertices(self, vertex_type, select=""): return [] + def runInterpretedQuery(self, query): + return [{"res": []}] + monkeypatch.setattr( sync_module, "jira_schema_status", @@ -546,7 +664,7 @@ def test_mapper_writes_small_schema_and_searchable_document(): "issuelinks": [], }, } - mapped = JiraIssueMapper(source(), "cloud-1").map(issue) + mapped = JiraIssueMapper(source()).map(issue) assert {vertex.vertex_type for vertex in mapped.vertices} == { "JiraComment", "JiraIssue", @@ -562,17 +680,178 @@ def test_mapper_writes_small_schema_and_searchable_document(): JIRA_COMMENT_REPLY_EDGE, JIRA_COMMENT_AFTER_EDGE, } - assert mapped.document["doc_type"] == "jira" - assert "PAY-123" in mapped.document["content"] - assert "Increase the gateway timeout." not in mapped.document["content"] + fact_text = "\n".join(fact.chunk.text for fact in mapped.facts) + assert fact_text.startswith("Issue: PAY-123") + assert "Login timeout" in fact_text + assert "Increase the gateway timeout." not in fact_text + assert "1970" not in fact_text + assert "Fix versions" not in fact_text + comment_vertex = next( + vertex + for vertex in mapped.vertices + if vertex.vertex_type == "JiraComment" + and vertex.attributes.get("comment_id") == "9001" + ) + assert comment_vertex.attributes["body"] == "Increase the gateway timeout." assert len(mapped.comments) == 2 assert len(mapped.comments[0].chunks) == 1 assert mapped.comments[0].chunks[0].chunk_id.startswith( - "jira:cloud-1:comment:9001:chunk:0:" + "jira:comment:9001:chunk:0:" ) + assert mapped.comments[0].chunks[0].text.startswith("Issue: PAY-123") assert "Increase the gateway timeout." in mapped.comments[0].chunks[0].text +def test_issue_changes_are_separate_facts_and_events(): + issue = { + "id": "10422", + "key": "GML-2192", + "fields": { + "summary": "Extend the LLM judge", + "project": {"id": "10001", "key": "GML", "name": "GraphRAG"}, + "status": {"name": "Done", "statusCategory": {"key": "done"}}, + "assignee": {"accountId": "prins", "displayName": "Prins Kumar"}, + "updated": "2026-09-24T10:00:00.000+0000", + "description": "Judge output now accepts gemini-3.5-flash.", + }, + "changelog": { + "histories": [ + { + "id": "7001", + "created": "2026-09-16T09:00:00.000+0000", + "author": { + "accountId": "prins", + "displayName": "Prins Kumar", + }, + "items": [ + { + "field": "status", + "fromString": "To Do", + "toString": "In Progress", + }, + { + "field": "Rank", + "fromString": "1", + "toString": "2", + }, + ], + }, + { + "id": "7002", + "created": "2026-09-24T10:00:00.000+0000", + "author": { + "accountId": "automation", + "displayName": "Automation for Jira", + "accountType": "app", + }, + "items": [ + { + "field": "status", + "fromString": "In Progress", + "toString": "Done", + } + ], + }, + ] + }, + } + + mapped = JiraIssueMapper(source()).map(issue) + changes = [ + vertex + for vertex in mapped.vertices + if vertex.vertex_type == "JiraChange" + ] + assert len(changes) == 2 + assert {vertex.attributes["field"] for vertex in changes} == {"status"} + assert {edge.edge_type for edge in mapped.edges} >= { + JIRA_CHANGE_EDGE, + JIRA_CHANGE_AUTHOR_EDGE, + } + fact_text = "\n".join(fact.chunk.text for fact in mapped.facts) + assert "Issue: GML-2192" in fact_text + assert 'changed status from "To Do" to "In Progress"' in fact_text + assert 'changed status from "In Progress" to "Done"' in fact_text + assert "Rank" not in fact_text + assert "1970" not in fact_text + assert "Fix versions" not in fact_text + change_facts = [ + fact for fact in mapped.facts if fact.entities + ] + assert len(change_facts) == 2 + assert all( + entity[0] == "JiraChange" for fact in change_facts for entity in fact.entities + ) + assert mapped.facts[0].chunk.chunk_id.startswith("jira:gml-2192:issue:fact:0:") + assert all( + fact.chunk.text.startswith("Issue: GML-2192") for fact in mapped.facts + ) + + +def test_issue_facts_are_upserted_directly_without_document(tmp_path, monkeypatch): + issue = { + "id": "10422", + "key": "GML-2192", + "fields": { + "summary": "Extend the LLM judge", + "project": {"id": "10001", "key": "GML", "name": "GraphRAG"}, + "status": {"name": "Done"}, + "description": "Judge output now accepts gemini-3.5-flash.", + }, + "changelog": { + "histories": [ + { + "id": "7001", + "created": "2026-09-16T09:00:00.000+0000", + "author": { + "accountId": "prins", + "displayName": "Prins Kumar", + }, + "items": [ + { + "field": "status", + "fromString": "To Do", + "toString": "In Progress", + } + ], + } + ] + }, + } + mapped = JiraIssueMapper(source()).map(issue) + service = JiraSyncService( + "TestGraph", + source(), + object(), + store=JiraSourceStore(str(tmp_path)), + client=object(), + ) + captured: dict = {} + monkeypatch.setattr( + service, + "_upsert_records", + lambda vertices, edges: captured.update(vertices=vertices, edges=edges), + ) + + service._upsert_issue_facts([mapped]) + + assert {vertex.vertex_type for vertex in captured["vertices"]} == { + "DocumentChunk", + "Content", + } + edge_types = { + (edge.source_type, edge.edge_type, edge.target_type) + for edge in captured["edges"] + } + assert ("DocumentChunk", "CONTAINS_ENTITY", "JiraIssue") in edge_types + assert ("DocumentChunk", "CONTAINS_ENTITY", "JiraChange") in edge_types + assert ("DocumentChunk", "HAS_CONTENT", "Content") in edge_types + assert ("DocumentChunk", "IS_AFTER", "DocumentChunk") in edge_types + assert "Document" not in { + vertex.vertex_type for vertex in captured["vertices"] + } + + def test_long_jira_comment_drops_log_heavy_blocks_before_chunking(): log_lines = "\n".join( f"2026-09-28 12:00:{index:02d} ERROR request failed" @@ -603,7 +882,7 @@ def test_long_jira_comment_drops_log_heavy_blocks_before_chunking(): }, } - mapped = JiraIssueMapper(source(), "cloud-1").map(issue) + mapped = JiraIssueMapper(source()).map(issue) content = "\n".join( chunk.text for chunk in mapped.comments[0].chunks ) @@ -634,7 +913,7 @@ def test_comment_chunks_are_upserted_directly_without_document(tmp_path, monkeyp }, }, } - comment = JiraIssueMapper(source(), "cloud-1").map(issue).comments[0] + comment = JiraIssueMapper(source()).map(issue).comments[0] service = JiraSyncService( "TestGraph", source(), @@ -696,7 +975,7 @@ def test_comment_chunks_are_embedded_by_existing_store(tmp_path, monkeypatch): }, }, } - comment = JiraIssueMapper(source(), "cloud-1").map(issue).comments[0] + comment = JiraIssueMapper(source()).map(issue).comments[0] embedded: list[tuple[list, list]] = [] processed: list[tuple[str, str, dict]] = [] @@ -720,7 +999,7 @@ def upsertVertex(self, vertex_type, vertex_id, attributes): client=object(), ) - service._embed_comment_chunks([comment]) + service._embed_chunks(comment.chunks) assert embedded == [ ( @@ -843,6 +1122,7 @@ def getVertices(self, vertex_type, select=""): def test_schema_is_bounded(): proposal = jira_schema_proposal() assert {vertex.name for vertex in proposal.vertices} == { + "JiraChange", "JiraComment", "JiraIssue", "JiraProject", @@ -858,6 +1138,8 @@ def test_schema_is_bounded(): JIRA_COMMENT_AUTHOR_EDGE, JIRA_COMMENT_REPLY_EDGE, JIRA_COMMENT_AFTER_EDGE, + JIRA_CHANGE_EDGE, + JIRA_CHANGE_AUTHOR_EDGE, } @@ -957,6 +1239,8 @@ def __init__(self): {"From": "DocumentChunk", "To": "JiraIssue"}, {"From": "Document", "To": "JiraComment"}, {"From": "DocumentChunk", "To": "JiraComment"}, + {"From": "Document", "To": "JiraChange"}, + {"From": "DocumentChunk", "To": "JiraChange"}, ], } self.edges["HAS_CHILD"] = { From d4f8583b28a494fc078b9e0593cd072df18bd8f1 Mon Sep 17 00:00:00 2001 From: Prins Kumar Date: Thu, 8 Oct 2026 02:13:32 +0530 Subject: [PATCH 19/19] Guard unsupported TigerGraph Cypher --- .../src/pages/setup/ConnectorPromptGuide.tsx | 13 +++- graphrag/app/tools/generate_cypher.py | 47 +++++++++++++++ graphrag/tests/test_generate_cypher.py | 59 +++++++++++++++++++ 3 files changed, 118 insertions(+), 1 deletion(-) create mode 100644 graphrag/tests/test_generate_cypher.py diff --git a/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx b/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx index 24d6c9f1..d1e4a236 100644 --- a/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx +++ b/graphrag-ui/src/pages/setup/ConnectorPromptGuide.tsx @@ -34,12 +34,17 @@ Once one issue is identified, plan both retrievals. Put the full issue key and t - History, timeline, progress, or who did what needs the JiraChange events and the comment text, together with the current status, assignee, and resolution. - Release, fix version, due date, or when it will be fixed needs fix_versions, due, status, and resolution, plus any release or fix mentioned in the description or comments. -For the structural step, say which match to use: an exact case-insensitive match on issue_key when the full key is known, or issue_key ending with the number when only a number is known. Name the attributes and neighbour names to return. Returning the vertex alone is not enough. +For the structural step, say which match to use: an exact case-insensitive match on issue_key when the full key is known, or issue_key ending with the number when only a number is known. Name the attributes and neighbour names to return. Returning the vertex alone is not enough. For a broad issue summary, request current fields, assignee, reporter, and project only. Request parent or linked issues only when the user asks about them, using a separate structural step for each relationship. A count, list, or filter across many issues stays a structural query. Add hybrid search only when that question also needs description, comment, or change-log text. The final answer step depends on every retrieval step.`; +const JIRA_QUERY_GUIDANCE_EXAMPLE = `## Jira issue queries +- For a broad single-issue lookup, retrieve the JiraIssue fields, assignee, reporter, and project. Do not add parent, issue-link, or comment relationships unless the question asks for them. +- Handle JIRA_HAS_PARENT and JIRA_LINKS_TO in a separate, focused graph query. If the relationship is requested, prefer MATCH over OPTIONAL MATCH. +- Return the related issue_key and requested edge attributes such as link_type. Do not return the parent or link edge object itself.`; + const JIRA_AGENT_EXAMPLE = `## Jira issues Each issue has a current record in the graph and short document chunks. For one issue, call both graphrag__structural_retrieve and graphrag__hybrid_search before you answer. Use the conversation to resolve which issue is meant when this message does not repeat the key. @@ -80,6 +85,12 @@ const GUIDES: Record = { "Do not put a real ticket key in the prompt. The model will treat that key as the ticket to retrieve. If fix version or due date is empty, say it is not available.", ], examples: [ + { + id: "query_guidance", + name: "Query Guidance", + where: "Customize Prompts → Query Guidance, on this graph", + text: JIRA_QUERY_GUIDANCE_EXAMPLE, + }, { id: "agentic_planner", name: "Agentic Planner", diff --git a/graphrag/app/tools/generate_cypher.py b/graphrag/app/tools/generate_cypher.py index d9199de4..b9f3dbac 100644 --- a/graphrag/app/tools/generate_cypher.py +++ b/graphrag/app/tools/generate_cypher.py @@ -13,6 +13,7 @@ # along with this program. If not, see . import logging +import re from typing import Iterable from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import PromptTemplate @@ -27,6 +28,51 @@ logger = logging.getLogger(__name__) +def validate_tigergraph_cypher(query: str, schema: str) -> None: + """Reject the OpenCypher shape TigerGraph cannot compile.""" + directed_self_edges: set[str] = set() + edge_name = from_type = to_type = None + for line in schema.partition("Edge Types:")[2].splitlines() + ["END"]: + if line and not line[0].isspace(): + edge_name = line.strip() + from_type = to_type = None + elif edge_name and line.strip().startswith("From Vertex:"): + from_type = line.split(":", 1)[1].strip() + elif edge_name and line.strip().startswith("To Vertex:"): + to_type = line.split(":", 1)[1].strip() + elif ( + edge_name + and from_type == to_type + and line.strip() == "Edge direction: Directed" + ): + directed_self_edges.add(edge_name) + + returned = re.search(r"\bRETURN\b(.*)", query, re.I | re.S) + if not returned: + return + return_items = returned.group(1).split(",") + for edge_var, edge_type in re.findall( + r"OPTIONAL\s+MATCH\s*\([^)]*\)\s*-\s*" + r"\[\s*(\w+)\s*:\s*(\w+)[^]]*]\s*-\s*\([^)]*\)", + query, + re.I, + ): + returns_edge = any( + re.fullmatch( + rf"`?{re.escape(edge_var)}`?(?:\s+AS\s+\w+)?", + item.strip(), + re.I, + ) + for item in return_items + ) + if edge_type in directed_self_edges and returns_edge: + raise ValueError( + f"TigerGraph cannot return {edge_var} from an undirected " + f"OPTIONAL MATCH on directed self-edge {edge_type}. Use MATCH " + "or return only the related vertex and requested edge attributes." + ) + + class GenerateCypher(BaseTool): """GenerateCypher Tool. Tool to generate and execute the appropriate Cypher query for the question. @@ -105,6 +151,7 @@ def generate_cypher(self, question: str, history: Iterable[str]) -> str: if not any(kw in out_upper for kw in ("MATCH", "RETURN", "WITH", "UNWIND", "CALL")): LogWriter.info(f"request_id={req_id_cv.get()} EXIT generate_cypher - LLM did not produce a valid Cypher query") raise ValueError(f"LLM did not produce a valid Cypher query: {out[:200]}") + validate_tigergraph_cypher(out, schema) query_header = "USE GRAPH " + self.conn.graphname + " "+ "\n" + "INTERPRET OPENCYPHER QUERY () {" + "\n" query_footer = "\n}" diff --git a/graphrag/tests/test_generate_cypher.py b/graphrag/tests/test_generate_cypher.py new file mode 100644 index 00000000..bc7fcd66 --- /dev/null +++ b/graphrag/tests/test_generate_cypher.py @@ -0,0 +1,59 @@ +import pytest + +from app.tools.generate_cypher import validate_tigergraph_cypher + + +SCHEMA = """Edge Types: +REPORTS_TO + From Vertex: Person + To Vertex: Person + Edge direction: Directed + Attributes: + since of type DATETIME +WORKS_FOR + From Vertex: Person + To Vertex: Company + Edge direction: Directed + Attributes: + No attributes +""" + + +def test_rejects_optional_undirected_directed_self_edge_object(): + query = """ +MATCH (person:Person) +OPTIONAL MATCH (person)-[manager_edge:REPORTS_TO]-(manager:Person) +RETURN person, manager_edge, manager +""" + + with pytest.raises(ValueError, match="cannot return manager_edge"): + validate_tigergraph_cypher(query, SCHEMA) + + +@pytest.mark.parametrize( + "query", + [ + """ + OPTIONAL MATCH (person:Person)-[manager_edge:REPORTS_TO]->(manager:Person) + RETURN person, manager_edge, manager + """, + """ + MATCH (person:Person)-[manager_edge:REPORTS_TO]-(manager:Person) + RETURN person, manager_edge, manager + """, + """ + OPTIONAL MATCH (person:Person)-[employment:WORKS_FOR]-(company:Company) + RETURN person, employment, company + """, + """ + OPTIONAL MATCH (person:Person)-[manager_edge:REPORTS_TO]-(manager:Person) + RETURN person, manager + """, + """ + OPTIONAL MATCH (person:Person)-[manager_edge:REPORTS_TO]-(manager:Person) + RETURN person, manager_edge.since, manager + """, + ], +) +def test_accepts_other_query_shapes(query): + validate_tigergraph_cypher(query, SCHEMA)