diff --git a/changai/changai/api/v1/prediction_pipeline_v1.py b/changai/changai/api/v1/prediction_pipeline_v1.py index 2c3dce7..d1836e6 100644 --- a/changai/changai/api/v1/prediction_pipeline_v1.py +++ b/changai/changai/api/v1/prediction_pipeline_v1.py @@ -13,9 +13,8 @@ import jinja2 from typing import Any, Dict, Optional from symspellpy.symspellpy import SymSpell, Verbosity -from changai.changai.api.v2.text2sql_pipeline import get_settings +# from changai.changai.api.v2.text2sql_pipeline import get_settings -CONFIG=get_settings() pleasantry_file_path = frappe.get_app_path("changai", "changai", "api", "pleasantry.json") business_keywords_file = frappe.get_app_path("changai", "changai", "api", "business_keywords.json") custom_dictionary = frappe.get_app_path("changai", "changai", "api", "erp_dictionary.txt") diff --git a/changai/changai/api/v2/ai_translate.py b/changai/changai/api/v2/ai_translate.py index 14a5a5e..958f31e 100644 --- a/changai/changai/api/v2/ai_translate.py +++ b/changai/changai/api/v2/ai_translate.py @@ -125,5 +125,5 @@ def translate_and_store(docname: str, doctype: str, from_field: str, to_field: s if not hasattr(doc, to_field): frappe.throw(f"Field '{to_field}' does not exist on Item") doc.set(to_field, translated_text) - doc.save(ignore_permissions=True) + doc.save(ignore_permissions=False) return to_field diff --git a/changai/changai/api/v2/assets/business_keywords_v1.json b/changai/changai/api/v2/assets/business_keywords_v1.json index da3e06a..fd68f9b 100644 --- a/changai/changai/api/v2/assets/business_keywords_v1.json +++ b/changai/changai/api/v2/assets/business_keywords_v1.json @@ -134,8 +134,8 @@ "show fields", "child", "show all fields", - "show customer names", "employee name", + "name", "invoice number", "quotation number", "purchase order number", diff --git a/changai/changai/api/v2/fvs_stores/erpnext/emb_dir/field_docs.pkl b/changai/changai/api/v2/fvs_stores/erpnext/emb_dir/field_docs.pkl index 2650d8b..5e48b21 100644 Binary files a/changai/changai/api/v2/fvs_stores/erpnext/emb_dir/field_docs.pkl and b/changai/changai/api/v2/fvs_stores/erpnext/emb_dir/field_docs.pkl differ diff --git a/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.faiss b/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.faiss index 4921997..6e98485 100644 Binary files a/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.faiss and b/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.faiss differ diff --git a/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.pkl b/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.pkl index c01dabe..2f240b9 100644 Binary files a/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.pkl and b/changai/changai/api/v2/fvs_stores/erpnext/report_fvs/index.pkl differ diff --git a/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.faiss b/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.faiss index 491dec9..9c14c2c 100644 Binary files a/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.faiss and b/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.faiss differ diff --git a/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.pkl b/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.pkl index 7cc13aa..7f2d9a1 100644 Binary files a/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.pkl and b/changai/changai/api/v2/fvs_stores/erpnext/table_fvs/index.pkl differ diff --git a/changai/changai/api/v2/schema_utils.py b/changai/changai/api/v2/schema_utils.py index ac1a385..b44a585 100644 --- a/changai/changai/api/v2/schema_utils.py +++ b/changai/changai/api/v2/schema_utils.py @@ -264,7 +264,7 @@ def is_master_data_changed(last_sync: str, stored_data: list): return True return False - +@frappe.whitelist(allow_guest=True) def check_file_updates(file_name: str): RAG_FOLDER = "Home/RAG Sources" from changai.changai.api.v2.build_cards_faiss_index_v2 import _read_file_doc @@ -542,7 +542,7 @@ class ChangAIConfig: def get(cls): if not hasattr(frappe.local, "_changai_config"): frappe.clear_document_cache(CHANGAI_SETTINGS) - frappe.local._changai_config = get_settings() + frappe.local._changai_config = _get_internal_settings_config() return frappe.local._changai_config @@ -555,21 +555,7 @@ def _build_frontend_settings_config() -> Dict[str, Any]: or getattr(settings, "aws_default_region", None) or "us-east-1" ) - return { - "RETAIN_MEM": settings.retain_memory, - "LLM_VERSION_ID": settings.llm_version_id, - "EMBED_VERSION_ID": settings.embedder_version_id, - "REMOTE": bool(settings.remote), - "deploy_url": settings.deploy_url, - "entity_retriever": settings.entity_retriever, - "support_api_url": settings.support_url, - "get_ticket_details_url": settings.get_ticket_details_url, - "llm": settings.llm, - "location": settings.gemini_location, - "retriever_structure": settings.retriever_structure, - "gemini_project_id": settings.gemini_project_id, - "gemini_json_content": settings.gemini_json_content, "enable_voice_chat": bool(settings.enable_voice_chat), "aws_region": aws_region, "polly_voice_id": "Zayd", @@ -578,8 +564,7 @@ def _build_frontend_settings_config() -> Dict[str, Any]: } -@frappe.whitelist(allow_guest=False) -def get_settings() -> Dict[str, Any]: +def _get_internal_settings_config() -> Dict[str, Any]: settings = frappe.get_single(CHANGAI_SETTINGS) config = { "RETAIN_MEM": settings.retain_memory, @@ -602,9 +587,14 @@ def get_settings() -> Dict[str, Any]: } return config +@frappe.whitelist(allow_guest=False) +def get_settings() -> Dict[str, Any]: + frappe.has_permission(CHANGAI_SETTINGS, "read", throw=True) + return _get_internal_settings_config() @frappe.whitelist(allow_guest=False) def get_frontend_settings() -> Dict[str, Any]: + frappe.has_permission(CHANGAI_SETTINGS, "read", throw=True) return _build_frontend_settings_config() def clean_sql(s: Any) -> str: diff --git a/changai/changai/api/v2/store_chats.py b/changai/changai/api/v2/store_chats.py index f6eb73f..5ca1a0f 100644 --- a/changai/changai/api/v2/store_chats.py +++ b/changai/changai/api/v2/store_chats.py @@ -244,7 +244,7 @@ def find_similar_log_question(new_question: str, threshold: int = 98): score = fuzz.token_set_ratio( new_question, - log.user_question + log.rewritten_question ) if score > best_score: diff --git a/changai/changai/api/v2/text2sql_pipeline_v2.py b/changai/changai/api/v2/text2sql_pipeline_v2.py index 976dfa7..9c1c5d5 100644 --- a/changai/changai/api/v2/text2sql_pipeline_v2.py +++ b/changai/changai/api/v2/text2sql_pipeline_v2.py @@ -77,6 +77,7 @@ mapping_data = read_asset("metaschema_clean_v2.json", base="assets") CONVERSATION_TEMPLATE = read_asset("conversation_template_v2.j2", base="assets") SQL_SYS_PROMPT = read_asset("sql_system_prompt.txt", base="prompts") +GDOC_SYS_PROMPT = read_asset("gdoc_sql_prompt.txt", base="prompts") SQL_PROMPT = read_asset("sql_user_prompt.txt", base="prompts") FILTER_TABLES = read_asset("filter_tables.txt", base="prompts") filter_fields = read_asset("filter_fields.txt", base="prompts") @@ -226,6 +227,8 @@ def _safe_strip(v): # Shared State class SQLState(TypedDict, total=False): + table : str + source :str payload_res :dict is_cud:bool cud_type:str @@ -237,6 +240,7 @@ class SQLState(TypedDict, total=False): report_name:str entity_name:str doc:str + clarification:str reports_filter_before_call:list entity_type:str final_prompt:str @@ -268,7 +272,44 @@ class SQLState(TypedDict, total=False): message:str stop_followup:bool - +TURN_RESET: SQLState = { + "clarification": "", + "sql": "", + "orm": "", + "payload": {}, + "payload_res": None, + "error": None, + "validation": {}, + "tries": 0, + "sql_prompt": "", + "final_prompt": "", + "formatted_q": "", + "non_erp_res": "", + "message": "", + "stop_followup": False, + "create_entity": False, + "open_report": False, + "report_name": "", + "filters": "", + "doc": "", + "entity_name": "", + "is_cud": False, + "cud_type": "", + "entity_cards": [], + "entity_words": [], + "contains_values": False, + "selected_tables": [], + "selected_fields": "", + "query_type":"", + "top_tables": [], + "top_fields": {}, + "hits": [], + "context": "", + "entity_raw":None, + "question":"", + "formatting_prompt":"", + "session_id":"" +} def route_action(state: SQLState) -> str: if state.get("stop_followup"): return "STOP_FOLLOW" @@ -322,9 +363,12 @@ def guardrail_router(state: SQLState) -> SQLState: request_id = state.get("request_id") chat_id = state.get("session_id") raw_q = state.get("question") or "" + source = state.get("source") try: - is_erp= is_erp_query(False,raw_q,BUSINESS_KEYWORDS,98) - if is_erp: + is_erp= is_erp_query(False,raw_q,BUSINESS_KEYWORDS,95) + if source == "gdoc": + query_type = "ERP" + elif is_erp : query_type = "ERP" elif is_thread_erp(raw_q, chat_id): query_type = "ERP" @@ -623,6 +667,7 @@ def generate_sql(state:SQLState) -> SQLState: # if state.get("context") == "" or state.get("context") == None: # state,context = hits_to_prompt_context(state) request_id = state.get("request_id") + source = state.get("source") fields = _safe_strip(state.get("selected_fields") or "") entity_cards = state.get("entity_cards") or [] entity_block = "" @@ -639,7 +684,7 @@ def generate_sql(state:SQLState) -> SQLState: else: prompt=fill_sql_prompt(formatted_q,state["context"]) try: - response=call_model(prompt,"llm",SQL_SYS_PROMPT) + response=call_model(prompt,"llm",SQL_SYS_PROMPT if source == "erp" else GDOC_SYS_PROMPT) if not response: return {**state, "error": "Empty response from LLM", "sql_prompt": prompt} if isinstance(response, str): @@ -651,13 +696,20 @@ def generate_sql(state:SQLState) -> SQLState: "error": "Invalid JSON returned by LLM" } sql = response.get("sql", "") + table = response.get("table", "") + clarification = response.get("clarification") or None + if clarification: + publish_pipeline_update(request_id, "clarification_needed", clarification) + return {**state, "sql_prompt": prompt, "sql": "", "payload": {}, + "payload_res": None, "error": None, + "clarification": clarification} payload = response.get("payload", {}) publish_pipeline_update( request_id, "sql_generated", "SQL generated" ) - return {**state,"sql_prompt":prompt,"sql":sql,"payload":payload,"error":None,"payload_res": None} + return {**state,"sql_prompt":prompt,"sql":sql,"payload":payload,"error":None,"payload_res": None,"table":table} except frappe.exceptions.ValidationError: raise except Exception as e: @@ -711,7 +763,8 @@ def repair_sqlquery(state: SQLState) -> SQLState: patched_prompt =sql_prompt + "\n\n#VALIDATION HINTS\n" + "\n".join(f"-{h}" for h in hints) try: - response = call_model(patched_prompt,"llm",SQL_SYS_PROMPT) + sys_prompt = GDOC_SYS_PROMPT if state.get("source") == "gdoc" else SQL_SYS_PROMPT + response = call_model(patched_prompt,"llm",sys_prompt) if isinstance(response, str): try: response = json.loads(response) @@ -941,8 +994,10 @@ def execute_query(sql: str, doctypes: List[str]) -> Any: return {"error": f"SQL Execution Failed: {e}\n Check Quick Start Guide Here 👇:\n {CHANGAI_GUIDE_LINK}"} -def _invoke_pipeline(user_question: str, chat_id: str, request_id: str,sendNonErptoAI: bool = False): +def _invoke_pipeline(source:str,user_question: str, chat_id: str, request_id: str,sendNonErptoAI: bool = False): initial_state: SQLState = { + **TURN_RESET, + "source":source, "question": user_question or "", "session_id": chat_id, "request_id": request_id, @@ -1011,16 +1066,58 @@ def _handle_sql_result( extracted_tables=[] formatted_result ="" payload = {} + source ="" + source=final.get("source") + context = (final.get("context") or final.get("selected_fields") or fields or "")[:800] + clarification = final.get("clarification") or "" + if clarification: + publish_pipeline_update( + request_id, + "clarification_needed", + "Need clarification from user", + done=True # close the pipeline UI on this turn + ) + save_turn_2( + session_id=chat_id, + user_text=formatted_q, + bot_text=clarification, + type_="erp" + ) + return { + "Model returned SQL": "", + "context": context, + "payload": payload, + "entity_words": final.get("entity_words") or [], + "Question": user_question, + "payload_res": None, + "Formated Question": formatted_q, + "Cleaned SQL": "", + "Tables": selected_tables, + "Fields": fields, + "Entity Values present ?": "", + "Validation": val, + "Error": None, + "result": [], + "EntityDebug": entity_debug if entity_debug.get("contains_values") else None, + "Bot": clarification + } try: request_id = request_id or final.get("request_id") org_sql = final.get("sql") or sql payload = final.get("payload") or {} payload_res = final.get("payload_res") + table = final.get("table") if org_sql: extracted_tables = extract_tables_from_sql(org_sql) try: if sql: sql_result = execute_query(sql, extracted_tables) + if source == "gdoc": + return { + "sql_out":sql_result, + "table":table + } + except Exception as e: # err = str(e) final["error"] = str(e) @@ -1029,7 +1126,6 @@ def _handle_sql_result( except Exception as e: final["error"] = str(e) return {"ok": False, "error": f"SQL Execution Failed: {e}"} - context = (final.get("context") or final.get("selected_fields") or fields or "")[:800] contains_values = final.get("contains_values") or entity_debug.get("contains_values") or "" err = final.get("error") @@ -1091,7 +1187,7 @@ def _handle_sql_result( # ✅ Normal success case formatted_result = format_data(user_question, payload_res) - elif sql: + elif sql: formatted_result = format_data( user_question, sql_result @@ -1142,6 +1238,7 @@ def _handle_sql_result( entity_debug=entity_debug, type_="ERP" ) + return { "Model returned SQL": org_sql, "context": context, @@ -1162,8 +1259,9 @@ def _handle_sql_result( } RETRY_PROMPT = read_asset("retry_sys_prompt.txt",base="prompts") RETRY_USER_PROMPT = read_asset("retry_user_prompt.txt",base="prompts") -def retry_sql(sql, error, formatted_q, sql_prompt): - retry_prompt = SQL_SYS_PROMPT + RETRY_PROMPT +def retry_sql(sql, error, formatted_q, sql_prompt,source="erp"): + base = GDOC_SYS_PROMPT if source == "gdoc" else SQL_SYS_PROMPT + retry_prompt = base + RETRY_PROMPT user_prompt = sql_prompt + RETRY_USER_PROMPT.format(sql=sql,error=error,formatted_q=formatted_q) try: rewritten = call_gemini(user_prompt, sys_prompt=retry_prompt) @@ -1184,45 +1282,45 @@ def is_thread_erp(q:str,chat_id:str): else: return False -@frappe.whitelist(allow_guest=False) +@frappe.whitelist(allow_guest=True) def run_text2sql_pipeline( user_question: str, - chat_id: str, - request_id: str, + chat_id: str = "None", + request_id: str = "None", + source:str = "erp", sendNonErptoAI: bool = False ) -> Dict: memory_status = check_memory_status() - logs = find_similar_log_question(user_question) - if logs.get("matched"): - publish_pipeline_update( - request_id, - "cache_hit", - "Using cached result" - ) - formatted_q = logs.get("rewritten_question") - sql = logs.get("sql") - tables = json.loads(logs.get("tables") or "[]") - fields = logs.get("fields") or "" - entity_debug = json.loads(logs.get("entity_debug") or "{}") - return _handle_sql_result( - memory_status, - request_id, - None, - {}, - sql, - formatted_q, - fields, - tables, - {"ok": True, "from_cache": True}, - entity_debug, - user_question, - chat_id - ) + if source!="gdoc": + logs = find_similar_log_question(user_question) + if logs.get("matched"): + publish_pipeline_update( + request_id, + "cache_hit", + "Using cached result" + ) + formatted_q = logs.get("rewritten_question") + sql = logs.get("sql") + tables = json.loads(logs.get("tables") or "[]") + fields = logs.get("fields") or "" + entity_debug = json.loads(logs.get("entity_debug") or "{}") + return _handle_sql_result( + memory_status, + request_id, + None, + {}, + sql, + formatted_q, + fields, + tables, + {"ok": True, "from_cache": True}, + entity_debug, + user_question, + chat_id + ) - # -------------------------------------------------- - # PIPELINE INVOCATION - # -------------------------------------------------- final, err_response = _invoke_pipeline( + source, user_question, chat_id, request_id, @@ -1321,6 +1419,7 @@ def run_text2sql_pipeline( payload = final.get("payload") or {} payload_res = final.get("payload_res") sql = clean_sql(final.get("sql")) or "" + clarification = final.get("clarification") or "" formatted_q = _safe_strip(final.get("formatted_q") or "") context = final.get("context") or "" selected_tables = final.get("selected_tables") or [] @@ -1347,6 +1446,13 @@ def run_text2sql_pipeline( user_question, chat_id ) + clarification = final.get("clarification") or "" + if clarification: + return _handle_sql_result( + memory_status, request_id, sql_prompt, final, "", + formatted_q, fields, selected_tables, {}, entity_debug, + user_question, chat_id + ) if sql: res = validate_sql_schema(sql) publish_pipeline_update( @@ -1374,7 +1480,8 @@ def run_text2sql_pipeline( sql, res.get("error"), formatted_q, - sql_prompt + sql_prompt, + source ) if retry2_val_res.get("ok"): @@ -1397,7 +1504,8 @@ def run_text2sql_pipeline( retried_sql2, retry2_val_res.get("error"), formatted_q, - sql_prompt + sql_prompt, + source ) if retry3_val_res.get("ok"): @@ -1450,37 +1558,3 @@ def run_text2sql_pipeline( "No SQL or Payload generated", err ) - - - -@frappe.whitelist(allow_guest=False) -def guardrail_router_test( - question: str, - chat_id: str = None, - request_id: str = "test_001" -): - try: - is_erp = is_erp_query(False, question, BUSINESS_KEYWORDS, 98) - if is_erp: - query_type = "ERP" - elif chat_id and is_thread_erp(question, chat_id): - query_type = "ERP" - else: - query_type = "NON_ERP" - - return { - "success": True, - "question": question, - "query_type": query_type, - "is_erp_match": bool(is_erp), - "is_thread_erp": bool(chat_id and is_thread_erp(question, chat_id)) - } - - except Exception as e: - frappe.log_error(frappe.get_traceback(), "Guardrail Router Test Error") - return { - "success": False, - "question": question, - "query_type": "NON_ERP", - "error": str(e) - } diff --git a/changai/changai/prompts/gdoc_sql_prompt.txt b/changai/changai/prompts/gdoc_sql_prompt.txt new file mode 100644 index 0000000..6b4acca --- /dev/null +++ b/changai/changai/prompts/gdoc_sql_prompt.txt @@ -0,0 +1,110 @@ +You are a RESTRICTED MariaDB SQL generator for the GDoc Document retrieval pipeline. +You have NO knowledge of ERPNext, databases, or any schema beyond what is written in SCHEMA CONTEXT. +SCHEMA CONTEXT is your entire universe. Nothing outside it exists. +ABSOLUTE LAW: Only use fields and tables from SCHEMA CONTEXT. +Empty SQL = critical failure. + +═══ GDOC PURPOSE (READ FIRST) ═══ +Your ONLY job is to identify DOCUMENT RECORDS that match the user's question. +The caller will use the returned record names to fetch file links. +Therefore: +- The SELECT clause MUST return ONLY the `name` field of the main table. +- Correct form: SELECT `tabX`.`name` FROM `tabX` WHERE ... +- NEVER select any other field in the final output (no file names, no dates, + no vendor fields in SELECT). Other fields may appear ONLY in WHERE, + JOIN, ORDER BY, GROUP BY, or HAVING clauses — never in SELECT. +- If the query requires DISTINCT to avoid duplicate names, use + SELECT DISTINCT `tabX`.`name`. +- Aggregations (COUNT, SUM, etc.) in SELECT are forbidden. This pipeline + only fetches record names. If the user asks "how many", still return + the names of the matching records — the caller will count them. + +═══ FIELD & TABLE LAW (ABSOLUTE — ZERO EXCEPTIONS) ═══ +BEFORE writing any field or table name, locate it physically in SCHEMA CONTEXT. + FOUND → you may use it, only under the table it is listed. + NOT FOUND → it does not exist. Do not write it. Do not guess it. Do not remember it. + +- A field listed under one table CANNOT appear in a query on another table unless + an explicit JOIN exists via a link field in SCHEMA CONTEXT. +- JOIN is only allowed if SCHEMA CONTEXT explicitly shows a link field (→) between the two tables. +- Do NOT assume any relationship between tables. If no link field exists → no JOIN. +- SELECT * is forbidden. +- Never assume fields. Only use the fields and tables that exist in SCHEMA CONTEXT. +- NEVER return empty sql. Always generate the best possible SELECT query using the + closest available fields and tables from SCHEMA CONTEXT. + +═══ SELF-CHECK PROTOCOL (MANDATORY BEFORE OUTPUT) ═══ +For every field in your SQL, answer internally: + Q: Is this field listed under THIS exact table in SCHEMA CONTEXT? + YES → keep it. NO → remove it immediately. No exceptions. +For every table in your SQL, answer internally: + Q: Is this table present in SCHEMA CONTEXT? + YES → keep it. NO → remove it immediately. No exceptions. +Final structural check: + Q: Does the SELECT clause contain ONLY the main table's `name` field? + NO → rewrite it so it does. No exceptions. + +═══ ENTITY FILTERING ═══ +- If ENTITY_CARDS exist, use ONLY those exact entity values (no spelling/case changes). +- NEVER use entity string literals directly from the user question in SQL. +- Match user-mentioned names to the closest ENTITY_CARD using typo/phonetic/fuzzy/ + semantic matching, abbreviations, and partial names. +- Always use the entity card's exact filter_field and filter_value in the WHERE clause. +- Do NOT default to the `name` field for filtering unless the entity card + filter_field is `name`. +CRITICAL ENTITY FILTER OVERRIDE RULE: +If a matching ENTITY_CARD is found for a user-mentioned entity, the WHERE clause +MUST use ONLY the ENTITY_CARD filter_field and filter_value. + +═══ MARIADB COMPATIBILITY (ZERO TOLERANCE) ═══ +FORBIDDEN — NEVER USE: STRFTIME, DATE_TRUNC, ::, ILIKE, TO_CHAR, NOW()::, EXTRACT, INTERVAL 'x' +ALLOWED DATE PATTERNS ONLY: + This month : MONTH(col) = MONTH(CURDATE()) AND YEAR(col) = YEAR(CURDATE()) + Date range : col BETWEEN DATE_SUB(CURDATE(), INTERVAL n DAY) AND CURDATE() + Date math : DATE_SUB(date, INTERVAL n UNIT) + Filtering : YEAR(), MONTH(), QUARTER(), LAST_DAY(), CURDATE() +If you are about to write a forbidden token → STOP → rewrite using MariaDB equivalents. + +═══ SCHEMA RANKING AWARENESS ═══ +- SCHEMA CONTEXT is retrieved via semantic search — relevant tables/fields may appear + low in the list. Rank is a retrieval artifact, not relevance. +- Scan the ENTIRE SCHEMA CONTEXT before selecting any table or field. +- Choose the most semantically correct table/field even if it appears last. + +═══ STATUS MAPPING ═══ +- Map user status words to the status options provided in the field schema. +- Match the closest option to the user's intent. +- If multiple options match, use IN (...). + +═══ SYNTAX LAW ═══ +- Multiple values on same field → always IN (...), never chained OR. +- OR combined with AND → always wrap OR in parentheses. +- Wrap EVERY table name in backticks exactly as written in SCHEMA CONTEXT + (never "" or '' — those fail in MariaDB). +- Uppercase all SQL keywords. +- SQL must start with SELECT and end with ; +- Only SELECT queries. INSERT, UPDATE, DELETE are strictly forbidden, + even if the user explicitly asks. +- Verify operator precedence, no missing filters, and that the logic + matches the question exactly before output. + +═══ OUTPUT LAW ═══ +Output ONLY raw JSON. No explanation. No markdown. No extra keys. +"table" must be the exact main table name used in FROM, exactly as written +in SCHEMA CONTEXT (with the tab prefix, without backticks). + +Output Format: +{ + "sql": "", + "table": "" +} + +NEVER return empty sql. If uncertain, output the best partial SELECT query +using the closest available fields — but the SELECT clause must still +contain ONLY the main table's `name` field. + +FINAL CHECK: +1. Every field exists in SCHEMA CONTEXT under the exact table used? +2. SELECT clause contains ONLY `name` of the main table? +3. "table" key matches the FROM table exactly? +4. Raw JSON only, both keys present? \ No newline at end of file diff --git a/changai/changai/prompts/sql_rewrite_sys_prompt.txt b/changai/changai/prompts/sql_rewrite_sys_prompt.txt index b978b7a..8448f55 100644 --- a/changai/changai/prompts/sql_rewrite_sys_prompt.txt +++ b/changai/changai/prompts/sql_rewrite_sys_prompt.txt @@ -3,34 +3,34 @@ Return ONLY valid JSON: {{"standalone_question":"...","contains_values":true/false,"entity_words":["..."],"is_cud":false}} TASK 1 — FOLLOW-UP -- Analyze the last user message and last AI reply together. -- If the last AI reply ended with a question and the user message is a reply to it, - rewrite using the AI’s last question intent. -- If the query depends on chat history, rewrite it as a complete standalone question by analysing the chat history . - Otherwise keep it unchanged, except Arabic ERP questions which must be rewritten into English according to the Arabic translation rules. -TASK 1.5 — FOLLOW-UP TERMINATION DETECTION -If the last AI reply ended with a question, suggestion, recommendation, or offer, and the user replies with a message indicating they do not want to continue the conversation flow, such as: -"no", "no thanks", "no thank you", "thanks", "thank you", "that's all", "nothing else", "ok", "okay", "fine", "not now", "later", "never mind", "لا", "شكرا", "تمام" -then do NOT rewrite the message into an ERP question, do NOT continue the previous context, and do NOT perform ERP, report, entity, or creation detection. -Return ONLY: -{ - "standalone_question": "", - "contains_values": false, - "entity_words": [], - "stop_followup": true, - "message": "You're welcome! Let me know if you need anything else." -} -If the user replies positively to the AI's last question or suggestion, such as: -"yes", "yes please", "sure", "go ahead", "continue", "show me", "proceed", "ok show me", "نعم" -then continue with the normal follow-up rewriting rules and return: -{ - "standalone_question": "", - "contains_values": true, - "stop_followup": false, - "message":"", - "entity_words": [], - "is_cud":true, - "cud_type": "" -} +- If the current message is a complete question (names its own subject/doctype) + → STANDALONE. Ignore history entirely — never merge. +- Treat as follow-up ONLY if it's short and incomplete alone ("yes", + "the submitted ones", "by revenue") → rewrite using history. +- CLARIFICATION MERGE: if the last AI turn was a clarifying question and the reply picks an option → rebuild as ORIGINAL question + + chosen interpretation. Keep doctype, period, and conditions EXACTLY as + the original; never add anything new. + Example: + User: "count of completed Purchase Orders?" + AI: "status', or submitted?" + User: "submitted ones" + → "What is the count of submitted Purchase Orders (docstatus = 1)?" +- Arabic ERP questions are always rewritten into English per the Arabic rules. +TASK 1.5 — FOLLOW-UP TERMINATION +Applies only when the last AI reply ended with a question/offer: +- DECLINE — user's reply only closes the conversation ("no", "no thanks", + "thanks", "that's all", "ok", "not now", "never mind", "لا", "شكرا", "تمام") + → do NOT rewrite, do NOT detect ERP/report/entity/CUD. Return ONLY: + {"standalone_question":"", "contains_values":false, "entity_words":[], + "stop_followup":true, "is_cud":false, "cud_type":"", + "message":"You're welcome! Let me know if you need anything else."} +- ACCEPT — user's reply agrees to the AI's offer ("yes", "sure", "go ahead", + "show me", "نعم") → rewrite the AI's offered action as the standalone + question per TASK 1. Set stop_followup:false. Set contains_values, + entity_words, is_cud, and cud_type from the REWRITTEN question — is_cud is + true ONLY if the offered action was itself a create/update/delete. +- If the reply both closes and asks something new ("ok thanks, and show + suppliers too") → it is NOT a termination; process the new question normally. TASK 2 — ENTITY DETECTION contains_values = TRUE: Any noun that refers to a specific named master record (item name, customer name, supplier name, warehouse name, employee name) @@ -68,7 +68,6 @@ Use natural phrasing ("based on", "using"): stock → actual_qty production → produced_qty finance → debit / credit - status → status 5. Time fields: Sales/Stock/Finance → posting_date @@ -82,9 +81,7 @@ Use natural phrasing ("based on", "using"): STYLE: - Natural business language - No SQL, no tab* names -EXAMPLES: -"total sales amount last month" -→ What is the total sales amount from Sales Invoices last month based on grand_total and posting_date? +ex: "stock in warehouse a" → What is the stock quantity in Warehouse A based on actual_qty from Bin? "who worked today" @@ -106,7 +103,7 @@ STRICT RULES: - Choose fields based on business meaning and user intent, not rank position. - Never invent schema elements. - Always return any one clear user-readable business field, not only technical IDs, unless explicitly requested. -- If the query is ambiguous, ask for clarification and set "clarify": true. +- If the query is ambiguous, ask for clarification and set "clarify": true. - this is very important.Do not map user intent to non-existent fields. If the user asks for something like "status" or "active customers" but the schema does not have a column literally named status, do not substitute it with a guessed equivalent. Instead, look at what columns actually exist in the schema and either use the correct matching column if it clearly corresponds, or inform the user that the exact field is not in the schema. - Preserve ERPNext business semantics during rewriting and avoid converting ERPNext document lifecycle terms into generic workflow or approval terminology that may change the original meaning. - Note ** - If the user question is in Arabic, you MUST rewrite the question in English but entity values must be in the language that user mentioned in the question if the entty values are mentioned in arabic preserve arabic only for entity values remaining part u can translate into english. @@ -181,7 +178,6 @@ and return }} otherwise if question is like the user is asking for analysis, summary, trends, top records, totals, comparisons, or custom reporting, generate SQL and show the results in ChangAI. - Other Examples: - Show sales report this month - Give me a monthly sales summary for last year diff --git a/changai/changai/prompts/sql_rewrite_user_prompt.txt b/changai/changai/prompts/sql_rewrite_user_prompt.txt index 4001a11..35a37c4 100644 --- a/changai/changai/prompts/sql_rewrite_user_prompt.txt +++ b/changai/changai/prompts/sql_rewrite_user_prompt.txt @@ -1,5 +1,4 @@ Chat History: {rows} - User Question: {qstn} \ No newline at end of file