diff --git a/README.md b/README.md index 43469cb..a45a9a9 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ sf provar auth login claude mcp add provar -s user -- sf provar mcp start --allowed-paths /path/to/your/provar/project ``` -📖 **[docs/mcp.md](https://github.com/ProvarTesting/provardx-cli/blob/main/docs/mcp.md) — full setup, all 42 tools, 6 resources, 11 MCP prompts, troubleshooting.** +📖 **[docs/mcp.md](https://github.com/ProvarTesting/provardx-cli/blob/main/docs/mcp.md) — full setup, all 43 tools, 6 resources, 11 MCP prompts, troubleshooting.** --- @@ -251,7 +251,7 @@ DESCRIPTION Note: --json is not available on this command — stdout is reserved for MCP traffic. TOOLS EXPOSED - 42 tools across: project inspection & org describe, Page Object and test-case + 43 tools across: project inspection & org describe, Page Object and test-case authoring/validation, test-suite/plan validation, properties files, Quality Hub (test runs, defects, corpus examples), Provar Automation, ANT build, and NitroX components. See docs/mcp.md for the full catalogue with schemas and examples. diff --git a/docs/VALIDATION_RULE_REGISTRY.md b/docs/VALIDATION_RULE_REGISTRY.md index 2269776..79a5be2 100644 --- a/docs/VALIDATION_RULE_REGISTRY.md +++ b/docs/VALIDATION_RULE_REGISTRY.md @@ -11,216 +11,220 @@ Provar test-case validation runs in two layers. This registry is the single cano **The validity bridge (PDX-509):** a `critical` best-practice violation is surfaced into `issues[]` as an `ERROR` and therefore gates `is_valid` — EXCEPT where a Layer-1 check already owns the concept (then it is suppressed to avoid double-reporting). `major`/`minor`/`info` affect `quality_score` (and the `needs_improvement` status) only. The `status` field is tri-state: `invalid` (a critical) / `needs_improvement` (loads but `quality_score < quality_threshold`) / `valid`. -**Counts:** Layer 1 — 23 rules (18 gating). Layer 2 — 179 rules (critical 64 / major 68 / minor 29 / info 18; 58 bridged to `is_valid`). +**Counts:** Layer 1 — 26 rules (19 gating). Layer 2 — 180 rules (critical 64 / major 69 / minor 29 / info 18; 58 bridged to `is_valid`). ## Layer 1 — Structural validity rules -| Rule ID | Severity | Gates is_valid? | Applies to | Checks | -| ------------------------- | -------- | --------------- | ---------- | ---------------------------------------------------------------------------------------------- | -| `TC_001` | ERROR | Yes | document | XML declaration present ( first line). | -| `TC_002` | ERROR | Yes | document | XML is well-formed (parses without error). | -| `TC_003` | ERROR | Yes | document | Root element is . | -| `TC_010` | ERROR | Yes | testCase | testCase id, when present, is a non-negative integer (id is optional; guid is the identifier). | -| `TC_011` | ERROR | Yes | testCase | testCase has a guid attribute. | -| `TC_012` | ERROR | Yes | testCase | testCase guid is a valid UUID v4. | -| `TC_020` | ERROR | Yes | testCase | testCase has a element. | -| `TC_030` | ERROR | Yes | apiCall | Each apiCall has a guid attribute. | -| `TC_031` | ERROR | Yes | apiCall | Each apiCall guid is a valid UUID v4. | -| `TC_032` | ERROR | Yes | apiCall | Each apiCall has an apiId attribute. | -| `TC_033` | WARNING | No | apiCall | Each apiCall has a descriptive name attribute. | -| `TC_034` | ERROR | Yes | apiCall | Each apiCall has a testItemId attribute. | -| `TC_035` | ERROR | Yes | apiCall | apiCall testItemId is a whole number. | -| `DATA-001` | WARNING | No | testCase | only iterates under a test plan; flags direct testCase-mode execution. | -| `VAR-REF-001` | WARNING | No | argument | A whole-token {Var} stored as valueClass="string" (use class="variable"). | -| `VAR-REF-002` | WARNING | No | argument | {Var} tokens embedded in a plain string (use class="compound"). | -| `UI-TARGET-001` | ERROR | Yes | apiCall | UiWithScreen/UiWithRow target uses class="uiTarget". | -| `UI-LOCATOR-001` | ERROR | Yes | apiCall | UI action locator uses class="uiLocator". | -| `UI-INTERACTION-001` | ERROR | Yes | apiCall | UiDoAction interaction uses class="uiInteraction". | -| `UI-ASSERT-STRUCTURE-001` | ERROR | Yes | apiCall | UiAssert uses nested field/column/page assertion containers, not a flat argument. | -| `SETVALUES-STRUCTURE-001` | ERROR | Yes | apiCall | SetValues values argument uses class="valueList" with . | -| `ASSERT-001` | WARNING | No | apiCall | AssertValues namedValues format flagged for variable/Apex comparisons. | -| `COMPARISON-TYPE-001` | ERROR | Yes | apiCall | comparisonType is within the step-scoped enum subset (load-blocking otherwise). | +| Rule ID | Severity | Gates is_valid? | Applies to | Checks | +| ------- | -------- | --------------- | ---------- | ------ | +| `TC_001` | ERROR | Yes | document | XML declaration present ( first line). | +| `TC_002` | ERROR | Yes | document | XML is well-formed (parses without error). | +| `TC_003` | ERROR | Yes | document | Root element is . | +| `TC_010` | ERROR | Yes | testCase | testCase id, when present, is a non-negative integer (id is optional; guid is the identifier). | +| `TC_011` | ERROR | Yes | testCase | testCase has a guid attribute. | +| `TC_012` | ERROR | Yes | testCase | testCase guid is a valid UUID v4. | +| `TC_020` | ERROR | Yes | testCase | testCase has a element. | +| `TC_030` | ERROR | Yes | apiCall | Each apiCall has a guid attribute. | +| `TC_031` | ERROR | Yes | apiCall | Each apiCall guid is a valid UUID v4. | +| `TC_032` | ERROR | Yes | apiCall | Each apiCall has an apiId attribute. | +| `TC_033` | WARNING | No | apiCall | Each apiCall has a descriptive name attribute. | +| `TC_034` | ERROR | Yes | apiCall | Each apiCall has a testItemId attribute. | +| `TC_035` | ERROR | Yes | apiCall | apiCall testItemId is a whole number. | +| `DATA-001` | WARNING | No | testCase | only iterates under a test plan; flags direct testCase-mode execution. | +| `VAR-REF-001` | WARNING | No | argument | A whole-token {Var} stored as valueClass="string" (use class="variable"). | +| `VAR-REF-002` | WARNING | No | argument | {Var} tokens embedded in a plain string (use class="compound"). | +| `UI-TARGET-001` | ERROR | Yes | apiCall | UiWithScreen/UiWithRow target uses class="uiTarget". | +| `UI-LOCATOR-001` | ERROR | Yes | apiCall | UI action locator uses class="uiLocator". | +| `UI-INTERACTION-001` | ERROR | Yes | apiCall | UiDoAction interaction uses class="uiInteraction". | +| `UI-INTERACTION-002` | ERROR | Yes | apiCall | UiDoAction interaction name is a real Provar interaction (action/set/file), not a hallucination like click/type. | +| `UI-ASSERT-STRUCTURE-001` | ERROR | Yes | apiCall | UiAssert field assertions use a nested , not a flat argument or a block. | +| `SF-CONNECT-TYPE-001` | WARNING | No | testCase | A Salesforce UI test connects via ApexConnect (API + UI), not a UiConnect with no ApexConnect. | +| `CONNECT-REF-CONSISTENCY-001` | WARNING | No | apiCall | Every connection reference (uiConnectionName/apexConnectionName/…) matches a connect step's resultName. | +| `SETVALUES-STRUCTURE-001` | ERROR | Yes | apiCall | SetValues values argument uses class="valueList" with . | +| `ASSERT-001` | WARNING | No | apiCall | AssertValues namedValues format flagged for variable/Apex comparisons. | +| `COMPARISON-TYPE-001` | ERROR | Yes | apiCall | comparisonType is within the step-scoped enum subset (load-blocking otherwise). | ## Layer 2 — Best-practice rules -| Rule ID | Category | Severity | Weight | Gates is_valid? | Checks | -| ---------------------------------------- | -------------------------- | -------- | ------ | --------------- | ---------------------------------------------------------------------------------------------------- | -| `APEX-ASSERT-LAYOUT-001` | ApexAPI | major | 5 | No | ApexAssertLayout must have object and expected file. | -| `APEX-CONNECTION-REF-001` | ApexAPI | major | 5 | No | Apex API steps must reference a valid connection. | -| `APEX-CREATE-FIELDS-001` | ApexAPI | major | 5 | No | ApexCreateObject with fields must populate at least one field. | -| `APEX-CREATE-METADATA-001` | ApexAPI | major | 5 | No | ApexCreateObject and ApexUpdateObject must include parameter metadata. | -| `APEX-CREATE-UPDATE-STRUCTURE-001` | ApexAPI | major | 5 | No | ApexUpdateObject/ApexCreateObject field arguments must be direct, not nested in uiObjectFieldValue. | -| `APEX-DELETE-ID-001` | ApexAPI | major | 5 | No | ApexDeleteObject must have valid record ID. | -| `APEX-EXTRACT-LAYOUT-001` | ApexAPI | major | 5 | No | ApexExtractLayout must have object, file type, and path. | -| `APEX-OBJECT-TYPE-001` | ApexAPI | major | 5 | No | Apex CRUD operations must have valid object types. | -| `APEX-PARAM-GEN-URI-001` | ApexAPI | minor | 2 | No | Apex CRUD operations should include parameterGeneratorUri. | -| `APEX-READ-ASSERTIONS-001` | ApexAPI | minor | 3 | No | ApexReadObject should use resultAssertions instead of separate AssertValues. | -| `APEX-READ-FIELDS-STRUCTURE-001` | ApexAPI | major | 5 | No | ApexReadObject must use generatedParameters for field references, not fields argument with textType. | -| `APEX-READ-ID-001` | ApexAPI | major | 5 | No | ApexReadObject must have valid record ID. | -| `APEX-UPDATE-FIELDS-001` | ApexAPI | major | 5 | No | ApexUpdateObject must specify fields to update. | -| `APEX-UPDATE-ID-001` | ApexAPI | major | 5 | No | ApexUpdateObject must have valid record ID. | -| `CONN-ARG-001` | ApexAPI | minor | 2 | No | Connection arguments must use correct naming convention. | -| `BUILD-PLAN-001` | BuildAndCI | major | 5 | No | Regression Test Plan exists for CI. | -| `APEX-AUTOCLEANUP-001` | ConnectionsAndEnvironments | minor | 2 | No | Prefer autoCleanup over manual ApexDeleteObject steps. | -| `CONN-APEX-001` | ConnectionsAndEnvironments | critical | 8 | Yes | Apex API calls reference valid connections. | -| `CONN-DB-001` | ConnectionsAndEnvironments | critical | 8 | Yes | Database operations reference valid connections. | -| `CONN-DB-002` | ConnectionsAndEnvironments | major | 5 | No | DbConnect resultName must match dbConnectionName in DB operations. | -| `CONN-UI-001` | ConnectionsAndEnvironments | critical | 8 | Yes | UI operations reference valid connections. | -| `DB-CONNECT-001` | ConnectionsAndEnvironments | critical | 8 | Yes | DbConnect has connectionName. | -| `DB-CONNECT-002` | ConnectionsAndEnvironments | critical | 8 | Yes | DbConnect has resultName. | -| `ENV-CONN-001` | ConnectionsAndEnvironments | major | 5 | No | Admin connection supports Login-As. | -| `ENV-CONN-002` | ConnectionsAndEnvironments | minor | 2 | No | Connection names should not contain environment specifiers. | -| `REST-CONN-001` | ConnectionsAndEnvironments | critical | 8 | Yes | WebConnect has connectionName. | -| `REST-CONN-002` | ConnectionsAndEnvironments | critical | 8 | Yes | WebConnect has resultName. | -| `UI-CONN-LITERAL-001` | ConnectionsAndEnvironments | critical | 8 | Yes | uiConnectionName must be a literal string. | -| `UI-CONNECT-ARGS-001` | ConnectionsAndEnvironments | critical | 10 | Yes | UiConnect has invalid arguments (ApexConnect arguments used). | -| `UI-NITROX-CONNECT-ARGS-001` | ConnectionsAndEnvironments | critical | 10 | Yes | NitroX MS connect step has invalid arguments. | -| `UI-NITROX-VARIANT-ARG-001` | ConnectionsAndEnvironments | minor | 2 | No | NitroX MS connect step missing variant-specific argument. | -| `DDT-EXCEL-001` | DataDrivenTesting | major | 5 | No | Excel headers match field label or API name. | -| `DDT-NO-FUNC-001` | DataDrivenTesting | major | 5 | No | No Excel functions in data. | -| `DDT-VAR-001` | DataDrivenTesting | minor | 3 | No | No hardcoded values in steps. | -| `VAR-NAMING-001` | DataDrivenTesting | major | 5 | No | Variable names must use valid identifiers. | -| `VAR-USAGE-001` | DataDrivenTesting | minor | 2 | No | Variable references use correct syntax. | -| `UI-BINDING-ORDER-001` | LocatorPatterns | major | 5 | No | UI binding parameter order must have object= first. | -| `MAINT-FOLDER-001` | MaintenanceAndFolders | minor | 3 | No | Folder-level setup test per application segment. | -| `MAINT-VERSION-001` | MaintenanceAndFolders | info | 1 | No | Consistent Provar/OS/browser versions. | -| `APEX-RESULTNAME-001` | NamingConventions | minor | 2 | No | ApexConnect resultName is unique. | -| `CUSTOM-FIELD-001` | NamingConventions | major | 5 | No | Custom fields end with \_\_c. | -| `NC-FIELD-001` | NamingConventions | minor | 2 | No | Field names use camelCase. | -| `NC-FOLDER-001` | NamingConventions | major | 5 | No | Folder names are modular and title-cased. | -| `NC-PARAM-001` | NamingConventions | major | 5 | No | Parameters and variables use camelCase. | -| `NC-PO-001` | NamingConventions | major | 5 | No | Page Objects use PascalCase. | -| `NC-TESTCASE-001` | NamingConventions | minor | 2 | No | Test case names use consistent naming convention. | -| `SETVALUES-NAME-001` | NamingConventions | critical | 8 | Yes | SetValues namedValue elements have name attribute. | -| `CALLABLE-VISIBILITY-001` | ReusabilityAndCallables | critical | 8 | Yes | Called test cases are marked as Callable. | -| `REUSE-CALL-001` | ReusabilityAndCallables | minor | 2 | No | Callable tests reside in Callables folder. | -| `REUSE-CALL-002` | ReusabilityAndCallables | minor | 2 | No | Callable tests are parameterized. | -| `REUSE-CALL-003` | ReusabilityAndCallables | minor | 2 | No | Callable tests executable in isolation. | -| `ASSERT-STR-VAR-001` | StructureAndGrouping | major | 5 | No | AssertValues must not use string literal to reference a variable. | -| `BDD-AND-LIMIT-001` | StructureAndGrouping | info | 1 | No | Limit And/But chain length. | -| `BDD-GIVEN-FIRST-001` | StructureAndGrouping | info | 1 | No | BDD scenario should start with Given. | -| `BDD-ORDER-001` | StructureAndGrouping | info | 1 | No | BDD steps should follow logical order. | -| `CONTROL-FINALLY-001` | StructureAndGrouping | major | 5 | No | Finally block should be at end of test. | -| `RENDER-ARG-001` | StructureAndGrouping | critical | 10 | Yes | All arguments must have value elements. | -| `RENDER-BOOL-001` | StructureAndGrouping | critical | 10 | Yes | Boolean values must use lowercase. | -| `RENDER-CASE-001` | StructureAndGrouping | critical | 10 | Yes | valueClass attributes must use lowercase. | -| `RENDER-ROOT-001` | StructureAndGrouping | minor | 3 | No | Test case root element should not have unknown attributes. | -| `SETVALUES-FUNC-STR-001` | StructureAndGrouping | major | 5 | No | SetValues must not use string interpolation for function calls. | -| `SETVALUES-INVALID-ELEMENT-001` | StructureAndGrouping | critical | 10 | Yes | SetValues must not contain invalid child elements. | -| `SETVALUES-ZERO-IDX-001` | StructureAndGrouping | major | 5 | No | SetValues string expression must not use [0] index. | -| `STEP-NAMES-001` | StructureAndGrouping | minor | 2 | No | Custom step names for UI asserts and sets. | -| `STRUCT-GROUP-001` | StructureAndGrouping | minor | 2 | No | All steps are inside Group steps or BDD structure. | -| `STRUCT-SUMMARY-001` | StructureAndGrouping | info | 1 | No | Test case has top-level summary. | -| `UI-ASSERT-STRUCT-001` | StructureAndGrouping | critical | 8 | Yes | UiAssert steps must include all required arguments. | -| `UI-ASSERT-STRUCT-002` | StructureAndGrouping | critical | 10 | Yes | UiAssert steps must NOT contain generatedParameters. | -| `VALUE-CLASS-001` | StructureAndGrouping | critical | 10 | Yes | Value elements must use valid class attribute. | -| `AI-CONVERSATION-SESSION-001` | TestCaseDesign | critical | 8 | Yes | AIAgentConversation requires valid session. | -| `AI-IMAGE-CONFIDENCE-001` | TestCaseDesign | major | 5 | No | ImageValidator confidence should be 0.0-1.0. | -| `AI-SESSION-WEBCONNECT-001` | TestCaseDesign | critical | 8 | Yes | AIAgentSession requires WebConnect first. | -| `AI-UTTERANCE-COUNT-001` | TestCaseDesign | info | 1 | No | GenerateUtterance count should be reasonable. | -| `APEX-BULK-LIMIT-001` | TestCaseDesign | info | 1 | No | ApexBulk should be used for large data volumes. | -| `APEX-EXECUTE-SYNTAX-001` | TestCaseDesign | critical | 8 | Yes | ApexExecute code should be valid Apex syntax. | -| `APEX-REUSE-CONN-001` | TestCaseDesign | major | 5 | No | ApexConnect reuseConnectionName should be left blank. | -| `ASSERT-ACTUAL-001` | TestCaseDesign | critical | 8 | Yes | AssertValues has actualValue. | -| `ASSERT-API-001` | TestCaseDesign | critical | 8 | Yes | Must use AssertValues API, not deprecated Assert API. | -| `ASSERT-ARG-ORDER-001` | TestCaseDesign | info | 1 | No | AssertValues arguments must be in correct order. | -| `ASSERT-COMPARISON-001` | TestCaseDesign | critical | 8 | Yes | AssertValues has comparisonType. | -| `ASSERT-DATE-FORMAT-001` | TestCaseDesign | minor | 4 | No | Date/DateTime assertions should use proper format functions. | -| `ASSERT-EXPECTED-001` | TestCaseDesign | critical | 8 | Yes | AssertValues has expectedValue. | -| `ASSERT-VALUES-COMPARISON-001` | TestCaseDesign | major | 5 | No | AssertValues should have meaningful expected values. | -| `BDD-GIVEN-001` | TestCaseDesign | major | 5 | No | Given steps have description. | -| `BDD-THEN-001` | TestCaseDesign | major | 5 | No | Then steps have description. | -| `BDD-WHEN-001` | TestCaseDesign | major | 5 | No | When steps have description. | -| `CLEANUP-CONSISTENCY-001` | TestCaseDesign | major | 5 | No | Manual cleanup matches object creation. | -| `CLEANUP-ORDER-001` | TestCaseDesign | minor | 2 | No | Cleanup deletes objects in reverse order. | -| `CONTROL-FINALLY-001` | TestCaseDesign | major | 5 | No | Finally block must have description and be at end. | -| `CONTROL-FOREACH-001` | TestCaseDesign | major | 4 | No | ForEach loops have valid source collection. | -| `CONTROL-FOREACH-002` | TestCaseDesign | critical | 8 | Yes | ForEach loops have valueName to store current item. | -| `CONTROL-IF-001` | TestCaseDesign | critical | 8 | Yes | If statements have conditions. | -| `CONTROL-SLEEP-001` | TestCaseDesign | major | 5 | No | Sleep step duration and frequency issues. | -| `CONTROL-SLEEP-001` | TestCaseDesign | info | 1 | No | Sleep duration should be under 5 seconds. | -| `CONTROL-SLEEP-002` | TestCaseDesign | critical | 8 | Yes | Sleep steps have duration specified. | -| `CONTROL-SWITCH-001` | TestCaseDesign | critical | 8 | Yes | Switch statements have value expression. | -| `CONTROL-WAITFOR-001` | TestCaseDesign | critical | 8 | Yes | WaitFor steps have condition. | -| `CONTROL-WAITFOR-002` | TestCaseDesign | major | 5 | No | WaitFor steps have max iterations limit. | -| `CONTROL-WHILE-001` | TestCaseDesign | critical | 8 | Yes | While loops have exit conditions. | -| `CONTROL-WHILE-MAX-001` | TestCaseDesign | major | 5 | No | While loop must have termination condition. | -| `CREATE-RESULT-001` | TestCaseDesign | major | 5 | No | ApexCreateObject steps specify resultIdName. | -| `DATA-DB-WHERE-001` | TestCaseDesign | critical | 8 | Yes | DbDelete and DbUpdate should have WHERE clause. | -| `DATA-REST-BODY-001` | TestCaseDesign | major | 5 | No | POST/PUT/PATCH should have request body. | -| `DATA-REST-METHOD-001` | TestCaseDesign | critical | 8 | Yes | RestRequest method should be valid HTTP method. | -| `DATA-REST-STATUS-001` | TestCaseDesign | info | 1 | No | Validate REST response status. | -| `DATA-SOAP-XML-001` | TestCaseDesign | critical | 8 | Yes | SOAP request body should be well-formed XML. | -| `DATA-TYPE-BOOL-001` | TestCaseDesign | critical | 8 | Yes | Boolean values are 'true' or 'false'. | -| `DATA-TYPE-NUMBER-001` | TestCaseDesign | info | 0 | No | Numeric values are valid numbers. | -| `DESIGN-APIUI-001` | TestCaseDesign | minor | 3 | No | Prefer API for setup where possible. | -| `DESIGN-GROUP-001` | TestCaseDesign | minor | 2 | No | Use Group Steps or BDD structure for logical phases. | -| `FILE-READ-PATH-001` | TestCaseDesign | critical | 8 | Yes | Read dataUrl should be valid file path. | -| `FILE-WRITE-PATH-001` | TestCaseDesign | critical | 8 | Yes | Write dataUrl should be writable. | -| `LOG-LEVEL-001` | TestCaseDesign | info | 1 | No | Log messages use appropriate log levels. | -| `MESSAGING-SUBSCRIBE-BEFORE-RECEIVE-001` | TestCaseDesign | critical | 8 | Yes | Subscribe before ReceiveMessage. | -| `MESSAGING-TIMEOUT-001` | TestCaseDesign | info | 1 | No | ReceiveMessage timeout should be reasonable. | -| `PICKLIST-001` | TestCaseDesign | major | 7 | No | Picklist values should match Salesforce metadata. | -| `PO-FIELD-EXISTS-001` | TestCaseDesign | major | 5 | No | Page Object locator references non-existent field. | -| `REST-REQUEST-001` | TestCaseDesign | critical | 8 | Yes | RestRequest has connectionName. | -| `SETVALUES-STRUCTURE-001` | TestCaseDesign | critical | 8 | Yes | SetValues steps have namedValues container. | -| `SETVALUES-VALUE-001` | TestCaseDesign | critical | 8 | Yes | SetValues namedValue elements have value element. | -| `SF-CONVERT-LEAD-STATUS-001` | TestCaseDesign | critical | 8 | Yes | ConvertLead status must be valid. | -| `SF-LAYOUT-EXTRACT-BEFORE-ASSERT-001` | TestCaseDesign | minor | 2 | No | ExtractSalesforceLayout before AssertSalesforceLayout. | -| `SOQL-IN-LOOP-001` | TestCaseDesign | major | 5 | No | SOQL queries must not be inside loops. | -| `SOQL-QUERY-001` | TestCaseDesign | critical | 8 | Yes | ApexSoqlQuery has soqlQuery argument. | -| `SOQL-RESULT-001` | TestCaseDesign | critical | 8 | Yes | SOQL queries specify resultListName. | -| `SOQL-SELECT-ID-001` | TestCaseDesign | minor | 2 | No | SOQL queries include Id and Name. | -| `SOQL-STRUCTURE-001` | TestCaseDesign | critical | 8 | Yes | SOQL queries have SELECT and FROM clauses. | -| `SOQL-WHERE-001` | TestCaseDesign | major | 5 | No | SOQL queries include WHERE or LIMIT clause. | -| `SQL-QUERY-001` | TestCaseDesign | critical | 8 | Yes | SqlQuery has query argument. | -| `SQL-QUERY-002` | TestCaseDesign | critical | 8 | Yes | SqlQuery has dbConnectionName. | -| `STEP-DISABLED-001` | TestCaseDesign | minor | 2 | No | Disabled test steps should be removed. | -| `STEP-ITEMID-001` | TestCaseDesign | critical | 8 | No | testItemId values are whole numbers. _(Layer-1 owns this concept; not bridged)_ | -| `TEST-LENGTH-001` | TestCaseDesign | minor | 3 | No | Test case should not be excessively long. | -| `UI-ALERT-HANDLE-001` | TestCaseDesign | info | 1 | No | UiHandleAlert should capture alert text. | -| `UI-ASSERT-COMPOUND-001` | TestCaseDesign | major | 6 | No | UiAssert must use compound fields for component field assertions. | -| `UI-ASSERT-FIELDLOCATOR-001` | TestCaseDesign | major | 5 | No | UiAssert fieldLocator uses object+field binding. | -| `UI-ASSERT-FIELDLOCATOR-002` | TestCaseDesign | major | 5 | No | UiAssert fieldAssertion must not wrap fieldLocator in uiLocator. | -| `UI-ASSERT-FIELDLOCATOR-003` | TestCaseDesign | critical | 10 | Yes | UiAssert bare locator in Salesforce metadata context causes render failure. | -| `UI-ASSERT-TYPE-001` | TestCaseDesign | minor | 2 | No | UiAssert steps specify assertion type. | -| `UI-DOACTION-VALUE-001` | TestCaseDesign | critical | 8 | Yes | UiDoAction Set requires value argument. | -| `UI-FIELD-METADATA-001` | TestCaseDesign | major | 5 | No | UiDoAction/UiAssert fields should exist in Salesforce metadata. | -| `UI-FILL-VERIFY-001` | TestCaseDesign | info | 1 | No | Verify fields after UiFill. | -| `UI-LOCATOR-ACTION-001` | TestCaseDesign | major | 5 | No | UiDoAction locator URIs must use valid patterns. | -| `UI-LOCATOR-BINDING-001` | TestCaseDesign | major | 5 | No | Ui locator built-in actions use object binding. | -| `UI-LOCATOR-BUTTON-CASING-001` | TestCaseDesign | major | 5 | No | Standard Salesforce flow buttons must use correct locator pattern. | -| `UI-LOCATOR-RECORDTYPE-001` | TestCaseDesign | major | 5 | No | Record Type field locator must use name=RecordType not name=recordTypeId. | -| `UI-LOCATOR-SAVE-001` | TestCaseDesign | major | 5 | No | Save button locator must use correct pattern. | -| `UI-LOOKUP-ID-001` | TestCaseDesign | major | 6 | No | UiDoAction lookup fields should use Name values, not IDs. | -| `UI-NAVIGATE-PREFER-SCREEN-001` | TestCaseDesign | info | 1 | No | Prefer UiWithScreen over UiNavigate for Salesforce. | -| `UI-SCREEN-CONTEXT-001` | TestCaseDesign | major | 5 | No | UI verification or post-save step left under the wrong screen context. | -| `UI-SCREEN-NAV-001` | TestCaseDesign | major | 5 | No | First UiWithScreen must use navigate=Always or IfNeccessary. | -| `UI-SCREEN-NAV-002` | TestCaseDesign | minor | 2 | No | First UiWithScreen should prefer navigate=Always over IfNeccessary. | -| `UI-SCREEN-OBJID-001` | TestCaseDesign | major | 5 | No | UiWithScreen with navigate=Always for Edit/View must have sfUiTargetObjectId. | -| `UI-SCREEN-TARGET-001` | TestCaseDesign | major | 5 | No | UiWithScreen target URIs must use valid patterns. | -| `UI-TARGET-ACTION-001` | TestCaseDesign | major | 5 | No | UiWithScreen target uses invalid action value. | -| `UI-WAIT-VALUECLASS-001` | TestCaseDesign | major | 5 | No | Wait arguments must use uiWait value class. | -| `UTIL-MATCH-REGEX-001` | TestCaseDesign | critical | 8 | Yes | Match regex pattern should be valid. | -| `UTIL-REPLACE-EMPTY-001` | TestCaseDesign | major | 5 | No | Replace searchString should not be empty. | -| `UTIL-SPLIT-DELIMITER-001` | TestCaseDesign | major | 5 | No | Split delimiter should not be empty. | -| `VALID-GUID-001` | TestCaseDesign | critical | 8 | No | Test case has valid identifier. _(Layer-1 owns this concept; not bridged)_ | -| `VALID-STEPS-001` | TestCaseDesign | critical | 8 | No | Test case has steps element. _(Layer-1 owns this concept; not bridged)_ | -| `VAR-PROPERTY-001` | TestCaseDesign | major | 6 | No | Variable property references must be valid. | -| `VAR-REFERENCE-001` | TestCaseDesign | major | 5 | No | Variables are defined before use. | -| `VAR-STRING-LITERAL-001` | TestCaseDesign | major | 5 | No | Variable reference stored as plain string. | -| `APEX-APIPARAM-HALLUCINATION-001` | XMLSchema | critical | 10 | Yes | Apex CRUD apiParam elements must be self-closing without summary/type children. | -| `APEX-CONNECT-ARGS-001` | XMLSchema | critical | 10 | Yes | ApexConnect - Only valid argument IDs allowed. | -| `APEX-CONNECT-CONNID-001` | XMLSchema | critical | 10 | Yes | ApexConnect connectionId must use valueClass='id'. | -| `API-UNKNOWN-001` | XMLSchema | critical | 10 | Yes | API identifier must be a valid Provar API. | -| `FUNCCALL-VALID-001` | XMLSchema | major | 6 | No | funcCall id must be a valid Provar function. | -| `RENDER-DATE-VALUECLASS-001` | XMLSchema | critical | 10 | Yes | valueClass='date' requires epoch timestamp, not date string. | -| `SCHEMA-EMPTY-001` | XMLSchema | minor | 2 | No | Test case should not be empty. | -| `SCHEMA-ID-001` | XMLSchema | critical | 10 | No | Test case must have valid identifier. _(Layer-1 owns this concept; not bridged)_ | -| `SCHEMA-LEGACY-001` | XMLSchema | info | 1 | No | Consider migrating from registryId to id or guid. | -| `SCHEMA-ROOT-001` | XMLSchema | critical | 10 | No | Test case root element must be testCase. _(Layer-1 owns this concept; not bridged)_ | -| `SCHEMA-STEPS-001` | XMLSchema | critical | 10 | No | Test case must have steps element. _(Layer-1 owns this concept; not bridged)_ | -| `SCHEMA-URI-001` | XMLSchema | critical | 10 | Yes | URI attributes must properly encode ampersands. | -| `SCHEMA-VALUE-001` | XMLSchema | critical | 10 | Yes | Value elements must not use text attribute. | -| `STRUCT-ATTR-001` | XMLSchema | info | 1 | No | Test case should have failureBehaviour attribute. | -| `UI-NEST-STRUCT-001` | XMLSchema | major | 7 | No | UI action steps must be nested inside a UiWithScreen substeps clause. | +| Rule ID | Category | Severity | Weight | Gates is_valid? | Checks | +| ------- | -------- | -------- | ------ | --------------- | ------ | +| `APEX-ASSERT-LAYOUT-001` | ApexAPI | major | 5 | No | ApexAssertLayout must have object and expected file. | +| `APEX-CONNECTION-REF-001` | ApexAPI | major | 5 | No | Apex API steps must reference a valid connection. | +| `APEX-CREATE-FIELDS-001` | ApexAPI | major | 5 | No | ApexCreateObject with fields must populate at least one field. | +| `APEX-CREATE-METADATA-001` | ApexAPI | major | 5 | No | ApexCreateObject and ApexUpdateObject must include parameter metadata. | +| `APEX-CREATE-UPDATE-STRUCTURE-001` | ApexAPI | major | 5 | No | ApexUpdateObject/ApexCreateObject field arguments must be direct, not nested in uiObjectFieldValue. | +| `APEX-DELETE-ID-001` | ApexAPI | major | 5 | No | ApexDeleteObject must have valid record ID. | +| `APEX-EXTRACT-LAYOUT-001` | ApexAPI | major | 5 | No | ApexExtractLayout must have object, file type, and path. | +| `APEX-OBJECT-TYPE-001` | ApexAPI | major | 5 | No | Apex CRUD operations must have valid object types. | +| `APEX-PARAM-GEN-URI-001` | ApexAPI | minor | 2 | No | Apex CRUD operations should include parameterGeneratorUri. | +| `APEX-READ-ASSERTIONS-001` | ApexAPI | minor | 3 | No | ApexReadObject should use resultAssertions instead of separate AssertValues. | +| `APEX-READ-FIELDS-STRUCTURE-001` | ApexAPI | major | 5 | No | ApexReadObject must use generatedParameters for field references, not fields argument with textType. | +| `APEX-READ-ID-001` | ApexAPI | major | 5 | No | ApexReadObject must have valid record ID. | +| `APEX-UPDATE-FIELDS-001` | ApexAPI | major | 5 | No | ApexUpdateObject must specify fields to update. | +| `APEX-UPDATE-ID-001` | ApexAPI | major | 5 | No | ApexUpdateObject must have valid record ID. | +| `CONN-ARG-001` | ApexAPI | minor | 2 | No | Connection arguments must use correct naming convention. | +| `BUILD-PLAN-001` | BuildAndCI | major | 5 | No | Regression Test Plan exists for CI. | +| `APEX-AUTOCLEANUP-001` | ConnectionsAndEnvironments | minor | 2 | No | Prefer autoCleanup over manual ApexDeleteObject steps. | +| `CONN-APEX-001` | ConnectionsAndEnvironments | critical | 8 | Yes | Apex API calls reference valid connections. | +| `CONN-DB-001` | ConnectionsAndEnvironments | critical | 8 | Yes | Database operations reference valid connections. | +| `CONN-DB-002` | ConnectionsAndEnvironments | major | 5 | No | DbConnect resultName must match dbConnectionName in DB operations. | +| `CONN-UI-001` | ConnectionsAndEnvironments | critical | 8 | Yes | UI operations reference valid connections. | +| `DB-CONNECT-001` | ConnectionsAndEnvironments | critical | 8 | Yes | DbConnect has connectionName. | +| `DB-CONNECT-002` | ConnectionsAndEnvironments | critical | 8 | Yes | DbConnect has resultName. | +| `ENV-CONN-001` | ConnectionsAndEnvironments | major | 5 | No | Admin connection supports Login-As. | +| `ENV-CONN-002` | ConnectionsAndEnvironments | minor | 2 | No | Connection names should not contain environment specifiers. | +| `REST-CONN-001` | ConnectionsAndEnvironments | critical | 8 | Yes | WebConnect has connectionName. | +| `REST-CONN-002` | ConnectionsAndEnvironments | critical | 8 | Yes | WebConnect has resultName. | +| `UI-CONN-LITERAL-001` | ConnectionsAndEnvironments | critical | 8 | Yes | uiConnectionName must be a literal string. | +| `UI-CONNECT-ARGS-001` | ConnectionsAndEnvironments | critical | 10 | Yes | UiConnect has invalid arguments (ApexConnect arguments used). | +| `UI-NITROX-CONNECT-ARGS-001` | ConnectionsAndEnvironments | critical | 10 | Yes | NitroX MS connect step has invalid arguments. | +| `UI-NITROX-VARIANT-ARG-001` | ConnectionsAndEnvironments | minor | 2 | No | NitroX MS connect step missing variant-specific argument. | +| `DDT-EXCEL-001` | DataDrivenTesting | major | 5 | No | Excel headers match field label or API name. | +| `DDT-NO-FUNC-001` | DataDrivenTesting | major | 5 | No | No Excel functions in data. | +| `DDT-VAR-001` | DataDrivenTesting | minor | 3 | No | No hardcoded values in steps. | +| `VAR-NAMING-001` | DataDrivenTesting | major | 5 | No | Variable names must use valid identifiers. | +| `VAR-USAGE-001` | DataDrivenTesting | minor | 2 | No | Variable references use correct syntax. | +| `UI-BINDING-ORDER-001` | LocatorPatterns | major | 5 | No | UI binding parameter order must have object= first. | +| `MAINT-FOLDER-001` | MaintenanceAndFolders | minor | 3 | No | Folder-level setup test per application segment. | +| `MAINT-VERSION-001` | MaintenanceAndFolders | info | 1 | No | Consistent Provar/OS/browser versions. | +| `APEX-RESULTNAME-001` | NamingConventions | minor | 2 | No | ApexConnect resultName is unique. | +| `CUSTOM-FIELD-001` | NamingConventions | major | 5 | No | Custom fields end with __c. | +| `NC-FIELD-001` | NamingConventions | minor | 2 | No | Field names use camelCase. | +| `NC-FOLDER-001` | NamingConventions | major | 5 | No | Folder names are modular and title-cased. | +| `NC-PARAM-001` | NamingConventions | major | 5 | No | Parameters and variables use camelCase. | +| `NC-PO-001` | NamingConventions | major | 5 | No | Page Objects use PascalCase. | +| `NC-TESTCASE-001` | NamingConventions | minor | 2 | No | Test case names use consistent naming convention. | +| `SETVALUES-NAME-001` | NamingConventions | critical | 8 | Yes | SetValues namedValue elements have name attribute. | +| `CALLABLE-VISIBILITY-001` | ReusabilityAndCallables | critical | 8 | Yes | Called test cases are marked as Callable. | +| `REUSE-CALL-001` | ReusabilityAndCallables | minor | 2 | No | Callable tests reside in Callables folder. | +| `REUSE-CALL-002` | ReusabilityAndCallables | minor | 2 | No | Callable tests are parameterized. | +| `REUSE-CALL-003` | ReusabilityAndCallables | minor | 2 | No | Callable tests executable in isolation. | +| `ASSERT-STR-VAR-001` | StructureAndGrouping | major | 5 | No | AssertValues must not use string literal to reference a variable. | +| `BDD-AND-LIMIT-001` | StructureAndGrouping | info | 1 | No | Limit And/But chain length. | +| `BDD-GIVEN-FIRST-001` | StructureAndGrouping | info | 1 | No | BDD scenario should start with Given. | +| `BDD-ORDER-001` | StructureAndGrouping | info | 1 | No | BDD steps should follow logical order. | +| `CONTROL-FINALLY-001` | StructureAndGrouping | major | 5 | No | Finally block should be at end of test. | +| `RENDER-ARG-001` | StructureAndGrouping | critical | 10 | Yes | All arguments must have value elements. | +| `RENDER-BOOL-001` | StructureAndGrouping | critical | 10 | Yes | Boolean values must use lowercase. | +| `RENDER-CASE-001` | StructureAndGrouping | critical | 10 | Yes | valueClass attributes must use lowercase. | +| `RENDER-ROOT-001` | StructureAndGrouping | minor | 3 | No | Test case root element should not have unknown attributes. | +| `SETVALUES-FUNC-STR-001` | StructureAndGrouping | major | 5 | No | SetValues must not use string interpolation for function calls. | +| `SETVALUES-INVALID-ELEMENT-001` | StructureAndGrouping | critical | 10 | Yes | SetValues must not contain invalid child elements. | +| `SETVALUES-ZERO-IDX-001` | StructureAndGrouping | major | 5 | No | SetValues string expression must not use [0] index. | +| `STEP-NAMES-001` | StructureAndGrouping | minor | 2 | No | Custom step names for UI asserts and sets. | +| `STEP-REQUIRED-ARGS-001` | StructureAndGrouping | major | 5 | No | Steps must declare all schema-required arguments. | +| `STRUCT-GROUP-001` | StructureAndGrouping | minor | 2 | No | All steps are inside Group steps or BDD structure. | +| `STRUCT-SUMMARY-001` | StructureAndGrouping | info | 1 | No | Test case has top-level summary. | +| `UI-ASSERT-STRUCT-001` | StructureAndGrouping | critical | 8 | Yes | UiAssert steps must include all required arguments. | +| `UI-ASSERT-STRUCT-002` | StructureAndGrouping | critical | 10 | Yes | UiAssert steps must NOT contain generatedParameters. | +| `VALUE-CLASS-001` | StructureAndGrouping | critical | 10 | Yes | Value elements must use valid class attribute. | +| `AI-CONVERSATION-SESSION-001` | TestCaseDesign | critical | 8 | Yes | AIAgentConversation requires valid session. | +| `AI-IMAGE-CONFIDENCE-001` | TestCaseDesign | major | 5 | No | ImageValidator confidence should be 0.0-1.0. | +| `AI-SESSION-WEBCONNECT-001` | TestCaseDesign | critical | 8 | Yes | AIAgentSession requires WebConnect first. | +| `AI-UTTERANCE-COUNT-001` | TestCaseDesign | info | 1 | No | GenerateUtterance count should be reasonable. | +| `APEX-BULK-LIMIT-001` | TestCaseDesign | info | 1 | No | ApexBulk should be used for large data volumes. | +| `APEX-EXECUTE-SYNTAX-001` | TestCaseDesign | critical | 8 | Yes | ApexExecute code should be valid Apex syntax. | +| `APEX-REUSE-CONN-001` | TestCaseDesign | major | 5 | No | ApexConnect reuseConnectionName should be left blank. | +| `ASSERT-ACTUAL-001` | TestCaseDesign | critical | 8 | Yes | AssertValues has actualValue. | +| `ASSERT-API-001` | TestCaseDesign | critical | 8 | Yes | Must use AssertValues API, not deprecated Assert API. | +| `ASSERT-ARG-ORDER-001` | TestCaseDesign | info | 1 | No | AssertValues arguments must be in correct order. | +| `ASSERT-COMPARISON-001` | TestCaseDesign | critical | 8 | Yes | AssertValues has comparisonType. | +| `ASSERT-DATE-FORMAT-001` | TestCaseDesign | minor | 4 | No | Date/DateTime assertions should use proper format functions. | +| `ASSERT-EXPECTED-001` | TestCaseDesign | critical | 8 | Yes | AssertValues has expectedValue. | +| `ASSERT-VALUES-COMPARISON-001` | TestCaseDesign | major | 5 | No | AssertValues should have meaningful expected values. | +| `BDD-GIVEN-001` | TestCaseDesign | major | 5 | No | Given steps have description. | +| `BDD-THEN-001` | TestCaseDesign | major | 5 | No | Then steps have description. | +| `BDD-WHEN-001` | TestCaseDesign | major | 5 | No | When steps have description. | +| `CLEANUP-CONSISTENCY-001` | TestCaseDesign | major | 5 | No | Manual cleanup matches object creation. | +| `CLEANUP-ORDER-001` | TestCaseDesign | minor | 2 | No | Cleanup deletes objects in reverse order. | +| `CONTROL-FINALLY-001` | TestCaseDesign | major | 5 | No | Finally block must have description and be at end. | +| `CONTROL-FOREACH-001` | TestCaseDesign | major | 4 | No | ForEach loops have valid source collection. | +| `CONTROL-FOREACH-002` | TestCaseDesign | critical | 8 | Yes | ForEach loops have valueName to store current item. | +| `CONTROL-IF-001` | TestCaseDesign | critical | 8 | Yes | If statements have conditions. | +| `CONTROL-SLEEP-001` | TestCaseDesign | major | 5 | No | Sleep step duration and frequency issues. | +| `CONTROL-SLEEP-001` | TestCaseDesign | info | 1 | No | Sleep duration should be under 5 seconds. | +| `CONTROL-SLEEP-002` | TestCaseDesign | critical | 8 | Yes | Sleep steps have duration specified. | +| `CONTROL-SWITCH-001` | TestCaseDesign | critical | 8 | Yes | Switch statements have value expression. | +| `CONTROL-WAITFOR-001` | TestCaseDesign | critical | 8 | Yes | WaitFor steps have condition. | +| `CONTROL-WAITFOR-002` | TestCaseDesign | major | 5 | No | WaitFor steps have max iterations limit. | +| `CONTROL-WHILE-001` | TestCaseDesign | critical | 8 | Yes | While loops have exit conditions. | +| `CONTROL-WHILE-MAX-001` | TestCaseDesign | major | 5 | No | While loop must have termination condition. | +| `CREATE-RESULT-001` | TestCaseDesign | major | 5 | No | ApexCreateObject steps specify resultIdName. | +| `DATA-DB-WHERE-001` | TestCaseDesign | critical | 8 | Yes | DbDelete and DbUpdate should have WHERE clause. | +| `DATA-REST-BODY-001` | TestCaseDesign | major | 5 | No | POST/PUT/PATCH should have request body. | +| `DATA-REST-METHOD-001` | TestCaseDesign | critical | 8 | Yes | RestRequest method should be valid HTTP method. | +| `DATA-REST-STATUS-001` | TestCaseDesign | info | 1 | No | Validate REST response status. | +| `DATA-SOAP-XML-001` | TestCaseDesign | critical | 8 | Yes | SOAP request body should be well-formed XML. | +| `DATA-TYPE-BOOL-001` | TestCaseDesign | critical | 8 | Yes | Boolean values are 'true' or 'false'. | +| `DATA-TYPE-NUMBER-001` | TestCaseDesign | info | 0 | No | Numeric values are valid numbers. | +| `DESIGN-APIUI-001` | TestCaseDesign | minor | 3 | No | Prefer API for setup where possible. | +| `DESIGN-GROUP-001` | TestCaseDesign | minor | 2 | No | Use Group Steps or BDD structure for logical phases. | +| `FILE-READ-PATH-001` | TestCaseDesign | critical | 8 | Yes | Read dataUrl should be valid file path. | +| `FILE-WRITE-PATH-001` | TestCaseDesign | critical | 8 | Yes | Write dataUrl should be writable. | +| `LOG-LEVEL-001` | TestCaseDesign | info | 1 | No | Log messages use appropriate log levels. | +| `MESSAGING-SUBSCRIBE-BEFORE-RECEIVE-001` | TestCaseDesign | critical | 8 | Yes | Subscribe before ReceiveMessage. | +| `MESSAGING-TIMEOUT-001` | TestCaseDesign | info | 1 | No | ReceiveMessage timeout should be reasonable. | +| `PICKLIST-001` | TestCaseDesign | major | 7 | No | Picklist values should match Salesforce metadata. | +| `PO-FIELD-EXISTS-001` | TestCaseDesign | major | 5 | No | Page Object locator references non-existent field. | +| `REST-REQUEST-001` | TestCaseDesign | critical | 8 | Yes | RestRequest has connectionName. | +| `SETVALUES-STRUCTURE-001` | TestCaseDesign | critical | 8 | Yes | SetValues steps have namedValues container. | +| `SETVALUES-VALUE-001` | TestCaseDesign | critical | 8 | Yes | SetValues namedValue elements have value element. | +| `SF-CONVERT-LEAD-STATUS-001` | TestCaseDesign | critical | 8 | Yes | ConvertLead status must be valid. | +| `SF-LAYOUT-EXTRACT-BEFORE-ASSERT-001` | TestCaseDesign | minor | 2 | No | ExtractSalesforceLayout before AssertSalesforceLayout. | +| `SOQL-IN-LOOP-001` | TestCaseDesign | major | 5 | No | SOQL queries must not be inside loops. | +| `SOQL-QUERY-001` | TestCaseDesign | critical | 8 | Yes | ApexSoqlQuery has soqlQuery argument. | +| `SOQL-RESULT-001` | TestCaseDesign | critical | 8 | Yes | SOQL queries specify resultListName. | +| `SOQL-SELECT-ID-001` | TestCaseDesign | minor | 2 | No | SOQL queries include Id and Name. | +| `SOQL-STRUCTURE-001` | TestCaseDesign | critical | 8 | Yes | SOQL queries have SELECT and FROM clauses. | +| `SOQL-WHERE-001` | TestCaseDesign | major | 5 | No | SOQL queries include WHERE or LIMIT clause. | +| `SQL-QUERY-001` | TestCaseDesign | critical | 8 | Yes | SqlQuery has query argument. | +| `SQL-QUERY-002` | TestCaseDesign | critical | 8 | Yes | SqlQuery has dbConnectionName. | +| `STEP-DISABLED-001` | TestCaseDesign | minor | 2 | No | Disabled test steps should be removed. | +| `STEP-ITEMID-001` | TestCaseDesign | critical | 8 | No | testItemId values are whole numbers. _(Layer-1 owns this concept; not bridged)_ | +| `TEST-LENGTH-001` | TestCaseDesign | minor | 3 | No | Test case should not be excessively long. | +| `UI-ALERT-HANDLE-001` | TestCaseDesign | info | 1 | No | UiHandleAlert should capture alert text. | +| `UI-ASSERT-COMPOUND-001` | TestCaseDesign | major | 6 | No | UiAssert must use compound fields for component field assertions. | +| `UI-ASSERT-FIELDLOCATOR-001` | TestCaseDesign | major | 5 | No | UiAssert fieldLocator uses object+field binding. | +| `UI-ASSERT-FIELDLOCATOR-002` | TestCaseDesign | major | 5 | No | UiAssert fieldAssertion must not wrap fieldLocator in uiLocator. | +| `UI-ASSERT-FIELDLOCATOR-003` | TestCaseDesign | critical | 10 | Yes | UiAssert bare locator in Salesforce metadata context causes render failure. | +| `UI-ASSERT-TYPE-001` | TestCaseDesign | minor | 2 | No | UiAssert steps specify assertion type. | +| `UI-DOACTION-VALUE-001` | TestCaseDesign | critical | 8 | Yes | UiDoAction Set requires value argument. | +| `UI-FIELD-METADATA-001` | TestCaseDesign | major | 5 | No | UiDoAction/UiAssert fields should exist in Salesforce metadata. | +| `UI-FILL-VERIFY-001` | TestCaseDesign | info | 1 | No | Verify fields after UiFill. | +| `UI-LOCATOR-ACTION-001` | TestCaseDesign | major | 5 | No | UiDoAction locator URIs must use valid patterns. | +| `UI-LOCATOR-BINDING-001` | TestCaseDesign | major | 5 | No | Ui locator built-in actions use object binding. | +| `UI-LOCATOR-BUTTON-CASING-001` | TestCaseDesign | major | 5 | No | Standard Salesforce flow buttons must use correct locator pattern. | +| `UI-LOCATOR-RECORDTYPE-001` | TestCaseDesign | major | 5 | No | Record Type field locator must use name=RecordType not name=recordTypeId. | +| `UI-LOCATOR-SAVE-001` | TestCaseDesign | major | 5 | No | Save button locator must use correct pattern. | +| `UI-LOOKUP-ID-001` | TestCaseDesign | major | 6 | No | UiDoAction lookup fields should use Name values, not IDs. | +| `UI-NAVIGATE-PREFER-SCREEN-001` | TestCaseDesign | info | 1 | No | Prefer UiWithScreen over UiNavigate for Salesforce. | +| `UI-SCREEN-CONTEXT-001` | TestCaseDesign | major | 5 | No | UI verification or post-save step left under the wrong screen context. | +| `UI-SCREEN-NAV-001` | TestCaseDesign | major | 5 | No | First UiWithScreen must use navigate=Always or IfNeccessary. | +| `UI-SCREEN-NAV-002` | TestCaseDesign | minor | 2 | No | First UiWithScreen should prefer navigate=Always over IfNeccessary. | +| `UI-SCREEN-OBJID-001` | TestCaseDesign | major | 5 | No | UiWithScreen with navigate=Always for Edit/View must have sfUiTargetObjectId. | +| `UI-SCREEN-TARGET-001` | TestCaseDesign | major | 5 | No | UiWithScreen target URIs must use valid patterns. | +| `UI-TARGET-ACTION-001` | TestCaseDesign | major | 5 | No | UiWithScreen target uses invalid action value. | +| `UI-WAIT-VALUECLASS-001` | TestCaseDesign | major | 5 | No | Wait arguments must use uiWait value class. | +| `UTIL-MATCH-REGEX-001` | TestCaseDesign | critical | 8 | Yes | Match regex pattern should be valid. | +| `UTIL-REPLACE-EMPTY-001` | TestCaseDesign | major | 5 | No | Replace searchString should not be empty. | +| `UTIL-SPLIT-DELIMITER-001` | TestCaseDesign | major | 5 | No | Split delimiter should not be empty. | +| `VALID-GUID-001` | TestCaseDesign | critical | 8 | No | Test case has valid identifier. _(Layer-1 owns this concept; not bridged)_ | +| `VALID-STEPS-001` | TestCaseDesign | critical | 8 | No | Test case has steps element. _(Layer-1 owns this concept; not bridged)_ | +| `VAR-PROPERTY-001` | TestCaseDesign | major | 6 | No | Variable property references must be valid. | +| `VAR-REFERENCE-001` | TestCaseDesign | major | 5 | No | Variables are defined before use. | +| `VAR-STRING-LITERAL-001` | TestCaseDesign | major | 5 | No | Variable reference stored as plain string. | +| `APEX-APIPARAM-HALLUCINATION-001` | XMLSchema | critical | 10 | Yes | Apex CRUD apiParam elements must be self-closing without summary/type children. | +| `APEX-CONNECT-ARGS-001` | XMLSchema | critical | 10 | Yes | ApexConnect - Only valid argument IDs allowed. | +| `APEX-CONNECT-CONNID-001` | XMLSchema | critical | 10 | Yes | ApexConnect connectionId must use valueClass='id'. | +| `API-UNKNOWN-001` | XMLSchema | critical | 10 | Yes | API identifier must be a valid Provar API. | +| `FUNCCALL-VALID-001` | XMLSchema | major | 6 | No | funcCall id must be a valid Provar function. | +| `RENDER-DATE-VALUECLASS-001` | XMLSchema | critical | 10 | Yes | valueClass='date' requires epoch timestamp, not date string. | +| `SCHEMA-EMPTY-001` | XMLSchema | minor | 2 | No | Test case should not be empty. | +| `SCHEMA-ID-001` | XMLSchema | critical | 10 | No | Test case must have valid identifier. _(Layer-1 owns this concept; not bridged)_ | +| `SCHEMA-LEGACY-001` | XMLSchema | info | 1 | No | Consider migrating from registryId to id or guid. | +| `SCHEMA-ROOT-001` | XMLSchema | critical | 10 | No | Test case root element must be testCase. _(Layer-1 owns this concept; not bridged)_ | +| `SCHEMA-STEPS-001` | XMLSchema | critical | 10 | No | Test case must have steps element. _(Layer-1 owns this concept; not bridged)_ | +| `SCHEMA-URI-001` | XMLSchema | critical | 10 | Yes | URI attributes must properly encode ampersands. | +| `SCHEMA-VALUE-001` | XMLSchema | critical | 10 | Yes | Value elements must not use text attribute. | +| `STRUCT-ATTR-001` | XMLSchema | info | 1 | No | Test case should have failureBehaviour attribute. | +| `UI-NEST-STRUCT-001` | XMLSchema | major | 7 | No | UI action steps must be nested inside a UiWithScreen substeps clause. | diff --git a/docs/mcp.md b/docs/mcp.md index b350704..ad0f878 100644 --- a/docs/mcp.md +++ b/docs/mcp.md @@ -61,6 +61,7 @@ The Provar DX CLI ships with a built-in **Model Context Protocol (MCP) server** - [provar_nitrox_patch](#provar_nitrox_patch) - [Quality Hub API tools](#quality-hub-api-tools) - [provar_qualityhub_examples_retrieve](#provar_qualityhub_examples_retrieve) + - [provar_step_schema](#provar_step_schema) - [Org metadata via Salesforce Hosted MCP](#org-metadata-via-salesforce-hosted-mcp) - [MCP Prompts](#mcp-prompts) - [Migration prompts](#migration-prompts) @@ -565,7 +566,7 @@ Paste the [standard config](#the-standard-config-recommended) into either file u } ``` -> **Tool limit:** Agentforce Vibes loads approximately 20 tools per MCP server at runtime. The Provar MCP server exposes 42 tools — you may need to restart or re-enable the server between tasks if the active tool list gets out of date. Salesforce is tracking this limit; consult the [Agentforce Vibes MCP documentation](https://developer.salesforce.com/docs/platform/einstein-for-devs/guide/devagent-mcp.html) for the latest guidance. +> **Tool limit:** Agentforce Vibes loads approximately 20 tools per MCP server at runtime. The Provar MCP server exposes 43 tools — you may need to restart or re-enable the server between tasks if the active tool list gets out of date. Salesforce is tracking this limit; consult the [Agentforce Vibes MCP documentation](https://developer.salesforce.com/docs/platform/einstein-for-devs/guide/devagent-mcp.html) for the latest guidance. @@ -672,7 +673,7 @@ Restricts which tool groups are registered when the server starts. Only the grou | `automation` | `provar_automation_setup`, `provar_automation_config_load`, `provar_automation_metadata_download`, `provar_automation_compile`, `provar_automation_testrun` | | `qualityhub` | `provar_qualityhub_connect`, `provar_qualityhub_display`, `provar_qualityhub_testrun`, `provar_qualityhub_testrun_abort`, `provar_qualityhub_testrun_report`, `provar_qualityhub_examples_retrieve`, `provar_qualityhub_testcase_retrieve`, `provar_qualityhub_defect_create` | | `validation` | `provar_project_validate`, `provar_ant_generate`, `provar_ant_validate`, `provar_properties_*`, `provar_testcase_validate`, `provar_testsuite_validate`, `provar_testplan_validate`, `provar_pageobject_validate` | -| `authoring` | `provar_testcase_generate`, `provar_pageobject_generate`, `provar_testcase_step_edit`, `provar_testplan_*` | +| `authoring` | `provar_testcase_generate`, `provar_pageobject_generate`, `provar_testcase_step_edit`, `provar_testplan_*`, `provar_step_schema` | | `inspect` | `provar_project_inspect` | | `connection` | `provar_connection_list` | | `rca` | `provar_testrun_rca`, `provar_testrun_report_locate` | @@ -1058,7 +1059,7 @@ Validates an XML test case for schema correctness (validity score) and best prac - **UI-TARGET-001** — A UiWithScreen or UiWithRow `target` argument uses the wrong XML class (e.g. `class="value"`). Must be `class="uiTarget"` or the screen binding is silently ignored at runtime. - **UI-LOCATOR-001** — A locator-bearing UI step (`UiDoAction`, `UiAssert`, `UiRead`, `UiFill`) has a `locator` argument that uses the wrong XML class. Must be `class="uiLocator"` or Provar cannot resolve the element. - **UI-INTERACTION-001** (ERROR) — A UI action step (e.g. `UiDoAction`) has an `interaction` argument that uses the wrong XML class (e.g. `class="value"`). Must be `class="uiInteraction"` (``). A plain string runs green from the CLI but renders the Action field blank in the Provar IDE step editor. The `interaction` attribute is converted automatically by `provar_testcase_generate`. -- **UI-ASSERT-STRUCTURE-001** (ERROR) — A `UiAssert` step carries a flat top-level field-assertion argument (`fieldLocator`, `attributeName`, `comparisonType`, or `expectedValue`) instead of the nested `fieldAssertions` → `uiFieldAssertion` structure the Provar IDE Result Assertions tab binds from. The flat shape runs green from the CLI but renders the Result Assertions tab blank in the IDE. The correct form nests a `` containing a **bare** `` element (NOT `class="uiLocator"`) and ``, plus empty `columnAssertions`/`pageAssertions`. `provar_testcase_generate` builds this structure automatically when you pass `fieldLocator`/`attributeName`/`comparisonType`/`expectedValue` as flat attributes. +- **UI-ASSERT-STRUCTURE-001** (ERROR) — A `UiAssert` step expresses its field assertion in a wrong shape instead of the nested `fieldAssertions` → `uiFieldAssertion` structure the Provar IDE Result Assertions tab binds from. Two wrong shapes are caught: (a) flat top-level field-assertion arguments (`fieldLocator`, `attributeName`, `comparisonType`, or `expectedValue`); and (b) a `fieldAssertions` argument filled with a ``/`` block (the shape an LLM reaches for by analogy with `UiFill`/`SetValues`). Both run green from the CLI but render the Result Assertions tab blank in the IDE. This rule now runs on **nested** UI steps too (UI actions/asserts live inside a `UiWithScreen` ``), not just top-level steps. The correct form nests a `` containing a **bare** `` element (NOT `class="uiLocator"`) and ``, plus empty `columnAssertions`/`pageAssertions`. `provar_testcase_generate` builds this structure automatically when you pass `fieldLocator`/`attributeName`/`comparisonType`/`expectedValue` as flat attributes. - **SETVALUES-STRUCTURE-001** (ERROR) — A `SetValues` step's `values` argument uses `class="value"` (plain string) instead of `class="valueList"` with `` children. This causes an immediate `ClassCastException` at runtime. - **COMPARISON-TYPE-001** (ERROR) — A `comparisonType` value is used outside the subset its step type allows. `comparisonType` is a single Provar enum but each step type accepts only a subset: **AssertValues** accepts the 16-value set (`EqualTo, NotEqualTo, GreaterThan, GreaterThanOrEqualTo, LessThan, LessThanOrEqualTo, IsPresent, IsEmpty, Matches, NotMatches, Contains, NotContains, StartsWith, NotStartsWith, EndsWith, NotEndsWith`); a **UI Assert** (`uiAttributeAssertion`) accepts only the 6-value set (`EqualTo, Contains, StartsWith, EndsWith, Matches, None`). A value outside the step's subset (e.g. `NotEqualTo` on a UI Assert) is load-blocking — the whole test case fails to load at runtime with `IllegalArgumentException: No enum constant com.provar.core.model.base.java.ComparisonType.`. This local check runs even offline / in `local_fallback`, so the load-blocker is caught without the Quality Hub back-end. Only literal `comparisonType` values are checked; variable / compound references are skipped. See [`provar://docs/step-reference`](#resources) for the full step-scoped tables. - **UI-NEST-STRUCT-001** (severity `major`, weight 7, category `XMLSchema`) — A UI action step (`UiDoAction`, `UiAssert`, `UiRead`, `UiFill`, `UiNavigate`, `UiWithRow`, or `UiHandleAlert`) is emitted outside a screen ancestor. To pass, every UI action must descend from a `UiWithScreen` or `UiWithRow` `apiCall` through a `` path. Control-flow wrappers (`If`/`ForEach`/`DoWhile`/`WaitFor`/`Switch`) between the screen ancestor and the UI action are allowed; steps inside `` are exempt (disabled / settings blocks). One violation is emitted per offending step, so `(rule_id, test_item_id)` de-duplicates cleanly against the Quality Hub API. Provar IDE cannot bind flat-emitted UI actions to a screen context and they will not render in the editor. Wrap each offending step in the canonical chain: @@ -1108,8 +1109,12 @@ Validates an XML test case for schema correctness (validity score) and best prac - **`SETVALUES-VALUE-001`** (`namedValueValue`, critical) — a `` inside a `SetValues` step is missing its child `` element. - **`UI-ASSERT-STRUCT-002`** (`uiAssertHallucinatedGeneratedParameters`, critical) — a `UiAssert` step contains a `` element, which is never valid on `UiAssert` and blocks validation. - **`UI-ASSERT-STRUCT-001`** (`uiAssertMissingArguments`, critical) — a `UiAssert` step is missing one or more required arguments (`fieldAssertions`, `columnAssertions`, `pageAssertions`, `resultScope`, `captureAfter`, `beforeWait`, `autoRetry` — present even if empty). + - **`STEP-REQUIRED-ARGS-001`** (`schemaRequiredArguments`, major) — generalises the UiAssert-only check above to **every** step type: any step missing an argument that `provar_test_step_schema.json` lists as required for its `apiId` is flagged (one violation per step). This closes the gap where a test case could score 100 while omitting required arguments on `UiConnect`, `UiWithScreen`, `UiDoAction`, `UiFill`, DB/Apex/REST steps, etc. The required-argument set is read from the schema at runtime (the same data the `provar://schema/test-step` resource serves), so correcting the schema automatically corrects this rule — nothing is hardcoded. It is **major** (docks `quality_score`, does not gate `is_valid`) because the schema is a bundled reference rather than the authoritative Quality Hub ruleset. `UiAssert` (covered by `UI-ASSERT-STRUCT-001`) and the NitroX MS connect variants (dedicated rules) are excluded to avoid double-reporting. - **`UI-BINDING-ORDER-001`** (`bindingParameterOrder`, critical) — a `uiLocator` binding URI lists `action=`/`field=` before `object=`; the corpus-majority convention is `object=` first. - **`UI-CONN-LITERAL-001`** (`uiConnectionNameLiteral`, critical) — a UI step's `uiConnectionName` uses a `class="variable"` value; it must be a literal connection name. + - **`UI-INTERACTION-002`** (local, ERROR) — a `UiDoAction` `interaction` uses a name that is not a real Provar interaction. The valid names are `action` (click a button/link), `set` (fill a field / pick a picklist value) and `file` (upload). Common LLM hallucinations (`click`, `type`, `select`, …) render the IDE Action widget blank and do not run; the rule names the correct replacement (e.g. `click` → `action`). Complements `UI-INTERACTION-001`, which checks the value **class**. + - **`SF-CONNECT-TYPE-001`** (local, WARNING) — the test drives the Salesforce UI (`sf:ui:` targets) but connects with `UiConnect` and **no** `ApexConnect`. `UiConnect` opens a browser only and has no Salesforce credential of its own; a Salesforce UI test should connect via `ApexConnect` (which opens the API connection and, with `quickUiLogin=true`/`uiApplicationName=LightningSales`, the Lightning UI). Add `UiConnect` only for a second, browser-only session that reuses the `ApexConnect` result. WARNING, not ERROR, because the connection could in rare setups be supplied by a parent callable test. + - **`CONNECT-REF-CONSISTENCY-001`** (local, WARNING) — a step references a connection (`uiConnectionName` / `apexConnectionName` / `dbConnectionName` / `webConnectionName`) whose name no connect step in the test produces. Downstream steps reference the connect step's `resultName` (which defaults to `connectionName` when omitted); a reference that matches nothing is a dangling connection. Only fires when the test contains at least one connect step (so inherited connections do not false-fire). - **`FUNCCALL-VALID-001`** (`validFuncCallId`, major) — a `` uses an `id` that is not one of Provar's built-in functions (e.g. the hallucinated `Concatenate`/`Substring`); use the documented set (`Count`, `DateAdd`, `StringReplace`, …) or `class="compound"` for concatenation. - **`RENDER-ROOT-001`** (`rootAttributes`, minor) — the root `` element carries an attribute outside the allowed set (`guid`, `id`, `name`, `visibility`, `registryId`, `failureBehaviour`). @@ -2378,7 +2383,7 @@ These tools call the Quality Hub HTTP API directly (no `sf` subprocess). They re Retrieve N similar Provar test case examples from the Quality Hub corpus (1000+ tests indexed in Bedrock). Use this **before** `provar_testcase_generate` to provide few-shot grounding examples. -If retrieval fails for any reason (no key, invalid key, rate limit, network error), the tool returns `{ examples: [], count: 0, warning: "..." }` with `isError: false` so the generation workflow can continue without grounding. It **never** hard-errors on API failure. +When **no API key** is configured (or the key is **invalid**), the tool returns a small set of offline, known-correct **bundled** examples (`source: "bundled"`, `quality_tier: "bundled"`) instead of an empty list, so first-test-case generation in a brand-new project still has a validated structural pattern to follow. On **transient** failures (rate limit, network error) it returns `{ examples: [], count: 0, warning: "..." }` so a retry is the obvious next move. In all cases `isError: false` — it **never** hard-errors on API failure. Bundled examples are validated in CI (`quality_score === 100`, zero violations) against the same rules `provar_testcase_validate` applies. | Input | Type | Required | Default | Description | | --------------------- | ------- | -------- | ------- | ------------------------------------------------------------------------------ | @@ -2404,13 +2409,37 @@ Each element in `examples`: | `xml` | Full Provar XML test case content | | `similarity_score` | Similarity score in [0, 1] | | `salesforce_object` | Primary Salesforce object the test exercises | -| `quality_tier` | Corpus tier (`tier4`, `tier3`, `tier2`, `tier1`) | +| `quality_tier` | Corpus tier (`tier4`, `tier3`, `tier2`, `tier1`), or `bundled` for offline examples | | `full_content` | `true` when the full XML was returned (not truncated server-side) | +| `source` | `bundled` on offline examples (absent for corpus matches) | **Error codes:** `INVALID_QUERY` (empty query — only error that sets `isError: true`) --- +### `provar_step_schema` + +Returns the argument schema for Provar test-case step types, read from the bundled step reference (`provar_test_step_schema.json`). This is the **tool** form of the `provar://schema/test-step` MCP resource — agents that cannot read MCP resources can still reach the schema to ground test-case generation and hand-editing in exact argument names. Call it before `provar_testcase_generate` / `provar_testcase_step_edit`, or when `provar_testcase_validate` reports a structural error, to get a step type's required/optional arguments. Read-only and offline. + +| Input | Type | Required | Default | Description | +| ---------- | ------ | -------- | ------- | ------------------------------------------------------------------------------------------------------------ | +| `api_id` | string | no | — | Full apiId (`com.provar.plugins.forcedotcom.core.ui.UiConnect`) or short name (`UiConnect`). Returns that step's schema. | +| `category` | string | no | — | Category to list (`UI`, `Salesforce`, `Control`, `Data`, `Utility`, `ProvarAI`, `ProvarLabs`). | + +Precedence: `api_id` → single step; else `category` → steps in that category; else the full index. + +| Output field | Description | +| ------------ | -------------------------------------------------------------------------------------------------- | +| `step` | (with `api_id`) `{ apiId, description, category, required_arguments[], optional_arguments[], validation_rules[], best_practices[] }`. Each argument is `{ id, type, default?, description? }`. | +| `steps` | (with `category` or index) array of `{ name, apiId, category, required_arguments[] }` | +| `count` | Number of step types returned (category/index modes) | + +**Example** — `{ "api_id": "UiConnect" }` returns `step.required_arguments = [{ "id": "connectionName", "type": "string" }]`. + +**Error codes:** `STEP_TYPE_NOT_FOUND` (no step matches `api_id`; includes a `suggestion` with near matches), `STEP_SCHEMA_NOT_FOUND` (bundled schema unreadable — reinstall the plugin). + +--- + ### Org metadata via Salesforce Hosted MCP Provar MCP does not include a built-in org introspection tool. Instead, connect the **Salesforce Hosted MCP Server** (`platform/sobject-reads`) alongside Provar MCP and call `getObjectSchema` to retrieve sObject field metadata. Pass the result as additional context in your `provar_qualityhub_examples_retrieve` query. diff --git a/scripts/mcp-smoke.cjs b/scripts/mcp-smoke.cjs index 467df36..993f89e 100644 --- a/scripts/mcp-smoke.cjs +++ b/scripts/mcp-smoke.cjs @@ -191,6 +191,9 @@ async function runTests() { output_path: path.join(TMP, 'steps-required-smoke-rejected.testcase'), }); + // ── 6c. provar_step_schema (by api_id) ──────────────────────────────────── + if (inGroup('authoring')) await callTool('provar_step_schema', { api_id: 'UiConnect' }); + // ── 7. provar_testcase_validate ─────────────────────────────────────────── if (inGroup('validation')) await callTool('provar_testcase_validate', { content: '' }); diff --git a/src/mcp/examples/bundledExamples.ts b/src/mcp/examples/bundledExamples.ts new file mode 100644 index 0000000..a86e2c3 --- /dev/null +++ b/src/mcp/examples/bundledExamples.ts @@ -0,0 +1,200 @@ +/* + * Copyright (c) 2024 Provar Limited. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.md file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +/* eslint-disable camelcase */ + +/** + * Offline, known-correct example test cases bundled with the MCP server. + * + * `provar_qualityhub_examples_retrieve` returns these (clearly labelled `bundled`) + * when no Provar API key is configured, so first-test-case generation in a brand-new + * project has *some* grounded XML to pattern-match against instead of falling back + * entirely to model recollection. + * + * Every example here is validated by test/unit/mcp/bundledExamples.test.ts, which runs + * it through the same best-practices + structural validators the server exposes and + * asserts quality_score === 100 with zero violations. If you add or edit an example, + * that test guards it against drifting out of correctness — it is what caught the + * original example using UiConnect / interaction=click / flat field assertions. + */ + +export interface BundledExample { + id: string; + name: string; + /** Short scenario key used for lightweight query matching. */ + scenario: string; + salesforce_object: string; + keywords: string[]; + xml: string; +} + +const UI_CREATE_VALIDATE_ACCOUNT = ` + + Creates an Account through the Salesforce UI (mandatory Name field), saves it, and verifies the Name on the record detail page. + + + + MyConnection + MyConnection + Test + LightningSales + true + + + + + + + MyConnection + + Always + On SF Account Home screen + false + false + + + + + + + + + Click the New button + false + false + + + + + + + + + MyConnection + + Dont + NewAccountId + false + false + + + + + + + + + Set the Account Name field + false + false + Provar Automation Account + + + + + + + Click the Save button + false + false + + + + + + + + + MyConnection + + Dont + On SF Account View screen + false + false + + + + + + + + + + + + + + Provar Automation Account + + + + + + + + AccountNameAssertion + Test + false + + + + + + + + + + +`; + +export const BUNDLED_EXAMPLES: BundledExample[] = [ + { + id: 'bundled-ui-create-validate-account', + name: 'Create and verify an Account via the Salesforce UI', + scenario: 'ui-create-validate', + salesforce_object: 'Account', + keywords: ['ui', 'create', 'new', 'account', 'validate', 'verify', 'assert', 'save', 'record', 'detail'], + xml: UI_CREATE_VALIDATE_ACCOUNT, + }, +]; + +export interface BundledExampleResult { + id: string; + name: string; + xml: string; + similarity_score: number; + salesforce_object: string; + quality_tier: string; + full_content: boolean; + source: 'bundled'; +} + +/** + * Pick up to `n` bundled examples for a query, ranked by keyword overlap. With a small + * bundled set this mostly returns everything, but it keeps the most on-topic example + * first. Returned objects mirror the corpus `CorpusExample` shape plus `source: 'bundled'`. + */ +export function selectBundledExamples(query: string, n?: number): BundledExampleResult[] { + const q = query.toLowerCase(); + // Tolerate a missing/invalid n (callers that bypass the tool's zod default): return all. + const limit = typeof n === 'number' && Number.isFinite(n) && n > 0 ? Math.floor(n) : BUNDLED_EXAMPLES.length; + const scored = BUNDLED_EXAMPLES.map((ex) => { + const hits = ex.keywords.filter((k) => q.includes(k)).length; + return { ex, hits }; + }).sort((a, b) => b.hits - a.hits); + + return scored.slice(0, Math.max(1, limit)).map(({ ex, hits }) => ({ + id: ex.id, + name: ex.name, + xml: ex.xml, + similarity_score: hits > 0 ? Math.min(1, 0.5 + hits * 0.05) : 0, + salesforce_object: ex.salesforce_object, + quality_tier: 'bundled', + full_content: true, + source: 'bundled', + })); +} diff --git a/src/mcp/rules/provar_best_practices_rules.json b/src/mcp/rules/provar_best_practices_rules.json index 80248df..7eac13d 100644 --- a/src/mcp/rules/provar_best_practices_rules.json +++ b/src/mcp/rules/provar_best_practices_rules.json @@ -2624,6 +2624,22 @@ "notes": "Based on corpus analysis of 1451 test files with 329,424 elements. Common AI hallucination: class='null' (never valid). Empty arguments should have no child.", "source": "Corpus analysis: All valid value class patterns from InternalProjects" }, + { + "id": "STEP-REQUIRED-ARGS-001", + "category": "StructureAndGrouping", + "name": "Steps must declare all schema-required arguments", + "description": "Every step type has a set of arguments Provar requires it to declare (even if empty). This rule derives that set for each apiId from provar_test_step_schema.json and flags any step missing one. It generalises the UiAssert-only UI-ASSERT-STRUCT-001 check to all step types (UiConnect, UiWithScreen, UiDoAction, UiFill, DB/Apex/REST steps, etc.), so a test case can no longer score 100 while omitting required arguments on non-UiAssert steps. Runs at major severity because the schema is a bundled reference, not the authoritative Quality Hub ruleset; UiAssert and the NitroX MS connect variants are excluded here as they have dedicated rules.", + "appliesTo": ["Step"], + "severity": "major", + "weight": 5, + "recommendation": "Add the missing argument(s) to the step. Arguments that carry no value should still be declared as empty self-closing tags (e.g. ). See the provar://schema/test-step resource for each step type's required arguments.", + "check": { + "type": "schemaRequiredArguments", + "target": "step" + }, + "notes": "Schema-driven: the required-argument list is read from provar_test_step_schema.json at runtime, so correcting the schema automatically corrects this rule. No per-argument list is hardcoded here.", + "source": "Derived from provar_test_step_schema.json (validated against docs/PROVAR_TEST_STEP_REFERENCE.md)" + }, { "id": "UI-ASSERT-STRUCT-001", "category": "StructureAndGrouping", diff --git a/src/mcp/rules/provar_layer1_rules.json b/src/mcp/rules/provar_layer1_rules.json index 289eb69..3160586 100644 --- a/src/mcp/rules/provar_layer1_rules.json +++ b/src/mcp/rules/provar_layer1_rules.json @@ -122,11 +122,29 @@ "applies_to": "apiCall", "description": "UiDoAction interaction uses class=\"uiInteraction\"." }, + { + "id": "UI-INTERACTION-002", + "severity": "ERROR", + "applies_to": "apiCall", + "description": "UiDoAction interaction name is a real Provar interaction (action/set/file), not a hallucination like click/type." + }, { "id": "UI-ASSERT-STRUCTURE-001", "severity": "ERROR", "applies_to": "apiCall", - "description": "UiAssert uses nested field/column/page assertion containers, not a flat argument." + "description": "UiAssert field assertions use a nested , not a flat argument or a block." + }, + { + "id": "SF-CONNECT-TYPE-001", + "severity": "WARNING", + "applies_to": "testCase", + "description": "A Salesforce UI test connects via ApexConnect (API + UI), not a UiConnect with no ApexConnect." + }, + { + "id": "CONNECT-REF-CONSISTENCY-001", + "severity": "WARNING", + "applies_to": "apiCall", + "description": "Every connection reference (uiConnectionName/apexConnectionName/…) matches a connect step's resultName." }, { "id": "SETVALUES-STRUCTURE-001", diff --git a/src/mcp/rules/provar_test_step_schema.json b/src/mcp/rules/provar_test_step_schema.json index 27761cb..863f3b6 100644 --- a/src/mcp/rules/provar_test_step_schema.json +++ b/src/mcp/rules/provar_test_step_schema.json @@ -1861,7 +1861,12 @@ "default": "Chrome_Headless" } ], - "validation_rules": ["Should be first UI step", "connectionName must be unique"], + "validation_rules": [ + "Browser-only: UiConnect opens a UI session but has NO Salesforce credential of its own — it reuses an existing ApexConnect result. For a Salesforce UI test use ApexConnect (with quickUiLogin=true, uiApplicationName=LightningSales), NOT UiConnect (rule SF-CONNECT-TYPE-001).", + "Add UiConnect only for a SECOND, browser-only session alongside an ApexConnect.", + "connectionName must be unique", + "Downstream steps reference this connection via uiConnectionName = this step's resultName (defaults to connectionName)." + ], "best_practices": [ "Use headless mode for CI/CD", "Reuse connections when possible", @@ -2049,11 +2054,6 @@ "description": "Navigate to and interact with Salesforce screen", "category": "UI", "required_arguments": [ - { - "id": "screenName", - "type": "string", - "description": "Screen name (e.g., 'Account Home', 'Account New')" - }, { "id": "uiConnectionName", "type": "string" @@ -2153,7 +2153,8 @@ "parent_required": "UiWithScreen", "validation_rules": [ "Must be inside UiWithScreen substeps", - "value required for set/select interactions", + "interaction name MUST be one of: action (click a button or link), set (fill a field or pick a picklist value), file (upload a file). NEVER use name=click/type/select/enter — those are not valid Provar interactions and render the IDE Action widget blank (rule UI-INTERACTION-002).", + "value required for set interactions (a literal or a class=variable reference); omit value for action interactions", "Picklist: use ui:interaction?name=set", "Lead Conversion buttons: MUST use name=submitConvert and action=submitConvert (NOT Convert, ConvertButton, or action=convert)", "Dialog fields (e.g., OpportunityName): MUST bind to the TARGET object (object=Opportunity with field=name, NOT object=Lead with field=OpportunityName)" @@ -2192,29 +2193,52 @@ "category": "UI", "required_arguments": [ { - "id": "locator", - "type": "uiLocator" + "id": "fieldAssertions", + "type": "valueList", + "description": "Field-level assertions as namedValues (empty valueList when unused)" }, { - "id": "assertionType", - "type": "string", - "description": "Type of assertion (EqualTo, Contains, etc.)" - } - ], - "optional_arguments": [ + "id": "columnAssertions", + "type": "valueList", + "description": "Column-level assertions (empty valueList when unused)" + }, { - "id": "expectedValue", - "type": "value" + "id": "pageAssertions", + "type": "valueList", + "description": "Page-level assertions (empty valueList when unused)" }, { - "id": "continueOnFailure", + "id": "resultScope", + "type": "string", + "default": "Test" + }, + { + "id": "captureAfter", "type": "boolean", "default": "false" + }, + { + "id": "beforeWait", + "type": "string", + "description": "Include even if empty ()" + }, + { + "id": "autoRetry", + "type": "string", + "description": "Include even if empty ()" + } + ], + "optional_arguments": [ + { + "id": "resultName", + "type": "string" } ], "parent_required": "UiWithScreen", "validation_rules": [ "Must be inside UiWithScreen substeps", + "fieldAssertions MUST nest a containing a BARE (NOT class=\"uiLocator\") and an with a holding the expected . Do NOT use a / block (that is the UiFill/SetValues shape) or flat fieldLocator/attributeName arguments — either renders the IDE Result Assertions tab blank (rule UI-ASSERT-STRUCTURE-001).", + "comparisonType (UI Assert) must be one of: EqualTo, Contains, StartsWith, EndsWith, Matches, None.", "Currency fields: format as $1,000.00", "Phone fields: format as (123) 456-7890", "Date/time: use Contains or EqualTo based on format" @@ -2230,14 +2254,13 @@ "apiId": "com.provar.plugins.forcedotcom.core.ui.UiNavigate", "description": "Navigate to URL", "category": "UI", - "required_arguments": [ + "required_arguments": [], + "optional_arguments": [ { "id": "url", "type": "string", "description": "URL to navigate to" - } - ], - "optional_arguments": [ + }, { "id": "uiConnectionName", "type": "string" @@ -2257,16 +2280,21 @@ "category": "UI", "required_arguments": [ { - "id": "formLocator", + "id": "locator", "type": "uiLocator", - "description": "Form container locator" + "description": "Form/field locator" } ], "optional_arguments": [ { "id": "fieldValues", - "type": "object", - "description": "Field name/value pairs" + "type": "valueList", + "description": "Field name/value pairs as namedValues" + }, + { + "id": "value", + "type": "value", + "description": "Single value when filling one field via locator" } ], "parent_required": "UiWithScreen", @@ -2280,14 +2308,14 @@ "category": "UI", "required_arguments": [ { - "id": "tableLocator", + "id": "locator", "type": "uiLocator", "description": "Table/list locator" }, { - "id": "rowSelector", - "type": "string", - "description": "Row selection criteria" + "id": "uiRowLocator", + "type": "uiLocator", + "description": "Row selection locator" } ], "optional_arguments": [], @@ -2311,9 +2339,9 @@ "category": "UI", "required_arguments": [ { - "id": "action", - "type": "string", - "description": "Action to take (Accept, Dismiss, GetText)" + "id": "alerts", + "type": "valueList", + "description": "Alert handling entries (action per alert: Accept, Dismiss, GetText)" } ], "optional_arguments": [ @@ -2678,8 +2706,8 @@ "category": "UI", "steps": [ { - "step": "UiConnect", - "purpose": "Establish UI connection" + "step": "ApexConnect (quickUiLogin=true, uiApplicationName=LightningSales)", + "purpose": "Establish the Salesforce connection (API + Lightning UI). Use ApexConnect for Salesforce UI tests — NOT UiConnect, which is browser-only and needs an existing ApexConnect result as its credential. Downstream UI steps set uiConnectionName to this step's resultName (which defaults to connectionName)." }, { "step": "UiWithScreen (navigate=Always)", diff --git a/src/mcp/server.ts b/src/mcp/server.ts index 309d3e0..9cbd6f7 100644 --- a/src/mcp/server.ts +++ b/src/mcp/server.ts @@ -35,6 +35,7 @@ import { registerAllNitroXTools } from './tools/nitroXTools.js'; import { registerAllTestCaseStepTools } from './tools/testCaseStepTools.js'; import { registerAllConnectionTools } from './tools/connectionTools.js'; import { registerAllOrgDescribeTools } from './tools/orgDescribeTools.js'; +import { registerAllStepSchemaTools } from './tools/stepSchemaTools.js'; import { registerAllPrompts } from './prompts/index.js'; import { createDepthGuardState, @@ -64,6 +65,7 @@ const TOOL_GROUPS: Record = new Set([UI_ASSERT_API_ID]); + +/** Build apiId → required-argument-id[] from the bundled step schema. Cached after first read. */ +let schemaRequiredArgs: Map | null = null; +function getSchemaRequiredArgs(): Map { + if (schemaRequiredArgs) return schemaRequiredArgs; + const map = new Map(); + try { + const raw = readFileSync(join(dirPath, '..', 'rules', 'provar_test_step_schema.json'), 'utf-8'); + const parsed = JSON.parse(raw) as { apiCalls?: Record }; + const categories = parsed.apiCalls ?? {}; + for (const category of Object.values(categories)) { + if (!category || typeof category !== 'object') continue; + for (const entry of Object.values(category as Record)) { + if (!entry || typeof entry !== 'object') continue; + const e = entry as { apiId?: unknown; required_arguments?: unknown }; + if (typeof e.apiId !== 'string') continue; + const req = Array.isArray(e.required_arguments) ? e.required_arguments : []; + const ids = req + .map((r) => (r && typeof r === 'object' ? (r as { id?: unknown }).id : undefined)) + .filter((id): id is string => typeof id === 'string'); + if (ids.length) map.set(e.apiId, ids); + } + } + } catch { + // Schema missing/corrupt → empty map → rule silently passes (graceful degradation). + } + schemaRequiredArgs = map; + return schemaRequiredArgs; +} + +/** STEP-REQUIRED-ARGS-001 — one violation per step missing a schema-required argument. */ +function validateSchemaRequiredArguments(tc: XmlNode, rule: BPRule): BPViolation[] { + const required = getSchemaRequiredArgs(); + if (required.size === 0) return []; + const violations: BPViolation[] = []; + for (const call of getAllApiCalls(tc)) { + const apiId = call['@_apiId'] as string | undefined; + if (!apiId || REQUIRED_ARGS_EXCLUDED_API_IDS.has(apiId) || apiId.includes('NitroXConnect')) continue; + const req = required.get(apiId); + if (!req || req.length === 0) continue; + const present = new Set(); + for (const a of getCallArguments(call)) { + const id = a['@_id'] as string | undefined; + if (id) present.add(id); + } + const missing = req.filter((r) => !present.has(r)); + if (!missing.length) continue; + const ctx = stepContext(call); + violations.push( + makeViolation( + rule, + `${ctx.apiName} step '${ctx.title}' is missing required argument(s): ${missing.join(', ')}. ` + + `The Provar step schema lists these as required for ${apiId} (testItemId=${ctx.tid})`, + undefined, + ctx.tid + ) + ); + } + return violations; +} + // UI-BINDING-ORDER-001 — binding URIs must list object= before action=/field= (percent-encoded). const BINDING_WRONG_ACTION_FIRST = /object%3Faction%3D[^%]+%26object%3D/; const BINDING_WRONG_FIELD_FIRST = /object%3Ffield%3D[^%]+%26object%3D/; @@ -2428,6 +2504,7 @@ const VALIDATOR_REGISTRY: Record = { type MultiValidatorFn = (tc: XmlNode, rule: BPRule) => BPViolation[]; const MULTI_VALIDATOR_REGISTRY: Record = { + schemaRequiredArguments: validateSchemaRequiredArguments, uiActionNestingStructure: validateUiActionNestingStructure, uiAssertScreenContext: validateUiAssertScreenContext, nitroxConnectInvalidArgs: validateNitroxConnectInvalidArgs, diff --git a/src/mcp/tools/qualityHubApiTools.ts b/src/mcp/tools/qualityHubApiTools.ts index 1fc3d8e..9c75111 100644 --- a/src/mcp/tools/qualityHubApiTools.ts +++ b/src/mcp/tools/qualityHubApiTools.ts @@ -18,21 +18,26 @@ import { QualityHubRateLimitError, REQUEST_ACCESS_URL, } from '../../services/qualityHub/client.js'; +import { selectBundledExamples } from '../examples/bundledExamples.js'; import { desc } from './descHelper.js'; const CORPUS_FALLBACK_HINT = - 'Fallback: read the provar://docs/step-reference MCP resource for step types and attribute formats, then continue.'; + 'Fallback: call the provar_step_schema tool (api_id or category) for step types and their required/optional arguments, then continue.'; + +const BUNDLED_NOTE = + 'These are offline, bundled reference examples (quality_tier="bundled", source="bundled"), NOT corpus-retrieved ' + + 'matches — use them as a structural pattern, adapting object/field/connection names to your project.'; const CORPUS_ONBOARDING_WARNING = - 'Corpus retrieval skipped — no Provar API key configured. Continuing without example grounding.\n' + - 'To enable corpus retrieval: run sf provar auth login\n' + + 'Corpus retrieval skipped — no Provar API key configured. Returning bundled offline examples instead.\n' + + 'To enable full corpus retrieval: run sf provar auth login\n' + `No account? Request access at: ${REQUEST_ACCESS_URL}\n` + - CORPUS_FALLBACK_HINT; + BUNDLED_NOTE; const CORPUS_AUTH_WARNING = - 'Corpus retrieval skipped — API key is invalid or expired. Continuing without example grounding.\n' + + 'Corpus retrieval skipped — API key is invalid or expired. Returning bundled offline examples instead.\n' + `Run sf provar auth login to get a new key, or request access at: ${REQUEST_ACCESS_URL}\n` + - CORPUS_FALLBACK_HINT; + BUNDLED_NOTE; const CORPUS_RATE_LIMIT_WARNING = 'Corpus retrieval skipped — rate limit reached. Continuing without example grounding. Try again shortly.\n' + @@ -56,14 +61,16 @@ export function registerCorpusExamplesRetrieve(server: McpServer): void { 'Use this BEFORE writing any Provar .testcase XML — whether via provar_testcase_generate, Write, or Edit.', 'Pass a user story, requirement, source test file content, or step type keywords as the query.', 'Returns up to N example Provar XML test cases ordered by similarity score.', - 'If retrieval fails (no auth, network error, rate limit), returns empty examples with a warning — the', - 'generation workflow can still continue without grounding. Never hard-errors on API failure.', + 'If no API key is configured (or the key is invalid), returns a small set of offline, known-correct', + 'BUNDLED examples (quality_tier="bundled", source="bundled") so first-test-case generation still has', + 'a structural pattern to follow. On transient failures (rate limit, network) returns empty with a', + 'warning. Never hard-errors on API failure — the generation workflow can always continue.', '', 'For org-specific field metadata: first call getObjectSchema from the Salesforce Hosted MCP', '(platform/sobject-reads — https://api.salesforce.com/platform/mcp/v1/platform/sobject-reads),', 'then include key field names in your query (e.g. "Opportunity: CloseDate, Amount, StageName").', '', - 'Requires a Provar API key (sf provar auth login). Without a key, returns empty examples with onboarding instructions.', + 'A Provar API key (sf provar auth login) unlocks full corpus retrieval; without one you still get bundled offline examples plus onboarding instructions.', ].join('\n'), 'Retrieve similar Provar test case examples from the Quality Hub corpus.' ), @@ -126,12 +133,14 @@ export function registerCorpusExamplesRetrieve(server: McpServer): void { const apiKey = credentialsService.resolveApiKey(); if (!apiKey) { - log('warn', 'provar_qualityhub_examples_retrieve: no api key', { requestId }); + log('warn', 'provar_qualityhub_examples_retrieve: no api key — returning bundled examples', { requestId }); + const examples = selectBundledExamples(query, n); const result = { requestId, - examples: [], - count: 0, + examples, + count: examples.length, query_truncated: false, + source: 'bundled' as const, warning: CORPUS_ONBOARDING_WARNING, }; return { content: [{ type: 'text' as const, text: JSON.stringify(result) }], structuredContent: result }; @@ -160,11 +169,25 @@ export function registerCorpusExamplesRetrieve(server: McpServer): void { const result = { requestId, ...response }; return { content: [{ type: 'text' as const, text: JSON.stringify(result) }], structuredContent: result }; } catch (err: unknown) { - let warning: string; + // On a bad/expired key, bundled examples are still useful grounding. On transient + // failures (rate limit, network) return empty — retrying is the right move, and + // bundled examples would mask a temporary outage as "no matches". if (err instanceof QualityHubAuthError) { - warning = CORPUS_AUTH_WARNING; - log('warn', 'provar_qualityhub_examples_retrieve: auth error', { requestId }); - } else if (err instanceof QualityHubRateLimitError) { + log('warn', 'provar_qualityhub_examples_retrieve: auth error — returning bundled examples', { requestId }); + const examples = selectBundledExamples(query, n); + const result = { + requestId, + examples, + count: examples.length, + query_truncated: false, + source: 'bundled' as const, + warning: CORPUS_AUTH_WARNING, + }; + return { content: [{ type: 'text' as const, text: JSON.stringify(result) }], structuredContent: result }; + } + + let warning: string; + if (err instanceof QualityHubRateLimitError) { warning = CORPUS_RATE_LIMIT_WARNING; log('warn', 'provar_qualityhub_examples_retrieve: rate limited', { requestId }); } else { diff --git a/src/mcp/tools/stepSchemaTools.ts b/src/mcp/tools/stepSchemaTools.ts new file mode 100644 index 0000000..8b5cacb --- /dev/null +++ b/src/mcp/tools/stepSchemaTools.ts @@ -0,0 +1,183 @@ +/* + * Copyright (c) 2024 Provar Limited. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.md file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +/* eslint-disable camelcase */ +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join } from 'node:path'; +import { z } from 'zod'; +import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; +import { makeError, makeRequestId } from '../schemas/common.js'; +import { log } from '../logging/logger.js'; +import { desc } from './descHelper.js'; + +// ── Schema shape (only the fields this tool surfaces) ───────────────────────── + +interface SchemaArgument { + id: string; + type?: string; + default?: string; + description?: string; +} + +interface StepEntry { + apiId: string; + description?: string; + category?: string; + required_arguments?: SchemaArgument[]; + optional_arguments?: SchemaArgument[]; + validation_rules?: string[]; + best_practices?: string[]; +} + +interface StepIndexEntry { + name: string; + apiId: string; + category: string; + required_arguments: string[]; +} + +const dirPath = dirname(fileURLToPath(import.meta.url)); + +// ── Schema loading (lazy, module-level singleton) ───────────────────────────── + +interface LoadedSchema { + /** short name (lower) → entry */ byName: Map; + /** apiId (lower) → entry */ byApiId: Map; + index: StepIndexEntry[]; +} + +let loaded: LoadedSchema | null = null; + +function loadSchema(): LoadedSchema | null { + if (loaded) return loaded; + let parsed: { apiCalls?: Record }; + try { + const raw = readFileSync(join(dirPath, '..', 'rules', 'provar_test_step_schema.json'), 'utf-8'); + parsed = JSON.parse(raw) as { apiCalls?: Record }; + } catch { + return null; + } + const byName = new Map(); + const byApiId = new Map(); + const index: StepIndexEntry[] = []; + for (const [categoryName, category] of Object.entries(parsed.apiCalls ?? {})) { + if (!category || typeof category !== 'object') continue; + for (const [stepName, entry] of Object.entries(category as Record)) { + if (!entry || typeof entry !== 'object') continue; + const e = entry as StepEntry; + if (typeof e.apiId !== 'string') continue; + const reqIds = (e.required_arguments ?? []).map((a) => a.id).filter((id): id is string => typeof id === 'string'); + byName.set(stepName.toLowerCase(), e); + byApiId.set(e.apiId.toLowerCase(), e); + index.push({ name: stepName, apiId: e.apiId, category: e.category ?? categoryName, required_arguments: reqIds }); + } + } + loaded = { byName, byApiId, index }; + return loaded; +} + +/** Resolve an api_id argument (full apiId or short step name, e.g. "UiConnect") to an entry. */ +function resolveStep(schema: LoadedSchema, apiId: string): StepEntry | null { + const key = apiId.trim().toLowerCase(); + return schema.byApiId.get(key) ?? schema.byName.get(key) ?? schema.byName.get(key.split('.').pop() ?? key) ?? null; +} + +// ── Tool: provar_step_schema ────────────────────────────────────────────────── + +function registerStepSchema(server: McpServer): void { + server.registerTool( + 'provar_step_schema', + { + title: 'Provar Step Schema', + description: desc( + [ + 'Return the argument schema for Provar test-case step types from the bundled step reference.', + 'Grounds test-case generation and hand-editing in exact argument names — pass api_id (full apiId like', + '"com.provar.plugins.forcedotcom.core.ui.UiConnect" or the short name "UiConnect") to get that step type\'s', + 'required_arguments, optional_arguments (each with id/type/default), validation_rules, and best_practices.', + 'Pass category (e.g. "UI", "Salesforce", "Control") to list every step type in that category.', + 'Omit both to get an index of all step types (name, apiId, category, required-argument ids).', + 'Read-only and offline — this is the tool form of the provar://schema/test-step resource, so agents that', + 'cannot read MCP resources can still reach the schema. Use it before provar_testcase_generate / _step_edit.', + ].join(' '), + 'Return required/optional arguments for a Provar step type (by api_id), a category, or the full index.' + ), + inputSchema: { + api_id: z + .string() + .optional() + .describe( + desc( + 'Full apiId or short step name (e.g. "UiConnect"). Returns that step type\'s full argument schema.', + 'string, optional; full apiId or short step name' + ) + ), + category: z + .string() + .optional() + .describe( + desc( + 'Category name (UI, Salesforce, Control, Data, Utility, ProvarAI, ProvarLabs). Lists steps in it.', + 'string, optional; category to list' + ) + ), + }, + }, + ({ api_id, category }) => { + const requestId = makeRequestId(); + log('info', 'provar_step_schema', { requestId, api_id, category }); + + const schema = loadSchema(); + if (!schema) { + const err = makeError( + 'STEP_SCHEMA_NOT_FOUND', + 'provar_test_step_schema.json could not be read. Reinstall or upgrade the plugin and try again.', + requestId, + false + ); + return { isError: true, content: [{ type: 'text' as const, text: JSON.stringify(err) }] }; + } + + if (api_id) { + const entry = resolveStep(schema, api_id); + if (!entry) { + const near = schema.index + .filter((s) => s.name.toLowerCase().includes(api_id.trim().toLowerCase())) + .slice(0, 8) + .map((s) => s.name); + const err = makeError( + 'STEP_TYPE_NOT_FOUND', + `No step type matches api_id "${api_id}".`, + requestId, + false, + near.length + ? { suggestion: `Did you mean one of: ${near.join(', ')}? Or omit api_id to list all step types.` } + : { suggestion: 'Omit api_id to get the full index of step types.' } + ); + return { isError: true, content: [{ type: 'text' as const, text: JSON.stringify(err) }] }; + } + const result = { requestId, step: entry }; + return { content: [{ type: 'text' as const, text: JSON.stringify(result) }], structuredContent: result }; + } + + if (category) { + const cat = category.trim().toLowerCase(); + const steps = schema.index.filter((s) => s.category.toLowerCase() === cat); + const result = { requestId, category, count: steps.length, steps }; + return { content: [{ type: 'text' as const, text: JSON.stringify(result) }], structuredContent: result }; + } + + const result = { requestId, count: schema.index.length, steps: schema.index }; + return { content: [{ type: 'text' as const, text: JSON.stringify(result) }], structuredContent: result }; + } + ); +} + +export function registerAllStepSchemaTools(server: McpServer): void { + registerStepSchema(server); +} diff --git a/src/mcp/tools/testCaseGenerate.ts b/src/mcp/tools/testCaseGenerate.ts index 4a7b380..74be923 100644 --- a/src/mcp/tools/testCaseGenerate.ts +++ b/src/mcp/tools/testCaseGenerate.ts @@ -241,7 +241,7 @@ const TOOL_DESCRIPTION = [ 'Cleanup warning: ApexDeleteObject steps near end of test will be skipped if an earlier step fails (stopOnError=false). Use a TearDown callable.', 'Validation: when validate_after_edit=true (default) the response includes a validation field and returns TESTCASE_INVALID if the generated XML fails structural checks.', 'Grounding: call provar_qualityhub_examples_retrieve before generating to get corpus examples for the scenario — correct XML structure for the step types you need.', - 'If the response has count: 0 with a warning field (API unavailable or not configured), fall back: read the provar://docs/step-reference MCP resource for step types and attribute formats, then continue.', + 'If the response has count: 0 with a warning field (API unavailable or not configured), fall back: call the provar_step_schema tool (api_id or category) for step types and their required/optional arguments, then continue.', ].join(' '); export function registerTestCaseGenerate(server: McpServer, config: ServerConfig): void { diff --git a/src/mcp/tools/testCaseStepTools.ts b/src/mcp/tools/testCaseStepTools.ts index bafb4e7..07198bc 100644 --- a/src/mcp/tools/testCaseStepTools.ts +++ b/src/mcp/tools/testCaseStepTools.ts @@ -113,7 +113,7 @@ export function registerTestCaseStepEdit(server: McpServer, config: ServerConfig 'Returns STEP_NOT_FOUND (with all_test_item_ids list) when the target step is absent.', 'Returns INVALID_STEP_XML when step_xml cannot be parsed or contains ≠1 elements.', 'Returns INVALID_XML_AFTER_EDIT (backup restored) when the mutated file fails validation.', - 'Grounding for step_xml: call provar_qualityhub_examples_retrieve for corpus examples of the step type you need; if the response has count: 0 with a warning field, fall back: read the provar://docs/step-reference MCP resource.', + 'Grounding for step_xml: call provar_qualityhub_examples_retrieve for corpus examples of the step type you need; if the response has count: 0 with a warning field, fall back: call the provar_step_schema tool (api_id or category) for the step type\'s required/optional arguments.', ].join(' '), 'AMENDMENT-ONLY: add or remove a single apiCall step in an existing Provar test case (not for constructing new ones).' ), diff --git a/src/mcp/tools/testCaseValidate.ts b/src/mcp/tools/testCaseValidate.ts index c99307c..75ba9e0 100644 --- a/src/mcp/tools/testCaseValidate.ts +++ b/src/mcp/tools/testCaseValidate.ts @@ -165,7 +165,7 @@ export function registerTestCaseValidate(server: McpServer, config: ServerConfig { title: 'Validate Test Case', description: desc( - "Validate a Provar XML test case for structural correctness and quality. Checks XML declaration, root element, required attributes (guid UUID v4, testItemId integer), presence, and applies best-practice rules. When a Provar API key is configured (via sf provar auth login or PROVAR_API_KEY env var), calls the Quality Hub API for full 170-rule scoring. Falls back to local validation if no key is set or the API is unavailable. Returns validity_score (schema compliance), quality_score (best practices, 0–100), and validation_source indicating which ruleset was applied. Returns a tri-state status: 'invalid' (a critical defect — the test will not load in Provar, is_valid=false), 'needs_improvement' (loads but quality_score is below quality_threshold), or 'valid' (loads and clears the bar); meets_quality_threshold and the effective quality_threshold are also returned. Note: a critical best-practice violation (e.g. an unknown apiId) now gates is_valid the same way a structural error does — it surfaces in issues[] as an ERROR. major/minor/info best-practice violations affect quality_score (and the status verdict) only. Every response includes run_id — pass it as baseline_run_id in the next call to receive only new/resolved issues. Data-driven note (DATA-001): when file_path is supplied and the project's provardx-properties.json references the test case directly via top-level `testCase` / `testCases` rather than via a `.testinstance` inside a plan, the validator emits DATA-001 warning a declaration will resolve all variables to null in direct testCase-mode — wire the test into a plan via provar_testplan_add-instance to enable data-driven iteration. When structural errors are returned, consult the provar://docs/step-reference MCP resource for correct step attribute schemas.", + "Validate a Provar XML test case for structural correctness and quality. Checks XML declaration, root element, required attributes (guid UUID v4, testItemId integer), presence, and applies best-practice rules. When a Provar API key is configured (via sf provar auth login or PROVAR_API_KEY env var), calls the Quality Hub API for full 170-rule scoring. Falls back to local validation if no key is set or the API is unavailable. Returns validity_score (schema compliance), quality_score (best practices, 0–100), and validation_source indicating which ruleset was applied. Returns a tri-state status: 'invalid' (a critical defect — the test will not load in Provar, is_valid=false), 'needs_improvement' (loads but quality_score is below quality_threshold), or 'valid' (loads and clears the bar); meets_quality_threshold and the effective quality_threshold are also returned. Note: a critical best-practice violation (e.g. an unknown apiId) now gates is_valid the same way a structural error does — it surfaces in issues[] as an ERROR. major/minor/info best-practice violations affect quality_score (and the status verdict) only. Every response includes run_id — pass it as baseline_run_id in the next call to receive only new/resolved issues. Data-driven note (DATA-001): when file_path is supplied and the project's provardx-properties.json references the test case directly via top-level `testCase` / `testCases` rather than via a `.testinstance` inside a plan, the validator emits DATA-001 warning a declaration will resolve all variables to null in direct testCase-mode — wire the test into a plan via provar_testplan_add-instance to enable data-driven iteration. When structural errors are returned, call the provar_step_schema tool (api_id or category) for correct step attribute schemas.", 'Validate a Provar XML test case: structure, UUIDs, steps, quality scoring; run_id for baseline diff.' ), inputSchema: { @@ -566,7 +566,15 @@ export function validateTestCase( ? ((Array.isArray(rawApiCalls) ? rawApiCalls : [rawApiCalls]) as Array>) : []; - for (const call of apiCalls) { + // Validate EVERY apiCall in the tree, not just top-level ones. UI action and + // assertion steps (UiDoAction, UiFill, UiAssert, …) live nested inside a + // UiWithScreen ``, so a top-level-only walk silently + // skipped every real UI step — the per-call structural rules (UI-INTERACTION-*, + // UI-ASSERT-STRUCTURE-001, UI-LOCATOR-001) never fired on them. Collect + // recursively so those rules apply wherever the step actually sits. + const allApiCalls: Array> = []; + collectNodesByKey(tc, 'apiCall', allApiCalls); + for (const call of allApiCalls) { validateApiCall(call, issues); } @@ -576,6 +584,11 @@ export function validateTestCase( // covered too, not just top-level apiCalls. validateComparisonTypes(tc, issues); + // SF-CONNECT-TYPE-001 / CONNECT-REF-CONSISTENCY-001: connection-topology checks + // that need the whole test (all connect steps + all downstream references). + validateSalesforceConnectType(tc, issues); + validateConnectionReferenceConsistency(tc, issues); + // VAR-REF-001 / VAR-REF-002: detect {VarName} tokens inside valueClass="string" elements. // Provar does not interpolate {…} tokens in plain string values at runtime — they must use // class="variable" (pure reference) or class="compound" (embedded in surrounding text). @@ -739,9 +752,44 @@ function checkUiInteraction(call: Record, stepName: string, iss 'Emit the interaction as: . ' + 'In provar_testcase_generate the "interaction" attribute is converted automatically.', }); + return; + } + // UI-INTERACTION-002: the interaction NAME must be a real Provar interaction. + // The valid UiDoAction interactions are `action` (click a button/link), `set` + // (fill a field or pick a picklist value) and `file` (upload). Common LLM + // hallucinations like `click`/`type`/`select` render the IDE Action widget + // blank and do not run. Denylist the known-wrong names (mapping each to the + // correct one) rather than allow-listing, so a valid-but-rare name never + // false-fires. + const uri = (interactionNode['@_uri'] as string | undefined) ?? ''; + const nameMatch = /[?&]name=([^&"]+)/.exec(uri); + const interactionName = nameMatch ? nameMatch[1] : ''; + const correction = INVALID_INTERACTION_NAMES[interactionName.toLowerCase()]; + if (correction) { + issues.push({ + rule_id: 'UI-INTERACTION-002', + severity: 'ERROR', + message: `"${stepName}" uses interaction name="${interactionName}", which is not a valid Provar UI interaction — use name="${correction}" instead.`, + applies_to: 'apiCall', + suggestion: + 'Valid UiDoAction interactions are: name=action (click a button/link), name=set (fill a field or select a ' + + `picklist value), name=file (upload). Emit .`, + }); } } +// Known-wrong interaction names → the correct Provar interaction they map to. +const INVALID_INTERACTION_NAMES: Record = { + click: 'action', + tap: 'action', + press: 'action', + type: 'set', + fill: 'set', + enter: 'set', + input: 'set', + select: 'set', +}; + // Flat-form argument ids that the broken/CLI-only UiAssert shape carries at the // top level. The IDE-renderable contract nests these inside a uiFieldAssertion. const UI_ASSERT_FLAT_ARG_IDS = ['fieldLocator', 'attributeName', 'comparisonType', 'expectedValue']; @@ -755,9 +803,32 @@ const UI_ASSERT_FLAT_ARG_IDS = ['fieldLocator', 'attributeName', 'comparisonType // (Named "-STRUCTURE-001" to match the local SETVALUES-STRUCTURE-001 convention // and avoid colliding with the best-practice JSON rule UI-ASSERT-STRUCT-001.) function checkUiAssertStructure(call: Record, stepName: string, issues: ValidationIssue[]): void { - const flatArg = getArgList(call).find((a) => - UI_ASSERT_FLAT_ARG_IDS.includes((a['@_id'] as string | undefined) ?? '') - ); + const args = getArgList(call); + + // Shape B: fieldAssertions present but filled with / + // instead of the required . This is the shape an LLM reaches + // for by analogy with UiFill/SetValues; it renders the IDE Result Assertions + // tab blank exactly like the flat form. Detected here because the flat-arg + // check below only covers top-level fieldLocator-style arguments. + const fieldAssertionsArg = args.find((a) => (a['@_id'] as string | undefined) === 'fieldAssertions'); + const faValue = fieldAssertionsArg?.['value'] as Record | undefined; + if (faValue && typeof faValue === 'object' && faValue['namedValues'] != null && faValue['uiFieldAssertion'] == null) { + issues.push({ + rule_id: 'UI-ASSERT-STRUCTURE-001', + severity: 'ERROR', + message: `UiAssert step "${stepName}" fieldAssertions uses — field assertions must be nested inside a (with a bare ), or the Provar IDE Result Assertions tab renders blank.`, + applies_to: 'apiCall', + suggestion: + 'Replace the namedValues block with: ' + + '' + + '' + + '. In provar_testcase_generate pass fieldLocator/attributeName/' + + 'comparisonType/expectedValue as flat attributes — the generator builds this structure automatically.', + }); + return; + } + + const flatArg = args.find((a) => UI_ASSERT_FLAT_ARG_IDS.includes((a['@_id'] as string | undefined) ?? '')); if (!flatArg) return; const offendingId = (flatArg['@_id'] as string | undefined) ?? ''; issues.push({ @@ -917,6 +988,105 @@ function literalValueText(valueNode: Record | undefined): strin * ...ComparisonType.`), so this is emitted at ERROR tier — not as a * best-practices quality warning — to mirror the runtime failure offline. */ +/** Read an argument's literal string value; returns undefined for missing/variable/non-literal values. */ +function literalArgValue(call: Record, id: string): string | undefined { + const arg = getArgList(call).find((a) => (a['@_id'] as string | undefined) === id); + const v = arg?.['value'] as Record | undefined; + if (!v || typeof v !== 'object' || v['@_class'] !== 'value') return undefined; + const text = v['#text']; + if (typeof text === 'string') return text; + if (typeof text === 'number' || typeof text === 'boolean') return String(text); + return undefined; +} + +const CONNECT_REFERENCE_ARG_IDS = ['uiConnectionName', 'apexConnectionName', 'dbConnectionName', 'webConnectionName']; + +/** True when this apiId opens a connection (its result is what later steps reference). */ +function isConnectStep(apiId: string): boolean { + return /Connect(?::|$)/.test(apiId); +} + +// SF-CONNECT-TYPE-001: a Salesforce UI test authenticates through ApexConnect +// (which opens an API connection AND, via quickUiLogin, the Lightning UI). +// UiConnect opens a browser only and needs an existing ApexConnect result as its +// Salesforce credential — so a test that drives Salesforce UI (sf:ui:target) but +// has UiConnect and NO ApexConnect has no Salesforce credential. WARNING, not +// ERROR: UiConnect is syntactically valid and the connection could, in rare +// setups, be supplied by a parent callable test. +function validateSalesforceConnectType(tc: Record, issues: ValidationIssue[]): void { + const calls: Array> = []; + collectNodesByKey(tc, 'apiCall', calls); + const apiIds = calls.map((c) => (c['@_apiId'] as string | undefined) ?? ''); + const hasUiConnect = apiIds.some((a) => a.endsWith('.UiConnect')); + const hasApexConnect = apiIds.some((a) => a.endsWith('.ApexConnect')); + if (!hasUiConnect || hasApexConnect) return; + const hasSalesforceUi = calls.some((c) => + getArgList(c).some((a) => { + const v = a['value'] as Record | undefined; + const uri = (v?.['@_uri'] as string | undefined) ?? ''; + return uri.startsWith('sf:ui:'); + }) + ); + if (!hasSalesforceUi) return; + issues.push({ + rule_id: 'SF-CONNECT-TYPE-001', + severity: 'WARNING', + message: + 'This test drives the Salesforce UI (sf:ui: targets) but connects with UiConnect and no ApexConnect. ' + + 'UiConnect opens a browser only and has no Salesforce credential of its own.', + applies_to: 'testCase', + suggestion: + 'Use ApexConnect as the Salesforce connection step (it opens the API connection and, with quickUiLogin=true ' + + 'and uiApplicationName=LightningSales, the Lightning UI too). Add UiConnect only when you need a second, ' + + 'browser-only session that reuses the ApexConnect result.', + }); +} + +// CONNECT-REF-CONSISTENCY-001: every literal connection reference +// (uiConnectionName / apexConnectionName / …) must match the resultName produced +// by a connect step in this test (a connect step's result defaults to its +// connectionName when resultName is omitted). A reference to a name no connect +// step produced is a dangling connection. WARNING, since the connection could be +// established by a parent callable test. +function validateConnectionReferenceConsistency(tc: Record, issues: ValidationIssue[]): void { + const calls: Array> = []; + collectNodesByKey(tc, 'apiCall', calls); + + const available = new Set(); + let connectStepCount = 0; + for (const c of calls) { + const apiId = (c['@_apiId'] as string | undefined) ?? ''; + if (!isConnectStep(apiId)) continue; + connectStepCount++; + const resultName = literalArgValue(c, 'resultName'); + const connectionName = literalArgValue(c, 'connectionName'); + if (resultName) available.add(resultName); + else if (connectionName) available.add(connectionName); + } + if (connectStepCount === 0) return; // connections may be inherited from a parent test + + for (const c of calls) { + const apiId = (c['@_apiId'] as string | undefined) ?? ''; + if (isConnectStep(apiId)) continue; + for (const refId of CONNECT_REFERENCE_ARG_IDS) { + const ref = literalArgValue(c, refId); + if (ref && !available.has(ref)) { + const stepName = (c['@_name'] as string | undefined) ?? '(unnamed)'; + issues.push({ + rule_id: 'CONNECT-REF-CONSISTENCY-001', + severity: 'WARNING', + message: `Step "${stepName}" references ${refId}="${ref}", but no connect step in this test produces that connection result (available: ${ + [...available].map((n) => `"${n}"`).join(', ') || 'none' + }).`, + applies_to: 'apiCall', + suggestion: `Set the connect step's resultName to "${ref}", or change ${refId} to match a connect step's resultName. Downstream steps reference the connect step's resultName — which defaults to the connectionName when resultName is omitted.`, + }); + break; // one violation per step is enough + } + } + } +} + function validateComparisonTypes(tc: Record, issues: ValidationIssue[]): void { // AssertValues steps → the 16-value AssertValues subset. const apiCalls: Array> = []; diff --git a/test/fixtures/account-ui-gaps-bad.testcase b/test/fixtures/account-ui-gaps-bad.testcase new file mode 100644 index 0000000..187cd85 --- /dev/null +++ b/test/fixtures/account-ui-gaps-bad.testcase @@ -0,0 +1,204 @@ + + + Creates an Account through the Salesforce Lightning UI with the mandatory Name field, saves it, and validates the created record. This fixture reproduces the four generation gaps (wrong connect type, dangling connection reference, invalid interaction name, flat field assertions). + + + + + AdminOauth + + + UiConnection + + + Test + + + + fc530f8d-1a88-4ac9-91ce-4686f43e42da + + + + + + + + + AdminOauth + + + + + + Always + + + On SF Account Home screen + + + Default + + + Default + + + false + + + false + + + + + + + + + + + + + + + + Click the New button + + + false + + + false + + + + + + + + + + + + + + AdminOauth + + + + + + Dont + + + NewAccountId + + + On SF Account New screen + + + false + + + false + + + + + + + + + + + + + + + Provar MCP Claude Account + + + + + + + + + + + + + + + + Click Save + + + false + + + false + + + + + + + + + + + AdminOauth + + + + + + Dont + + + On SF Account View screen + + + false + + + false + + + + + + + + + + + + Provar MCP Claude Account + + + + + + + + + + + + Test + + + false + + + + + + + + + + + + diff --git a/test/fixtures/account-ui-gaps-good.testcase b/test/fixtures/account-ui-gaps-good.testcase new file mode 100644 index 0000000..36bca33 --- /dev/null +++ b/test/fixtures/account-ui-gaps-good.testcase @@ -0,0 +1,242 @@ + + + Creates an Account through the Salesforce Lightning UI, saves it, and validates the created record. Corrected shape: ApexConnect, consistent connection references, valid interaction names, nested field assertions. + + + + + AdminOauth + + + fc530f8d-1a88-4ac9-91ce-4686f43e42da + + + AdminOauth + + + Test + + + LightningSales + + + true + + + true + + + Fail + + + + + + + + + + + + + AccountName + + + TestAccount + + + Test + + + + + + + + + + AdminOauth + + + + + + Always + + + On SF Account Home screen + + + false + + + false + + + + + + + + + + + + + + + Click the New button + + + false + + + false + + + + + + + + + + + AdminOauth + + + + + + Dont + + + NewAccountId + + + false + + + false + + + + + + + + + + + + + + + Set the Account Name field + + + false + + + false + + + + + + + + + + + + + + + + + + Click the Save button + + + false + + + false + + + + + + + + + + + AdminOauth + + + + + + Dont + + + On SF Account View screen + + + false + + + false + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + AccountNameAssertion + + + Test + + + false + + + + + + + + + + + diff --git a/test/unit/mcp/bestPracticesEngine.test.ts b/test/unit/mcp/bestPracticesEngine.test.ts index 67df39a..0b97039 100644 --- a/test/unit/mcp/bestPracticesEngine.test.ts +++ b/test/unit/mcp/bestPracticesEngine.test.ts @@ -177,16 +177,14 @@ describe('runBestPractices', () => { it('passes for a valid SF target URI (sf:ui:target?object=Account&action=view)', () => { const result = runBestPractices(buildUwsXml('sf:ui:target?object=Account&action=view')); - const uwsViolation = result.violations.find( - (v) => v.rule_id.includes('UI-SCREEN') || v.message.includes('UiWithScreen') - ); + const uwsViolation = result.violations.find((v) => v.rule_id.includes('UI-SCREEN-TARGET')); assert.ok(!uwsViolation, `Expected no uiWithScreenTarget violation, got: ${uwsViolation?.message}`); }); it('passes for a valid page object target URI (ui:pageobject:target?pageId=pageobjects.LoginPage)', () => { const result = runBestPractices(buildUwsXml('ui:pageobject:target?pageId=pageobjects.LoginPage')); const uwsViolation = result.violations.find( - (v) => v.message.includes('UiWithScreen') || v.message.includes('pageId') + (v) => v.rule_id.includes('UI-SCREEN-TARGET') || v.message.includes('pageId') ); assert.ok(!uwsViolation, `Expected no uiWithScreenTarget violation, got: ${uwsViolation?.message}`); }); @@ -2027,3 +2025,106 @@ describe('reference doc ⇄ validator apiId parity (PDX-512)', () => { ); }); }); + +// ── STEP-REQUIRED-ARGS-001 (schema-driven required arguments, all step types) ── + +describe('STEP-REQUIRED-ARGS-001 schema-driven required arguments', () => { + const TC = '550e8400-e29b-41d4-a716-4466554409a0'; + const G = (n: number): string => `550e8400-e29b-41d4-a716-4466554409${String(n).padStart(2, '0')}`; + + const stepArgsViolations = (xml: string): BPViolation[] => + runBestPractices(xml).violations.filter((v) => v.rule_id === 'STEP-REQUIRED-ARGS-001'); + + it('fires when UiConnect uses `connection` instead of the required `connectionName` (the Appendix A bug)', () => { + const xml = ` + + + + AdminOauth + + +`; + const vs = stepArgsViolations(xml); + assert.equal(vs.length, 1, 'UiConnect missing connectionName should raise exactly one violation'); + assert.ok(vs[0].message.includes('connectionName'), `should name the missing arg: ${vs[0].message}`); + assert.equal(vs[0].severity, 'major', 'rule must be major (does not gate is_valid)'); + }); + + it('fires per-step: UiWithScreen missing uiConnectionName AND UiDoAction missing interaction', () => { + const xml = ` + + + + + + + + + + + +`; + const vs = stepArgsViolations(xml); + assert.equal(vs.length, 2, 'both incomplete steps should be flagged (one violation each)'); + assert.ok(vs.some((v) => v.message.includes('uiConnectionName')), 'UiWithScreen missing uiConnectionName'); + assert.ok(vs.some((v) => v.message.includes('interaction')), 'UiDoAction missing interaction'); + }); + + it('does NOT fire when every required argument is present (no false positive on a correct file)', () => { + const xml = ` + + + + AdminOauth + + + + UiConnection + + + + + + + + + +`; + assert.deepEqual(stepArgsViolations(xml), [], 'complete steps must not be flagged'); + }); + + it('does NOT resurrect the removed phantom args (screenName on UiWithScreen, formLocator on UiFill)', () => { + // A UiWithScreen with target+uiConnectionName but NO screenName, and a UiFill with locator (not formLocator), + // are correct Provar steps — the pre-fix schema would have flagged both. Regression guard. + const xml = ` + + + + + UiConnection + + + + + + + + + +`; + const vs = stepArgsViolations(xml); + assert.equal(vs.length, 0, `must not flag screenName/formLocator: ${vs.map((v) => v.message).join(' | ')}`); + }); + + it('excludes UiAssert (covered by the dedicated UI-ASSERT-STRUCT-001 rule)', () => { + const xml = ` + + + + + + +`; + assert.deepEqual(stepArgsViolations(xml), [], 'UiAssert must not be double-reported by STEP-REQUIRED-ARGS-001'); + }); +}); diff --git a/test/unit/mcp/bundledExamples.test.ts b/test/unit/mcp/bundledExamples.test.ts new file mode 100644 index 0000000..1aa8c0f --- /dev/null +++ b/test/unit/mcp/bundledExamples.test.ts @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2024 Provar Limited. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.md file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +/* eslint-disable camelcase */ +import { strict as assert } from 'node:assert'; +import { describe, it } from 'mocha'; +import { BUNDLED_EXAMPLES, selectBundledExamples } from '../../../src/mcp/examples/bundledExamples.js'; +import { runBestPractices } from '../../../src/mcp/tools/bestPracticesEngine.js'; +import { validateTestCase } from '../../../src/mcp/tools/testCaseValidate.js'; + +// Every bundled example MUST pass the same validators the server exposes — otherwise +// we would be shipping the exact "looks right but is wrong" XML the fix set out to prevent. + +describe('bundled examples', () => { + it('ships at least one example', () => { + assert.ok(BUNDLED_EXAMPLES.length >= 1); + }); + + for (const ex of BUNDLED_EXAMPLES) { + describe(`example: ${ex.id}`, () => { + it('is structurally valid (validity_score 100, is_valid)', () => { + const meta = validateTestCase(ex.xml, undefined, {}); + assert.equal(meta.is_valid, true, `structural errors: ${JSON.stringify(meta.issues ?? [])}`); + assert.equal(meta.validity_score, 100); + }); + + it('scores 100 with zero best-practice violations', () => { + const bp = runBestPractices(ex.xml); + assert.equal( + bp.violations.length, + 0, + `violations: ${bp.violations.map((v) => `${v.rule_id}: ${v.message}`).join(' | ')}` + ); + assert.equal(bp.quality_score, 100); + }); + }); + } + + it('selectBundledExamples returns up to n, labelled bundled, with full XML', () => { + const picked = selectBundledExamples('create and validate an account via ui', 5); + assert.ok(picked.length >= 1 && picked.length <= 5); + assert.ok(picked.every((p) => p.source === 'bundled' && p.quality_tier === 'bundled')); + assert.ok(picked[0].xml.includes(' { + const onTopic = selectBundledExamples('ui account create validate', 1)[0]; + const offTopic = selectBundledExamples('zzz nothing matches here', 1)[0]; + assert.ok(onTopic.similarity_score > offTopic.similarity_score); + }); +}); diff --git a/test/unit/mcp/qualityHubApiTools.test.ts b/test/unit/mcp/qualityHubApiTools.test.ts index 0f453c0..575b9ba 100644 --- a/test/unit/mcp/qualityHubApiTools.test.ts +++ b/test/unit/mcp/qualityHubApiTools.test.ts @@ -121,30 +121,41 @@ describe('qualityHubApiTools', () => { assert.equal(opts.prefer_high_quality, false); }); - it('returns empty examples with no isError when API key is not configured', async () => { + it('returns BUNDLED examples (not empty) with no isError when API key is not configured', async () => { resolveKeyStub.returns(null); - const result = await server.call('provar_qualityhub_examples_retrieve', { query: 'Create an Opportunity' }); + const result = await server.call('provar_qualityhub_examples_retrieve', { + query: 'Create and validate an Account via UI', + }); // CRITICAL: must NOT be isError:true — the LLM workflow must continue assert.equal(isError(result), false, 'Must not set isError:true when key missing'); const body = parseBody(result); - assert.deepEqual(body.examples, []); - assert.equal(body.count, 0); + const examples = body.examples as Array<{ source?: string; quality_tier?: string; xml?: string }>; + assert.ok(examples.length > 0, 'Should return bundled examples instead of empty'); + assert.equal(body.count, examples.length); + assert.equal(body.source, 'bundled'); + assert.ok( + examples.every((e) => e.source === 'bundled' && e.quality_tier === 'bundled'), + 'Every example must be labelled bundled' + ); + assert.ok(examples[0].xml && examples[0].xml.includes(' 0, 'Should include warning message'); assert.ok(warning.includes('sf provar auth login'), 'Warning should mention auth login'); + assert.ok(warning.toLowerCase().includes('bundled'), 'Warning should flag examples as bundled'); }); - it('returns empty examples with no isError on 401 auth error', async () => { + it('returns BUNDLED examples with no isError on 401 auth error', async () => { retrieveStub.rejects(new QualityHubAuthError('Key invalid')); - const result = await server.call('provar_qualityhub_examples_retrieve', { query: 'Create an Opportunity' }); + const result = await server.call('provar_qualityhub_examples_retrieve', { query: 'Create an Account via UI' }); // CRITICAL: must NOT be isError:true — graceful degrade assert.equal(isError(result), false, 'Must not set isError:true on auth failure'); const body = parseBody(result); - assert.deepEqual(body.examples, []); + const examples = body.examples as Array<{ source?: string }>; + assert.ok(examples.length > 0, 'Bad key should still yield bundled examples'); + assert.equal(body.source, 'bundled'); assert.ok(typeof body.warning === 'string', 'Should include warning'); }); diff --git a/test/unit/mcp/stepSchemaTools.test.ts b/test/unit/mcp/stepSchemaTools.test.ts new file mode 100644 index 0000000..846c563 --- /dev/null +++ b/test/unit/mcp/stepSchemaTools.test.ts @@ -0,0 +1,144 @@ +/* + * Copyright (c) 2024 Provar Limited. + * All rights reserved. + * Licensed under the BSD 3-Clause license. + * For full license text, see LICENSE.md file in the repo root or https://opensource.org/licenses/BSD-3-Clause + */ + +/* eslint-disable camelcase */ +import { strict as assert } from 'node:assert'; +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { describe, it } from 'mocha'; +import { registerAllStepSchemaTools } from '../../../src/mcp/tools/stepSchemaTools.js'; + +// ── Minimal McpServer mock ──────────────────────────────────────────────────── + +type ToolHandler = (args: Record) => unknown; + +class MockMcpServer { + private handlers = new Map(); + public registerTool(name: string, _config: unknown, handler: ToolHandler): void { + this.handlers.set(name, handler); + } + public call(name: string, args: Record): ReturnType { + const h = this.handlers.get(name); + if (!h) throw new Error(`Tool not registered: ${name}`); + return h(args); + } +} + +function parseText(result: unknown): Record { + const r = result as { content: Array<{ type: string; text: string }> }; + return JSON.parse(r.content[0].text) as Record; +} +function isError(result: unknown): boolean { + return (result as { isError?: boolean }).isError === true; +} + +interface StepArg { + id: string; +} +interface StepEntry { + apiId: string; + required_arguments?: StepArg[]; + optional_arguments?: StepArg[]; +} + +function makeServer(): MockMcpServer { + const server = new MockMcpServer(); + registerAllStepSchemaTools(server as unknown as Parameters[0]); + return server; +} + +describe('provar_step_schema', () => { + it('returns the schema for a step type by short name (UiConnect)', () => { + const res = makeServer().call('provar_step_schema', { api_id: 'UiConnect' }); + assert.ok(!isError(res)); + const step = parseText(res).step as StepEntry; + assert.equal(step.apiId, 'com.provar.plugins.forcedotcom.core.ui.UiConnect'); + const req = (step.required_arguments ?? []).map((a) => a.id); + assert.deepEqual(req, ['connectionName'], 'UiConnect requires connectionName'); + }); + + it('resolves by full apiId as well as short name', () => { + const res = makeServer().call('provar_step_schema', { + api_id: 'com.provar.plugins.forcedotcom.core.ui.UiDoAction', + }); + const step = parseText(res).step as StepEntry; + const req = (step.required_arguments ?? []).map((a) => a.id); + assert.deepEqual(req, ['locator', 'interaction']); + }); + + it('reflects the corrected schema — no phantom screenName/formLocator', () => { + const server = makeServer(); + const uws = (parseText(server.call('provar_step_schema', { api_id: 'UiWithScreen' })).step as StepEntry) + .required_arguments!.map((a) => a.id); + assert.ok(!uws.includes('screenName'), `UiWithScreen must not require phantom screenName: ${uws.join(',')}`); + assert.deepEqual(uws, ['uiConnectionName', 'target']); + + const fill = (parseText(server.call('provar_step_schema', { api_id: 'UiFill' })).step as StepEntry) + .required_arguments!.map((a) => a.id); + assert.ok(!fill.includes('formLocator'), `UiFill must not require phantom formLocator: ${fill.join(',')}`); + assert.deepEqual(fill, ['locator']); + }); + + it('lists all step types in a category', () => { + const res = makeServer().call('provar_step_schema', { category: 'UI' }); + const body = parseText(res); + assert.ok((body.count as number) > 0); + const steps = body.steps as Array<{ apiId: string }>; + assert.ok(steps.every((s) => s.apiId.length > 0)); + assert.ok(steps.some((s) => s.apiId.endsWith('.UiConnect'))); + }); + + it('returns the full index when no arguments are given', () => { + const body = parseText(makeServer().call('provar_step_schema', {})); + assert.ok((body.count as number) >= 60, 'index should cover the full step catalog'); + const steps = body.steps as Array<{ name: string; apiId: string; category: string }>; + assert.ok(steps.every((s) => s.name && s.apiId && s.category)); + }); + + it('returns STEP_TYPE_NOT_FOUND with a suggestion for an unknown api_id', () => { + const res = makeServer().call('provar_step_schema', { api_id: 'UiTotallyMadeUp' }); + assert.ok(isError(res)); + const err = parseText(res); + assert.equal(err.error_code, 'STEP_TYPE_NOT_FOUND'); + assert.ok(err.details, 'should include a suggestion'); + }); +}); + +// Drift guard: the schema is the single source of truth for generation guidance. +// These assertions lock the corrections that steer generation away from the four +// gaps, so the guidance can never silently regress (e.g. back to recommending +// UiConnect or interaction=click). +describe('step schema generation guidance (drift guard)', () => { + const schema = JSON.parse( + readFileSync(join(process.cwd(), 'src', 'mcp', 'rules', 'provar_test_step_schema.json'), 'utf-8') + ) as { + apiCalls: { UI: Record }; + common_patterns: { ui_interaction_pattern: { steps: Array<{ step: string }> } }; + }; + const rulesText = (step: string): string => (schema.apiCalls.UI[step].validation_rules ?? []).join(' | '); + + it('ui_interaction_pattern opens with ApexConnect, not UiConnect (gap 1)', () => { + const first = schema.common_patterns.ui_interaction_pattern.steps[0].step; + assert.ok(first.includes('ApexConnect'), `pattern should open with ApexConnect, got: ${first}`); + assert.ok(!first.startsWith('UiConnect'), 'pattern must not open with UiConnect'); + }); + + it('UiConnect guidance steers Salesforce UI tests to ApexConnect (gap 1)', () => { + const t = rulesText('UiConnect'); + assert.ok(t.includes('ApexConnect') && /browser-only/i.test(t), `UiConnect guidance missing steer: ${t}`); + }); + + it('UiDoAction guidance lists valid interactions and forbids click (gap 3)', () => { + const t = rulesText('UiDoAction'); + assert.ok(t.includes('name=click') && t.includes('action') && t.includes('set'), `interaction vocab missing: ${t}`); + }); + + it('UiAssert guidance requires nested uiFieldAssertion and warns off namedValues (gap 4)', () => { + const t = rulesText('UiAssert'); + assert.ok(t.includes('uiFieldAssertion') && t.includes('namedValues'), `assert nesting guidance missing: ${t}`); + }); +}); diff --git a/test/unit/mcp/testCaseGenerate.test.ts b/test/unit/mcp/testCaseGenerate.test.ts index 2616b99..edb56bf 100644 --- a/test/unit/mcp/testCaseGenerate.test.ts +++ b/test/unit/mcp/testCaseGenerate.test.ts @@ -86,7 +86,7 @@ afterEach(() => { // ── tool description ────────────────────────────────────────────────────────── describe('provar_testcase_generate description', () => { - it('references corpus tool and step-reference fallback', () => { + it('references corpus tool and step-schema fallback', () => { const reg = server.registrations.find((r) => r.name === 'provar_testcase_generate'); assert.ok(reg, 'tool should be registered'); assert.ok( @@ -94,8 +94,8 @@ describe('provar_testcase_generate description', () => { 'description should reference corpus tool' ); assert.ok( - reg.description.includes('provar://docs/step-reference'), - 'description should include step-reference fallback' + reg.description.includes('provar_step_schema'), + 'description should include the provar_step_schema fallback' ); }); diff --git a/test/unit/mcp/testCaseStepTools.test.ts b/test/unit/mcp/testCaseStepTools.test.ts index c353d26..fc3a1bf 100644 --- a/test/unit/mcp/testCaseStepTools.test.ts +++ b/test/unit/mcp/testCaseStepTools.test.ts @@ -94,7 +94,7 @@ afterEach(() => { // ── tool description ────────────────────────────────────────────────────────── describe('provar_testcase_step_edit description', () => { - it('references corpus tool and step-reference fallback', () => { + it('references corpus tool and step-schema fallback', () => { const reg = server.registrations.find((r) => r.name === 'provar_testcase_step_edit'); assert.ok(reg, 'tool should be registered'); assert.ok( @@ -102,8 +102,8 @@ describe('provar_testcase_step_edit description', () => { 'description should reference corpus tool' ); assert.ok( - reg.description.includes('provar://docs/step-reference'), - 'description should include step-reference fallback' + reg.description.includes('provar_step_schema'), + 'description should include the provar_step_schema fallback' ); }); diff --git a/test/unit/mcp/testCaseValidate.test.ts b/test/unit/mcp/testCaseValidate.test.ts index c3ca6d6..5cf002e 100644 --- a/test/unit/mcp/testCaseValidate.test.ts +++ b/test/unit/mcp/testCaseValidate.test.ts @@ -955,6 +955,120 @@ describe('validateTestCase', () => { 'UI-ASSERT-STRUCTURE-001 must only apply to UiAssert steps' ); }); + + // Recursion coverage: UI steps live nested under UiWithScreen substeps. The + // per-call structural rules must run there too, not just at the top level. + it('fires for a NESTED UiAssert with the flat namedValues shape', () => { + const r = validateTestCase( + ` + + + + + + + + X + + + + + +` + ); + assert.ok( + r.issues.some((i) => i.rule_id === 'UI-ASSERT-STRUCTURE-001'), + 'nested UiAssert must be reached by the structural rule' + ); + }); + }); + + // Phase-2 generation-gap rules, dogfooded against the account-ui fixtures. + describe('generation-gap rules (account-ui fixtures)', () => { + const bad = fs.readFileSync(path.resolve(process.cwd(), 'test', 'fixtures', 'account-ui-gaps-bad.testcase'), 'utf-8'); + const good = fs.readFileSync(path.resolve(process.cwd(), 'test', 'fixtures', 'account-ui-gaps-good.testcase'), 'utf-8'); + const GAP_RULES = ['SF-CONNECT-TYPE-001', 'CONNECT-REF-CONSISTENCY-001', 'UI-INTERACTION-002', 'UI-ASSERT-STRUCTURE-001']; + + it('bad fixture: all four gap rules fire and is_valid is false', () => { + const r = validateTestCase(bad); + for (const rule of GAP_RULES) { + assert.ok(r.issues.some((i) => i.rule_id === rule), `expected ${rule} to fire on the bad fixture`); + } + assert.equal(r.is_valid, false, 'ERROR-tier gap rules must gate is_valid'); + }); + + it('good fixture: none of the gap rules fire and is_valid is true', () => { + const r = validateTestCase(good); + const fired = r.issues.filter((i) => GAP_RULES.includes(i.rule_id)).map((i) => i.rule_id); + assert.deepEqual([...new Set(fired)], [], `no gap rule should fire on the good fixture, got: ${fired.join(', ')}`); + assert.equal(r.is_valid, true); + }); + }); + + describe('UI-INTERACTION-002 (invalid interaction name)', () => { + it('errors on interaction name="click" and suggests "action"', () => { + const r = validateTestCase( + ` + + + + + + + + + +` + ); + const issue = r.issues.find((i) => i.rule_id === 'UI-INTERACTION-002'); + assert.ok(issue, 'expected UI-INTERACTION-002'); + assert.equal(issue.severity, 'ERROR'); + assert.ok(issue.message.includes('action'), 'should steer to name="action"'); + }); + + it('does not fire for valid interaction names (action, set)', () => { + for (const name of ['action', 'set']) { + const r = validateTestCase( + ` + + + + + + + + + +` + ); + assert.ok( + !r.issues.some((i) => i.rule_id === 'UI-INTERACTION-002'), + `UI-INTERACTION-002 must not fire for valid name="${name}"` + ); + } + }); + }); + + describe('CONNECT-REF-CONSISTENCY-001', () => { + it('does not fire when no connect step exists (connection inherited from parent)', () => { + const r = validateTestCase( + ` + + + + + Inherited + + + + +` + ); + assert.ok( + !r.issues.some((i) => i.rule_id === 'CONNECT-REF-CONSISTENCY-001'), + 'must not fire without a connect step in the test' + ); + }); }); describe('SETVALUES-STRUCTURE-001', () => { @@ -1842,13 +1956,13 @@ describe('provar_testcase_validate description', () => { } } - it('includes step-reference guidance', () => { + it('includes step-schema tool guidance', () => { const srv = new DescriptionCapturingServer(); registerTestCaseValidate(srv as unknown as McpServer, { allowedPaths: [] }); assert.ok(srv.capturedDescription, 'description should be captured'); assert.ok( - String(srv.capturedDescription).includes('provar://docs/step-reference'), - 'description should include step-reference guidance' + String(srv.capturedDescription).includes('provar_step_schema'), + 'description should point at the provar_step_schema tool for step attribute schemas' ); }); });