TRCLI-230: Support for passing quality rating values using --result-fields option

CHANGELOG.MD

@@ -12,6 +12,7 @@ _released 04--2026
 
 ### Added
  - **AI Evaluation Template Support**: Uploading test result support for TestRail's AI Evaluation Template with multi-dimensional quality ratings. See README "AI Evaluation Template Support" section for complete examples.
+ - **Global Quality Rating via `--result-fields`**: Added support for applying quality ratings to all test results using `--result-fields quality_rating:'{"category": value}'`. Test-specific quality ratings in XML/JSON properties take precedence over CLI global ratings.
 
 ## [1.14.1]
 

README.md

@@ -576,6 +576,70 @@ Traces: https://logs.example.com/trace/123
 Latency: 0.8 seconds
 ```
 
+### Using `--result-fields` for Quality Rating
+
+In addition to specifying quality ratings in XML/JSON properties, you can apply a **global quality rating** to all test results using the `--result-fields` command-line option:
+
+```shell
+trcli parse_junit \
+  -f sample_results.xml \
+  --project-id 1 \
+  --suite-id 2 \
+  --result-fields quality_rating:'{"factual_accuracy": 4, "reliability": 5, "performance": 3}'
+```
+
+#### Behavior
+
+- **Global Application**: The quality rating specified via `--result-fields` is applied to **all test results** that don't already have one
+- **Test-Specific Override**: Quality ratings specified in test properties/metadata **always take precedence** over `--result-fields`
+- **Validation**: The same validation rules apply (max 15 categories, 0-5 stars, at least one ≥ 1)
+
+#### Example: Mixed Quality Ratings
+
+```xml
+<testsuites>
+  <testsuite name="API Tests">
+    <!-- Test 1: Uses CLI global quality_rating (no rating in XML) -->
+    <testcase name="C100_test_payment_success" time="2.5">
+      <properties>
+        <property name="testrail_result_field" value="custom_api_endpoint:/api/v1/payment"/>
+      </properties>
+    </testcase>
+
+    <!-- Test 2: Uses test-specific quality_rating (overrides CLI) -->
+    <testcase name="C101_test_refund_success" time="3.1">
+      <properties>
+        <property name="quality_rating" value='{"factual_accuracy": 5, "response_time": 5}'/>
+      </properties>
+    </testcase>
+  </testsuite>
+</testsuites>
+```
+
+CLI command:
+```shell
+trcli parse_junit \
+  -f report.xml \
+  --project-id 1 \
+  --suite-id 2 \
+  --result-fields quality_rating:'{"factual_accuracy": 4, "reliability": 5}'
+```
+
+**Result:**
+- **C100** gets the CLI quality rating: `{"factual_accuracy": 4, "reliability": 5}`
+- **C101** gets its test-specific quality rating: `{"factual_accuracy": 5, "response_time": 5}`
+
+#### Error Handling with --result-fields
+
+If the quality_rating value in `--result-fields` is invalid, TRCLI will exit with an error before uploading:
+
+```
+ERROR: Unable to parse quality_rating in --result-fields property.
+Star values must be between 0 and 5, got 10 for category 'accuracy'
+```
+
+**Note:** This is different from invalid property-based quality ratings, which log a warning and continue. CLI validation is stricter because it affects all results.
+
 ### Robot Framework Support
 
 Robot Framework test results fully support AI Evaluation Template features. Quality ratings and AI context fields are specified in the test's documentation section using special markers.

tests/test_junit_parser.py

@@ -164,6 +164,14 @@ def test_junit_xml_parser_invalid_empty_file(self):
         with pytest.raises(ParseError):
             file_reader.parse_file()
 
+    def __remove_none_quality_ratings(self, result_json: dict) -> dict:
+        """Remove quality_rating fields that are None for backward compatibility with existing tests"""
+        for section in result_json.get("testsections", []):
+            for testcase in section.get("testcases", []):
+                if testcase.get("result", {}).get("quality_rating") is None:
+                    testcase["result"].pop("quality_rating", None)
+        return result_json
+
     @pytest.mark.parse_junit
     def test_junit_xml_parser_file_not_found(self):
         with pytest.raises(FileNotFoundError):

tests/test_result_fields_quality_rating.py

@@ -0,0 +1,166 @@
+"""Unit tests for quality_rating support via --result-fields"""
+
+import pytest
+from trcli.data_classes.dataclass_testrail import TestRailResult
+from trcli.data_classes.validation_exception import ValidationException
+
+
+class TestResultFieldsQualityRating:
+    """Test quality_rating handling in --result-fields (CLI global result fields)"""
+
+    def test_quality_rating_via_result_fields_valid(self):
+        """Test that valid quality_rating JSON string via --result-fields is parsed and set"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": '{"factual_accuracy": 5, "relevance": 4}', "custom_field": "value1"}
+
+        result.add_global_result_fields(global_fields)
+
+        # quality_rating should be parsed and set on the attribute
+        assert result.quality_rating == {"factual_accuracy": 5, "relevance": 4}
+        # Other fields should be in result_fields dict
+        assert result.result_fields["custom_field"] == "value1"
+        # quality_rating should NOT be in result_fields dict
+        assert "quality_rating" not in result.result_fields
+
+    def test_quality_rating_via_result_fields_invalid_json(self):
+        """Test that invalid JSON in quality_rating raises ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": "{not valid json}"}
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "must be valid JSON" in str(exc_info.value)
+
+    def test_quality_rating_via_result_fields_too_many_categories(self):
+        """Test that quality_rating with >15 categories raises ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        # Create 16 categories (exceeds MAX_CATEGORIES=15)
+        categories = {f"category_{i}": 3 for i in range(16)}
+        global_fields = {"quality_rating": str(categories).replace("'", '"')}
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "at most 15 categories" in str(exc_info.value)
+
+    def test_quality_rating_via_result_fields_invalid_star_value(self):
+        """Test that quality_rating with invalid star values raises ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": '{"factual_accuracy": 6}'}  # 6 exceeds MAX_STAR_VALUE=5
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "must be between 0 and 5" in str(exc_info.value)
+
+    def test_quality_rating_via_result_fields_all_zeros(self):
+        """Test that quality_rating with all zero values raises ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": '{"factual_accuracy": 0, "relevance": 0}'}
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "at least one category with a star value >= 1" in str(exc_info.value)
+
+    def test_quality_rating_test_specific_overrides_global(self):
+        """Test that test-specific quality_rating (from properties) takes precedence over --result-fields"""
+        # Simulate test-specific quality_rating already set (from XML properties)
+        result = TestRailResult(case_id=1, status_id=1, quality_rating={"test_specific": 5, "accuracy": 4})
+
+        # Attempt to apply global quality_rating via --result-fields
+        global_fields = {"quality_rating": '{"global_rating": 3}'}
+
+        result.add_global_result_fields(global_fields)
+
+        # Test-specific rating should be preserved (not overridden by global)
+        assert result.quality_rating == {"test_specific": 5, "accuracy": 4}
+        assert result.quality_rating != {"global_rating": 3}
+
+    def test_quality_rating_via_result_fields_empty_string(self):
+        """Test that empty string quality_rating raises ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": ""}
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "cannot be empty" in str(exc_info.value)
+
+    def test_quality_rating_via_result_fields_empty_object(self):
+        """Test that empty JSON object quality_rating raises ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": "{}"}
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "cannot be an empty object" in str(exc_info.value)
+
+    def test_quality_rating_via_result_fields_non_integer_value(self):
+        """Test that non-integer star values raise ValidationException"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": '{"factual_accuracy": 4.5}'}  # float instead of int
+
+        with pytest.raises(ValidationException) as exc_info:
+            result.add_global_result_fields(global_fields)
+
+        assert "Unable to parse quality_rating in --result-fields" in str(exc_info.value)
+        assert "must be integers" in str(exc_info.value)
+
+    def test_quality_rating_via_result_fields_mixed_with_other_fields(self):
+        """Test that quality_rating works alongside other result fields"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {
+            "quality_rating": '{"factual_accuracy": 5, "relevance": 4, "completeness": 3}',
+            "custom_field_1": "value1",
+            "custom_field_2": "value2",
+            "custom_priority": "3",
+        }
+
+        result.add_global_result_fields(global_fields)
+
+        # quality_rating should be on the attribute
+        assert result.quality_rating == {"factual_accuracy": 5, "relevance": 4, "completeness": 3}
+        # Other fields should be in result_fields dict
+        assert result.result_fields["custom_field_1"] == "value1"
+        assert result.result_fields["custom_field_2"] == "value2"
+        assert result.result_fields["custom_priority"] == "3"
+        # quality_rating should NOT be in result_fields dict
+        assert "quality_rating" not in result.result_fields
+
+    def test_quality_rating_to_dict_serialization(self):
+        """Test that quality_rating is properly serialized in to_dict()"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"quality_rating": '{"factual_accuracy": 5, "security": 4}', "custom_field": "value1"}
+
+        result.add_global_result_fields(global_fields)
+        result_dict = result.to_dict()
+
+        # quality_rating should be at root level (not nested)
+        assert "quality_rating" in result_dict
+        assert result_dict["quality_rating"] == {"factual_accuracy": 5, "security": 4}
+        # Other fields should also be present
+        assert result_dict["custom_field"] == "value1"
+        assert result_dict["case_id"] == 1
+        assert result_dict["status_id"] == 1
+
+    def test_no_quality_rating_in_result_fields_no_error(self):
+        """Test that absence of quality_rating doesn't cause issues"""
+        result = TestRailResult(case_id=1, status_id=1)
+        global_fields = {"custom_field_1": "value1", "custom_field_2": "value2"}
+
+        result.add_global_result_fields(global_fields)
+
+        # No quality_rating should be set
+        assert result.quality_rating is None
+        # Other fields should be in result_fields dict
+        assert result.result_fields["custom_field_1"] == "value1"
+        assert result.result_fields["custom_field_2"] == "value2"

trcli/cli.py

@@ -17,7 +17,7 @@
     TOOL_VERSION,
     COMMAND_FAULT_MAPPING,
 )
-from trcli.data_classes.data_parsers import FieldsParser
+from trcli.data_classes.data_parsers import FieldsParser, QualityRatingParser
 from trcli.settings import DEFAULT_API_CALL_TIMEOUT, DEFAULT_BATCH_SIZE
 
 # Import structured logging infrastructure
@@ -123,6 +123,23 @@ def result_fields(self, result_fields: Union[List[str], dict]):
         if error:
             self.elog(error)
             exit(1)
+
+        # Validate quality_rating if present in result_fields
+        if "quality_rating" in fields_dict:
+            quality_rating_value = fields_dict["quality_rating"]
+            _, validation_error = QualityRatingParser.parse_quality_rating(quality_rating_value)
+            if validation_error:
+                self.elog(
+                    f"ERROR: Invalid quality_rating provided in --result-fields parameter:\n"
+                    f"{validation_error}\n\n"
+                    f"Quality rating requirements:\n"
+                    f"  - Maximum 15 categories\n"
+                    f"  - Star values must be integers 0-5\n"
+                    f"  - At least one category must have a value >= 1\n"
+                    f"  - Must be valid JSON object format"
+                )
+                exit(1)
+
         self._result_fields = fields_dict
 
     def log(self, msg: str, new_line=True, *args):

trcli/commands/cmd_parse_junit.py

@@ -76,7 +76,23 @@ def cli(environment: Environment, context: click.Context, *args, **kwargs):
     settings.ALLOW_ELAPSED_MS = environment.allow_ms
     print_config(environment)
     try:
-        parsed_suites = JunitParser(environment).parse_file()
+        junit_parser = JunitParser(environment)
+        parsed_suites = junit_parser.parse_file()
+
+        # Check if any invalid quality ratings were found during parsing
+        if junit_parser.invalid_quality_ratings_found:
+            environment.elog(
+                "\nERROR: One or more test results have invalid quality_rating values that were rejected.\n"
+                "Cannot proceed with upload as quality_rating is required for tests that specify it.\n\n"
+                "Please fix the invalid quality ratings in your test report and try again.\n\n"
+                "Quality rating requirements:\n"
+                "  - Maximum 15 categories\n"
+                "  - Star values must be integers 0-5\n"
+                "  - At least one category must have a value >= 1\n"
+                "  - Must be valid JSON object format"
+            )
+            exit(1)
+
         run_id = None
         case_update_results = {}
 

trcli/commands/cmd_parse_robot.py

@@ -23,7 +23,23 @@ def cli(environment: Environment, context: click.Context, *args, **kwargs):
     settings.ALLOW_ELAPSED_MS = environment.allow_ms
     print_config(environment)
     try:
-        parsed_suites = RobotParser(environment).parse_file()
+        robot_parser = RobotParser(environment)
+        parsed_suites = robot_parser.parse_file()
+
+        # Check if any invalid quality ratings were found during parsing
+        if robot_parser.invalid_quality_ratings_found:
+            environment.elog(
+                "\nERROR: One or more test results have invalid quality_rating values that were rejected.\n"
+                "Cannot proceed with upload as quality_rating is required for tests that specify it.\n\n"
+                "Please fix the invalid quality ratings in your test report and try again.\n\n"
+                "Quality rating requirements:\n"
+                "  - Maximum 15 categories\n"
+                "  - Star values must be integers 0-5\n"
+                "  - At least one category must have a value >= 1\n"
+                "  - Must be valid JSON object format"
+            )
+            exit(1)
+
         for suite in parsed_suites:
             result_uploader = ResultsUploader(environment=environment, suite=suite)
             result_uploader.upload_results()