feat(memory):feedback-ranking-tie-breaker-evaluation

kun/src/memory/fixtures/kun-memory-feedback-tiebreaker-v1.json

@@ -0,0 +1,103 @@
+{
+  "schemaVersion": 1,
+  "evaluationVersion": "p3-feedback-tiebreaker-v1",
+  "decisionId": "kun-memory-feedback-tiebreaker-v1",
+  "status": "final",
+  "artifactHashes": {
+    "fixtureSha256": "a6a37e35a5da6afdfbdae42b9abe8a9f182dcad19167fc82361f662873d7e259",
+    "manifestSha256": "17ab0ea2186d11bbdfb38e0f76b75f3547144c74412183e8118cb2e525d6124a",
+    "calibrationSha256": "b34b6d995a2230aac543885d4173c07f8e3024b4fd8603637d5f49e4258a7964",
+    "decisionPlanSha256": "27dfc5852afa9d86e0ff52bf549fb154e5b788ac00690758c0c8accd15460ab7",
+    "candidateLockSha256": "03716e77bd081515b098e1844e3bb3e4121194fc2323df8cb74cf5ee9c9cfe8f"
+  },
+  "selectedCandidateId": "foundation-control",
+  "holdoutRunCount": 1,
+  "development": {
+    "foundation": {
+      "pairAccuracy": 0,
+      "pairCount": 4,
+      "recallAtK": 0.4375,
+      "precisionAtK": 0.4375,
+      "meanReciprocalRank": 0.4375,
+      "abstentionAccuracy": 1,
+      "explicitForbiddenSelections": 6,
+      "authorizationOrLifecycleViolations": 0,
+      "rankedCaseCount": 16,
+      "noResultCaseCount": 2,
+      "caseCount": 18
+    },
+    "candidate": {
+      "pairAccuracy": 0,
+      "pairCount": 4,
+      "recallAtK": 0.4375,
+      "precisionAtK": 0.4375,
+      "meanReciprocalRank": 0.4375,
+      "abstentionAccuracy": 1,
+      "explicitForbiddenSelections": 6,
+      "authorizationOrLifecycleViolations": 0,
+      "rankedCaseCount": 16,
+      "noResultCaseCount": 2,
+      "caseCount": 18
+    },
+    "bootstrapLowerBounds": {
+      "pairAccuracyGain": 0,
+      "recallGain": 0,
+      "mrrGain": 0
+    },
+    "gates": {
+      "localBenefit": false,
+      "globalNonRegression": true,
+      "safety": false,
+      "privacy": true,
+      "determinism": true,
+      "resource": true,
+      "passed": false
+    }
+  },
+  "holdout": {
+    "foundation": {
+      "pairAccuracy": 0,
+      "pairCount": 4,
+      "recallAtK": 0.25,
+      "precisionAtK": 0.25,
+      "meanReciprocalRank": 0.25,
+      "abstentionAccuracy": 1,
+      "explicitForbiddenSelections": 1,
+      "authorizationOrLifecycleViolations": 0,
+      "rankedCaseCount": 16,
+      "noResultCaseCount": 2,
+      "caseCount": 18
+    },
+    "candidate": {
+      "pairAccuracy": 0,
+      "pairCount": 4,
+      "recallAtK": 0.25,
+      "precisionAtK": 0.25,
+      "meanReciprocalRank": 0.25,
+      "abstentionAccuracy": 1,
+      "explicitForbiddenSelections": 1,
+      "authorizationOrLifecycleViolations": 0,
+      "rankedCaseCount": 16,
+      "noResultCaseCount": 2,
+      "caseCount": 18
+    },
+    "bootstrapLowerBounds": {
+      "pairAccuracyGain": 0,
+      "recallGain": 0,
+      "mrrGain": 0
+    },
+    "gates": {
+      "localBenefit": false,
+      "globalNonRegression": true,
+      "safety": false,
+      "privacy": true,
+      "determinism": true,
+      "resource": true,
+      "passed": false
+    }
+  },
+  "decision": "no-go",
+  "reasons": [
+    "foundation-control is the pre-registered fallback after all development candidates failed the safety gate."
+  ]
+}

kun/src/memory/fixtures/memory-feedback-tiebreaker-calibration.v1.json

@@ -0,0 +1,34 @@
+{
+  "schemaVersion": 1,
+  "evaluationVersion": "p3-feedback-tiebreaker-v1",
+  "fixtureSha256": "a6a37e35a5da6afdfbdae42b9abe8a9f182dcad19167fc82361f662873d7e259",
+  "foundationVersion": "post-1308-foundation-v1",
+  "procedure": {
+    "source": "development-foundation-only",
+    "gapFormula": "higher-score-minus-lower-score",
+    "quantiles": [0.25, 0.5, 0.75],
+    "quantileMethod": "floor-index",
+    "maximumGridSize": 4,
+    "boundaryInclusive": true,
+    "grouping": "leader-relative",
+    "maximumReorderWindow": 3,
+    "exactTieControl": true,
+    "noRerankControl": true,
+    "stableFallback": ["foundation-score", "memory-id"]
+  },
+  "observedGaps": [
+    {"caseId":"case_dev_confirmed_01","higherId":"mem_tie_list","lowerId":"mem_tie_table","gap":0.00375},
+    {"caseId":"case_dev_corrected_01","higherId":"mem_corr_distractor","lowerId":"mem_corr_new","gap":0.0075},
+    {"caseId":"case_dev_corrected_02","higherId":"mem_corr_distractor","lowerId":"mem_corr_new","gap":0.0075},
+    {"caseId":"case_dev_frequent_01","higherId":"mem_freq_wrong","lowerId":"mem_freq_right","gap":0.0075},
+    {"caseId":"case_dev_misleading_01","higherId":"mem_misleading_confirmed","lowerId":"mem_misleading_expected","gap":0.01125},
+    {"caseId":"case_dev_misleading_02","higherId":"mem_misleading_confirmed","lowerId":"mem_misleading_expected","gap":0.01125},
+    {"caseId":"case_dev_neutral_01","higherId":"mem_neutral_blue","lowerId":"mem_neutral_green","gap":0.0075},
+    {"caseId":"case_dev_wide_01","higherId":"mem_wide_primary","lowerId":"mem_wide_confirmed","gap":0.2}
+  ],
+  "boundaryGrid": [
+    {"id":"gap_0","maximumGap":0,"inclusive":true},
+    {"id":"gap_1","maximumGap":0.0075,"inclusive":true},
+    {"id":"gap_2","maximumGap":0.01125,"inclusive":true}
+  ]
+}

kun/src/memory/fixtures/memory-feedback-tiebreaker-checksums.v1.json

@@ -0,0 +1,9 @@
+{
+  "schemaVersion": 1,
+  "evaluationVersion": "p3-feedback-tiebreaker-v1",
+  "algorithm": "sha256",
+  "files": {
+    "fixture": "a6a37e35a5da6afdfbdae42b9abe8a9f182dcad19167fc82361f662873d7e259",
+    "manifest": "17ab0ea2186d11bbdfb38e0f76b75f3547144c74412183e8118cb2e525d6124a"
+  }
+}

kun/src/memory/fixtures/memory-feedback-tiebreaker-development.v1.json

@@ -0,0 +1,67 @@
+{
+  "schemaVersion": 1,
+  "evaluationVersion": "p3-feedback-tiebreaker-v1",
+  "partition": "development",
+  "artifactHashes": {
+    "fixtureSha256": "a6a37e35a5da6afdfbdae42b9abe8a9f182dcad19167fc82361f662873d7e259",
+    "manifestSha256": "17ab0ea2186d11bbdfb38e0f76b75f3547144c74412183e8118cb2e525d6124a",
+    "calibrationSha256": "b34b6d995a2230aac543885d4173c07f8e3024b4fd8603637d5f49e4258a7964",
+    "decisionPlanSha256": "27dfc5852afa9d86e0ff52bf549fb154e5b788ac00690758c0c8accd15460ab7"
+  },
+  "foundation": {
+    "pairAccuracy":0,"pairCount":4,"recallAtK":0.4375,"precisionAtK":0.4375,
+    "meanReciprocalRank":0.4375,"abstentionAccuracy":1,"explicitForbiddenSelections":6,
+    "authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18
+  },
+  "configurations": [
+    {
+      "candidateId":"foundation-control","boundaryId":null,"maximumGap":0,"signalRule":"foundation-only",
+      "metrics":{"pairAccuracy":0,"pairCount":4,"recallAtK":0.4375,"precisionAtK":0.4375,"meanReciprocalRank":0.4375,"abstentionAccuracy":1,"explicitForbiddenSelections":6,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0,"recallGain":0,"mrrGain":0},
+      "gates":{"localBenefit":false,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"867715da1bd988bdf02bc50eef39d6440e7d633ee12f5018056a386402455540"
+    },
+    {
+      "candidateId":"confirmation-gap_0","boundaryId":"gap_0","maximumGap":0,"signalRule":"confirmation",
+      "metrics":{"pairAccuracy":0,"pairCount":4,"recallAtK":0.4375,"precisionAtK":0.4375,"meanReciprocalRank":0.4375,"abstentionAccuracy":1,"explicitForbiddenSelections":6,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0,"recallGain":0,"mrrGain":0},
+      "gates":{"localBenefit":false,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"867715da1bd988bdf02bc50eef39d6440e7d633ee12f5018056a386402455540"
+    },
+    {
+      "candidateId":"confirmation-correction-gap_0","boundaryId":"gap_0","maximumGap":0,"signalRule":"confirmation-correction",
+      "metrics":{"pairAccuracy":0,"pairCount":4,"recallAtK":0.4375,"precisionAtK":0.4375,"meanReciprocalRank":0.4375,"abstentionAccuracy":1,"explicitForbiddenSelections":6,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0,"recallGain":0,"mrrGain":0},
+      "gates":{"localBenefit":false,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"867715da1bd988bdf02bc50eef39d6440e7d633ee12f5018056a386402455540"
+    },
+    {
+      "candidateId":"confirmation-gap_1","boundaryId":"gap_1","maximumGap":0.0075,"signalRule":"confirmation",
+      "metrics":{"pairAccuracy":0.25,"pairCount":4,"recallAtK":0.5,"precisionAtK":0.5,"meanReciprocalRank":0.5,"abstentionAccuracy":1,"explicitForbiddenSelections":5,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0,"recallGain":0,"mrrGain":0},
+      "gates":{"localBenefit":false,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"839628f204c80ed5e5b39ee20066e70b92126dfe08c5e934890eb1aefa19c1fc"
+    },
+    {
+      "candidateId":"confirmation-correction-gap_1","boundaryId":"gap_1","maximumGap":0.0075,"signalRule":"confirmation-correction",
+      "metrics":{"pairAccuracy":0.75,"pairCount":4,"recallAtK":0.625,"precisionAtK":0.625,"meanReciprocalRank":0.625,"abstentionAccuracy":1,"explicitForbiddenSelections":3,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0.25,"recallGain":0.0625,"mrrGain":0.0625},
+      "gates":{"localBenefit":true,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"3e386fe6b11bdc8c7dd68c43e210533bdc70557f25901c168c3d2ca78f5fbc57"
+    },
+    {
+      "candidateId":"confirmation-gap_2","boundaryId":"gap_2","maximumGap":0.01125,"signalRule":"confirmation",
+      "metrics":{"pairAccuracy":0.25,"pairCount":4,"recallAtK":0.5,"precisionAtK":0.5,"meanReciprocalRank":0.5,"abstentionAccuracy":1,"explicitForbiddenSelections":5,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0,"recallGain":0,"mrrGain":0},
+      "gates":{"localBenefit":false,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"839628f204c80ed5e5b39ee20066e70b92126dfe08c5e934890eb1aefa19c1fc"
+    },
+    {
+      "candidateId":"confirmation-correction-gap_2","boundaryId":"gap_2","maximumGap":0.01125,"signalRule":"confirmation-correction",
+      "metrics":{"pairAccuracy":0.75,"pairCount":4,"recallAtK":0.625,"precisionAtK":0.625,"meanReciprocalRank":0.625,"abstentionAccuracy":1,"explicitForbiddenSelections":3,"authorizationOrLifecycleViolations":0,"rankedCaseCount":16,"noResultCaseCount":2,"caseCount":18},
+      "bootstrapLowerBounds":{"pairAccuracyGain":0.25,"recallGain":0.0625,"mrrGain":0.0625},
+      "gates":{"localBenefit":true,"globalNonRegression":true,"safety":false,"passed":false},
+      "selectedIdsSha256":"3e386fe6b11bdc8c7dd68c43e210533bdc70557f25901c168c3d2ca78f5fbc57"
+    }
+  ]
+}