diff --git a/Sources/Implementation/DefaultDecisionService.swift b/Sources/Implementation/DefaultDecisionService.swift index 5cb71497..a0165684 100644 --- a/Sources/Implementation/DefaultDecisionService.swift +++ b/Sources/Implementation/DefaultDecisionService.swift @@ -20,6 +20,11 @@ struct HoldoutDecision { let experiment: ExperimentCore let variation: Variation + init(experiment: ExperimentCore, variation: Variation) { + self.experiment = experiment + self.variation = variation + } + init?(from featureDecision: FeatureDecision?) { guard let experiment = featureDecision?.experiment, let variation = featureDecision?.variation else { return nil } @@ -34,7 +39,7 @@ struct FeatureDecision { let source: String var cmabUUID: String? var error = false - var holdoutDecision: HoldoutDecision? = nil + var holdoutDecisions: [HoldoutDecision] = [] } struct VariationDecision { @@ -436,21 +441,27 @@ class DefaultDecisionService: OPTDecisionService { } } + var holdoutDecisions = [HoldoutDecision(from: storedHoldoutDecision)].compactMap { $0 } + if storedHoldoutDecision == nil { let flagExpDecision = getVariationForFeatureExperiments(config: config, featureFlag: featureFlag, user: user, userProfileTracker: userProfileTracker, isAsync: isAsync, options: options) reasons.merge(flagExpDecision.reasons) if let decision = flagExpDecision.result { - return DecisionResponse(result: decision, reasons: reasons) + if decision.variation != nil || decision.error { + return DecisionResponse(result: decision, reasons: reasons) + } + holdoutDecisions += decision.holdoutDecisions } } let rolloutDecision = getVariationForFeatureRollout(config: config, featureFlag: featureFlag, user: user, options: options) reasons.merge(rolloutDecision.reasons) + holdoutDecisions += rolloutDecision.result?.holdoutDecisions ?? [] - if var decision: FeatureDecision = rolloutDecision.result { - decision.holdoutDecision = HoldoutDecision(from: storedHoldoutDecision) + if var decision: FeatureDecision = rolloutDecision.result, decision.variation != nil { + decision.holdoutDecisions = holdoutDecisions let info = LogMessage.userBucketedIntoRollout(user.userId, featureFlag.key) logger.i(info) reasons.addInfo(info) @@ -459,9 +470,9 @@ class DefaultDecisionService: OPTDecisionService { let info = LogMessage.userNotBucketedIntoRollout(user.userId, featureFlag.key) logger.i(info) reasons.addInfo(info) - if let holdout = HoldoutDecision(from: storedHoldoutDecision) { + if !holdoutDecisions.isEmpty { var emptyDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.rollout.rawValue) - emptyDecision.holdoutDecision = holdout + emptyDecision.holdoutDecisions = holdoutDecisions return DecisionResponse(result: emptyDecision, reasons: reasons) } return DecisionResponse(result: nil, reasons: reasons) @@ -492,6 +503,8 @@ class DefaultDecisionService: OPTDecisionService { reasons.addInfo(info) } + var holdoutDecisions = [HoldoutDecision]() + // Check if there are any experiment IDs inside feature flag // Evaluate each experiment ID and return the first bucketed experiment variation for experimentId in experimentIds { @@ -508,15 +521,13 @@ class DefaultDecisionService: OPTDecisionService { if result.cmabError { // For CMAB - we're supposed to get decision from the server. // If failed, return decision with nil variation, so the client can take care of them. - let featureDecision = FeatureDecision(experiment: experiment, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, error: true) + let featureDecision = FeatureDecision(experiment: experiment, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, error: true, holdoutDecisions: holdoutDecisions) return DecisionResponse(result: featureDecision, reasons: reasons) } else if let variation = result.variation { - // Check if this variation came from a holdout if let holdout = result.holdout { - let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue, cmabUUID: result.cmabUUID) - return DecisionResponse(result: featureDecision, reasons: reasons) + holdoutDecisions.append(HoldoutDecision(experiment: holdout, variation: variation)) } else { - let featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID) + let featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID, holdoutDecisions: holdoutDecisions) return DecisionResponse(result: featureDecision, reasons: reasons) } } @@ -524,6 +535,10 @@ class DefaultDecisionService: OPTDecisionService { } } + if !holdoutDecisions.isEmpty { + let featureDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, holdoutDecisions: holdoutDecisions) + return DecisionResponse(result: featureDecision, reasons: reasons) + } return DecisionResponse(result: nil, reasons: reasons) } @@ -564,6 +579,7 @@ class DefaultDecisionService: OPTDecisionService { return DecisionResponse(result: nil, reasons: reasons) } + var holdoutDecisions = [HoldoutDecision]() var index = 0 while index < rolloutRules.count { let decisionResponse = getVariationFromDeliveryRule(config: config, @@ -576,21 +592,23 @@ class DefaultDecisionService: OPTDecisionService { let result = decisionResponse.result! if let variation = result.variation { - let rule = rolloutRules[index] - // Check if this variation came from a holdout if let holdout = result.holdout { - let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue) - return DecisionResponse(result: featureDecision, reasons: reasons) - } else { - let featureDecision = FeatureDecision(experiment: rule, variation: variation, source: Constants.DecisionSource.rollout.rawValue) - return DecisionResponse(result: featureDecision, reasons: reasons) + holdoutDecisions.append(HoldoutDecision(experiment: holdout, variation: variation)) + index += 1 + continue } + let featureDecision = FeatureDecision(experiment: rolloutRules[index], variation: variation, source: Constants.DecisionSource.rollout.rawValue, holdoutDecisions: holdoutDecisions) + return DecisionResponse(result: featureDecision, reasons: reasons) } // the last rule is special for "Everyone Else" index = result.skipToEveryoneElse ? (rolloutRules.count - 1) : (index + 1) } + if !holdoutDecisions.isEmpty { + let featureDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.rollout.rawValue, holdoutDecisions: holdoutDecisions) + return DecisionResponse(result: featureDecision, reasons: reasons) + } return DecisionResponse(result: nil, reasons: reasons) } diff --git a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift index ee6cbc0d..e8286949 100644 --- a/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift +++ b/Sources/Optimizely+Decide/OptimizelyClient+Decide.swift @@ -324,7 +324,7 @@ extension OptimizelyClient { cmabUUID: flagDecision?.cmabUUID) decisionEventDispatched = true } - if let holdoutInfo = flagDecision?.holdoutDecision { + for holdoutInfo in flagDecision?.holdoutDecisions ?? [] { sendImpressionEvent(experiment: holdoutInfo.experiment, variation: holdoutInfo.variation, userId: userId, diff --git a/Tests/OptimizelyTests-Common/BatchEventBuilderTests_Events.swift b/Tests/OptimizelyTests-Common/BatchEventBuilderTests_Events.swift index 653d4e65..680e50e4 100644 --- a/Tests/OptimizelyTests-Common/BatchEventBuilderTests_Events.swift +++ b/Tests/OptimizelyTests-Common/BatchEventBuilderTests_Events.swift @@ -531,7 +531,7 @@ extension BatchEventBuilderTests_Events { func testImpressionEvent_UserInHoldout_IncludedFlags() { let eventDispatcher2 = MockEventDispatcher() - var optimizely: OptimizelyClient! = OptimizelyClient(sdkKey: "12345", eventDispatcher: eventDispatcher2) + var optimizely: OptimizelyClient! = OptimizelyClient(sdkKey: OTUtils.randomSdkKey, eventDispatcher: eventDispatcher2) try! optimizely.start(datafile: datafile) @@ -552,11 +552,14 @@ extension BatchEventBuilderTests_Events { let result = XCTWaiter.wait(for: [exp], timeout: 0.2) if result == XCTWaiter.Result.completed { - let event = getFirstEventJSON(client: optimizely)! + optimizely.eventLock.sync {} + // No rule serves and sendFlagDecisions is off, so only the holdout event is sent. + XCTAssertEqual(eventDispatcher2.events.count, 1) + let event = getFirstEventJSON(dispatcher: eventDispatcher2)! let visitor = (event["visitors"] as! Array>)[0] let snapshot = (visitor["snapshots"] as! Array>)[0] let decision = (snapshot["decisions"] as! Array>)[0] - + let metaData = decision["metadata"] as! Dictionary XCTAssertEqual(metaData["rule_type"] as! String, Constants.DecisionSource.holdout.rawValue) XCTAssertEqual(metaData["rule_key"] as! String, "holdout_key") diff --git a/Tests/OptimizelyTests-Common/DecisionListenerTest_Holdouts.swift b/Tests/OptimizelyTests-Common/DecisionListenerTest_Holdouts.swift index e9a329ec..0c72485e 100644 --- a/Tests/OptimizelyTests-Common/DecisionListenerTest_Holdouts.swift +++ b/Tests/OptimizelyTests-Common/DecisionListenerTest_Holdouts.swift @@ -132,8 +132,9 @@ class DecisionListenerTests_Holdouts: XCTestCase { XCTAssertEqual(type, Constants.DecisionType.flag.rawValue) XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.flagKey] as! String, self.kFeatureKey) XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.enabled] as! Bool, false) - XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.variationKey] as! String, "key_holdout_variation") - XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.ruleKey] as! String, "key_holdout") + XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.variationKey] is NSNull) + XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.ruleKey] is NSNull) + XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.decisionEventDispatched] as! Bool, true) exp.fulfill() } @@ -183,8 +184,9 @@ class DecisionListenerTests_Holdouts: XCTestCase { XCTAssertEqual(type, Constants.DecisionType.flag.rawValue) XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.flagKey] as! String, self.kFeatureKey) XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.enabled] as! Bool, false) - XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.variationKey] as! String, "key_holdout_variation") - XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.ruleKey] as! String, "holdout_key_2") + XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.variationKey] is NSNull) + XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.ruleKey] is NSNull) + XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.decisionEventDispatched] as! Bool, true) exp.fulfill() } diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift index bae26ac4..6bffd637 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_ExcludeTargetedDeliveries.swift @@ -69,10 +69,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - XCTAssertEqual(decision.variationKey, "holdout_variation_key") - XCTAssertFalse(decision.enabled) + assertHeldOutThenServedByEveryoneElse(decision) } // MARK: - excludeTargetedDeliveries = true with TD rule @@ -92,10 +91,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - XCTAssertEqual(decision.variationKey, "holdout_variation_key") - XCTAssertFalse(decision.enabled) + assertHeldOutThenServedByEveryoneElse(decision) } // MARK: - excludeTargetedDeliveries = true with AB rule @@ -115,10 +113,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - XCTAssertEqual(decision.variationKey, "holdout_variation_key") - XCTAssertFalse(decision.enabled) + assertHeldOutThenServedByEveryoneElse(decision) } // MARK: - Backward compatibility (field missing from JSON) @@ -148,10 +145,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - XCTAssertEqual(decision.variationKey, "holdout_variation_key") - XCTAssertFalse(decision.enabled) + assertHeldOutThenServedByEveryoneElse(decision) } // MARK: - Global holdout with excludeTargetedDeliveries true and TD rule @@ -196,10 +192,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - XCTAssertEqual(decision.variationKey, "holdout_variation_key") - XCTAssertFalse(decision.enabled) + assertHeldOutThenServedByEveryoneElse(decision) } // MARK: - Forced decision beats 100% traffic local holdout @@ -278,10 +273,10 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { // Delivery (rollout) rules are evaluated — Everyone Else matches XCTAssertNotNil(response.result?.variation) XCTAssertEqual(response.result?.source, Constants.DecisionSource.rollout.rawValue) - // Holdout is queued so its impression fires via holdoutDecision - XCTAssertNotNil(response.result?.holdoutDecision) - XCTAssertEqual(response.result?.holdoutDecision?.experiment.key, "holdout_test_key") - XCTAssertEqual(response.result?.holdoutDecision?.variation.key, "holdout_variation_key") + // Holdout is queued so its impression fires via holdoutDecisions + XCTAssertEqual(response.result?.holdoutDecisions.count, 1) + XCTAssertEqual(response.result?.holdoutDecisions.first?.experiment.key, "holdout_test_key") + XCTAssertEqual(response.result?.holdoutDecisions.first?.variation.key, "holdout_variation_key") } // MARK: - Local holdout with excludeTrue still applies @@ -301,10 +296,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - XCTAssertEqual(decision.variationKey, "holdout_variation_key") - XCTAssertFalse(decision.enabled) + assertHeldOutThenServedByEveryoneElse(decision) } // MARK: - Decision reason for excludeTargetedDeliveries bypass @@ -363,11 +357,11 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { wait(for: [exp], timeout: 1) } - // MARK: - Global holdout excludeTrue, TD matches => holdoutDecision populated + // MARK: - Global holdout excludeTrue, TD matches => holdoutDecisions populated // When excludeTargetedDeliveries=true, experiment rules are skipped and the // decision comes from the delivery (rollout) path. The holdout is queued via - // holdoutDecision so its impression fires alongside the rollout impression. + // holdoutDecisions so its impression fires alongside the rollout impression. func testGlobalHoldout_ExcludeTrue_HoldoutEventSent() { var experiment = config.getExperiment(id: experimentRuleId)! experiment.type = .targetedDelivery @@ -393,8 +387,15 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase { XCTAssertNotNil(response.result) // Experiments are skipped; decision comes from delivery (rollout) rules XCTAssertEqual(response.result?.source, Constants.DecisionSource.rollout.rawValue) - XCTAssertNotNil(response.result?.holdoutDecision) - XCTAssertEqual(response.result?.holdoutDecision?.experiment.key, "holdout_test_key") - XCTAssertEqual(response.result?.holdoutDecision?.variation.key, "holdout_variation_key") + XCTAssertEqual(response.result?.holdoutDecisions.count, 1) + XCTAssertEqual(response.result?.holdoutDecisions.first?.experiment.key, "holdout_test_key") + XCTAssertEqual(response.result?.holdoutDecisions.first?.variation.key, "holdout_variation_key") + } + + private func assertHeldOutThenServedByEveryoneElse(_ decision: OptimizelyDecision, file: StaticString = #filePath, line: UInt = #line) { + XCTAssert(decision.reasons.contains(LogMessage.userBucketedIntoVariationInHoldout(userId, "holdout_test_key", "holdout_variation_key").reason), file: file, line: line) + XCTAssertEqual(decision.ruleKey, "18322080788", file: file, line: line) + XCTAssertEqual(decision.variationKey, "18257766532", file: file, line: line) + XCTAssertTrue(decision.enabled, file: file, line: line) } } diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_Holdouts.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_Holdouts.swift index 87bf4a98..c94361d5 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_Holdouts.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_Holdouts.swift @@ -1,5 +1,5 @@ // -// Copyright 2022, Optimizely, Inc. and contributors +// Copyright 2022, 2026, Optimizely, Inc. and contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -355,9 +355,11 @@ extension DecisionServiceTests_Holdouts { ).result XCTAssertNotNil(decision, "Decision should not be nil") - XCTAssertEqual(decision?.experiment?.id, holdout.id, "Should return holdout experiment") - XCTAssertEqual(decision?.variation?.key, "holdout_a", "Should return holdout variation") - XCTAssertEqual(decision?.source, Constants.DecisionSource.holdout.rawValue, "Source should be holdout") + XCTAssertNil(decision?.experiment, "Held-out rule should not be served") + XCTAssertNil(decision?.variation) + XCTAssertEqual(decision?.holdoutDecisions.count, 1) + XCTAssertEqual(decision?.holdoutDecisions.first?.experiment.id, holdout.id) + XCTAssertEqual(decision?.holdoutDecisions.first?.variation.key, "holdout_a") } func testGetVariationForFeatureExperiment_HoldoutAudienceMismatch() { @@ -531,11 +533,11 @@ extension DecisionServiceTests_Holdouts { user: optimizely.createUserContext(userId: kUserId, attributes: kAttributesCountryMatch) ).result - // Global holdout fails bucketing, should select included holdout XCTAssertNotNil(decision) - XCTAssertEqual(decision?.experiment?.id, includedHoldout.id, "Should select included holdout") - XCTAssertEqual(decision?.variation?.key, "included_variation") - XCTAssertEqual(decision?.source, Constants.DecisionSource.holdout.rawValue) + XCTAssertNil(decision?.experiment, "Held-out rule should not be served") + XCTAssertNil(decision?.variation) + XCTAssertEqual(decision?.holdoutDecisions.map { $0.experiment.id }, [includedHoldout.id]) + XCTAssertEqual(decision?.holdoutDecisions.first?.variation.key, "included_variation") } func testGetVariationForFeatureExperiment_AllHoldoutsFailThenExperiment() { @@ -648,14 +650,13 @@ extension DecisionServiceTests_Holdouts { user: optimizely.createUserContext(userId: kUserId, attributes: kAttributesCountryMatch) ).result - // Should consistently return global holdout XCTAssertNotNil(decision1) XCTAssertNotNil(decision2) - XCTAssertEqual(decision1?.experiment?.id, includedHoldout.id) - XCTAssertEqual(decision2?.experiment?.id, includedHoldout.id) - XCTAssertEqual(decision1?.variation?.key, "included_variation") - XCTAssertEqual(decision2?.variation?.key, "included_variation") - XCTAssertEqual(decision1?.source, Constants.DecisionSource.holdout.rawValue) - XCTAssertEqual(decision2?.source, Constants.DecisionSource.holdout.rawValue) + XCTAssertNil(decision1?.variation) + XCTAssertNil(decision2?.variation) + XCTAssertEqual(decision1?.holdoutDecisions.map { $0.experiment.id }, [includedHoldout.id]) + XCTAssertEqual(decision2?.holdoutDecisions.map { $0.experiment.id }, [includedHoldout.id]) + XCTAssertEqual(decision1?.holdoutDecisions.first?.variation.key, "included_variation") + XCTAssertEqual(decision2?.holdoutDecisions.first?.variation.key, "included_variation") } } diff --git a/Tests/OptimizelyTests-Common/DecisionServiceTests_LocalHoldouts.swift b/Tests/OptimizelyTests-Common/DecisionServiceTests_LocalHoldouts.swift index 183521a0..b9bae857 100644 --- a/Tests/OptimizelyTests-Common/DecisionServiceTests_LocalHoldouts.swift +++ b/Tests/OptimizelyTests-Common/DecisionServiceTests_LocalHoldouts.swift @@ -24,11 +24,14 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { var optimizely: OptimizelyClient! var config: ProjectConfig! var decisionService: DefaultDecisionService! + var eventDispatcher: MockEventDispatcher! let userId = "test_user" let flagKey = "feature_1" let experimentRuleId = "10390977673" // From decide_datafile let deliveryRuleId = "3332020515" // From decide_datafile rollout + let everyoneElseRuleKey = "18322080788" + let everyoneElseVariationKey = "18257766532" // variation id and trafficAllocation entityId must be numeric strings. // Holdout id uses numeric too for fixture uniformity. Keys are arbitrary. @@ -56,8 +59,10 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { super.setUp() // Load a real datafile for testing + eventDispatcher = MockEventDispatcher() optimizely = OTUtils.createOptimizely(datafileName: "decide_datafile", - clearUserProfileService: true) + clearUserProfileService: true, + eventDispatcher: eventDispatcher) config = optimizely.config! decisionService = optimizely.decisionService as? DefaultDecisionService } @@ -124,11 +129,9 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - // Assert user got the holdout variation - XCTAssertEqual(decision.variationKey, "holdout_variation_key", "User should be bucketed into local holdout") - XCTAssertFalse(decision.enabled, "Holdout variation should have featureEnabled: false") + assertHeldOutThenServedByEveryoneElse(decision) } func testLocalHoldout_ExperimentRule_UserNotBucketed() { @@ -192,11 +195,9 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { optimizely.decisionService = mockDecisionService let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = user.decide(key: flagKey, options: [.includeReasons]) - // Assert user got the holdout variation - XCTAssertEqual(decision.variationKey, "holdout_variation_key", "User should be bucketed into local holdout for delivery rule") - XCTAssertFalse(decision.enabled, "Holdout variation should have featureEnabled: false") + assertHeldOutThenServedByEveryoneElse(decision) } func testLocalHoldout_DeliveryRule_UserNotBucketed() { @@ -239,19 +240,21 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { holdout2.includedRules = [experimentRuleId] holdout2.variations[0].id = "holdout_2_var_id" holdout2.variations[0].key = "holdout_2_variation" + holdout2.trafficAllocation[0].entityId = "holdout_2_var_id" config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout1, holdout2]) // Mock bucketer to ensure user buckets into both let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) - optimizely.decisionService = mockDecisionService - let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = mockDecisionService.getVariationForFeature(config: config, + featureFlag: config.getFeatureFlag(key: flagKey)!, + user: optimizely.createUserContext(userId: userId)).result - // Assert user got the FIRST holdout variation - XCTAssertEqual(decision.variationKey, "holdout_1_variation", "User should be in first matching holdout only") + XCTAssertEqual(decision?.holdoutDecisions.map { $0.experiment.key }, ["holdout_1"], "Only the first matching holdout is recorded for the rule") + XCTAssertEqual(decision?.holdoutDecisions.first?.variation.key, "holdout_1_variation") + XCTAssertEqual(decision?.experiment?.key, everyoneElseRuleKey) } func testMultipleLocalHoldouts_DifferentRules_EachEvaluated() { @@ -264,21 +267,25 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { var holdout2 = try! OTUtils.model(from: sampleHoldout) as Holdout holdout2.id = "holdout_2" + holdout2.key = "holdout_2" holdout2.includedRules = [deliveryRuleId] holdout2.variations[0].id = "holdout_2_var_id" + holdout2.trafficAllocation[0].entityId = "holdout_2_var_id" config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout1, holdout2]) // Mock bucketer let mockBucketer = MockBucketer(mockBucketValue: 2500) let mockDecisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), bucketer: mockBucketer) - optimizely.decisionService = mockDecisionService - let user = optimizely.createUserContext(userId: userId) - let decision = user.decide(key: flagKey) + let decision = mockDecisionService.getVariationForFeature(config: config, + featureFlag: config.getFeatureFlag(key: flagKey)!, + user: optimizely.createUserContext(userId: userId)).result - // Assert that one of the holdouts was evaluated (which one depends on rule evaluation order) - XCTAssertEqual(decision.variationKey, "holdout_variation_key", "User should be in one of the holdouts") + XCTAssertEqual(decision?.holdoutDecisions.map { $0.experiment.id }, [holdout1.id, "holdout_2"], "Each rule records its own holdout hit, in rule order") + XCTAssertEqual(decision?.experiment?.key, everyoneElseRuleKey) + XCTAssertEqual(decision?.variation?.key, everyoneElseVariationKey) + XCTAssertEqual(decision?.source, Constants.DecisionSource.rollout.rawValue) } // MARK: - Cross-Flag Local Holdouts @@ -300,8 +307,8 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { let user = optimizely.createUserContext(userId: userId) // Test feature_1 (should be affected) - let decision1 = user.decide(key: "feature_1") - XCTAssertEqual(decision1.variationKey, "holdout_variation_key", "feature_1 should be in holdout") + let decision1 = user.decide(key: "feature_1", options: [.includeReasons]) + assertHeldOutThenServedByEveryoneElse(decision1) // Test feature_2 (should NOT be affected - no rules targeted) let decision2 = user.decide(key: "feature_2") @@ -418,4 +425,327 @@ class DecisionServiceTests_LocalHoldouts: XCTestCase { holdout.includedRules = nil XCTAssertTrue(holdout.isGlobal, "Nil includedRules should be global") } + + // MARK: - Held-out rule continues to later rules + + func testLocalHoldout_HitOnExperimentRule_NextRuleServesAndHoldoutEventSent() { + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [experimentRuleId])]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId, attributes: ["gender": "f", "country": "US"]) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, deliveryRuleId) + XCTAssertEqual(decision.variationKey, "3324490633") + XCTAssertTrue(decision.enabled) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 2) + XCTAssertEqual(events.first?.metaData.ruleType, Constants.DecisionSource.rollout.rawValue) + XCTAssertEqual(events.first?.metaData.ruleKey, deliveryRuleId) + assertHoldoutEvent(events.last) + } + + func testLocalHoldout_TwoExperimentRules_HeldOutOfFirstServedBySecond() { + var flag = config.getFeatureFlag(key: flagKey)! + flag.experimentIds = [experimentRuleId, "10420810910"] + config.project.featureFlags = config.project.featureFlags.map { $0.key == flagKey ? flag : $0 } + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [experimentRuleId])]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId, attributes: ["gender": "f"]) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, "exp_no_audience") + XCTAssertEqual(decision.variationKey, "variation_with_traffic") + XCTAssertTrue(decision.enabled) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 2) + XCTAssertEqual(events.first?.metaData.ruleType, Constants.DecisionSource.featureTest.rawValue) + XCTAssertEqual(events.first?.metaData.ruleKey, "exp_no_audience") + assertHoldoutEvent(events.last) + } + + func testLocalHoldout_HitOnDeliveryRule_ContinuesToNextDeliveryRuleNotEveryoneElse() { + var rollout = config.getRollout(id: config.getFeatureFlag(key: flagKey)!.rolloutId)! + rollout.experiments[1].trafficAllocation[0].endOfRange = 10000 + config.project.rollouts = [rollout] + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [deliveryRuleId])]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId, attributes: ["country": "US", "browser": "safari"]) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, "3332020494") + XCTAssertEqual(decision.variationKey, "3324490562") + XCTAssertTrue(decision.enabled) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 2) + XCTAssertEqual(events.first?.metaData.ruleKey, "3332020494") + assertHoldoutEvent(events.last) + } + + func testLocalHoldout_OneHoldoutTargetingTwoRules_SendsTwoHoldoutEvents() { + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [experimentRuleId, deliveryRuleId])]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId, attributes: ["gender": "f", "country": "US"]) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, everyoneElseRuleKey) + XCTAssertEqual(decision.variationKey, everyoneElseVariationKey) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 3) + XCTAssertEqual(events.first?.metaData.ruleKey, everyoneElseRuleKey) + let holdoutEvents = Array(events.dropFirst()) + XCTAssertEqual(holdoutEvents.count, 2) + holdoutEvents.forEach { assertHoldoutEvent($0) } + } + + func testLocalHoldouts_HitsOnSeveralRules_SendOneHoldoutEventPerHit() { + let first = localHoldout(rules: [experimentRuleId]) + let second = localHoldout(rules: [deliveryRuleId], id: "9999900030", key: "holdout_test_key_2", variationId: "9999900040", variationKey: "holdout_variation_key_2") + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [first, second]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId, attributes: ["gender": "f", "country": "US"]) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, everyoneElseRuleKey) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 3) + assertHoldoutEvent(events.dropFirst().first) + assertHoldoutEvent(events.last, holdoutId: "9999900030", ruleKey: "holdout_test_key_2", variationId: "9999900040", variationKey: "holdout_variation_key_2") + } + + func testLocalHoldout_HitThenNoRuleServes_DecisionUnchangedAndHoldoutEventsSent() { + let allRules = [experimentRuleId, deliveryRuleId, "3332020494", everyoneElseRuleKey] + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: allRules)]) + useMockBucketer(2500) + + let decision = optimizely.createUserContext(userId: userId).decide(key: flagKey) + + XCTAssertNil(decision.variationKey) + XCTAssertNil(decision.ruleKey) + XCTAssertFalse(decision.enabled) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 5) + XCTAssertEqual(events.first?.metaData.ruleType, Constants.DecisionSource.rollout.rawValue) + XCTAssertEqual(events.first?.metaData.ruleKey, "") + XCTAssertEqual(events.first?.metaData.variationKey, "") + let holdoutEvents = Array(events.dropFirst()) + XCTAssertEqual(holdoutEvents.count, 4) + holdoutEvents.forEach { assertHoldoutEvent($0) } + } + + func testLocalHoldout_HitOnDeliveryRule_SendFlagDecisionsOff_HoldoutEventStillSent() { + config.project.sendFlagDecisions = false + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [deliveryRuleId])]) + useMockBucketer(2500) + + let decision = optimizely.createUserContext(userId: userId).decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, everyoneElseRuleKey) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 1) + assertHoldoutEvent(events.first) + } + + func testLocalHoldout_DisableDecisionEvent_SuppressesHoldoutEvents() { + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [experimentRuleId, deliveryRuleId])]) + useMockBucketer(2500) + + let decision = optimizely.createUserContext(userId: userId).decide(key: flagKey, options: [.disableDecisionEvent]) + + XCTAssertEqual(decision.ruleKey, everyoneElseRuleKey) + XCTAssertTrue(dispatchedDecisions().isEmpty) + } + + func testLocalHoldout_HitThenCmabErrorOnLaterRule_ErrorDecisionAndNoEventsSent() { + let cmabRuleId = "10420810910" + var flag = config.getFeatureFlag(key: flagKey)! + flag.experimentIds = [experimentRuleId, cmabRuleId] + config.project.featureFlags = config.project.featureFlags.map { $0.key == flagKey ? flag : $0 } + let cmab: Cmab = try! OTUtils.model(from: ["trafficAllocation": 10000, "attributeIds": []]) + config.project.experiments = config.project.experiments.map { + var exp = $0 + if exp.id == cmabRuleId { exp.cmab = cmab } + return exp + } + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [experimentRuleId])]) + optimizely.decisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), + bucketer: MockBucketer(mockBucketValue: 2500), + cmabService: FailingCmabService()) + + let user = optimizely.createUserContext(userId: userId, attributes: ["gender": "f"]) + let exp = expectation(description: "decideAsync") + var decision: OptimizelyDecision! + user.decideAsync(key: flagKey, options: [.includeReasons]) { + decision = $0 + exp.fulfill() + } + wait(for: [exp], timeout: 5) + + XCTAssertNil(decision.variationKey) + XCTAssertNil(decision.ruleKey) + XCTAssertFalse(decision.enabled) + XCTAssertTrue(decision.reasons.contains(LogMessage.userBucketedIntoVariationInHoldout(userId, "holdout_test_key", "holdout_variation_key").reason)) + XCTAssertTrue(decision.reasons.contains(LogMessage.cmabFetchFailed("exp_no_audience").reason)) + XCTAssertTrue(dispatchedDecisions().isEmpty) + } + + func testRuleForcedDecision_ExperimentRule_BeatsFullTrafficLocalHoldout() { + var holdout = localHoldout(rules: [experimentRuleId]) + holdout.trafficAllocation[0].endOfRange = 10000 + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId) + _ = user.setForcedDecision(context: OptimizelyDecisionContext(flagKey: flagKey, ruleKey: "exp_with_audience"), + decision: OptimizelyForcedDecision(variationKey: "b")) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, "exp_with_audience") + XCTAssertEqual(decision.variationKey, "b") + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.metaData.ruleKey, "exp_with_audience") + XCTAssertFalse(events.contains { $0.metaData.ruleType == Constants.DecisionSource.holdout.rawValue }) + } + + func testRuleForcedDecision_DeliveryRule_BeatsFullTrafficLocalHoldout() { + var holdout = localHoldout(rules: [deliveryRuleId]) + holdout.trafficAllocation[0].endOfRange = 10000 + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [holdout]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId) + _ = user.setForcedDecision(context: OptimizelyDecisionContext(flagKey: flagKey, ruleKey: deliveryRuleId), + decision: OptimizelyForcedDecision(variationKey: "3324490633")) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, deliveryRuleId) + XCTAssertEqual(decision.variationKey, "3324490633") + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.metaData.ruleKey, deliveryRuleId) + XCTAssertFalse(events.contains { $0.metaData.ruleType == Constants.DecisionSource.holdout.rawValue }) + } + + func testStoredGlobalHoldoutPlusLocalHoldoutOnDeliveryRule_LaterRuleServesAndBothHoldoutEventsSent() { + var global = localHoldout(rules: [], id: "9999900050", key: "global_holdout_key", variationId: "9999900060", variationKey: "global_variation_key") + global.includedRules = nil + global.excludeTargetedDeliveries = true + config.holdoutConfig = HoldoutConfig(globalHoldouts: [global], localHoldouts: [localHoldout(rules: [deliveryRuleId])]) + useMockBucketer(2500) + + let user = optimizely.createUserContext(userId: userId, attributes: ["country": "US"]) + let decision = user.decide(key: flagKey) + + XCTAssertEqual(decision.ruleKey, everyoneElseRuleKey) + XCTAssertEqual(decision.variationKey, everyoneElseVariationKey) + XCTAssertTrue(decision.enabled) + + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 3) + XCTAssertEqual(events.first?.metaData.ruleType, Constants.DecisionSource.rollout.rawValue) + XCTAssertEqual(events.first?.metaData.ruleKey, everyoneElseRuleKey) + assertHoldoutEvent(events.dropFirst().first, holdoutId: "9999900050", ruleKey: "global_holdout_key", variationId: "9999900060", variationKey: "global_variation_key") + assertHoldoutEvent(events.last) + } + + // MARK: - Legacy APIs + + func testLegacyIsFeatureEnabled_LocalHoldoutHit_SkipsRuleWithoutHoldoutEvent() { + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: [experimentRuleId])]) + useMockBucketer(2500) + + let enabled = optimizely.isFeatureEnabled(featureKey: flagKey, userId: userId, attributes: ["gender": "f", "country": "US"]) + + XCTAssertTrue(enabled) + let events = dispatchedDecisions() + XCTAssertEqual(events.count, 1) + XCTAssertEqual(events.first?.metaData.ruleType, Constants.DecisionSource.rollout.rawValue) + XCTAssertEqual(events.first?.metaData.ruleKey, deliveryRuleId) + } + + func testLegacyIsFeatureEnabled_AllRulesHeldOut_NoEventSent() { + config.project.sendFlagDecisions = false + let allRules = [experimentRuleId, deliveryRuleId, "3332020494", everyoneElseRuleKey] + config.holdoutConfig = HoldoutConfig(globalHoldouts: [], localHoldouts: [localHoldout(rules: allRules)]) + useMockBucketer(2500) + + XCTAssertFalse(optimizely.isFeatureEnabled(featureKey: flagKey, userId: userId)) + XCTAssertTrue(dispatchedDecisions().isEmpty) + } + + // MARK: - Helpers + + private func localHoldout(rules: [String], + id: String = "9999900010", + key: String = "holdout_test_key", + variationId: String = "9999900020", + variationKey: String = "holdout_variation_key") -> Holdout { + var holdout = try! OTUtils.model(from: sampleHoldout) as Holdout + holdout.id = id + holdout.key = key + holdout.variations[0].id = variationId + holdout.variations[0].key = variationKey + holdout.trafficAllocation[0].entityId = variationId + holdout.includedRules = rules + return holdout + } + + private func useMockBucketer(_ bucketValue: Int) { + optimizely.decisionService = DefaultDecisionService(userProfileService: OTUtils.createClearUserProfileService(), + bucketer: MockBucketer(mockBucketValue: bucketValue)) + } + + private func dispatchedDecisions() -> [Decision] { + optimizely.eventLock.sync {} + return eventDispatcher.events.map { + try! JSONDecoder().decode(BatchEvent.self, from: $0.body).visitors[0].snapshots[0].decisions![0] + } + } + + private func assertHoldoutEvent(_ decision: Decision?, + holdoutId: String = "9999900010", + ruleKey: String = "holdout_test_key", + variationId: String = "9999900020", + variationKey: String = "holdout_variation_key", + file: StaticString = #filePath, + line: UInt = #line) { + XCTAssertEqual(decision?.metaData.ruleType, Constants.DecisionSource.holdout.rawValue, file: file, line: line) + XCTAssertEqual(decision?.metaData.ruleKey, ruleKey, file: file, line: line) + XCTAssertEqual(decision?.metaData.variationKey, variationKey, file: file, line: line) + XCTAssertEqual(decision?.metaData.flagKey, flagKey, file: file, line: line) + XCTAssertEqual(decision?.metaData.enabled, false, file: file, line: line) + XCTAssertEqual(decision?.experimentID, holdoutId, file: file, line: line) + XCTAssertEqual(decision?.variationID, variationId, file: file, line: line) + } + + private func assertHeldOutThenServedByEveryoneElse(_ decision: OptimizelyDecision, file: StaticString = #filePath, line: UInt = #line) { + XCTAssert(decision.reasons.contains(LogMessage.userBucketedIntoVariationInHoldout(userId, "holdout_test_key", "holdout_variation_key").reason), file: file, line: line) + XCTAssertEqual(decision.ruleKey, everyoneElseRuleKey, file: file, line: line) + XCTAssertEqual(decision.variationKey, everyoneElseVariationKey, file: file, line: line) + XCTAssertTrue(decision.enabled, file: file, line: line) + } +} + +private struct FailingCmabService: CmabService { + func getDecision(config: ProjectConfig, userContext: OptimizelyUserContext, ruleId: String, + options: [OptimizelyDecideOption]) -> Result { + return .failure(CmabClientError.fetchFailed("test")) + } + + func getDecision(config: ProjectConfig, userContext: OptimizelyUserContext, ruleId: String, + options: [OptimizelyDecideOption], completion: @escaping CmabDecisionCompletionHandler) { + completion(.failure(CmabClientError.fetchFailed("test"))) + } } diff --git a/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_Holdouts.swift b/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_Holdouts.swift index e6c9922c..9ea421ee 100644 --- a/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_Holdouts.swift +++ b/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_Holdouts.swift @@ -175,10 +175,10 @@ class OptimizelyUserContextTests_Decide_Holdouts: XCTestCase { var user = optimizely.createUserContext(userId: kUserId, attributes: ["gender": "f"]) var decision = user.decide(key: featureKey) - XCTAssert(decision == OptimizelyDecision(variationKey: "key_holdout_variation", + XCTAssert(decision == OptimizelyDecision(variationKey: nil, enabled: false, variables: variablesExpected, - ruleKey: "key_holdout", + ruleKey: nil, flagKey: featureKey, userContext: user, reasons: [])) @@ -213,10 +213,10 @@ class OptimizelyUserContextTests_Decide_Holdouts: XCTestCase { let decision1 = user.decide(key: featureKey1) - XCTAssert(decision1 == OptimizelyDecision(variationKey: "key_holdout_variation", + XCTAssert(decision1 == OptimizelyDecision(variationKey: nil, enabled: false, variables: variablesExpected1, - ruleKey: "key_holdout", + ruleKey: nil, flagKey: featureKey1, userContext: user, reasons: [])) @@ -243,10 +243,10 @@ class OptimizelyUserContextTests_Decide_Holdouts: XCTestCase { XCTAssert(decisions.count == 2) - XCTAssert(decisions[featureKey1]! == OptimizelyDecision(variationKey: "key_holdout_variation", + XCTAssert(decisions[featureKey1]! == OptimizelyDecision(variationKey: nil, enabled: false, variables: variablesExpected1, - ruleKey: "key_holdout", + ruleKey: nil, flagKey: featureKey1, userContext: user, reasons: [])) @@ -336,10 +336,10 @@ extension OptimizelyUserContextTests_Decide_Holdouts { flagKey: featureKey1, userContext: user, reasons: [])) - XCTAssert(decisions[featureKey2]! == OptimizelyDecision(variationKey: "key_holdout_variation", + XCTAssert(decisions[featureKey2]! == OptimizelyDecision(variationKey: nil, enabled: false, variables: variablesExpected2, - ruleKey: "key_holdout", + ruleKey: nil, flagKey: featureKey2, userContext: user, reasons: [])) @@ -375,10 +375,10 @@ extension OptimizelyUserContextTests_Decide_Holdouts { XCTAssert(decisions.count == 3) - XCTAssert(decisions[featureKey1]! == OptimizelyDecision(variationKey: "key_holdout_variation", + XCTAssert(decisions[featureKey1]! == OptimizelyDecision(variationKey: nil, enabled: false, variables: variablesExpected1, - ruleKey: "key_holdout", + ruleKey: nil, flagKey: featureKey1, userContext: user, reasons: [])) @@ -435,10 +435,10 @@ extension OptimizelyUserContextTests_Decide_Holdouts { flagKey: feature1.key, userContext: user, reasons: [])) - XCTAssert(decisions[feature2.key]! == OptimizelyDecision(variationKey: "key_holdout_variation", + XCTAssert(decisions[feature2.key]! == OptimizelyDecision(variationKey: nil, enabled: false, variables: variablesExpected2, - ruleKey: "holdout_key_included", + ruleKey: nil, flagKey: feature2.key, userContext: user, reasons: [])) diff --git a/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift b/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift index 85ab261e..d7f90de6 100644 --- a/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift +++ b/Tests/OptimizelyTests-Common/OptimizelyUserContextTests_Decide_With_Holdouts_Reasons.swift @@ -91,11 +91,13 @@ class OptimizelyUserContextTests_Decide_With_Holdouts_Reasons: XCTestCase { let decision = user.decide(key: featureKey, options: [.includeReasons]) // Assertions XCTAssertEqual(decision.flagKey, "feature_1", "Expected flagKey to be 'feature_1'") - XCTAssertEqual(decision.variationKey, "key_holdout_variation", "Expected variationKey to be 'key_holdout_variation'") + XCTAssertNil(decision.variationKey, "Every rule is held out, so no rule serves") + XCTAssertNil(decision.ruleKey) XCTAssertFalse(decision.enabled, "Feature should be disabled in holdout") XCTAssert(decision.reasons.contains(LogMessage.userBucketedIntoVariationInHoldout(kUserId, "key_holdout", "key_holdout_variation").reason)) + XCTAssert(decision.reasons.contains(LogMessage.userNotBucketedIntoRollout(kUserId, featureKey).reason)) } - + /// Test when user is not bucketed into any holdout for feature_2 (excluded) func testDecideReasons_userNotBucketedIntoExcludedHoldout() { // Global holdout with 5% traffice