Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 36 additions & 18 deletions Sources/Implementation/DefaultDecisionService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,11 @@ struct HoldoutDecision {
let experiment: ExperimentCore
let variation: Variation

init(experiment: ExperimentCore, variation: Variation) {
self.experiment = experiment
self.variation = variation
}

init?(from featureDecision: FeatureDecision?) {
guard let experiment = featureDecision?.experiment,
let variation = featureDecision?.variation else { return nil }
Expand All @@ -34,7 +39,7 @@ struct FeatureDecision {
let source: String
var cmabUUID: String?
var error = false
var holdoutDecision: HoldoutDecision? = nil
var holdoutDecisions: [HoldoutDecision] = []
}

struct VariationDecision {
Expand Down Expand Up @@ -436,21 +441,27 @@ class DefaultDecisionService: OPTDecisionService {
}
}

var holdoutDecisions = [HoldoutDecision(from: storedHoldoutDecision)].compactMap { $0 }

if storedHoldoutDecision == nil {
let flagExpDecision = getVariationForFeatureExperiments(config: config, featureFlag: featureFlag, user: user, userProfileTracker: userProfileTracker, isAsync: isAsync, options: options)

reasons.merge(flagExpDecision.reasons)

if let decision = flagExpDecision.result {
return DecisionResponse(result: decision, reasons: reasons)
if decision.variation != nil || decision.error {
return DecisionResponse(result: decision, reasons: reasons)
}
holdoutDecisions += decision.holdoutDecisions
}
}

let rolloutDecision = getVariationForFeatureRollout(config: config, featureFlag: featureFlag, user: user, options: options)
reasons.merge(rolloutDecision.reasons)
holdoutDecisions += rolloutDecision.result?.holdoutDecisions ?? []

if var decision: FeatureDecision = rolloutDecision.result {
decision.holdoutDecision = HoldoutDecision(from: storedHoldoutDecision)
if var decision: FeatureDecision = rolloutDecision.result, decision.variation != nil {
decision.holdoutDecisions = holdoutDecisions
let info = LogMessage.userBucketedIntoRollout(user.userId, featureFlag.key)
logger.i(info)
reasons.addInfo(info)
Expand All @@ -459,9 +470,9 @@ class DefaultDecisionService: OPTDecisionService {
let info = LogMessage.userNotBucketedIntoRollout(user.userId, featureFlag.key)
logger.i(info)
reasons.addInfo(info)
if let holdout = HoldoutDecision(from: storedHoldoutDecision) {
if !holdoutDecisions.isEmpty {
var emptyDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.rollout.rawValue)
emptyDecision.holdoutDecision = holdout
emptyDecision.holdoutDecisions = holdoutDecisions
return DecisionResponse(result: emptyDecision, reasons: reasons)
}
return DecisionResponse(result: nil, reasons: reasons)
Expand Down Expand Up @@ -492,6 +503,8 @@ class DefaultDecisionService: OPTDecisionService {
reasons.addInfo(info)
}

var holdoutDecisions = [HoldoutDecision]()

// Check if there are any experiment IDs inside feature flag
// Evaluate each experiment ID and return the first bucketed experiment variation
for experimentId in experimentIds {
Expand All @@ -508,22 +521,24 @@ class DefaultDecisionService: OPTDecisionService {
if result.cmabError {
// For CMAB - we're supposed to get decision from the server.
// If failed, return decision with nil variation, so the client can take care of them.
let featureDecision = FeatureDecision(experiment: experiment, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, error: true)
let featureDecision = FeatureDecision(experiment: experiment, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, error: true, holdoutDecisions: holdoutDecisions)
return DecisionResponse(result: featureDecision, reasons: reasons)
} else if let variation = result.variation {
// Check if this variation came from a holdout
if let holdout = result.holdout {
let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue, cmabUUID: result.cmabUUID)
return DecisionResponse(result: featureDecision, reasons: reasons)
holdoutDecisions.append(HoldoutDecision(experiment: holdout, variation: variation))
} else {
let featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID)
let featureDecision = FeatureDecision(experiment: experiment, variation: variation, source: Constants.DecisionSource.featureTest.rawValue, cmabUUID: result.cmabUUID, holdoutDecisions: holdoutDecisions)
return DecisionResponse(result: featureDecision, reasons: reasons)
}
}
}
}
}

if !holdoutDecisions.isEmpty {
let featureDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.featureTest.rawValue, holdoutDecisions: holdoutDecisions)
return DecisionResponse(result: featureDecision, reasons: reasons)
}
return DecisionResponse(result: nil, reasons: reasons)
}

Expand Down Expand Up @@ -564,6 +579,7 @@ class DefaultDecisionService: OPTDecisionService {
return DecisionResponse(result: nil, reasons: reasons)
}

var holdoutDecisions = [HoldoutDecision]()
var index = 0
while index < rolloutRules.count {
let decisionResponse = getVariationFromDeliveryRule(config: config,
Expand All @@ -576,21 +592,23 @@ class DefaultDecisionService: OPTDecisionService {
let result = decisionResponse.result!

if let variation = result.variation {
let rule = rolloutRules[index]
// Check if this variation came from a holdout
if let holdout = result.holdout {
let featureDecision = FeatureDecision(experiment: holdout, variation: variation, source: Constants.DecisionSource.holdout.rawValue)
return DecisionResponse(result: featureDecision, reasons: reasons)
} else {
let featureDecision = FeatureDecision(experiment: rule, variation: variation, source: Constants.DecisionSource.rollout.rawValue)
return DecisionResponse(result: featureDecision, reasons: reasons)
holdoutDecisions.append(HoldoutDecision(experiment: holdout, variation: variation))
index += 1
continue
}
let featureDecision = FeatureDecision(experiment: rolloutRules[index], variation: variation, source: Constants.DecisionSource.rollout.rawValue, holdoutDecisions: holdoutDecisions)
return DecisionResponse(result: featureDecision, reasons: reasons)
}

// the last rule is special for "Everyone Else"
index = result.skipToEveryoneElse ? (rolloutRules.count - 1) : (index + 1)
}

if !holdoutDecisions.isEmpty {
let featureDecision = FeatureDecision(experiment: nil, variation: nil, source: Constants.DecisionSource.rollout.rawValue, holdoutDecisions: holdoutDecisions)
return DecisionResponse(result: featureDecision, reasons: reasons)
}
return DecisionResponse(result: nil, reasons: reasons)
}

Expand Down
2 changes: 1 addition & 1 deletion Sources/Optimizely+Decide/OptimizelyClient+Decide.swift
Original file line number Diff line number Diff line change
Expand Up @@ -324,7 +324,7 @@ extension OptimizelyClient {
cmabUUID: flagDecision?.cmabUUID)
decisionEventDispatched = true
}
if let holdoutInfo = flagDecision?.holdoutDecision {
for holdoutInfo in flagDecision?.holdoutDecisions ?? [] {
sendImpressionEvent(experiment: holdoutInfo.experiment,
variation: holdoutInfo.variation,
userId: userId,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -531,7 +531,7 @@ extension BatchEventBuilderTests_Events {

func testImpressionEvent_UserInHoldout_IncludedFlags() {
let eventDispatcher2 = MockEventDispatcher()
var optimizely: OptimizelyClient! = OptimizelyClient(sdkKey: "12345", eventDispatcher: eventDispatcher2)
var optimizely: OptimizelyClient! = OptimizelyClient(sdkKey: OTUtils.randomSdkKey, eventDispatcher: eventDispatcher2)

try! optimizely.start(datafile: datafile)

Expand All @@ -552,11 +552,14 @@ extension BatchEventBuilderTests_Events {

let result = XCTWaiter.wait(for: [exp], timeout: 0.2)
if result == XCTWaiter.Result.completed {
let event = getFirstEventJSON(client: optimizely)!
optimizely.eventLock.sync {}
// No rule serves and sendFlagDecisions is off, so only the holdout event is sent.
XCTAssertEqual(eventDispatcher2.events.count, 1)
let event = getFirstEventJSON(dispatcher: eventDispatcher2)!
let visitor = (event["visitors"] as! Array<Dictionary<String, Any>>)[0]
let snapshot = (visitor["snapshots"] as! Array<Dictionary<String, Any>>)[0]
let decision = (snapshot["decisions"] as! Array<Dictionary<String, Any>>)[0]

let metaData = decision["metadata"] as! Dictionary<String, Any>
XCTAssertEqual(metaData["rule_type"] as! String, Constants.DecisionSource.holdout.rawValue)
XCTAssertEqual(metaData["rule_key"] as! String, "holdout_key")
Expand Down
10 changes: 6 additions & 4 deletions Tests/OptimizelyTests-Common/DecisionListenerTest_Holdouts.swift
Original file line number Diff line number Diff line change
Expand Up @@ -132,8 +132,9 @@ class DecisionListenerTests_Holdouts: XCTestCase {
XCTAssertEqual(type, Constants.DecisionType.flag.rawValue)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.flagKey] as! String, self.kFeatureKey)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.enabled] as! Bool, false)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.variationKey] as! String, "key_holdout_variation")
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.ruleKey] as! String, "key_holdout")
XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.variationKey] is NSNull)
XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.ruleKey] is NSNull)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.decisionEventDispatched] as! Bool, true)
exp.fulfill()
}

Expand Down Expand Up @@ -183,8 +184,9 @@ class DecisionListenerTests_Holdouts: XCTestCase {
XCTAssertEqual(type, Constants.DecisionType.flag.rawValue)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.flagKey] as! String, self.kFeatureKey)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.enabled] as! Bool, false)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.variationKey] as! String, "key_holdout_variation")
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.ruleKey] as! String, "holdout_key_2")
XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.variationKey] is NSNull)
XCTAssertTrue(decisionInfo[Constants.DecisionInfoKeys.ruleKey] is NSNull)
XCTAssertEqual(decisionInfo[Constants.DecisionInfoKeys.decisionEventDispatched] as! Bool, true)
exp.fulfill()
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
optimizely.decisionService = mockDecisionService

let user = optimizely.createUserContext(userId: userId)
let decision = user.decide(key: flagKey)
let decision = user.decide(key: flagKey, options: [.includeReasons])

XCTAssertEqual(decision.variationKey, "holdout_variation_key")
XCTAssertFalse(decision.enabled)
assertHeldOutThenServedByEveryoneElse(decision)
}

// MARK: - excludeTargetedDeliveries = true with TD rule
Expand All @@ -92,10 +91,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
optimizely.decisionService = mockDecisionService

let user = optimizely.createUserContext(userId: userId)
let decision = user.decide(key: flagKey)
let decision = user.decide(key: flagKey, options: [.includeReasons])

XCTAssertEqual(decision.variationKey, "holdout_variation_key")
XCTAssertFalse(decision.enabled)
assertHeldOutThenServedByEveryoneElse(decision)
}

// MARK: - excludeTargetedDeliveries = true with AB rule
Expand All @@ -115,10 +113,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
optimizely.decisionService = mockDecisionService

let user = optimizely.createUserContext(userId: userId)
let decision = user.decide(key: flagKey)
let decision = user.decide(key: flagKey, options: [.includeReasons])

XCTAssertEqual(decision.variationKey, "holdout_variation_key")
XCTAssertFalse(decision.enabled)
assertHeldOutThenServedByEveryoneElse(decision)
}

// MARK: - Backward compatibility (field missing from JSON)
Expand Down Expand Up @@ -148,10 +145,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
optimizely.decisionService = mockDecisionService

let user = optimizely.createUserContext(userId: userId)
let decision = user.decide(key: flagKey)
let decision = user.decide(key: flagKey, options: [.includeReasons])

XCTAssertEqual(decision.variationKey, "holdout_variation_key")
XCTAssertFalse(decision.enabled)
assertHeldOutThenServedByEveryoneElse(decision)
}

// MARK: - Global holdout with excludeTargetedDeliveries true and TD rule
Expand Down Expand Up @@ -196,10 +192,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
optimizely.decisionService = mockDecisionService

let user = optimizely.createUserContext(userId: userId)
let decision = user.decide(key: flagKey)
let decision = user.decide(key: flagKey, options: [.includeReasons])

XCTAssertEqual(decision.variationKey, "holdout_variation_key")
XCTAssertFalse(decision.enabled)
assertHeldOutThenServedByEveryoneElse(decision)
}

// MARK: - Forced decision beats 100% traffic local holdout
Expand Down Expand Up @@ -278,10 +273,10 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
// Delivery (rollout) rules are evaluated — Everyone Else matches
XCTAssertNotNil(response.result?.variation)
XCTAssertEqual(response.result?.source, Constants.DecisionSource.rollout.rawValue)
// Holdout is queued so its impression fires via holdoutDecision
XCTAssertNotNil(response.result?.holdoutDecision)
XCTAssertEqual(response.result?.holdoutDecision?.experiment.key, "holdout_test_key")
XCTAssertEqual(response.result?.holdoutDecision?.variation.key, "holdout_variation_key")
// Holdout is queued so its impression fires via holdoutDecisions
XCTAssertEqual(response.result?.holdoutDecisions.count, 1)
XCTAssertEqual(response.result?.holdoutDecisions.first?.experiment.key, "holdout_test_key")
XCTAssertEqual(response.result?.holdoutDecisions.first?.variation.key, "holdout_variation_key")
}

// MARK: - Local holdout with excludeTrue still applies
Expand All @@ -301,10 +296,9 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
optimizely.decisionService = mockDecisionService

let user = optimizely.createUserContext(userId: userId)
let decision = user.decide(key: flagKey)
let decision = user.decide(key: flagKey, options: [.includeReasons])

XCTAssertEqual(decision.variationKey, "holdout_variation_key")
XCTAssertFalse(decision.enabled)
assertHeldOutThenServedByEveryoneElse(decision)
}

// MARK: - Decision reason for excludeTargetedDeliveries bypass
Expand Down Expand Up @@ -363,11 +357,11 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
wait(for: [exp], timeout: 1)
}

// MARK: - Global holdout excludeTrue, TD matches => holdoutDecision populated
// MARK: - Global holdout excludeTrue, TD matches => holdoutDecisions populated

// When excludeTargetedDeliveries=true, experiment rules are skipped and the
// decision comes from the delivery (rollout) path. The holdout is queued via
// holdoutDecision so its impression fires alongside the rollout impression.
// holdoutDecisions so its impression fires alongside the rollout impression.
func testGlobalHoldout_ExcludeTrue_HoldoutEventSent() {
var experiment = config.getExperiment(id: experimentRuleId)!
experiment.type = .targetedDelivery
Expand All @@ -393,8 +387,15 @@ class DecisionServiceTests_ExcludeTargetedDeliveries: XCTestCase {
XCTAssertNotNil(response.result)
// Experiments are skipped; decision comes from delivery (rollout) rules
XCTAssertEqual(response.result?.source, Constants.DecisionSource.rollout.rawValue)
XCTAssertNotNil(response.result?.holdoutDecision)
XCTAssertEqual(response.result?.holdoutDecision?.experiment.key, "holdout_test_key")
XCTAssertEqual(response.result?.holdoutDecision?.variation.key, "holdout_variation_key")
XCTAssertEqual(response.result?.holdoutDecisions.count, 1)
XCTAssertEqual(response.result?.holdoutDecisions.first?.experiment.key, "holdout_test_key")
XCTAssertEqual(response.result?.holdoutDecisions.first?.variation.key, "holdout_variation_key")
}

private func assertHeldOutThenServedByEveryoneElse(_ decision: OptimizelyDecision, file: StaticString = #filePath, line: UInt = #line) {
XCTAssert(decision.reasons.contains(LogMessage.userBucketedIntoVariationInHoldout(userId, "holdout_test_key", "holdout_variation_key").reason), file: file, line: line)
XCTAssertEqual(decision.ruleKey, "18322080788", file: file, line: line)
XCTAssertEqual(decision.variationKey, "18257766532", file: file, line: line)
XCTAssertTrue(decision.enabled, file: file, line: line)
}
}
Loading
Loading