@@ -29,10 +29,6 @@ and properties are also given in this file.
2929
3030 ## Main statements
3131
32- * `IsBayesAlgEnvSeq.prob_empMean_sub_actionMean_ge_le`,
33- `IsBayesAlgEnvSeq.prob_empMean_bestAction_sub_actionMean_le_le`: for sub-Gaussian rewards, the
34- empirical means of the actions are close to their means with high probability, uniformly in
35- time.
3632* `TS.integral_regret_le`: if Thompson sampling has the correct prior over environments, in every
3733 environment each of the `K` actions has a mean reward between `l` and `u`, and in almost every
3834 environment (under the prior) the reward distribution of each action is sub-Gaussian with
@@ -46,7 +42,9 @@ and the best action have the same conditional law given the history, the upper c
4642of the two actions have the same expectation, and the Bayesian regret is the sum of the expected
4743differences between the mean of the best action and its upper confidence bound, and between the
4844upper confidence bound and the mean of the action played (`TS.integral_regret_eq_add`). The first
49- sum is controlled by concentration, the second one by the widths of the confidence intervals.
45+ sum is controlled by concentration
46+ (`IsBayesAlgEnvSeq.prob_empMean_bestAction_sub_actionMean_le_le`), the second one by the widths of
47+ the confidence intervals.
5048
5149## References
5250
@@ -62,105 +60,6 @@ sum is controlled by concentration, the second one by the widths of the confiden
6260open MeasureTheory ProbabilityTheory Finset Learning
6361open scoped NNReal
6462
65- /-! ### Concentration of the empirical means in a Bayesian bandit -/
66-
67- namespace Learning.IsBayesAlgEnvSeq
68-
69- variable {𝓔 Ω : Type *} [MeasurableSpace 𝓔] [MeasurableSpace Ω]
70- variable {K : ℕ} [Nonempty (Fin K)]
71- variable {Q : Measure 𝓔} [IsProbabilityMeasure Q] {κ : Kernel (𝓔 × Fin K) ℝ} [IsMarkovKernel κ]
72- variable {alg : Algorithm Unit (Fin K) ℝ}
73- variable {E : Ω → 𝓔} {A : ℕ → Ω → (Fin K)} {R : ℕ → Ω → ℝ}
74- variable {P : Measure Ω} [IsProbabilityMeasure P]
75-
76- /-- Auxiliary lemma for `prob_empMean_sub_actionMean_ge_le`. -/
77- private lemma sqrt_two_mul_le_sub {k : ℕ} (hk : k ≠ 0 ) {s μ σ l : ℝ}
78- (h : √(2 * σ * l / k) ≤ s / k - μ) : √(2 * k * σ * l) ≤ s - k * μ := by
79- have hkp : (0 : ℝ) < k := by positivity
80- calc √(2 * k * σ * l)
81- _ = √(2 * σ * l / k * k ^ 2 ) := by
82- field_simp
83- _ = √(2 * σ * l / k) * k := by
84- rw [Real.sqrt_mul' _ (sq_nonneg _), Real.sqrt_sq hkp.le]
85- _ ≤ (s / k - μ) * k := by
86- nlinarith
87- _ = s - k * μ := by
88- field_simp
89-
90- lemma prob_empMean_sub_actionMean_ge_le (h : IsBayesAlgEnvSeq Q κ alg E A R P) {σ2 : ℝ≥0 }
91- (hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
92- (hσ2 : σ2 ≠ 0 ) {δ : ℝ} (hδ : 0 < δ) (n : ℕ) :
93- P {ω | ∃ t < n, ∃ a, pullCount A a t ω ≠ 0 ∧
94- √(2 * σ2 * Real.log (1 / δ) / pullCount A a t ω) ≤ empMean A R a t ω - actionMean κ E a ω}
95- ≤ ENNReal.ofReal (K * (n - 1 ) * δ) := by
96- have := h.measurable_param
97- have := h.measurable_action
98- have := h.measurable_feedback
99- let S : Set (𝓔 × (ℕ → Round Unit (Fin K) ℝ)) :=
100- {(e, τ) | ∃ a, ∃ t < n, pullCount IT.action a t τ ≠ 0 ∧
101- √(2 * pullCount IT.action a t τ * σ2 * Real.log (1 / δ)) ≤
102- sumRewards IT.action IT.feedback a t τ - pullCount IT.action a t τ * actionMean κ id a e}
103- calc
104- _ ≤ (P.map (fun ω ↦ (E ω, trajectory (noObs Ω) A R ω))) S := by
105- rw [Measure.map_apply (by fun_prop) (by measurability)]
106- apply measure_mono
107- intro ω ⟨t, ht, a, hpc, hle⟩
108- rw [empMean] at hle
109- exact ⟨a, t, ht, hpc, sqrt_two_mul_le_sub hpc hle⟩
110- _ = (P.map E ⊗ₘ condDistrib (trajectory (noObs Ω) A R) E P) S := by
111- rw [← compProd_map_condDistrib (by fun_prop) (by fun_prop)]
112- _ = ∫⁻ e, condDistrib (trajectory (noObs Ω) A R) E P e (Prod.mk e ⁻¹' S) ∂(P.map E) :=
113- Measure.compProd_apply (by measurability)
114- _ ≤ ∫⁻ e, ENNReal.ofReal (Fintype.card (Fin K) * (n - 1 ) * δ) ∂(P.map E) := by
115- apply lintegral_mono_ae
116- rw [h.hasLaw_env.map_eq]
117- filter_upwards [h.ae_isAlgEnvSeq, hs] with e he hse
118- exact Bandits.prob_exists_sumRewards_sub_pullCount_mul_ge_le he hse hσ2 hδ
119- _ = ENNReal.ofReal (K * (n - 1 ) * δ) := by
120- simp [Measure.map_apply h.measurable_param]
121-
122- /-- Auxiliary lemma for `prob_empMean_bestAction_sub_actionMean_le_le`. -/
123- private lemma sub_le_neg_sqrt_two_mul {k : ℕ} (hk : k ≠ 0 ) {s μ σ l : ℝ}
124- (h : s / k - μ ≤ -√(2 * σ * l / k)) : s - k * μ ≤ -√(2 * k * σ * l) := by
125- have : √(2 * k * σ * l) ≤ -s - k * -μ := sqrt_two_mul_le_sub hk (by grind)
126- linarith
127-
128- lemma prob_empMean_bestAction_sub_actionMean_le_le (h : IsBayesAlgEnvSeq Q κ alg E A R P)
129- {σ2 : ℝ≥0 } (hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
130- (hσ2 : σ2 ≠ 0 ) {δ : ℝ} (hδ : 0 < δ) (n : ℕ) :
131- P {ω | ∃ t < n, pullCount A (bestAction κ E ω) t ω ≠ 0 ∧
132- empMean A R (bestAction κ E ω) t ω - actionMean κ E (bestAction κ E ω) ω ≤
133- -√(2 * σ2 * Real.log (1 / δ) / (pullCount A (bestAction κ E ω) t ω))}
134- ≤ ENNReal.ofReal ((n - 1 ) * δ) := by
135- have := h.measurable_param
136- have := h.measurable_action
137- have := h.measurable_feedback
138- let S : Set (𝓔 × (ℕ → Round Unit (Fin K) ℝ)) :=
139- {(e, τ) | ∃ t < n, pullCount IT.action (bestAction κ id e) t τ ≠ 0 ∧
140- sumRewards IT.action IT.feedback (bestAction κ id e) t τ -
141- pullCount IT.action (bestAction κ id e) t τ * actionMean κ id (bestAction κ id e) e ≤
142- -√(2 * pullCount IT.action (bestAction κ id e) t τ * σ2 * Real.log (1 / δ))}
143- calc
144- _ ≤ (P.map (fun ω ↦ (E ω, trajectory (noObs Ω) A R ω))) S := by
145- rw [Measure.map_apply (by fun_prop) (by measurability)]
146- apply measure_mono
147- intro ω ⟨t, ht, hpc, hle⟩
148- rw [empMean] at hle
149- exact ⟨t, ht, hpc, sub_le_neg_sqrt_two_mul hpc hle⟩
150- _ = (P.map E ⊗ₘ condDistrib (trajectory (noObs Ω) A R) E P) S := by
151- rw [← compProd_map_condDistrib (by fun_prop) (by fun_prop)]
152- _ = ∫⁻ e, condDistrib (trajectory (noObs Ω) A R) E P e (Prod.mk e ⁻¹' S) ∂(P.map E) :=
153- Measure.compProd_apply (by measurability)
154- _ ≤ ∫⁻ e, ENNReal.ofReal ((n - 1 ) * δ) ∂(P.map E) := by
155- apply lintegral_mono_ae
156- rw [h.hasLaw_env.map_eq]
157- filter_upwards [h.ae_isAlgEnvSeq, hs] with e he hse
158- exact Bandits.prob_sumRewards_sub_pullCount_mul_le_le (ν := κ.sectR e) he (hse _) hσ2 hδ
159- _ = ENNReal.ofReal ((n - 1 ) * δ) := by
160- simp [Measure.map_apply h.measurable_param]
161-
162- end Learning.IsBayesAlgEnvSeq
163-
16463open IsBayesAlgEnvSeq (bestAction actionMean)
16564
16665namespace Bandits
@@ -437,7 +336,8 @@ lemma integral_sum_range_ucb_action_sub_actionMean_action_le {alg : Algorithm Un
437336 gcongr
438337 · have : (0 : ℝ) ≤ n - 1 := by simp [Nat.one_le_iff_ne_zero, hn]
439338 apply ENNReal.toReal_le_of_le_ofReal (by positivity)
440- exact h.prob_empMean_sub_actionMean_ge_le hs hσ2 hδ n
339+ exact (h.prob_empMean_sub_actionMean_ge_le hs hσ2 hδ n).trans_eq
340+ (by rw [Fintype.card_fin])
441341 · exact measureReal_le_one
442342 _ = _ := by
443343 ring
@@ -516,12 +416,8 @@ lemma integral_regret_eq_add (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R
516416 _ = _ := by
517417 rw [← integral_finsetSum _ (by fun_prop), ← integral_finsetSum _ (by fun_prop)]
518418
519- /-- **Bayesian regret of Thompson sampling.** If Thompson sampling has the correct prior over
520- environments, in every environment each of the `K` actions has a mean reward between `l` and `u`,
521- and in almost every environment (under the prior) the reward distribution of each action is
522- sub-Gaussian with variance proxy `σ2` after its mean is subtracted, then the Bayesian regret at
523- time `n` is at most `(2 * K + 1) * (u - l) + 8 * √(σ2 * K * n * Real.log n)`. -/
524- theorem integral_regret_le (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P)
419+ /-- The bound of `integral_regret_le` for a nonzero variance proxy `σ2`. -/
420+ private lemma integral_regret_le_of_ne_zero (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P)
525421 (hm : ∀ e a, (κ (e, a))[id] ∈ (Set.Icc l u))
526422 (hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
527423 (hσ2 : σ2 ≠ 0 ) (n : ℕ) :
@@ -555,6 +451,27 @@ theorem integral_regret_le (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P
555451 _ = _ := by
556452 ring
557453
454+ open Filter Topology in
455+ /-- **Bayesian regret of Thompson sampling.** If Thompson sampling has the correct prior over
456+ environments, in every environment each of the `K` actions has a mean reward between `l` and `u`,
457+ and in almost every environment (under the prior) the reward distribution of each action is
458+ sub-Gaussian with variance proxy `σ2` after its mean is subtracted, then the Bayesian regret at
459+ time `n` is at most `(2 * K + 1) * (u - l) + 8 * √(σ2 * K * n * Real.log n)`. -/
460+ theorem integral_regret_le (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P)
461+ (hm : ∀ e a, (κ (e, a))[id] ∈ (Set.Icc l u))
462+ (hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
463+ (n : ℕ) :
464+ P[IsBayesAlgEnvSeq.regret κ E A n]
465+ ≤ (2 * K + 1 ) * (u - l) + 8 * √(σ2 * K * n * Real.log n) := by
466+ -- The bound holds for every variance proxy `σ2' > σ2`, which is nonzero, and is continuous in
467+ -- the variance proxy: let `σ2'` tend to `σ2`.
468+ have h_cont : Continuous fun σ2 ' : ℝ≥0 ↦
469+ (2 * K + 1 ) * (u - l) + 8 * √(σ2 ' * K * n * Real.log n) := by fun_prop
470+ refine ge_of_tendsto (x := 𝓝[>] σ2 ) ((h_cont.tendsto σ2 ).mono_left nhdsWithin_le_nhds) ?_
471+ filter_upwards [self_mem_nhdsWithin] with σ2 ' hσ2 '
472+ exact integral_regret_le_of_ne_zero h hm (hs.mono fun e he a ↦ (he a).mono hσ2 '.le)
473+ (pos_of_gt (show σ2 < σ2 ' from hσ2 ')).ne' n
474+
558475end TS
559476
560477end Bandits
0 commit comments