Skip to content

Commit 61cd177

Browse files
authored
Remove concentration duplication (#289)
2 parents 4052810 + 7962935 commit 61cd177

5 files changed

Lines changed: 407 additions & 491 deletions

File tree

‎LeanMachineLearning/ForMathlib/Probability/Moments/SubGaussian.lean‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,9 @@ public import Mathlib.Probability.Moments.SubGaussian
1111
1212
## Main statements
1313
14+
* `Kernel.HasSubgaussianMGF.mono`, `HasCondSubgaussianMGF.mono`, `HasSubgaussianMGF.mono`: a
15+
sub-Gaussian moment-generating function with parameter `c` is sub-Gaussian with any larger
16+
parameter.
1417
* `HasSubgaussianMGF.measure_le_le`, `HasSubgaussianMGF.measure_sum_le_le_of_iIndepFun`: lower
1518
tail bounds (**Hoeffding's inequality** for the lower tail).
1619
* `HasSubgaussianMGF.measure_sum_le_sum_le`: bound on the probability that a sum of independent
@@ -24,11 +27,34 @@ open scoped ENNReal NNReal
2427

2528
namespace ProbabilityTheory
2629

30+
namespace Kernel.HasSubgaussianMGF
31+
32+
variable {Ω Ω' : Type*} {mΩ : MeasurableSpace Ω} {mΩ' : MeasurableSpace Ω'} {ν : Measure Ω'}
33+
{κ : Kernel Ω' Ω} {X : Ω → ℝ} {c c' : ℝ≥0}
34+
35+
/-- A random variable with a sub-Gaussian moment-generating function with parameter `c` also has
36+
a sub-Gaussian moment-generating function with any larger parameter. -/
37+
lemma mono (h : Kernel.HasSubgaussianMGF X c κ ν) (hcc' : c ≤ c') :
38+
Kernel.HasSubgaussianMGF X c' κ ν where
39+
integrable_exp_mul := h.integrable_exp_mul
40+
mgf_le := by
41+
filter_upwards [h.mgf_le] with ω' h t
42+
refine (h t).trans ?_
43+
gcongr
44+
45+
end Kernel.HasSubgaussianMGF
46+
2747
namespace HasCondSubgaussianMGF
2848

2949
variable {Ω : Type*} {m mΩ : MeasurableSpace Ω} {hm : m ≤ mΩ} [StandardBorelSpace Ω]
3050
{μ : Measure Ω} [IsFiniteMeasure μ] {X : Ω → ℝ} {c : ℝ≥0}
3151

52+
/-- A random variable with a conditionally sub-Gaussian moment-generating function with parameter
53+
`c` also has a conditionally sub-Gaussian moment-generating function with any larger parameter. -/
54+
lemma mono (h : HasCondSubgaussianMGF m hm X c μ) {c' : ℝ≥0} (hcc' : c ≤ c') :
55+
HasCondSubgaussianMGF m hm X c' μ :=
56+
Kernel.HasSubgaussianMGF.mono h hcc'
57+
3258
lemma ae_trim_condExp_exp_sub_le_one (h : HasCondSubgaussianMGF m hm X c μ) (t : ℝ) :
3359
∀ᵐ ω' ∂(μ.trim hm), (μ[fun ω ↦ exp (t * X ω - c * t ^ 2 / 2)|m]) ω' ≤ 1 := by
3460
have h_le : ∀ᵐ ω' ∂(μ.trim hm), μ[fun ω ↦ exp (t * X ω)|m] ω' ≤ exp (c * t ^ 2 / 2) :=
@@ -71,6 +97,13 @@ namespace HasSubgaussianMGF
7197

7298
variable {Ω : Type*} {m mΩ : MeasurableSpace Ω} {μ : Measure Ω} {X Y : Ω → ℝ} {c cX cY : ℝ≥0}
7399

100+
/-- A random variable with a sub-Gaussian moment-generating function with parameter `c` also has
101+
a sub-Gaussian moment-generating function with any larger parameter. -/
102+
lemma mono (h : HasSubgaussianMGF X c μ) {c' : ℝ≥0} (hcc' : c ≤ c') :
103+
HasSubgaussianMGF X c' μ where
104+
integrable_exp_mul := h.integrable_exp_mul
105+
mgf_le t := (h.mgf_le t).trans (by gcongr)
106+
74107
/-- Chernoff bound on the left tail of a sub-Gaussian random variable. -/
75108
lemma measure_le_le (h : HasSubgaussianMGF X c μ) {ε : ℝ} (hε : 0 ≤ ε) :
76109
μ.real {ω | X ω ≤ -ε} ≤ exp (-ε ^ 2 / (2 * c)) := by

‎LeanMachineLearning/Online/Bandit/Algorithms/Regret/BayesRegretTS.lean‎

Lines changed: 28 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,6 @@ and properties are also given in this file.
2929
3030
## Main statements
3131
32-
* `IsBayesAlgEnvSeq.prob_empMean_sub_actionMean_ge_le`,
33-
`IsBayesAlgEnvSeq.prob_empMean_bestAction_sub_actionMean_le_le`: for sub-Gaussian rewards, the
34-
empirical means of the actions are close to their means with high probability, uniformly in
35-
time.
3632
* `TS.integral_regret_le`: if Thompson sampling has the correct prior over environments, in every
3733
environment each of the `K` actions has a mean reward between `l` and `u`, and in almost every
3834
environment (under the prior) the reward distribution of each action is sub-Gaussian with
@@ -46,7 +42,9 @@ and the best action have the same conditional law given the history, the upper c
4642
of the two actions have the same expectation, and the Bayesian regret is the sum of the expected
4743
differences between the mean of the best action and its upper confidence bound, and between the
4844
upper confidence bound and the mean of the action played (`TS.integral_regret_eq_add`). The first
49-
sum is controlled by concentration, the second one by the widths of the confidence intervals.
45+
sum is controlled by concentration
46+
(`IsBayesAlgEnvSeq.prob_empMean_bestAction_sub_actionMean_le_le`), the second one by the widths of
47+
the confidence intervals.
5048
5149
## References
5250
@@ -62,105 +60,6 @@ sum is controlled by concentration, the second one by the widths of the confiden
6260
open MeasureTheory ProbabilityTheory Finset Learning
6361
open scoped NNReal
6462

65-
/-! ### Concentration of the empirical means in a Bayesian bandit -/
66-
67-
namespace Learning.IsBayesAlgEnvSeq
68-
69-
variable {𝓔 Ω : Type*} [MeasurableSpace 𝓔] [MeasurableSpace Ω]
70-
variable {K : ℕ} [Nonempty (Fin K)]
71-
variable {Q : Measure 𝓔} [IsProbabilityMeasure Q] {κ : Kernel (𝓔 × Fin K) ℝ} [IsMarkovKernel κ]
72-
variable {alg : Algorithm Unit (Fin K) ℝ}
73-
variable {E : Ω → 𝓔} {A : ℕ → Ω → (Fin K)} {R : ℕ → Ω → ℝ}
74-
variable {P : Measure Ω} [IsProbabilityMeasure P]
75-
76-
/-- Auxiliary lemma for `prob_empMean_sub_actionMean_ge_le`. -/
77-
private lemma sqrt_two_mul_le_sub {k : ℕ} (hk : k ≠ 0) {s μ σ l : ℝ}
78-
(h : √(2 * σ * l / k) ≤ s / k - μ) : √(2 * k * σ * l) ≤ s - k * μ := by
79-
have hkp : (0 : ℝ) < k := by positivity
80-
calc √(2 * k * σ * l)
81-
_ = √(2 * σ * l / k * k ^ 2) := by
82-
field_simp
83-
_ = √(2 * σ * l / k) * k := by
84-
rw [Real.sqrt_mul' _ (sq_nonneg _), Real.sqrt_sq hkp.le]
85-
_ ≤ (s / k - μ) * k := by
86-
nlinarith
87-
_ = s - k * μ := by
88-
field_simp
89-
90-
lemma prob_empMean_sub_actionMean_ge_le (h : IsBayesAlgEnvSeq Q κ alg E A R P) {σ2 : ℝ≥0}
91-
(hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
92-
(hσ2 : σ2 ≠ 0) {δ : ℝ} (hδ : 0 < δ) (n : ℕ) :
93-
P {ω | ∃ t < n, ∃ a, pullCount A a t ω ≠ 0 ∧
94-
√(2 * σ2 * Real.log (1 / δ) / pullCount A a t ω) ≤ empMean A R a t ω - actionMean κ E a ω}
95-
≤ ENNReal.ofReal (K * (n - 1) * δ) := by
96-
have := h.measurable_param
97-
have := h.measurable_action
98-
have := h.measurable_feedback
99-
let S : Set (𝓔 × (ℕ → Round Unit (Fin K) ℝ)) :=
100-
{(e, τ) | ∃ a, ∃ t < n, pullCount IT.action a t τ ≠ 0 ∧
101-
√(2 * pullCount IT.action a t τ * σ2 * Real.log (1 / δ)) ≤
102-
sumRewards IT.action IT.feedback a t τ - pullCount IT.action a t τ * actionMean κ id a e}
103-
calc
104-
_ ≤ (P.map (fun ω ↦ (E ω, trajectory (noObs Ω) A R ω))) S := by
105-
rw [Measure.map_apply (by fun_prop) (by measurability)]
106-
apply measure_mono
107-
intro ω ⟨t, ht, a, hpc, hle⟩
108-
rw [empMean] at hle
109-
exact ⟨a, t, ht, hpc, sqrt_two_mul_le_sub hpc hle⟩
110-
_ = (P.map E ⊗ₘ condDistrib (trajectory (noObs Ω) A R) E P) S := by
111-
rw [← compProd_map_condDistrib (by fun_prop) (by fun_prop)]
112-
_ = ∫⁻ e, condDistrib (trajectory (noObs Ω) A R) E P e (Prod.mk e ⁻¹' S) ∂(P.map E) :=
113-
Measure.compProd_apply (by measurability)
114-
_ ≤ ∫⁻ e, ENNReal.ofReal (Fintype.card (Fin K) * (n - 1) * δ) ∂(P.map E) := by
115-
apply lintegral_mono_ae
116-
rw [h.hasLaw_env.map_eq]
117-
filter_upwards [h.ae_isAlgEnvSeq, hs] with e he hse
118-
exact Bandits.prob_exists_sumRewards_sub_pullCount_mul_ge_le he hse hσ2 hδ
119-
_ = ENNReal.ofReal (K * (n - 1) * δ) := by
120-
simp [Measure.map_apply h.measurable_param]
121-
122-
/-- Auxiliary lemma for `prob_empMean_bestAction_sub_actionMean_le_le`. -/
123-
private lemma sub_le_neg_sqrt_two_mul {k : ℕ} (hk : k ≠ 0) {s μ σ l : ℝ}
124-
(h : s / k - μ ≤ -√(2 * σ * l / k)) : s - k * μ ≤ -√(2 * k * σ * l) := by
125-
have : √(2 * k * σ * l) ≤ -s - k * -μ := sqrt_two_mul_le_sub hk (by grind)
126-
linarith
127-
128-
lemma prob_empMean_bestAction_sub_actionMean_le_le (h : IsBayesAlgEnvSeq Q κ alg E A R P)
129-
{σ2 : ℝ≥0} (hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
130-
(hσ2 : σ2 ≠ 0) {δ : ℝ} (hδ : 0 < δ) (n : ℕ) :
131-
P {ω | ∃ t < n, pullCount A (bestAction κ E ω) t ω ≠ 0 ∧
132-
empMean A R (bestAction κ E ω) t ω - actionMean κ E (bestAction κ E ω) ω ≤
133-
-√(2 * σ2 * Real.log (1 / δ) / (pullCount A (bestAction κ E ω) t ω))}
134-
≤ ENNReal.ofReal ((n - 1) * δ) := by
135-
have := h.measurable_param
136-
have := h.measurable_action
137-
have := h.measurable_feedback
138-
let S : Set (𝓔 × (ℕ → Round Unit (Fin K) ℝ)) :=
139-
{(e, τ) | ∃ t < n, pullCount IT.action (bestAction κ id e) t τ ≠ 0 ∧
140-
sumRewards IT.action IT.feedback (bestAction κ id e) t τ -
141-
pullCount IT.action (bestAction κ id e) t τ * actionMean κ id (bestAction κ id e) e ≤
142-
-√(2 * pullCount IT.action (bestAction κ id e) t τ * σ2 * Real.log (1 / δ))}
143-
calc
144-
_ ≤ (P.map (fun ω ↦ (E ω, trajectory (noObs Ω) A R ω))) S := by
145-
rw [Measure.map_apply (by fun_prop) (by measurability)]
146-
apply measure_mono
147-
intro ω ⟨t, ht, hpc, hle⟩
148-
rw [empMean] at hle
149-
exact ⟨t, ht, hpc, sub_le_neg_sqrt_two_mul hpc hle⟩
150-
_ = (P.map E ⊗ₘ condDistrib (trajectory (noObs Ω) A R) E P) S := by
151-
rw [← compProd_map_condDistrib (by fun_prop) (by fun_prop)]
152-
_ = ∫⁻ e, condDistrib (trajectory (noObs Ω) A R) E P e (Prod.mk e ⁻¹' S) ∂(P.map E) :=
153-
Measure.compProd_apply (by measurability)
154-
_ ≤ ∫⁻ e, ENNReal.ofReal ((n - 1) * δ) ∂(P.map E) := by
155-
apply lintegral_mono_ae
156-
rw [h.hasLaw_env.map_eq]
157-
filter_upwards [h.ae_isAlgEnvSeq, hs] with e he hse
158-
exact Bandits.prob_sumRewards_sub_pullCount_mul_le_le (ν := κ.sectR e) he (hse _) hσ2 hδ
159-
_ = ENNReal.ofReal ((n - 1) * δ) := by
160-
simp [Measure.map_apply h.measurable_param]
161-
162-
end Learning.IsBayesAlgEnvSeq
163-
16463
open IsBayesAlgEnvSeq (bestAction actionMean)
16564

16665
namespace Bandits
@@ -437,7 +336,8 @@ lemma integral_sum_range_ucb_action_sub_actionMean_action_le {alg : Algorithm Un
437336
gcongr
438337
· have : (0 : ℝ) ≤ n - 1 := by simp [Nat.one_le_iff_ne_zero, hn]
439338
apply ENNReal.toReal_le_of_le_ofReal (by positivity)
440-
exact h.prob_empMean_sub_actionMean_ge_le hs hσ2 hδ n
339+
exact (h.prob_empMean_sub_actionMean_ge_le hs hσ2 hδ n).trans_eq
340+
(by rw [Fintype.card_fin])
441341
· exact measureReal_le_one
442342
_ = _ := by
443343
ring
@@ -516,12 +416,8 @@ lemma integral_regret_eq_add (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R
516416
_ = _ := by
517417
rw [← integral_finsetSum _ (by fun_prop), ← integral_finsetSum _ (by fun_prop)]
518418

519-
/-- **Bayesian regret of Thompson sampling.** If Thompson sampling has the correct prior over
520-
environments, in every environment each of the `K` actions has a mean reward between `l` and `u`,
521-
and in almost every environment (under the prior) the reward distribution of each action is
522-
sub-Gaussian with variance proxy `σ2` after its mean is subtracted, then the Bayesian regret at
523-
time `n` is at most `(2 * K + 1) * (u - l) + 8 * √(σ2 * K * n * Real.log n)`. -/
524-
theorem integral_regret_le (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P)
419+
/-- The bound of `integral_regret_le` for a nonzero variance proxy `σ2`. -/
420+
private lemma integral_regret_le_of_ne_zero (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P)
525421
(hm : ∀ e a, (κ (e, a))[id] ∈ (Set.Icc l u))
526422
(hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
527423
(hσ2 : σ2 ≠ 0) (n : ℕ) :
@@ -555,6 +451,27 @@ theorem integral_regret_le (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P
555451
_ = _ := by
556452
ring
557453

454+
open Filter Topology in
455+
/-- **Bayesian regret of Thompson sampling.** If Thompson sampling has the correct prior over
456+
environments, in every environment each of the `K` actions has a mean reward between `l` and `u`,
457+
and in almost every environment (under the prior) the reward distribution of each action is
458+
sub-Gaussian with variance proxy `σ2` after its mean is subtracted, then the Bayesian regret at
459+
time `n` is at most `(2 * K + 1) * (u - l) + 8 * √(σ2 * K * n * Real.log n)`. -/
460+
theorem integral_regret_le (h : IsBayesAlgEnvSeq Q κ (tsAlgorithm Q κ) E A R P)
461+
(hm : ∀ e a, (κ (e, a))[id] ∈ (Set.Icc l u))
462+
(hs : ∀ᵐ e ∂Q, ∀ a, HasSubgaussianMGF (fun x ↦ x - (κ (e, a))[id]) σ2 (κ (e, a)))
463+
(n : ℕ) :
464+
P[IsBayesAlgEnvSeq.regret κ E A n]
465+
≤ (2 * K + 1) * (u - l) + 8 * √(σ2 * K * n * Real.log n) := by
466+
-- The bound holds for every variance proxy `σ2' > σ2`, which is nonzero, and is continuous in
467+
-- the variance proxy: let `σ2'` tend to `σ2`.
468+
have h_cont : Continuous fun σ2' : ℝ≥0 ↦
469+
(2 * K + 1) * (u - l) + 8 * √(σ2' * K * n * Real.log n) := by fun_prop
470+
refine ge_of_tendsto (x := 𝓝[>] σ2) ((h_cont.tendsto σ2).mono_left nhdsWithin_le_nhds) ?_
471+
filter_upwards [self_mem_nhdsWithin] with σ2' hσ2'
472+
exact integral_regret_le_of_ne_zero h hm (hs.mono fun e he a ↦ (he a).mono hσ2'.le)
473+
(pos_of_gt (show σ2 < σ2' from hσ2')).ne' n
474+
558475
end TS
559476

560477
end Bandits

0 commit comments

Comments
 (0)