Skip to content

Commit 13aaedc

Browse files
cpplacppla
committed
fix: refresh SSL state after failures and config reloads
1 parent 050b107 commit 13aaedc

4 files changed

Lines changed: 335 additions & 21 deletions

File tree

‎server/ssl.go‎

Lines changed: 17 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,16 @@ func (a *App) reconcileCerts(configs []SSLCertConfig) {
3333
next := make(map[string]*CertState, len(configs))
3434
for _, config := range configs {
3535
key := certKey(config)
36+
// A reload requests a fresh check, even when the config is unchanged.
37+
// Replace the state so an in-flight check cannot publish an old result.
38+
state := &CertState{Config: config}
3639
if existing := a.certs[key]; existing != nil {
37-
existing.Config = config
38-
next[key] = existing
39-
} else {
40-
next[key] = &CertState{Config: config}
40+
state.LastAlarm7 = existing.LastAlarm7
41+
state.LastAlarm3 = existing.LastAlarm3
42+
state.LastAlarm1 = existing.LastAlarm1
43+
state.LastAlarmMismatch = existing.LastAlarmMismatch
4144
}
45+
next[key] = state
4246
}
4347
a.certs = next
4448
}
@@ -60,26 +64,27 @@ func (a *App) sslLoop() {
6064
func (a *App) runDueSSLChecks() {
6165
now := time.Now()
6266
type dueCheck struct {
63-
key string
64-
config SSLCertConfig
67+
key string
68+
state *CertState
6569
}
6670
due := make([]dueCheck, 0)
6771
a.certMu.Lock()
6872
for key, state := range a.certs {
6973
interval := secondsDuration(state.Config.Interval)
7074
if !state.Checking && (state.LastCheck.IsZero() || now.Sub(state.LastCheck) >= interval) {
7175
state.Checking = true
72-
due = append(due, dueCheck{key: key, config: state.Config})
76+
due = append(due, dueCheck{key: key, state: state})
7377
}
7478
}
7579
a.certMu.Unlock()
7680
for _, check := range due {
7781
check := check
78-
go a.executeSSLCheck(check.key, check.config)
82+
go a.executeSSLCheck(check.key, check.state)
7983
}
8084
}
8185

82-
func (a *App) executeSSLCheck(key string, config SSLCertConfig) {
86+
func (a *App) executeSSLCheck(key string, state *CertState) {
87+
config := state.Config
8388
expireTS, mismatch, err := checkCertificate(config)
8489
now := time.Now()
8590
type notification struct {
@@ -88,14 +93,15 @@ func (a *App) executeSSLCheck(key string, config SSLCertConfig) {
8893
alerts := make([]notification, 0, 2)
8994

9095
a.certMu.Lock()
91-
state := a.certs[key]
92-
if state == nil {
96+
if a.certs[key] != state {
9397
a.certMu.Unlock()
9498
return
9599
}
96100
state.Checking = false
97101
state.LastCheck = now
98102
if err != nil {
103+
state.ExpireTS = 0
104+
state.Mismatch = false
99105
state.LastError = err.Error()
100106
a.certMu.Unlock()
101107
return

‎server/ssl_test.go‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,232 @@
11
package main
22

33
import (
4+
"crypto/tls"
5+
"errors"
46
"net/http"
57
"net/http/httptest"
68
"net/url"
79
"strconv"
10+
"sync"
811
"sync/atomic"
912
"testing"
1013
"time"
1114
)
1215

16+
func sslTestServer(t *testing.T, handshake func(*tls.ClientHelloInfo) (*tls.Config, error)) (*httptest.Server, ConfigDocument) {
17+
t.Helper()
18+
server := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) }))
19+
server.TLS = &tls.Config{GetConfigForClient: handshake}
20+
server.StartTLS()
21+
t.Cleanup(server.Close)
22+
parsed, err := url.Parse(server.URL)
23+
if err != nil {
24+
t.Fatal(err)
25+
}
26+
port, err := strconv.Atoi(parsed.Port())
27+
if err != nil {
28+
t.Fatal(err)
29+
}
30+
doc := minimalTestConfig()
31+
doc["sslcerts"] = []any{map[string]any{
32+
"name": "local", "domain": server.URL, "port": port, "interval": 7200,
33+
}}
34+
return server, doc
35+
}
36+
37+
func sslTestState(app *App, key string) CertState {
38+
app.certMu.RLock()
39+
defer app.certMu.RUnlock()
40+
return *app.certs[key]
41+
}
42+
43+
func waitForSSLCheck(t *testing.T, app *App, config SSLCertConfig, previous time.Time) CertState {
44+
t.Helper()
45+
eventually(t, 2*time.Second, func() bool {
46+
state := sslTestState(app, certKey(config))
47+
return !state.Checking && state.LastCheck.After(previous)
48+
})
49+
return sslTestState(app, certKey(config))
50+
}
51+
52+
func TestSSLFailureAndRecovery(t *testing.T) {
53+
var rejectHandshake atomic.Bool
54+
server, doc := sslTestServer(t, func(*tls.ClientHelloInfo) (*tls.Config, error) {
55+
if rejectHandshake.Load() {
56+
return nil, errors.New("test TLS failure")
57+
}
58+
return nil, nil
59+
})
60+
app := newTestApp(t, doc)
61+
config := app.RuntimeSnapshot().SSLCerts[0]
62+
key := certKey(config)
63+
app.runDueSSLChecks()
64+
state := waitForSSLCheck(t, app, config, time.Time{})
65+
if state.ExpireTS != server.Certificate().NotAfter.Unix() || state.LastError != "" {
66+
t.Fatalf("initial certificate check: %+v", state)
67+
}
68+
69+
for _, phase := range []string{"TLS failure", "recovery", "connection failure"} {
70+
t.Run(phase, func(t *testing.T) {
71+
switch phase {
72+
case "TLS failure":
73+
rejectHandshake.Store(true)
74+
case "recovery":
75+
rejectHandshake.Store(false)
76+
case "connection failure":
77+
server.Close()
78+
}
79+
previous := state.LastCheck
80+
app.certMu.Lock()
81+
// Simulate the next scheduled interval without waiting two hours.
82+
app.certs[key].LastCheck = time.Time{}
83+
if phase != "recovery" {
84+
// A failed check must also discard any prior hostname mismatch.
85+
app.certs[key].Mismatch = true
86+
}
87+
app.certMu.Unlock()
88+
app.runDueSSLChecks()
89+
state = waitForSSLCheck(t, app, config, previous)
90+
entry := app.sslSnapshot([]SSLCertConfig{config}, time.Now())[0].(map[string]any)
91+
if phase == "recovery" {
92+
if state.LastError != "" || state.ExpireTS != server.Certificate().NotAfter.Unix() || state.Mismatch {
93+
t.Errorf("certificate did not recover: %+v", state)
94+
}
95+
if _, hasError := entry["error"]; hasError || entry["expire_days"].(int) <= 0 {
96+
t.Errorf("recovered snapshot: %#v", entry)
97+
}
98+
} else if state.LastError == "" || state.ExpireTS != 0 || state.Mismatch ||
99+
entry["error"] == nil || entry["expire_ts"] != int64(0) || entry["expire_days"] != 0 || entry["mismatch"] != false {
100+
t.Errorf("failed check retained a certificate result: state=%+v snapshot=%#v", state, entry)
101+
}
102+
})
103+
}
104+
}
105+
106+
func TestSSLConfigReloadRechecks(t *testing.T) {
107+
for _, change := range []string{"unchanged reload", "interval", "callback"} {
108+
t.Run(change, func(t *testing.T) {
109+
_, doc := sslTestServer(t, nil)
110+
app := newTestApp(t, doc)
111+
config := app.RuntimeSnapshot().SSLCerts[0]
112+
key := certKey(config)
113+
app.runDueSSLChecks()
114+
before := waitForSSLCheck(t, app, config, time.Time{})
115+
alarmTime := time.Now()
116+
app.certMu.Lock()
117+
app.certs[key].LastAlarm7 = alarmTime
118+
app.certs[key].LastAlarm3 = alarmTime
119+
app.certs[key].LastAlarm1 = alarmTime
120+
app.certs[key].LastAlarmMismatch = alarmTime
121+
app.certMu.Unlock()
122+
123+
if change == "unchanged reload" {
124+
if apiErr := app.ReloadConfig(); apiErr != nil {
125+
t.Fatal(apiErr)
126+
}
127+
} else {
128+
updated := app.ConfigSnapshot()
129+
cert := updated["sslcerts"].([]any)[0].(map[string]any)
130+
if change == "interval" {
131+
cert["interval"] = 14400
132+
} else {
133+
cert["callback"] = "http://127.0.0.1:1/?message="
134+
}
135+
if _, apiErr := app.ReplaceConfig(updated); apiErr != nil {
136+
t.Fatal(apiErr)
137+
}
138+
}
139+
state := sslTestState(app, key)
140+
if !state.LastCheck.IsZero() || state.Checking || state.ExpireTS != 0 || state.Mismatch || state.LastError != "" {
141+
t.Errorf("reload did not reset the certificate check: %+v", state)
142+
}
143+
if state.LastAlarm7 != alarmTime || state.LastAlarm3 != alarmTime || state.LastAlarm1 != alarmTime || state.LastAlarmMismatch != alarmTime {
144+
t.Errorf("reload reset notification cooldowns: %+v", state)
145+
}
146+
app.runDueSSLChecks()
147+
state = waitForSSLCheck(t, app, config, before.LastCheck)
148+
if state.ExpireTS == 0 || state.LastError != "" {
149+
t.Errorf("reload did not publish a fresh result: %+v", state)
150+
}
151+
})
152+
}
153+
}
154+
155+
func TestSSLDiscardInFlightResultAfterConfigChange(t *testing.T) {
156+
for _, change := range []string{"reload", "remove and readd"} {
157+
t.Run(change, func(t *testing.T) {
158+
oldStarted, newStarted := make(chan struct{}), make(chan struct{})
159+
releaseOld, releaseNew := make(chan struct{}), make(chan struct{})
160+
var oldOnce, newOnce sync.Once
161+
unblockOld := func() { oldOnce.Do(func() { close(releaseOld) }) }
162+
unblockNew := func() { newOnce.Do(func() { close(releaseNew) }) }
163+
var handshakes atomic.Int32
164+
_, doc := sslTestServer(t, func(*tls.ClientHelloInfo) (*tls.Config, error) {
165+
switch handshakes.Add(1) {
166+
case 1:
167+
close(oldStarted)
168+
<-releaseOld
169+
case 2:
170+
close(newStarted)
171+
<-releaseNew
172+
}
173+
return nil, nil
174+
})
175+
// Unblock handshakes before httptest.Server cleanup, including failures.
176+
t.Cleanup(unblockOld)
177+
t.Cleanup(unblockNew)
178+
app := newTestApp(t, doc)
179+
config := app.RuntimeSnapshot().SSLCerts[0]
180+
key := certKey(config)
181+
app.certMu.Lock()
182+
oldState := app.certs[key]
183+
oldState.Checking = true
184+
app.certMu.Unlock()
185+
oldDone := make(chan struct{})
186+
go func() {
187+
app.executeSSLCheck(key, oldState)
188+
close(oldDone)
189+
}()
190+
await := func(channel <-chan struct{}) {
191+
t.Helper()
192+
select {
193+
case <-channel:
194+
case <-time.After(2 * time.Second):
195+
t.Fatal("certificate handshake did not reach the expected stage")
196+
}
197+
}
198+
await(oldStarted)
199+
if change == "reload" {
200+
if apiErr := app.ReloadConfig(); apiErr != nil {
201+
t.Fatal(apiErr)
202+
}
203+
} else {
204+
withoutCert := app.ConfigSnapshot()
205+
withoutCert["sslcerts"] = []any{}
206+
if _, apiErr := app.ReplaceConfig(withoutCert); apiErr != nil {
207+
t.Fatal(apiErr)
208+
}
209+
if _, apiErr := app.ReplaceConfig(doc); apiErr != nil {
210+
t.Fatal(apiErr)
211+
}
212+
}
213+
app.runDueSSLChecks()
214+
await(newStarted)
215+
unblockOld()
216+
await(oldDone)
217+
state := sslTestState(app, key)
218+
if !state.Checking || !state.LastCheck.IsZero() || state.ExpireTS != 0 || state.Mismatch || state.LastError != "" {
219+
t.Errorf("old check changed the new in-flight state: %+v", state)
220+
}
221+
unblockNew()
222+
state = waitForSSLCheck(t, app, config, time.Time{})
223+
if state.ExpireTS == 0 || state.LastError != "" {
224+
t.Errorf("new check did not publish its result: %+v", state)
225+
}
226+
})
227+
}
228+
}
229+
13230
func TestCertificateCheckAndSnapshot(t *testing.T) {
14231
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(204) }))
15232
defer server.Close()

‎tests/webui/ssl.spec.js‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
const { test, expect } = require('@playwright/test');
2+
3+
const statsURL = /\/json\/stats\.json(?:\?|$)/;
4+
const error = 'lookup "example.test": <img src=x onerror="alert(1)"> & offline';
5+
const certificate = {
6+
name: 'certificate', domain: 'example.test', port: 443,
7+
expire_ts: 2_000_000_000, expire_days: 30, mismatch: false
8+
};
9+
10+
for (const compact of [false, true]) {
11+
test.describe(compact ? 'SSL cards' : 'SSL table', () => {
12+
test.use({ viewport: compact ? { width: 390, height: 844 } : { width: 1280, height: 800 } });
13+
14+
const items = page => page.locator(compact ? '#sslCards .card' : '#sslBody tr');
15+
const badge = item => item.locator(compact ? '.card-header .badge' : 'td:last-child .badge');
16+
const remaining = item => item.locator(compact ? '.kvlist > div:nth-child(3) > span:last-child' : 'td:nth-child(4)');
17+
const expiry = item => item.locator(compact ? '.kvlist > div:nth-child(4) > span:last-child' : 'td:nth-child(5)');
18+
const risk = page => page.locator('#overviewCards .overview-card', { hasText: '证书风险' }).locator('.value');
19+
20+
async function openSSL(page, getCertificates) {
21+
await page.route(statsURL, route => route.fulfill({
22+
contentType: 'application/json',
23+
body: JSON.stringify({ updated: String(Math.floor(Date.now() / 1000)), servers: [], sslcerts: getCertificates() })
24+
}));
25+
await page.goto('/');
26+
await page.locator('#navTabs button[data-tab="ssl"]').click();
27+
}
28+
29+
test('errors, pending checks and expiry states agree with the overview', async ({ page }) => {
30+
const cases = [
31+
{ name: 'healthy', expire_days: 8, status: '正常', cls: 'ok' },
32+
{ name: 'failed-stale', error, mismatch: true, status: '检测失败', cls: 'err', unknown: true },
33+
{ name: 'failed-empty', error, expire_ts: 0, expire_days: 0, status: '检测失败', cls: 'err', unknown: true },
34+
{ name: 'pending', expire_ts: 0, expire_days: 0, status: '待检测', cls: 'warn', unknown: true },
35+
{ name: 'expired', expire_days: 0, status: '已过期', cls: 'err' },
36+
{ name: 'mismatch', mismatch: true, status: '域名不匹配', cls: 'err' },
37+
{ name: 'expiring', expire_days: 7, status: '将到期', cls: 'warn' }
38+
];
39+
await openSSL(page, () => cases.map(({ status, cls, unknown, ...fields }) => ({ ...certificate, ...fields })));
40+
await expect(items(page)).toHaveCount(cases.length);
41+
await expect(risk(page)).toHaveText('6');
42+
for (const entry of cases) {
43+
const item = items(page).filter({ hasText: entry.name });
44+
await expect(badge(item)).toHaveText(entry.status);
45+
await expect(badge(item)).toHaveClass(`badge ${entry.cls}`);
46+
await expect(item).toHaveClass(entry.cls === 'ok' ? (compact ? 'card' : '') :
47+
`${compact ? 'card ' : ''}alert-${entry.cls === 'err' ? 'critical' : 'warning'}`);
48+
if (entry.unknown) {
49+
await expect(remaining(item)).toHaveText('-');
50+
await expect(expiry(item)).toHaveText('-');
51+
}
52+
if (entry.error) {
53+
await expect(badge(item)).toHaveAttribute('title', error);
54+
await expect(item.locator('img')).toHaveCount(0);
55+
}
56+
}
57+
});
58+
59+
test('polling updates success, failure and recovery without stale expiry or errors', async ({ page }) => {
60+
let current = { ...certificate };
61+
await openSSL(page, () => [current]);
62+
const item = items(page);
63+
await expect(badge(item)).toHaveText('正常');
64+
await expect(risk(page)).toHaveText('0');
65+
66+
current = { ...certificate, error };
67+
await expect(badge(item)).toHaveText('检测失败');
68+
await expect(remaining(item)).toHaveText('-');
69+
await expect(expiry(item)).toHaveText('-');
70+
await expect(risk(page)).toHaveText('1');
71+
72+
current = { ...certificate, expire_ts: 2_100_000_000, expire_days: 90 };
73+
await expect(badge(item)).toHaveText('正常');
74+
await expect(badge(item)).toHaveClass('badge ok');
75+
await expect(badge(item)).toHaveAttribute('title', '');
76+
await expect(remaining(item)).toHaveText(compact ? '90 天' : '90');
77+
const date = new Date(current.expire_ts * 1000).toISOString();
78+
await expect(expiry(item)).toHaveText(compact ? date.slice(0, 10) : date.replace('T', ' ').replace(/\.\d+Z/, ''));
79+
await expect(risk(page)).toHaveText('0');
80+
await expect(item).not.toHaveClass(/alert-/);
81+
});
82+
});
83+
}

0 commit comments

Comments
 (0)