Skip to content

Commit 44ba421

Browse files
committed
perf(query): resume edge drains without rediscovering the whole window
A resumed edge pass rebuilt its affected buckets over the entire frozen ingestion window before applying the event-time cursor, so draining an N-minute backlog repeated discovery N times. The first pass now stores the window's per-bucket affected-span counts in a private cache table, and later passes read them instead of rescanning Parquet. Summaries, output and cursor commit in one transaction and are cleared with the cursor on completion, skip-to-latest and restart rewind. The read-cache version moves to 5, so private read tables rebuild once. A 30-minute drain at 250k spans/minute went from 25.6 s to 5.4 s locally, with incoming discovery falling from 225M to 7.5M logical rows. The system-overview browser test now collects only responses for the final pinned document and reads their bodies while that document is alive, instead of reading discarded first-document bodies in a listener. Claude-Session: https://claude.ai/code/session_01XA52nWyQ3R5f5kK8kjpAQW
1 parent 846e101 commit 44ba421

7 files changed

Lines changed: 230 additions & 29 deletions

File tree

‎internal/query/batch_cache.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import (
1717

1818
// readCacheVersion covers both schema and aggregation semantics. These tables
1919
// are disposable; a mismatch rebuilds them together from immutable Parquet.
20-
const readCacheVersion = 4
20+
const readCacheVersion = 5
2121

2222
const (
2323
readCacheBatchLimit = 64
@@ -62,6 +62,12 @@ func createBatchCaches(db *sql.DB) error {
6262
return err
6363
}
6464
}
65+
// Bucket counts and the unfinished edge window are one disposable
66+
// unit. Restart that window from its unchanged ingestion watermark
67+
// when upgrading from a cache without these counts (or new semantics).
68+
if _, err := tx.Exec(`DELETE FROM rollup_state WHERE cache_key IN (?, ?)`, edgeRollupSubCursorKey, edgeRollupSubWindowKey); err != nil {
69+
return err
70+
}
6571
if _, err := tx.Exec(`DELETE FROM read_cache_version`); err != nil {
6672
return err
6773
}
@@ -70,6 +76,7 @@ func createBatchCaches(db *sql.DB) error {
7076
}
7177
}
7278
for _, stmt := range []string{
79+
`CREATE TABLE IF NOT EXISTS read_edge_buckets (namespace VARCHAR, bucket TIMESTAMPTZ_NS, spans BIGINT)`,
7380
`CREATE TABLE IF NOT EXISTS read_batches (batch_id VARCHAR PRIMARY KEY,replacement_id VARCHAR)`,
7481
`CREATE TABLE IF NOT EXISTS read_trace_candidates (trace_id VARCHAR PRIMARY KEY,namespace VARCHAR,service VARCHAR,min_start BIGINT,max_start BIGINT,max_end BIGINT,has_error INTEGER)`,
7582
`CREATE TABLE IF NOT EXISTS read_trace_parts (batch_id VARCHAR, namespace VARCHAR, service VARCHAR, trace_id VARCHAR, min_start BIGINT, max_start BIGINT, max_end BIGINT, has_error INTEGER)`,

‎internal/query/duck.go‎

Lines changed: 28 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -342,16 +342,14 @@ func (d *Duck) skipRollupToLatest(ctx context.Context) error {
342342
}{
343343
{serviceRollupStateKey, horizon},
344344
{edgeRollupStateKey, horizon},
345-
// Skipping discards any half-aggregated window, so its resume point
346-
// must go too or the next pass would reopen a window behind the
347-
// watermark it just advanced.
348-
{edgeRollupSubCursorKey, 0},
349-
{edgeRollupSubWindowKey, 0},
350345
} {
351346
if err := storeRollupWatermark(ctx, tx, w.key, w.watermark); err != nil {
352347
return err
353348
}
354349
}
350+
if err := clearEdgeRollupCursor(ctx, tx); err != nil {
351+
return err
352+
}
355353
if err := tx.Commit(); err != nil {
356354
return err
357355
}
@@ -960,12 +958,19 @@ func (d *Duck) refreshEdgeRollupPass(ctx context.Context) (int64, bool, error) {
960958
}
961959
newWatermark := windowEnd
962960

963-
// Build the incoming/recent-arrival repair relation once in this snapshot.
964-
// Count every affected span per minute so adaptive counting preserves repair
965-
// duplicates without retaining all of their individual rows.
966-
// TEMP is connection-local; creation rolls back on failure, and the table
967-
// is explicitly dropped before commit so no later pass can observe it.
968-
if _, err := tx.ExecContext(ctx, createEdgePassBucketsSQL, windowStart, windowEnd); err != nil {
961+
// Discover affected buckets once per frozen ingestion window, including
962+
// recent-arrival repairs. Persist only their counts, atomically with the
963+
// cursor. Resumes read these small summaries rather than the original spans.
964+
if !resuming {
965+
if _, err := tx.ExecContext(ctx, "DELETE FROM read_edge_buckets"); err != nil {
966+
return 0, false, err
967+
}
968+
if _, err := tx.ExecContext(ctx, "INSERT INTO read_edge_buckets "+edgePassBucketsSelectSQL, windowStart, windowEnd); err != nil {
969+
return 0, false, err
970+
}
971+
}
972+
// Keep the per-pass name connection-local. It never holds individual spans.
973+
if _, err := tx.ExecContext(ctx, "CREATE TEMP VIEW edge_pass_buckets AS SELECT * FROM read_edge_buckets"); err != nil {
969974
return 0, false, err
970975
}
971976

@@ -1037,14 +1042,17 @@ FROM edge_pass_buckets`).Scan(&minStartT, &maxStartT)
10371042
}
10381043
} else {
10391044
storedWatermark = lastWatermark
1045+
if _, err := tx.ExecContext(ctx, `DELETE FROM read_edge_buckets WHERE bucket < ?::TIMESTAMP_NS::TIMESTAMPTZ_NS`, time.Unix(0, nextCursor)); err != nil {
1046+
return 0, false, err
1047+
}
10401048
if err := storeRollupWatermark(ctx, tx, edgeRollupSubCursorKey, nextCursor); err != nil {
10411049
return 0, false, err
10421050
}
10431051
if err := storeRollupWatermark(ctx, tx, edgeRollupSubWindowKey, windowEnd); err != nil {
10441052
return 0, false, err
10451053
}
10461054
}
1047-
if _, err := tx.ExecContext(ctx, "DROP TABLE edge_pass_buckets"); err != nil {
1055+
if _, err := tx.ExecContext(ctx, "DROP VIEW edge_pass_buckets"); err != nil {
10481056
return 0, false, err
10491057
}
10501058
if err := tx.Commit(); err != nil {
@@ -1058,9 +1066,12 @@ FROM edge_pass_buckets`).Scan(&minStartT, &maxStartT)
10581066
return totalAffected, !completed || chunked, nil
10591067
}
10601068

1061-
// clearEdgeRollupCursor drops the resume point once its window is fully
1062-
// aggregated, so the next pass opens a fresh ingested window.
1069+
// clearEdgeRollupCursor discards the resume point and its bucket summaries
1070+
// together, so the next pass opens a fresh ingested window.
10631071
func clearEdgeRollupCursor(ctx context.Context, tx *sql.Tx) error {
1072+
if _, err := tx.ExecContext(ctx, "DELETE FROM read_edge_buckets"); err != nil {
1073+
return err
1074+
}
10641075
if err := storeRollupWatermark(ctx, tx, edgeRollupSubCursorKey, 0); err != nil {
10651076
return err
10661077
}
@@ -1413,8 +1424,9 @@ const edgeArrivalGap = 5 * time.Minute
14131424
// Only relationship keys are read from the recent ingestion range. Its stamp
14141425
// predicates allow Parquet footer pruning independently of event-time history.
14151426
// Messaging uses the aggregate's exact keys, including the shared minute.
1416-
var createEdgePassBucketsSQL = fmt.Sprintf(`
1417-
CREATE TEMP TABLE edge_pass_buckets AS
1427+
var createEdgePassBucketsSQL = "CREATE TEMP TABLE edge_pass_buckets AS " + edgePassBucketsSelectSQL
1428+
1429+
var edgePassBucketsSelectSQL = fmt.Sprintf(`
14181430
WITH incoming AS MATERIALIZED (
14191431
SELECT namespace, trace_id, span_id, start_time, kind,
14201432
messaging_destination, messaging_system

‎internal/query/edge_pass_test.go‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ func TestEdgePassDropsRelationOnEveryExit(t *testing.T) {
110110
d := &Duck{DB: db, writeDB: db, publicationBounds: func() (int64, int64, int64) { return stamp, stamp, stamp }}
111111
checkAbsent := func() {
112112
var count int
113-
if err := db.QueryRow("SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('edge_pass_buckets','edge_pass_parents','edge_pass_unmatched')").Scan(&count); err != nil || count != 0 {
113+
if err := db.QueryRow("SELECT count(*) FROM (SELECT table_name AS name FROM duckdb_tables() UNION ALL SELECT view_name AS name FROM duckdb_views()) WHERE name IN ('edge_pass_buckets','edge_pass_parents','edge_pass_unmatched')").Scan(&count); err != nil || count != 0 {
114114
t.Fatalf("pass relation survived: count=%d err=%v", count, err)
115115
}
116116
}
@@ -126,6 +126,10 @@ func TestEdgePassDropsRelationOnEveryExit(t *testing.T) {
126126
if watermark, cursor := readEdgeRollupState(t, d); watermark != 0 || cursor != 0 {
127127
t.Fatalf("failed pass advanced state: watermark=%d cursor=%d", watermark, cursor)
128128
}
129+
var buckets int
130+
if err := db.QueryRow("SELECT count(*) FROM read_edge_buckets").Scan(&buckets); err != nil || buckets != 0 {
131+
t.Fatalf("failed discovery retained buckets=%d err=%v", buckets, err)
132+
}
129133
if _, err := db.Exec("ALTER TABLE edge_rollup RENAME COLUMN saved_avg_ms TO avg_ms"); err != nil {
130134
t.Fatal(err)
131135
}
@@ -135,6 +139,25 @@ func TestEdgePassDropsRelationOnEveryExit(t *testing.T) {
135139
}
136140
checkAbsent()
137141
if !empty {
142+
watermark, cursor := readEdgeRollupState(t, d)
143+
// A failed resume must retain the previous cursor AND summaries.
144+
if _, err := db.Exec("ALTER TABLE edge_rollup RENAME COLUMN avg_ms TO saved_avg_ms"); err != nil {
145+
t.Fatal(err)
146+
}
147+
if _, _, err := d.refreshEdgeRollupPass(t.Context()); err == nil {
148+
t.Fatal("incompatible rollup schema did not fail the resume")
149+
}
150+
checkAbsent()
151+
if w, c := readEdgeRollupState(t, d); w != watermark || c != cursor {
152+
t.Fatalf("failed resume moved state: watermark=%d cursor=%d", w, c)
153+
}
154+
var buckets int
155+
if err := db.QueryRow("SELECT count(*) FROM read_edge_buckets").Scan(&buckets); err != nil || buckets != 1 {
156+
t.Fatalf("failed resume lost summaries: buckets=%d err=%v", buckets, err)
157+
}
158+
if _, err := db.Exec("ALTER TABLE edge_rollup RENAME COLUMN saved_avg_ms TO avg_ms"); err != nil {
159+
t.Fatal(err)
160+
}
138161
if _, pending, err := d.refreshEdgeRollupPass(t.Context()); err != nil || pending {
139162
t.Fatalf("resumed pass pending=%v err=%v", pending, err)
140163
}

‎internal/query/edge_resume_test.go‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
package query
2+
3+
import (
4+
"database/sql"
5+
"fmt"
6+
"path/filepath"
7+
"strings"
8+
"testing"
9+
"time"
10+
)
11+
12+
func TestEdgeResumeSetupIsLinearAcrossRestart(t *testing.T) {
13+
for _, minutes := range []int{5, 15, 30} {
14+
t.Run(fmt.Sprintf("%d_minutes", minutes), func(t *testing.T) {
15+
path := filepath.Join(t.TempDir(), "cache.duckdb")
16+
var db *sql.DB
17+
open := func() {
18+
var err error
19+
db, err = sql.Open("duckdb", path)
20+
if err != nil {
21+
t.Fatal(err)
22+
}
23+
db.SetMaxOpenConns(1)
24+
if _, err := db.Exec("SET TimeZone='UTC'"); err != nil {
25+
t.Fatal(err)
26+
}
27+
if err := CreateTables(db); err != nil {
28+
t.Fatal(err)
29+
}
30+
}
31+
open()
32+
defer func() { _ = db.Close() }()
33+
base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
34+
stamp := base.UnixNano()
35+
// N occupied event minutes, separated to force N resumable passes
36+
// without needing 250k spans per minute in the regular test suite.
37+
if _, err := db.Exec(`INSERT INTO telemetry.spans
38+
(namespace,trace_id,span_id,parent_span_id,service,kind,start_time,duration_ms,status,ingested_unix_nano)
39+
SELECT 'ns',(i//2)::VARCHAR,i::VARCHAR,CASE WHEN i%2=1 THEN (i-1)::VARCHAR END,
40+
CASE WHEN i%2=0 THEN 'gateway' ELSE 'checkout' END,'SPAN_KIND_SERVER',
41+
?::TIMESTAMP_NS + (i//2)*INTERVAL 31 MINUTE,5,'OK',? FROM range(?) t(i)`, base, stamp, minutes*2); err != nil {
42+
t.Fatal(err)
43+
}
44+
if _, err := db.Exec("CREATE SEQUENCE edge_setup_reads"); err != nil {
45+
t.Fatal(err)
46+
}
47+
// Only discovery uses ingestion stamps. Count actual source-row
48+
// evaluations there, rather than timing the drain or inspecting SQL.
49+
view := strings.Replace(viewSpans, " ingested_unix_nano,", " CASE WHEN nextval('edge_setup_reads')>0 THEN ingested_unix_nano END AS ingested_unix_nano,", 1)
50+
if _, err := db.Exec(view); err != nil {
51+
t.Fatal(err)
52+
}
53+
var setupReads int64
54+
for pass := 0; pass < minutes; pass++ {
55+
d := &Duck{DB: db, writeDB: db, publicationBounds: func() (int64, int64, int64) { return stamp, stamp, stamp }}
56+
_, pending, err := d.refreshEdgeRollupPass(t.Context())
57+
if err != nil || pending != (pass < minutes-1) {
58+
t.Fatalf("pass=%d pending=%v err=%v", pass, pending, err)
59+
}
60+
var reads int64
61+
if err := db.QueryRow("SELECT last_value FROM duckdb_sequences() WHERE sequence_name='edge_setup_reads'").Scan(&reads); err != nil {
62+
t.Fatal(err)
63+
}
64+
if pass == 0 {
65+
setupReads = reads
66+
if reads < int64(minutes*2) || reads > int64(minutes*8) {
67+
t.Fatalf("setup reads=%d, want a constant number per input row", reads)
68+
}
69+
} else if reads != setupReads {
70+
t.Fatalf("resume repeated discovery: reads=%d first pass=%d", reads, setupReads)
71+
}
72+
var remaining int
73+
if err := db.QueryRow("SELECT count(*) FROM read_edge_buckets").Scan(&remaining); err != nil || remaining != minutes-pass-1 {
74+
t.Fatalf("remaining=%d pass=%d err=%v", remaining, pass, err)
75+
}
76+
if pass == 0 {
77+
// Close the native database: summaries and cursor must survive,
78+
// while the connection-local pass view must have disappeared.
79+
if err := db.Close(); err != nil {
80+
t.Fatal(err)
81+
}
82+
open()
83+
}
84+
}
85+
var calls, buckets int
86+
if err := db.QueryRow("SELECT sum(calls),count(DISTINCT bucket) FROM edge_rollup").Scan(&calls, &buckets); err != nil || calls != minutes || buckets != minutes {
87+
t.Fatalf("calls=%d buckets=%d err=%v", calls, buckets, err)
88+
}
89+
t.Logf("occupied_minutes=%d setup_row_evaluations=%d passes=%d", minutes, setupReads, minutes)
90+
})
91+
}
92+
}
93+
94+
func TestEdgeResumeKeepsFrozenWindowAndRepairsNewArrivals(t *testing.T) {
95+
db := openTestDuck(t)
96+
if err := CreateTables(db); err != nil {
97+
t.Fatal(err)
98+
}
99+
if err := CreateViews(db); err != nil {
100+
t.Fatal(err)
101+
}
102+
base := time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)
103+
stamp := base.Add(time.Hour).UnixNano()
104+
horizon := stamp
105+
d := &Duck{DB: db, writeDB: db, publicationBounds: func() (int64, int64, int64) { return horizon, horizon, stamp }}
106+
for _, span := range []rollupTestSpan{
107+
{namespace: "ns", traceID: "late", spanID: "child", parentSpanID: "parent", service: "late-child", start: base, ingested: stamp},
108+
{namespace: "ns", traceID: "existing", spanID: "parent", service: "gateway", start: base, ingested: stamp},
109+
{namespace: "ns", traceID: "existing", spanID: "child", parentSpanID: "parent", service: "existing-child", start: base, ingested: stamp},
110+
{namespace: "ns", traceID: "tail", spanID: "parent", service: "gateway", start: base.Add(31 * time.Minute), ingested: stamp},
111+
{namespace: "ns", traceID: "tail", spanID: "child", parentSpanID: "parent", service: "tail-child", start: base.Add(31 * time.Minute), ingested: stamp},
112+
} {
113+
insertRollupTestSpan(t, db, span)
114+
}
115+
if _, pending, err := d.refreshEdgeRollupPass(t.Context()); err != nil || !pending {
116+
t.Fatalf("initial pass pending=%v err=%v", pending, err)
117+
}
118+
// Publish a parent for an already-completed event bucket while the old
119+
// ingestion window is paused. It belongs to the NEXT discovery window.
120+
horizon++
121+
insertRollupTestSpan(t, db, rollupTestSpan{namespace: "ns", traceID: "late", spanID: "parent", service: "late-gateway", start: base, ingested: horizon})
122+
if _, pending, err := d.refreshEdgeRollupPass(t.Context()); err != nil || !pending {
123+
t.Fatalf("resumed pass pending=%v err=%v", pending, err)
124+
}
125+
if watermark, cursor := readEdgeRollupState(t, d); watermark != stamp || cursor != 0 {
126+
t.Fatalf("resume changed frozen ingestion end: watermark=%d cursor=%d", watermark, cursor)
127+
}
128+
requireRowCount(t, db, "SELECT count(*) FROM edge_rollup WHERE callee=?", "late-child", 0)
129+
if _, pending, err := d.refreshEdgeRollupPass(t.Context()); err != nil || pending {
130+
t.Fatalf("repair pass pending=%v err=%v", pending, err)
131+
}
132+
// Repair replaces the complete bucket: both the new relationship and the
133+
// unrelated edge that was already aggregated must appear exactly once.
134+
for _, child := range []string{"late-child", "existing-child", "tail-child"} {
135+
requireRowCount(t, db, "SELECT count(*) FROM edge_rollup WHERE calls=1 AND callee=?", child, 1)
136+
}
137+
if watermark, cursor := readEdgeRollupState(t, d); watermark != horizon || cursor != 0 {
138+
t.Fatalf("repair did not reach publication horizon: watermark=%d cursor=%d", watermark, cursor)
139+
}
140+
}

‎internal/query/file_snapshot_test.go‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -321,20 +321,27 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) {
321321
if _, err := d.RefreshReadCaches(t.Context()); err != nil {
322322
t.Fatal(err)
323323
}
324-
for _, stmt := range []string{`UPDATE read_trace_candidates SET min_start=0`, `UPDATE read_cache_version SET version=3`, `CREATE TABLE read_endpoints (value INTEGER)`, `CREATE TABLE read_logs (value INTEGER)`, `CREATE TABLE read_obsolete (value INTEGER)`, `CREATE TABLE endpoint_rollup (value INTEGER)`, `INSERT INTO rollup_state VALUES ('endpoint_rollup:prod',1,now())`} {
324+
for _, stmt := range []string{`UPDATE read_trace_candidates SET min_start=0`, `UPDATE read_cache_version SET version=4`, `INSERT INTO read_edge_buckets VALUES ('ns',now(),10)`, `INSERT INTO rollup_state VALUES ('edge_rollup_v2_substart',1,now()),('edge_rollup_v2_subwindow_end',2,now())`, `CREATE TABLE read_endpoints (value INTEGER)`, `CREATE TABLE read_logs (value INTEGER)`, `CREATE TABLE read_obsolete (value INTEGER)`, `CREATE TABLE endpoint_rollup (value INTEGER)`, `INSERT INTO rollup_state VALUES ('endpoint_rollup:prod',1,now())`} {
325325
if _, err := d.DB.Exec(stmt); err != nil {
326326
t.Fatal(err)
327327
}
328328
}
329329
if err := CreateCacheTables(d.DB); err != nil {
330330
t.Fatal(err)
331331
}
332-
for _, table := range []string{"read_batches", "read_trace_parts", "read_trace_candidates"} {
332+
for _, table := range []string{"read_batches", "read_trace_parts", "read_trace_candidates", "read_edge_buckets"} {
333333
var count int
334334
if err := d.DB.QueryRow("SELECT count(*) FROM " + table).Scan(&count); err != nil || count != 0 {
335335
t.Fatalf("%s survived semantic change: %d %v", table, count, err)
336336
}
337337
}
338+
if watermark, cursor := readEdgeRollupState(t, d); watermark != 0 || cursor != 0 {
339+
t.Fatalf("obsolete edge cursor survived: watermark=%d cursor=%d", watermark, cursor)
340+
}
341+
var resumeEnd int64
342+
if err := d.DB.QueryRow(`SELECT coalesce(max(last_ingested_unix_nano),0) FROM rollup_state WHERE cache_key=?`, edgeRollupSubWindowKey).Scan(&resumeEnd); err != nil || resumeEnd != 0 {
343+
t.Fatalf("obsolete edge window survived: end=%d err=%v", resumeEnd, err)
344+
}
338345
var obsolete int
339346
if err := d.DB.QueryRow(`SELECT count(*) FROM duckdb_tables() WHERE table_name IN ('read_obsolete','endpoint_rollup','read_endpoints','read_logs')`).Scan(&obsolete); err != nil || obsolete != 0 {
340347
t.Fatalf("obsolete tables=%d: %v", obsolete, err)
@@ -350,7 +357,7 @@ func TestReadCacheSemanticVersionRebuildsAllContributions(t *testing.T) {
350357
t.Fatalf("rebuilt bound=%d: %v", minStart, err)
351358
}
352359
var version int
353-
if err := d.DB.QueryRow(`SELECT version FROM read_cache_version`).Scan(&version); err != nil || version != 4 {
360+
if err := d.DB.QueryRow(`SELECT version FROM read_cache_version`).Scan(&version); err != nil || version != 5 {
354361
t.Fatalf("version=%d: %v", version, err)
355362
}
356363

‎internal/query/sql_boundary_test.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ func TestSQLBoundaryRejectsHiddenReadsAndMutations(t *testing.T) {
3333
"SELECT * FROM read_traces",
3434
"SELECT * FROM read_trace_candidates",
3535
"SELECT * FROM read_cache_version",
36+
"SELECT * FROM read_edge_buckets",
3637
"SELECT * FROM endpoint_rollup",
3738
"SELECT current_setting('allowed_directories')",
3839
"SELECT getvariable('operator_secret')",
@@ -115,6 +116,7 @@ func TestExecuteSQLRejectsEngineMetadataReads(t *testing.T) {
115116
"SELECT * FROM read_traces",
116117
"SELECT * FROM read_trace_candidates",
117118
"SELECT * FROM read_cache_version",
119+
"SELECT * FROM read_edge_buckets",
118120
"SELECT * FROM endpoint_rollup",
119121
"SELECT current_setting('allowed_directories')",
120122
"SELECT getvariable('operator_secret')",

0 commit comments

Comments
 (0)