From 5d10d180d7a844d1c4df374eecaf690cd7434bde Mon Sep 17 00:00:00 2001 From: Olav Gjerde Date: Sun, 27 Sep 2026 15:23:40 +0200 Subject: [PATCH] Show what was skipped, and probe wide enough to see sparse damage The dashboard's OVER column becomes SKIP: how much --skip-slow has condemned without reading, which the weak count cannot show since a region never read has no sectors to count. --status shows the same; chunks over budget stay in --status-json and the report. REGRESSION guarded: the probes that step a gigabyte forward read eight chunks, and damage in one chunk of twenty-five slipped past most of them, so a real drive with 4% of its chunks bad was called clean a gigabyte on and nothing was ever skipped. They read 64 MiB now; the bisection narrows to 8 MiB probes once the gap is smaller than a wide one, so a dense region's edge stays sharp. Co-Authored-By: Claude Opus 5.5 (1M context) --- DESIGN.md | 24 +++---- README.md | 13 ++-- docs/img/cli-dm.svg | 2 +- docs/img/cli-report.svg | 2 +- docs/img/cli-status.svg | 4 +- docs/img/tui-dashboard.svg | 32 ++++------ hddscan.8 | 22 ++++--- hddscan.c | 128 +++++++++++++++++++------------------ tests/cli.sh | 23 ++++++- tests/tui.py | 12 ++-- tools/shots.py | 3 + 11 files changed, 143 insertions(+), 122 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 2e54ae7..08392c2 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -320,17 +320,12 @@ seek for every sector of it, finds almost nothing to call weak, and crawls: a real one did 83% of its surface at 390 KiB/s, with an ETA of 1822 hours and a WEAK count that had stopped moving. -Two things follow. The dashboard has an `OVER` column, the share of chunk -reads over budget across the whole scan, because on this drive it is the only -number that moves. It turns red at the same one-in-a-thousand line at which -the verdict holds it against the drive, so the screen and the report cannot -disagree about it. The opposite shape is as bad. Only a few chunks are over budget, but each is so full of weak sectors that its retries (twenty apiece, each behind a seek) cost minutes. Another real drive spent six and a half minutes on its first -forty megabytes that way: OVER at 4.4%, a climbing weak count, and an ETA of -four years. +forty megabytes that way: 4.4% of chunks over budget, a climbing weak count, +and an ETA of four years. `--skip-slow` (on under the repair profile) handles both. A region is too slow to scan when, over a trailing window of chunks, either: @@ -341,10 +336,17 @@ to scan when, over a trailing window of chunks, either: drill-down, retry and cache-busting seek counted, once there are at least 8 chunks and a minute of I/O to judge. -Then it stops reading. It probes a few chunks 1 GiB further on, and again, -until a probe comes back fast, then bisects between the last slow probe and -that one down to 16 MiB and carries on scanning from there. The first read of -each probe is thrown away because it carries the seek. A probe is slow if most +Then it stops reading. It probes 64 MiB 1 GiB further on, and again, until a +probe comes back fast, then bisects between the last slow probe and that one +down to 16 MiB and carries on scanning from there. The forward probes are +wide because damage in one chunk of twenty-five slips past an 8 MiB probe +more often than not, and the drive would be called clean a gigabyte on when +it is not. The bisection switches to 8 MiB probes once the gap is narrower +than a wide one, because a wide probe straddling the edge only says which +side most of it is on. The first read of +each probe is thrown away because it carries the seek. The dashboard's `SKIP` +column shows how much has been condemned so far, because the weak count +cannot: a skipped region was never read, so it has no sectors to count. A probe is slow if most of its chunks are over budget, or if drilling the ones that are would bring it under the floor. That cost is estimated from what drilling has cost on this drive so far; it is a cost, not a budget, and decides nothing about what counts diff --git a/README.md b/README.md index baa4e1d..85ba64f 100644 --- a/README.md +++ b/README.md @@ -169,12 +169,13 @@ that and says so, but the flags are the fix. A drive can also be slow across whole regions rather than in single sectors: every chunk there misses its budget, nearly every sector inside comes back in -time, and the scan crawls while its WEAK count stands still. The dashboard's -**OVER** column, the share of chunks over budget, is the number that moves. -`--skip-slow` (on under repair) stops reading such a region: it probes 1 GiB -on, and on, until the drive is fast again, bisects back to the edge, and -carries on there. What it jumped over is condemned unread, and the report says -so plainly. +time, and the scan crawls while its WEAK count stands still. Or only a few +chunks miss it, each so full of weak sectors that retrying them takes minutes. +`--skip-slow` (on under repair) stops reading a region that is too slow to +scan either way: it probes 1 GiB on, and on, until the drive is fast again, +bisects back to the edge, and carries on there. What it jumped over is +condemned unread, and the report says so plainly. The dashboard's **SKIP** +column shows how much that is. ## Putting a damaged drive back to work diff --git a/docs/img/cli-dm.svg b/docs/img/cli-dm.svg index fed3ee9..6bc5c6c 100644 --- a/docs/img/cli-dm.svg +++ b/docs/img/cli-dm.svg @@ -17,7 +17,7 @@ $ sudo hddscan dm status sdg.img device sdg.img 12.73 TiB name bb-sdg.img - created 2026-09-27 12:54, map generation 2 + created 2026-09-27 15:18, map generation 2 extent 1.00 MiB, 13348372 on the drive virtual device 12.72 TiB (13335018 extents) skipped 4 extents bad when the map was made (4.00 MiB) diff --git a/docs/img/cli-report.svg b/docs/img/cli-report.svg index f4d498a..9c072da 100644 --- a/docs/img/cli-report.svg +++ b/docs/img/cli-report.svg @@ -32,7 +32,7 @@ be told to keep what it stores off them: 1. write the list, in the block size the filesystem will use and relative to the partition it will live on: - hddscan --badblocks-from /var/lib/hddscan/runs/20260927-125407/sdh.img.ckpt \ + hddscan --badblocks-from /var/lib/hddscan/runs/20260927-151830/sdh.img.ckpt \ --badblocks-list /root/sdh.img.bb \ --badblocks-blocksize 4096 --badblocks-offset 1048576 (the offset is where the partition starts: 1 MiB is where diff --git a/docs/img/cli-status.svg b/docs/img/cli-status.svg index 6dd262f..da9cec6 100644 --- a/docs/img/cli-status.svg +++ b/docs/img/cli-status.svg @@ -14,10 +14,10 @@ 'hddscan --status <run>' for the drives in one, '--attach <run>' to watch it $ sudo hddscan --status 20260912-093044 | head -16 run 20260912-093044 format low level format, 4096 byte sectors, protection information kept 30 drives running elapsed 4h 08m - supervisor pid 111907 'hddscan --attach 20260912-093044' to watch, '--stop 20260912-093044' to stop it + supervisor pid 196552 'hddscan --attach 20260912-093044' to watch, '--stop 20260912-093044' to stop it overall 60.6% 28 running 0 queued 2 done reports in /var/log/hddscan/ records in /var/lib/hddscan/runs/20260912-093044 - DRIVE SIZE MODEL PCT BAD WEAK OVER STATE MESSAGE + DRIVE SIZE MODEL PCT BAD WEAK SKIP STATE MESSAGE sdaa 9.10 TiB HUH721010AL4200 55.3% 0 0 - formatting Format in progress, 55.33% done sdab 9.10 TiB HUH721010AL4200 49.3% 0 0 - formatting Format in progress, 49.28% done sdac 9.10 TiB HUH721010AL4200 66.8% 0 0 - formatting Format in progress, 66.78% done diff --git a/docs/img/tui-dashboard.svg b/docs/img/tui-dashboard.svg index 38a9eea..9cb910f 100644 --- a/docs/img/tui-dashboard.svg +++ b/docs/img/tui-dashboard.svg @@ -12,58 +12,52 @@ 905.75 MiB/s aggregate 472.63 TiB tested eta 7h 36m bad sectors 3 weak 1051 RATE, ETA and latency are measured over the last 30 seconds; latency in ms - DRIVE PCT RATE BAD WEAK OVER ETA STATE - sdb 71.0% 169.75 MiB/s 0 40 0% 5h 50m scanning + DRIVE PCT RATE BAD WEAK SKIP ETA STATE + sdb 71.0% 169.75 MiB/s 0 40 - 5h 50m scanning med 11.0 avg 12.8 min 7.5 max 26.7 wmed 21.4 - - - - sdc 75.1% 175.45 MiB/s 0 63 <0.1% 6h 16m scanning + + sdc 75.1% 175.45 MiB/s 0 63 - 6h 16m scanning med 11.4 avg 13.4 min 7.5 max 35.7 wmed 21.4 - sdd 79.2% 181.15 MiB/s 0 86 33% 6h 43m scanning + sdd 79.2% 181.15 MiB/s 0 8638.0G 6h 43m scanning med 11.8 avg 14.0 min 7.5 max 44.7 wmed 21.4 - - - - sde 83.3% 186.85 MiB/s 0 109 <0.1% 7h 10m scanning + + sde 83.3% 186.85 MiB/s 0 109 - 7h 10m scanning med 12.2 avg 14.6 min 7.5 max 53.7 wmed 21.4 - sdf 87.4% 192.55 MiB/s 0 132<0.1% 7h 36m scanning + sdf 87.4% 192.55 MiB/s 0 132 - 7h 36m scanning med 12.6 avg 15.2 min 7.5 max 62.7 wmed 21.4 - - - + - sdg 100.0% 0 B/s 3 612 <0.1% - SUSPECT + sdg 100.0% 0 B/s 3 612 - - SUSPECT HUH721010AL4200 9.10 TiB - sdi 100.0% 0 B/s 0 3 0% - SUSPECT + sdi 100.0% 0 B/s 0 3 - - SUSPECT HUH721010AL4200 9.10 TiB - sdp 100.0% 0 B/s 0 3 0% - SUSPECT + sdp 100.0% 0 B/s 0 3 - - SUSPECT HUH721010AL4200 9.10 TiB - sdw 100.0% 0 B/s 0 3 0% - SUSPECT + sdw 100.0% 0 B/s 0 3 - - SUSPECT ST14000NM0168 12.73 TiB page 1/3 < > pages g overview r runs n new test x stop q quit - the run keeps going diff --git a/hddscan.8 b/hddscan.8 index aa42085..245ad1e 100644 --- a/hddscan.8 +++ b/hddscan.8 @@ -124,13 +124,14 @@ bytes, and once either is not zero it is coloured, red and yellow. The end of scan report breaks the same sectors down by how many tries and how many errors each one took. .PP -.B OVER -is the share of chunk reads, over the whole scan so far, that missed their -budget. It is the number to watch on a drive that is slow across whole -regions: every chunk there is drilled into, nearly every sector inside comes -back in time, and WEAK barely moves while the scan crawls. It is yellow once -any chunk has been over, and red once more than one in a thousand have, which -is the same line at which the verdict turns SUSPECT for it. +.B SKIP +is how much of the drive +.B \-\-skip\-slow +has condemned without reading, in short units: 960M, 1.0G, 12T. A skipped +region was never read, so it has no sectors to count and WEAK does not include +it. Yellow once anything is skipped, since any skip makes the verdict at least +SUSPECT. How many chunk reads missed their budget is in the report and in +.BR \-\-status\-json . .SS Thresholds Budgets are calibrated per drive, using the same access pattern the scan will use. Timing scattered reads and then scanning sequentially would produce a @@ -381,9 +382,10 @@ apiece. A region counts as too slow once three quarters of the chunks over the last 64 MiB have missed their budget, or once scanning it \(em every drill-down, retry and seek included \(em has run below the .B \-\-min\-rate -floor for a minute of I/O. Then stop reading: probe a few chunks 1 GiB further -on, and again, until a probe comes back fast, then bisect between the last -slow probe and that one down to 16 MiB and carry on scanning there. Every +floor for a minute of I/O. Then stop reading: probe 64 MiB 1 GiB further on, +and again, until a probe comes back fast, then bisect between the last slow +probe and that one down to 16 MiB, with 8 MiB probes once the gap is narrower +than a wide one, and carry on scanning there. Every probe is judged against the same calibrated budget as the scan; what drilling a probe's over-budget chunks would cost is taken from what drilling has cost on this drive so far. Separately, a chunk whose drill-down has spent 30 s of diff --git a/hddscan.c b/hddscan.c index 697e620..797b359 100644 --- a/hddscan.c +++ b/hddscan.c @@ -442,11 +442,10 @@ typedef struct { double pct, rate, eta; uint64_t bad, weak, bytes; /* - * Chunk reads over their budget, out of how many were read. A drive - * whose slowness is spread across whole chunks rather than sitting in - * a few sectors drills every chunk and finds nothing to count as weak, - * so without this the one number that is climbing never reaches the - * screen. skipped is what --skip-slow jumped over without reading. + * Chunk reads over their budget, out of how many were read, for + * --status-json; and what --skip-slow condemned without reading, + * which the weak count cannot show: a region never read has no + * sectors to count. */ uint64_t slow, chunks, skipped; int too_slow; /* under the throughput floor, see rate_judge() */ @@ -5352,8 +5351,8 @@ static void check_temperature(ctx_t *c) * The other worst case is the opposite shape: only a few chunks over budget, * but each so full of weak sectors -- twenty retries apiece, each behind a * seek -- that one chunk costs minutes. A real drive spent six and a half - * minutes on its first forty megabytes that way, OVER at 4.4% and climbing - * weak count and all, with an ETA of four years. + * minutes on its first forty megabytes that way, 4.4% of chunks over + * budget and a climbing weak count, with an ETA of four years. * * So a region is slow when either of two things holds over a trailing * window of chunks: most of them were over budget, or scanning them -- every @@ -5389,9 +5388,10 @@ static void check_temperature(ctx_t *c) #define SKIP_OVER_DEN 4 #define SKIP_JUMP (1ull << 30) #define SKIP_RES (16ull << 20) -#define SKIP_PROBE_BYTES (8ull << 20) +#define SKIP_PROBE_BYTES (8ull << 20) /* to find an edge */ +#define SKIP_WIDE_BYTES (64ull << 20) /* to find damage: 4% of it, 94% */ #define SKIP_PROBE_MIN 4 -#define SKIP_PROBE_MAX 64 +#define SKIP_PROBE_MAX 512 #define SKIP_RATE_MIN_CHUNKS 8 /* the rate rule, once it has these */ #define SKIP_RATE_MIN_US (60ull * 1000000) /* and a minute of I/O */ #define SKIP_DRILL_CAP_US (30ull * 1000000) /* one chunk's drill-down */ @@ -5434,9 +5434,9 @@ static void skip_setup(ctx_t *c) c->sw_len = (int)w; } -static uint64_t skip_probe_len(const ctx_t *c) +static uint64_t skip_probe_len(const ctx_t *c, uint64_t bytes) { - uint64_t n = SKIP_PROBE_BYTES / (uint64_t)c->chunk; + uint64_t n = bytes / (uint64_t)c->chunk; if (n < SKIP_PROBE_MIN) n = SKIP_PROBE_MIN; @@ -5576,9 +5576,9 @@ static int skip_note(ctx_t *c, uint64_t step, int over) * a probe with no evidence is not slow, so the scan reads the place instead * of condemning it unseen. */ -static int skip_probe(ctx_t *c, uint64_t s, void *buf) +static int skip_probe(ctx_t *c, uint64_t s, uint64_t n, void *buf) { - uint64_t n = skip_probe_len(c), i, spent = 0, bytes = 0; + uint64_t i, spent = 0, bytes = 0; int over = 0, judged = 0; if (s + n > c->nchunks) @@ -5632,7 +5632,8 @@ static int skip_probe(ctx_t *c, uint64_t s, void *buf) static uint64_t skip_ahead(ctx_t *c, void *buf) { uint64_t t = c->step, L = t, F = 0, p, s0, s1, unread, off; - uint64_t pn = skip_probe_len(c); + uint64_t pn = skip_probe_len(c, SKIP_PROBE_BYTES); + uint64_t wn = skip_probe_len(c, SKIP_WIDE_BYTES); uint64_t jump = SKIP_JUMP / (uint64_t)c->chunk; uint64_t res = SKIP_RES / (uint64_t)c->chunk; int to_end = 0, i; @@ -5646,29 +5647,39 @@ static uint64_t skip_ahead(ctx_t *c, void *buf) jump = 1; if (res < pn) res = pn; - /* forward a gigabyte at a time until a probe comes back in budget */ + /* + * Forward a gigabyte at a time until a probe comes back fast. These + * probes are wide: damage in one chunk of twenty-five slips past an + * eight-chunk probe more often than not, and the drive would be called + * clean a gigabyte on when it is not. + */ while (!F && !to_end) { if (g_stop || c->dev_gone) return t + 1; p = L + jump; - if (p + pn > c->nchunks) - p = c->nchunks > pn ? c->nchunks - pn : 0; + if (p + wn > c->nchunks) + p = c->nchunks > wn ? c->nchunks - wn : 0; if (p <= L) { F = L + 1; /* too close to the end to bother */ break; } - if (!skip_probe(c, p, buf)) + if (!skip_probe(c, p, wn, buf)) F = p; - else if (p + pn >= c->nchunks) + else if (p + wn >= c->nchunks) to_end = 1; else L = p; } - /* and back: halve the gap until the edge is known to 16 MiB */ + /* + * And back: halve the gap until the edge is known to 16 MiB. Wide + * probes while the gap is wider than one, narrow ones to place the + * edge -- a wide probe straddling it says only which side most of it + * is on. + */ while (!to_end && F - L > res && !g_stop && !c->dev_gone) { uint64_t m = L + (F - L) / 2; - if (skip_probe(c, m, buf)) + if (skip_probe(c, m, F - L > wn ? wn : pn, buf)) L = m; else F = m; @@ -9227,47 +9238,40 @@ static int tui_count(uint64_t v, int width, const char *col, int striped) } /* - * OVER: the share of chunk reads that missed their budget, over the whole - * scan. Over the whole scan rather than the last thirty seconds because it - * is what the verdict holds against one in OVER_ONE_IN, and a column that - * turned red on a different rule from the verdict would be a screen - * contradicting its own report. It turns red exactly when that rule does. + * SKIP: how much of the drive --skip-slow has condemned without reading it. + * The weak count cannot say this -- a skipped region was never read, so it + * has no sectors to count -- and a scan whose percentage jumps a gigabyte at + * a time should say where that gigabyte went. Short units, so the column + * stays as narrow as a count: 960M, 1.0G, 12T. Yellow, because any skip + * makes the verdict at least SUSPECT. */ -static const char *over_str(uint64_t slow, uint64_t chunks, char *b, size_t n) +static const char *skip_str(uint64_t v, char *b, size_t n) { - double pct; + double m = (double)v / 1048576.0; - if (!chunks) { + if (!v) snprintf(b, n, "-"); - return b; - } - pct = 100.0 * (double)slow / (double)chunks; - if (!slow) - snprintf(b, n, "0%%"); - else if (pct < 0.1) - snprintf(b, n, "<0.1%%"); - else if (pct < 10) - snprintf(b, n, "%.1f%%", pct); + else if (m < 1) + snprintf(b, n, "<1M"); + else if (m < 1000) + snprintf(b, n, "%.0fM", m); + else if (m / 1024 < 100) + snprintf(b, n, "%.1fG", m / 1024); + else if (m / 1024 < 1000) + snprintf(b, n, "%.0fG", m / 1024); else - snprintf(b, n, "%.0f%%", pct); + snprintf(b, n, "%.1fT", m / 1048576); return b; } -static const char *over_col(uint64_t slow, uint64_t chunks) -{ - if (!slow) - return ""; - return over_too_often(slow, chunks) ? c_red() : c_yel(); -} - -static int tui_over(const jrec_t *x, int width, int striped) +static int tui_skip(const jrec_t *x, int width, int striped) { - const char *col = over_col(x->slow, x->chunks); + const char *col = x->skipped ? c_yel() : ""; char b[16]; int used; printf("%s", col); - used = printf("%*s", width, over_str(x->slow, x->chunks, b, sizeof(b))); + used = printf("%*s", width, skip_str(x->skipped, b, sizeof(b))); if (col[0]) printf("%s", striped ? c_plain() : c_off()); return used; @@ -9390,7 +9394,7 @@ static int tui_drive_tail(const jrec_t *x, int pw, int ident, int room, #define DBAR_MIN 10 typedef struct { - int rate, bad, weak, over, eta, state; + int rate, bad, weak, skip, eta, state; int size; /* width of the size column, with ident */ int one; /* each drive on a single row */ int ident; /* model and size columns after the name */ @@ -9405,7 +9409,7 @@ static void tui_dcols(const jrec_t *j, int n, int cols, dcols_t *dc) dc->rate = 11; dc->bad = 7; dc->weak = 8; - dc->over = 5; + dc->skip = 5; dc->eta = 10; dc->state = 5; dc->size = 4; @@ -9426,9 +9430,9 @@ static void tui_dcols(const jrec_t *j, int n, int cols, dcols_t *dc) w = snprintf(b1, sizeof(b1), "%" PRIu64, x->weak); if (w > dc->weak) dc->weak = w; - w = (int)strlen(over_str(x->slow, x->chunks, b1, sizeof(b1))); - if (w > dc->over) - dc->over = w; + w = (int)strlen(skip_str(x->skipped, b1, sizeof(b1))); + if (w > dc->skip) + dc->skip = w; if (x->state == JS_RUNNING) { w = (int)strlen(human_time(x->eta, b2, sizeof(b2))); if (w > dc->eta) @@ -9447,9 +9451,9 @@ static void tui_dcols(const jrec_t *j, int n, int cols, dcols_t *dc) if (w > tail_id) tail_id = w; } - /* " name(8) pct(6) rate bad weak over eta state" */ + /* " name(8) pct(6) rate bad weak skip eta state" */ left = 2 + 8 + 1 + 6 + 1 + dc->rate + 1 + dc->bad + 1 + dc->weak + - 1 + dc->over + 1 + dc->eta + 1 + dc->state; + 1 + dc->skip + 1 + dc->eta + 1 + dc->state; dc->one = left + tail <= cols; if (!dc->one) return; @@ -9583,7 +9587,7 @@ static void tui_dashboard(const run_t *r, const jrec_t *j, int n, int nruns, if (dc.bar) printf(" %*s", dc.bar + 2, ""); printf(" %*s %*s %*s %*s %*s %s", dc.rate, "RATE", dc.bad, - "BAD", dc.weak, "WEAK", dc.over, "OVER", dc.eta, "ETA", + "BAD", dc.weak, "WEAK", dc.skip, "SKIP", dc.eta, "ETA", "STATE"); } tui_eol(); @@ -9721,7 +9725,7 @@ static void tui_dashboard(const run_t *r, const jrec_t *j, int n, int nruns, used += printf(" "); used += tui_count(x->weak, dc.weak, c_yel(), striped); used += printf(" "); - used += tui_over(x, dc.over, striped); + used += tui_skip(x, dc.skip, striped); used += printf(" %*s ", dc.eta, x->state == JS_RUNNING ? human_time(x->eta, b2, sizeof(b2)) : "-"); printf("%s", col); @@ -14138,7 +14142,7 @@ static void status_one(const run_t *r, int json) out(" %s%d orphaned%s", c_yel(), t.orphaned, c_off()); out("\n reports in %s/ records in %s\n\n", r->outdir, r->dir); out(" %-10s %-9s %-20s %6s %9s %9s %6s %-10s %s\n", "DRIVE", "SIZE", - "MODEL", "PCT", "BAD", "WEAK", "OVER", "STATE", + "MODEL", "PCT", "BAD", "WEAK", "SKIP", "STATE", format ? "MESSAGE" : "ETA"); for (i = 0; i < n; i++) { const jrec_t *x = &j[i]; @@ -14149,9 +14153,9 @@ static void status_one(const run_t *r, int json) human_size(x->size, b1, sizeof(b1)), x->model[0] ? x->model : "?", x->state == JS_DONE ? 100.0 : x->pct, x->bad, x->weak, - over_col(x->slow, x->chunks), - over_str(x->slow, x->chunks, ob, sizeof(ob)), - x->slow ? c_off() : "", jrec_color(x), jrec_word(x, format), c_off(), + x->skipped ? c_yel() : "", + skip_str(x->skipped, ob, sizeof(ob)), + x->skipped ? c_off() : "", jrec_color(x), jrec_word(x, format), c_off(), format ? x->note : x->state == JS_RUNNING ? human_time(x->eta, b2, sizeof(b2)) : x->note); diff --git a/tests/cli.sh b/tests/cli.sh index d7e2ae5..8157239 100755 --- a/tests/cli.sh +++ b/tests/cli.sh @@ -571,7 +571,8 @@ assert_has "the checkpoint carries the region" "$(cat "$TMP/skip.ckpt")" \ "skip " rid=$(ls -t "$HDDSCAN_STATE_DIR/runs" | head -1) st=$($HDDSCAN --no-color --status "$rid" 2>&1) -assert_has "--status has an OVER column" "$st" "OVER" +assert_has "--status has a SKIP column" "$st" "SKIP" +assert_has "and it shows how much was skipped" "$st" "1.0G" assert_has "--status-json counts chunks over budget" \ "$($HDDSCAN --status-json "$rid" 2>&1)" '"chunks_over_budget": ' @@ -604,8 +605,9 @@ else "got ${roff:-?}..${rend:-?}" fi -# The other shape, the one a real drive showed with OVER at 4.4% and an ETA -# of four years: only some chunks over budget, but drilling each costs so +# The other shape, the one a real drive showed with 4.4% of chunks over +# budget and an ETA of four years: only some chunks over budget, but +# drilling each costs so # much that the scan crawls. One chunk in four between 1 and 2 GiB reads # slow, so three quarters are never over and only the rate rule can see it: # each slow chunk and its drill-down cost 40 s of I/O for 4 MiB covered. @@ -623,6 +625,21 @@ else bad "and the region found is the one with the damage in it" \ "got ${roff:-?}..${rend:-?}" fi +# REGRESSION: probes were eight chunks, and damage in one chunk of sixteen +# slipped past most of them, so a real drive with 4% of its chunks bad was +# called clean a gigabyte on and nothing was ever skipped. The probes that +# step forward are 64 MiB now. +out=$(HDDSCAN_SLOW=1G:1G:16M:1M HDDSCAN_SLOW_MS=20000 run --chunk 1M \ + --chunk-slow-ms 1000 --retries 1 --skip-slow \ + --json "$TMP/skipsparse2.json" "$f") +read -r roff rend <<< "$(json_region "$TMP/skipsparse2.json")" +if [ -n "$roff" ] && [ "$roff" -ge "$G" ] && [ "$rend" -le $((2 * G)) ] && + [ $((rend - roff)) -ge $((G * 3 / 4)) ]; then + ok "damage in one chunk of sixteen is still seen a gigabyte on" +else + bad "damage in one chunk of sixteen is still seen a gigabyte on" \ + "got ${roff:-?}..${rend:-?}" +fi assert_hasnt "without --skip-slow the rate rule does nothing" \ "$(HDDSCAN_SLOW=1G:1G:4M:1M HDDSCAN_SLOW_MS=20000 run --chunk 1M \ --chunk-slow-ms 1000 --retries 1 --end 1100M "$f")" "skipped" diff --git a/tests/tui.py b/tests/tui.py index 25d6818..a439aef 100755 --- a/tests/tui.py +++ b/tests/tui.py @@ -716,12 +716,10 @@ def rows_now(): # carries the tries and errors behind each sector. check("the table does not split weak sectors into two columns", "SLOW" not in lines[i], lines[i]) - # chunks over budget are a different number from weak sectors: - # a drive slow across whole regions drills every chunk and - # finds almost no sector to count, so WEAK sits still while the - # scan crawls, and this is the column that moves - check("the table has an OVER column for chunks over budget", - "OVER" in lines[i], lines[i]) + # what --skip-slow condemned is not in WEAK: it was never + # read, so it has no sectors to count + check("the table has a SKIP column for what was skipped", + "SKIP" in lines[i], lines[i]) # the header carries the first row of a record; the second is # labelled in place, and is checked against a running drive # further down @@ -1274,7 +1272,7 @@ def wide(cols): s.close() return ls, raw - # 140 is exactly what one row needs: the OVER column put it up six + # 140 is exactly what one row needs: the SKIP column put it up six lines, raw = wide(140) h, w = state_cols(lines) check("a wide terminal puts each drive on one row", diff --git a/tools/shots.py b/tools/shots.py index ad5a9ff..c5d2bad 100644 --- a/tools/shots.py +++ b/tools/shots.py @@ -351,6 +351,8 @@ def run(rid, kind, what, detail, profile, created, jobs): *j.get("lat", (0, 0, 0, 0)))) if j.get("chunks"): f.write("over %d %d\n" % (j.get("over", 0), j["chunks"])) + if j.get("skipped"): + f.write("skipped %d\n" % j["skipped"]) if j.get("wlat"): f.write("wlat %.2f\n" % j["wlat"]) if j.get("note"): @@ -375,6 +377,7 @@ def run(rid, kind, what, detail, profile, created, jobs): chunks=int(size * pct / 100) >> 20, over=(int(size * pct / 100) >> 20) // 3 if i == 2 else 5 * i, + skipped=38 << 30 if i == 2 else 0, lat=(11.0 + 0.4 * i, 12.8 + 0.6 * i, 7.5, 26.7 + 9 * i), wlat=21.4)) else: