fix(pipeline): align actions, repair filters, and classify no-data replies

README.md

@@ -358,6 +358,14 @@ Yes, with caveats:
 
 **The Pipeline view** (`eraser pipeline` or the web UI) shows you exactly which brokers need manual attention. It's not fully automated, but it's free—and it does the tedious work of sending 700+ emails and tracking responses for you.
 
+In the web Pipeline, select a status card to see matching requests or actions.
+The **Responses** tab filters by response type and searches broker names, subjects,
+and saved email bodies; **Action Required** also searches task notes. Filters stay
+in the URL for refresh and browser back/forward. Clear "no records for you" replies
+appear as **No data held**, including qualifying replies already saved before the
+upgrade.
+
+
 ---
 
 ## How It Compares

cmd/eraser/cmd_auto.go

@@ -168,7 +168,7 @@ func sentSince(cfgPath string, start time.Time) int {
 	if err != nil {
 		return 0
 	}
-	store, err := history.NewStore(history.DBPathFor(cfgPath))
+	store, err := openHistoryStore(history.DBPathFor(cfgPath))
 	if err != nil {
 		return 0
 	}

cmd/eraser/cmd_bounces.go

@@ -70,7 +70,7 @@ func runCleanupBounces(remove bool, days int) error {
 		return fmt.Errorf("failed to load brokers: %w", err)
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}
@@ -237,7 +237,7 @@ func runMarkBounced(brokerIDs []string, note string) error {
 		return err
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}

cmd/eraser/cmd_confirm.go

@@ -74,7 +74,7 @@ func runConfirm(confirmURL, brokerID string, pending, validateDomain, dryRun boo
 		return fmt.Errorf("failed to load brokers: %w", err)
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}

cmd/eraser/cmd_export.go

@@ -76,7 +76,7 @@ func runExport(opts exportOptions) error {
 		return err
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to open history: %w", err)
 	}

cmd/eraser/cmd_fill.go

@@ -128,7 +128,7 @@ func runFill(brokerID, formURL string, headless, headlessFlagSet, autoSubmit boo
 		screenshotDir = filepath.Join(home, ".eraser", "screenshots")
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}

cmd/eraser/cmd_mark_sent.go

@@ -83,7 +83,7 @@ func runMarkSent(args, regions, categories []string, dryRun bool) error {
 		return nil
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to open history: %w", err)
 	}

cmd/eraser/cmd_monitor.go

@@ -73,7 +73,7 @@ func runMonitor(days int, once bool, watch bool) error {
 		return fmt.Errorf("failed to load brokers: %w", err)
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}
@@ -151,6 +151,7 @@ func scanInbox(ctx context.Context, inboxCfg config.InboxConfig, brokerDB *broke
 	fmt.Printf("📊 Summary for %s:\n", inboxCfg.Email)
 	fmt.Printf("  Total responses:     %d\n", summary.Total)
 	fmt.Printf("  ✅ Success:          %d\n", summary.Success)
+	fmt.Printf("  ℹ️  No data held:     %d\n", summary.NoData)
 	fmt.Printf("  📝 Form required:    %d\n", summary.FormRequired)
 	fmt.Printf("  🔗 Confirm required: %d\n", summary.ConfirmRequired)
 	fmt.Printf("  ❌ Rejected:         %d\n", summary.Rejected)
@@ -182,7 +183,7 @@ func scanInbox(ctx context.Context, inboxCfg config.InboxConfig, brokerDB *broke
 func printClassifiedResponse(r inbox.ClassifiedResponse) {
 	var icon string
 	switch r.Type {
-	case inbox.ResponseSuccess:
+	case inbox.ResponseSuccess, inbox.ResponseNoData:
 		icon = "✅"
 	case inbox.ResponseFormRequired:
 		icon = "📝"

cmd/eraser/cmd_pipeline.go

@@ -32,7 +32,7 @@ func runPipelineStatus() error {
 		return err
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to open history: %w", err)
 	}
@@ -59,6 +59,7 @@ func runPipelineStatus() error {
 	fmt.Printf("  ✅ CAPTCHA solved:        %d\n", pipelineStats[history.PipelineCaptchaSolved])
 	fmt.Printf("  🔗 Awaiting confirmation: %d\n", pipelineStats[history.PipelineAwaitingConfirmation])
 	fmt.Printf("  ✅ Confirmed:             %d\n", pipelineStats[history.PipelineConfirmed])
+	fmt.Printf("  ℹ️  No data held:          %d\n", pipelineStats[history.PipelineNoData])
 	fmt.Printf("  ❌ Rejected:              %d\n", pipelineStats[history.PipelineRejected])
 	fmt.Printf("  💥 Failed:                %d\n", pipelineStats[history.PipelineFailed])
 

cmd/eraser/cmd_send.go

@@ -157,7 +157,7 @@ func runSend() error {
 
 	// Initialize history store early - needed for both the resend-cooldown
 	// skip and the daily send cap below.
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}

cmd/eraser/cmd_serve.go

@@ -63,7 +63,7 @@ func runServe(port int) error {
 		return fmt.Errorf("failed to load brokers: %w", err)
 	}
 
-	store, err := history.NewStore(history.DBPathFor(configPath))
+	store, err := openHistoryStore(history.DBPathFor(configPath))
 	if err != nil {
 		return fmt.Errorf("failed to initialize history: %w", err)
 	}

cmd/eraser/cmd_status.go

@@ -36,7 +36,7 @@ func runStatus(limit int) error {
 		return err
 	}
 
-	store, err := history.NewStore(history.DBPathFor(resolveConfigPath()))
+	store, err := openHistoryStore(history.DBPathFor(resolveConfigPath()))
 	if err != nil {
 		return fmt.Errorf("failed to open history: %w", err)
 	}

cmd/eraser/history.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/drumandbytes/eraser/internal/history"
+	"github.com/drumandbytes/eraser/internal/inbox"
+)
+
+// All production history openings share application-level data upgrades. The
+// history package remains independent of the classifier that uses it.
+func openHistoryStore(path string) (*history.Store, error) {
+	store, err := history.NewStore(path)
+	if err != nil {
+		return nil, err
+	}
+	if err := inbox.MigrateNoDataResponses(store); err != nil {
+		_ = store.Close()
+		return nil, fmt.Errorf("upgrade response classifications: %w", err)
+	}
+	return store, nil
+}

cmd/eraser/history_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/drumandbytes/eraser/internal/history"
+)
+
+func TestOpenHistoryStoreUpgradesSavedNoDataReply(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "history.db")
+	old, err := history.NewStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r := history.BrokerResponse{BrokerID: "acme", ResponseType: "rejected", EmailSubject: "Reply", EmailBody: "We do not have any records of you.", ReceivedAt: time.Now()}
+	if err := old.AddBrokerResponse(&r); err != nil {
+		t.Fatal(err)
+	}
+	_ = old.Close()
+	upgraded, err := openHistoryStore(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer func() { _ = upgraded.Close() }()
+	got, err := upgraded.GetBrokerResponseByID(r.ID, "default")
+	if err != nil || got.ResponseType != "no_data" || got.EmailBody != r.EmailBody {
+		t.Fatalf("upgrade=%+v err=%v", got, err)
+	}
+}

docs/broker-replies.md

@@ -26,14 +26,22 @@ Classifications match `ResponseType` in `internal/inbox/classifier.go`.
 | Reply says | Type | Action |
 |---|---|---|
 | Deletion done / suppressed / "we no longer hold your data" | `success` | none - archive the mail |
-| "No record / no match for you" from a genuine broker | `rejected` | none - they're still a real broker, a future re-scrape may list you again |
+| "No record / no match for you" from a genuine broker | `no_data` (No data held) | none - they're still a real broker, a future re-scrape may list you again |
 | "We are not a data broker" / "B2B only, not in scope" | `rejected` | remove the entry from `data/brokers.yaml`, then check the count floor (below) |
 | "We won't process this by email - use our web form / portal" | `form_required` | set `email: ""`, set/refresh `opt_out_url` to the form they name, add a dated `notes:` line |
 | Won't act without a government-ID document | - | set `category: requires-id`, add a dated `notes:` line (users exclude this category in bulk) |
 | Confirmation / identity link to click | `confirmation_required` | click it (or run `eraser confirm`) - no data change |
 | Bounce / dead address | `bounced` | `eraser mark-bounced <id>`, or set `email: ""` + dated note by hand. Never delete a real broker over a bounce - `MarkEmailUnreachable` in `internal/broker/broker.go` exists so the record survives for a later working address |
 | Refuses on a real statutory exemption (FCRA / GLBA / DPPA), genuine dead end | `rejected` | set `email: ""`, add a dated `notes:` line citing the basis; keep the row (see the `goodhire` entry) |
 
+Clear no-record replies are separate from deletion confirmations and refusals.
+Conditional replies or requests for identity verification still need review.
+On upgrade, Eraser reclassifies qualifying saved replies once, using their stored
+text without fetching mail. It only resolves the current request and superseded
+pending work when the latest reply is confident; newer requests, newer tasks,
+completed/skipped work, and manual pipeline progress are preserved. Original
+messages and task notes remain in history.
+
 "Remove the entry" vs. "blank the email" is a judgement call: delete when the
 company is genuinely not a broker and there's nothing to come back to; blank +
 note when they're a broker you just can't reach by email, or a dead end worth

docs/code-patterns.md

@@ -5,8 +5,10 @@
 - **Templates**: Go `text/template` with embedded files via `//go:embed`.
 - **CLI commands**: Each command has its own `cmd/eraser/cmd_*.go` file (e.g. `cmd_send.go`, `cmd_fill.go`) using Cobra; flags declared in the `*Cmd()` constructor, business logic in a separate `run*()` function so it's testable/callable independent of Cobra. `main.go` only holds root-command wiring and helpers shared by 3+ commands (config/profile/broker-path resolution).
 - **Regexes**: Compile once at package level (`var fooRe = regexp.MustCompile(...)`), never inside a function that runs per-email/per-loop-iteration. `internal/inbox/classifier.go` and `parser.go` follow this.
-- **Reply classifier**: `internal/inbox/classifier.go` holds the upstream English/CCPA keyword patterns; `classifier_eu.go` appends GDPR-vocabulary and German/French patterns to the same package-level slices in `init()`, so both `ClassifyResponse` and `ClassifyBySubjectOnly` pick them up. Add new EU/multilingual patterns there, keeping non-English ones high-signal to avoid matching English text. A `rejected`/`success` verdict below 0.7 confidence is flagged `NeedsReview` (a wrong one silently drops a broker from the `eraser export` overdue list).
+- **Reply classifier**: `internal/inbox/classifier.go` holds the upstream English/CCPA keyword patterns; `classifier_eu.go` appends GDPR-vocabulary and German/French patterns to the same package-level slices in `init()`, so both `ClassifyResponse` and `ClassifyBySubjectOnly` pick them up. Add new EU/multilingual patterns there, keeping non-English ones high-signal to avoid matching English text. A `rejected`/`success`/`no_data` verdict below 0.7 confidence is flagged `NeedsReview` (a wrong one silently drops a broker from the `eraser export` overdue list).
 - **SQLite**: `internal/history/history.go` has composite indexes (including `(profile_id, broker_id, sent_at DESC)`) for the "most recent record per broker, per profile" query pattern used throughout. `NewStore` opens the DB in WAL mode with `busy_timeout(5000)` so `eraser serve` and a CLI run can write at once; a copy/backup of `history.db` must include the `-wal` file (or be taken with the store closed), and WAL doesn't work on network filesystems.
+- **History startup**: CLI commands use `openHistoryStore`, which opens the schema and runs the versioned, transactional saved-reply migration via `inbox.MigrateNoDataResponses`. Keep new entry points on this helper so upgrades do not require visiting the web UI. The migration is local-only and retryable after a rollback.
+- **Pipeline UI**: counts and pending rows share `history.GetActionItems`; use it for Pipeline and Action Needed together. The Pipeline URL carries view, status, response type, search, review, and page filters. Response search filters the active profile in SQLite before pagination, including stored email bodies.
 - **Multi-profile scoping**: any new `history.Store` method that reads/writes per-user data should take a `profileID string` first parameter and filter by it (see [multi-profile.md](multi-profile.md)) - the two deliberate exceptions are `ClearBrokerResponses`/`GetAllBrokerResponses`, which operate on the one shared inbox across all profiles.
 
 ## Known quirks

internal/evidence/evidence.go

@@ -104,7 +104,7 @@ func legalBasisFor(templateName string) string {
 // actually engaging with the request (vs. silence, a bounce, or noise).
 func substantiveResponse(responseType string) bool {
 	switch responseType {
-	case "success", "rejected", "form_required", "confirmation_required":
+	case "success", "rejected", "form_required", "confirmation_required", "no_data":
 		return true
 	default:
 		return false
@@ -230,7 +230,7 @@ func Build(
 				ConfirmURL:  r.ConfirmURL,
 				NeedsReview: r.NeedsReview,
 			})
-			if substantiveResponse(r.ResponseType) {
+			if substantiveResponse(r.ResponseType) && (r.ResponseType != "no_data" || (!r.NeedsReview && r.Confidence >= 0.7)) {
 				gotSubstantive = true
 			}
 		}

internal/evidence/evidence_test.go

@@ -240,6 +240,18 @@ func TestJoinAddressSkipsBlankParts(t *testing.T) {
 	}
 }
 
+func TestNoDataEvidenceRequiresConfidentReply(t *testing.T) {
+	now := time.Now()
+	requests := []history.Record{{BrokerID: "acme", BrokerName: "Acme", Status: history.StatusSent, Template: "gdpr", SentAt: now.AddDate(0, -2, 0)}}
+	for _, review := range []bool{false, true} {
+		responses := []history.BrokerResponse{{BrokerID: "acme", ResponseType: "no_data", Confidence: 0.85, NeedsReview: review, ReceivedAt: now}}
+		report := Build(testProfile(), requests, responses, testBrokerDB(), testEngine(t), time.Time{}, now)
+		if got := findBroker(t, report, "acme"); got.PastDeadline != review || got.Responses[0].Type != "no_data" {
+			t.Fatalf("review=%v evidence=%+v", review, got)
+		}
+	}
+}
+
 func TestRenderHTML(t *testing.T) {
 	now := time.Date(2026, 6, 1, 12, 0, 0, 0, time.UTC)
 	p := testProfile()