From 71153c3fd6af1345df3cf142494b6b69ecfc4cde Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 13 Sep 2026 22:02:26 +0800 Subject: [PATCH 1/2] fix(portable): ignore store hydration cursor churn in identity --- CHANGELOG.md | 1 + internal/portable/identity.go | 3 ++ internal/portable/identity_test.go | 55 ++++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9426edc..07682a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Keep gitcrawl-store hydration checkpoints in portable exports without treating local cursor changes as new source data. - Add explicit cloud archive admission with immutable source, integrity, enrichment, and warning evidence while preserving strict publication defaults. Thanks @vincentkoc. - Compatibility: report portable publication time as `last_export_at`; `last_sync_at` now describes retained successful sync runs instead of old repository scan checkpoints. Thanks @obviyus. - Stop REST pagination when GitHub returns repeated or cyclic next links, preventing repeated requests and incomplete sync results. diff --git a/internal/portable/identity.go b/internal/portable/identity.go index 09cc3f9..d228a76 100644 --- a/internal/portable/identity.go +++ b/internal/portable/identity.go @@ -71,6 +71,9 @@ var currentStateSemanticPolicy = artifactIdentityPolicy{ "observation_convergence_allocator_delete", }, DroppedTables: []string{ + // gitcrawl-store owns these resumable hydration cursors, not source facts. + // Retain them in exports but do not republish unchanged data for cursor churn. + "gitcrawl_store_hydration_progress", "observation_schema_convergence", "repo_pipeline_state", "repo_sync_state", diff --git a/internal/portable/identity_test.go b/internal/portable/identity_test.go index a6a5483..63104d4 100644 --- a/internal/portable/identity_test.go +++ b/internal/portable/identity_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "encoding/binary" "errors" + "fmt" "os" "path/filepath" "slices" @@ -133,6 +134,7 @@ func TestSemanticArtifactIdentityIncludesMeaningfulState(t *testing.T) { {name: "workflow public state", sql: `update github_workflow_runs set status = 'in_progress'`}, {name: "public timestamp", sql: `update threads set updated_at_gh = '2026-08-09T00:00:00Z'`}, {name: "repository identity", sql: `update repositories set full_name = 'openclaw/renamed'`}, + {name: "unknown future table", sql: `create table gitcrawl_store_public_facts(value text); insert into gitcrawl_store_public_facts values('retained')`}, {name: "unknown future column", sql: `alter table threads add column future_public_value text; update threads set future_public_value = 'retained'`}, } for _, mutation := range mutations { @@ -156,6 +158,58 @@ func TestSemanticArtifactIdentityIncludesMeaningfulState(t *testing.T) { } } +func TestSemanticArtifactIdentityRetainsStoreHydrationProgressInExport(t *testing.T) { + ctx := context.Background() + dir := t.TempDir() + sourcePath := filepath.Join(dir, "source.db") + st := seedExportSource(t, ctx, sourcePath) + defer st.Close() + before, err := Export(ctx, testExportOptions(sourcePath, filepath.Join(dir, "before"))) + if err != nil { + t.Fatal(err) + } + if _, err := st.DB().ExecContext(ctx, ` + create table gitcrawl_store_hydration_progress( + repo_id integer not null references repositories(id) on delete cascade, + phase text not null, + cursor integer not null, + primary key(repo_id, phase) + ); + insert into gitcrawl_store_hydration_progress values(1, 'historical-pr-details', 10); + insert into gitcrawl_store_hydration_progress values(1, 'active-pr-context', 20); + `); err != nil { + t.Fatal(err) + } + for _, cursor := range []int{10, 30} { + if _, err := st.DB().ExecContext(ctx, `update gitcrawl_store_hydration_progress set cursor = ? where phase = 'historical-pr-details'`, cursor); err != nil { + t.Fatal(err) + } + result, err := Export(ctx, testExportOptions(sourcePath, filepath.Join(dir, fmt.Sprintf("cursor-%d", cursor)))) + if err != nil { + t.Fatal(err) + } + if result.ArtifactID != before.ArtifactID { + t.Fatalf("hydration progress changed semantic identity: got %s, want %s", result.ArtifactID, before.ArtifactID) + } + if result.SHA256 == before.SHA256 { + t.Fatal("checkpoint-bearing export retained the previous exact file hash") + } + exported := openRawDB(t, result.DatabasePath) + var historical, active int + err = exported.QueryRowContext(ctx, `select cursor from gitcrawl_store_hydration_progress where repo_id = 1 and phase = 'historical-pr-details'`).Scan(&historical) + if err == nil { + err = exported.QueryRowContext(ctx, `select cursor from gitcrawl_store_hydration_progress where repo_id = 1 and phase = 'active-pr-context'`).Scan(&active) + } + closeErr := exported.Close() + if err != nil || closeErr != nil { + t.Fatalf("read exported hydration progress: %v; close: %v", err, closeErr) + } + if historical != cursor || active != 20 { + t.Fatalf("exported hydration progress = (%d, %d), want (%d, 20)", historical, active, cursor) + } + } +} + func TestSemanticArtifactIdentityCanonicalizesCompositeKeyRowids(t *testing.T) { ctx := context.Background() dir := t.TempDir() @@ -388,6 +442,7 @@ func TestCurrentStateSemanticPolicyIsExplicitAndValid(t *testing.T) { t.Fatalf("current semantic identity policy: %v", err) } wantDropped := []string{ + "gitcrawl_store_hydration_progress", "observation_schema_convergence", "pull_request_review_thread_syncs", "repo_pipeline_state", From 1749b9632ca25f18dda94cfea1fdaf36dfe5dd0d Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 13 Sep 2026 22:06:42 +0800 Subject: [PATCH 2/2] test(portable): qualify hydration cursor identity semantics --- docs/portable-stores.md | 7 ++++++- internal/portable/identity_test.go | 20 +++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/portable-stores.md b/docs/portable-stores.md index cf747c7..34b1105 100644 --- a/docs/portable-stores.md +++ b/docs/portable-stores.md @@ -369,7 +369,7 @@ tables and columns are skipped safely. The `current-state-semantic-v1` policy is | Action | Tables or columns | | --- | --- | -| Delete local-only tables when present | `observation_schema_convergence`, `repo_pipeline_state`, `repo_sync_state`, `sqlite_stat1`, `sqlite_stat4`, `thread_observation_sequence`, `thread_child_observation_reservations`, `workflow_run_observation_reservations`, `pull_request_review_thread_syncs` | +| Delete local-only tables when present | `gitcrawl_store_hydration_progress`, `observation_schema_convergence`, `repo_pipeline_state`, `repo_sync_state`, `sqlite_stat1`, `sqlite_stat4`, `thread_observation_sequence`, `thread_child_observation_reservations`, `workflow_run_observation_reservations`, `pull_request_review_thread_syncs` | | Clear repository ingestion time | `repositories.updated_at` | | Clear thread ingestion/order fields | `threads.first_pulled_at`, `last_pulled_at`, `updated_at`, `observation_sequence`, `evidence_observation_sequence`, `evidence_source_updated_at` | | Clear revision/fingerprint record bookkeeping | `thread_revisions.observation_sequence`, `thread_revisions.created_at`, `thread_fingerprints.created_at` | @@ -377,6 +377,11 @@ tables and columns are skipped safely. The `current-state-semantic-v1` policy is | Clear PR/workflow fetch and local record times | `pull_request_details.fetched_at` and `updated_at`; `pull_request_files.fetched_at`; `pull_request_commits.fetched_at`; `pull_request_checks.fetched_at`; `pull_request_review_threads.fetched_at`; `pull_request_review_thread_revisions.fetched_at` and `recorded_at`; `github_workflow_runs.fetched_at` | | Preserve tombstone state without local observation time | Non-NULL `threads.closed_at_local` and `deleted_at` values on comments, PR commits, review threads, and review-thread revisions become an empty non-NULL marker; NULL remains NULL | +`gitcrawl_store_hydration_progress` belongs to the gitcrawl-store producer. +Its checkpoints remain in runtime caches and generated portable SQLite files. +Cursor-only changes do not require publishing a new archive, so a publisher's +semantic no-op can leave the published checkpoint behind the runtime cache. + The policy also removes the named observation-convergence triggers associated with the deleted allocator/reservation state and normalizes SQLite's transient schema cookie before compaction. To prevent insertion-order-only hidden rowids diff --git a/internal/portable/identity_test.go b/internal/portable/identity_test.go index 63104d4..bce43e5 100644 --- a/internal/portable/identity_test.go +++ b/internal/portable/identity_test.go @@ -164,10 +164,6 @@ func TestSemanticArtifactIdentityRetainsStoreHydrationProgressInExport(t *testin sourcePath := filepath.Join(dir, "source.db") st := seedExportSource(t, ctx, sourcePath) defer st.Close() - before, err := Export(ctx, testExportOptions(sourcePath, filepath.Join(dir, "before"))) - if err != nil { - t.Fatal(err) - } if _, err := st.DB().ExecContext(ctx, ` create table gitcrawl_store_hydration_progress( repo_id integer not null references repositories(id) on delete cascade, @@ -180,7 +176,11 @@ func TestSemanticArtifactIdentityRetainsStoreHydrationProgressInExport(t *testin `); err != nil { t.Fatal(err) } - for _, cursor := range []int{10, 30} { + before, err := Export(ctx, testExportOptions(sourcePath, filepath.Join(dir, "before"))) + if err != nil { + t.Fatal(err) + } + for _, cursor := range []int{30, 40} { if _, err := st.DB().ExecContext(ctx, `update gitcrawl_store_hydration_progress set cursor = ? where phase = 'historical-pr-details'`, cursor); err != nil { t.Fatal(err) } @@ -208,6 +208,16 @@ func TestSemanticArtifactIdentityRetainsStoreHydrationProgressInExport(t *testin t.Fatalf("exported hydration progress = (%d, %d), want (%d, 20)", historical, active, cursor) } } + if _, err := st.DB().ExecContext(ctx, `update threads set title = 'changed source fact' where id = 1`); err != nil { + t.Fatal(err) + } + changed, err := Export(ctx, testExportOptions(sourcePath, filepath.Join(dir, "source-change"))) + if err != nil { + t.Fatal(err) + } + if changed.ArtifactID == before.ArtifactID { + t.Fatal("hydration progress table hid a meaningful source change") + } } func TestSemanticArtifactIdentityCanonicalizesCompositeKeyRowids(t *testing.T) {