diff --git a/sei-db/state_db/giga/types/live_state_store.go b/sei-db/state_db/giga/types/live_state_store.go index 9a4854c9d7..78c2b5d58d 100644 --- a/sei-db/state_db/giga/types/live_state_store.go +++ b/sei-db/state_db/giga/types/live_state_store.go @@ -73,6 +73,10 @@ type LiveStateStore interface { // initialVersion <= 0. SetInitialVersion(initialVersion int64) error + // SeededVersion returns the version SetInitialVersion seeded this store at above genesis. The store holds + // no history at or below it. ok is false when no such version is recorded. + SeededVersion() (version int64, ok bool) + // Get returns the value for a key within the given module. // For EVM keys (moduleName == "evm"), the key is a memiavl EVM key // routed to account/storage/code/misc DBs internally. diff --git a/sei-db/state_db/sc/composite/store.go b/sei-db/state_db/sc/composite/store.go index 7e7b552370..e058dd46fe 100644 --- a/sei-db/state_db/sc/composite/store.go +++ b/sei-db/state_db/sc/composite/store.go @@ -451,7 +451,8 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type } } - if cs.flatKV != nil { + flatKVPredates := cs.flatKVPredates(targetVersion) + if cs.flatKV != nil && !flatKVPredates { fkv, err := cs.flatKV.LoadVersionReadOnly(targetVersion) if err != nil { return nil, fmt.Errorf("failed to load FlatKV version: %w", err) @@ -475,7 +476,9 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type return nil, err } } - if err := ro.resolveCurrentWriteMode(false); err != nil { + if flatKVPredates { + ro.currentWriteMode = types.MemiavlOnly + } else if err := ro.resolveCurrentWriteMode(false); err != nil { return nil, fmt.Errorf("failed to resolve effective write mode for read-only handle: %w", err) } if err := ro.buildRouter(); err != nil { @@ -487,6 +490,17 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type return ro, nil } +// flatKVPredates reports whether version is at or below the height flatKV was seeded at, so that a view or +// export there is served from memIAVL alone. Version 0 means latest and never predates. +func (cs *CompositeCommitStore) flatKVPredates(version int64) bool { + if cs.flatKV == nil || cs.memIAVL == nil || version <= 0 { + return false + } + // Up to the seeded height memIAVL alone made up the AppHash, and flatKV holds no history there. + seeded, ok := cs.flatKV.SeededVersion() + return ok && version <= seeded +} + // resolveCurrentWriteMode sets cs.currentWriteMode after the backends have been // opened. For a fixed configured mode this is a copy; for types.Auto the // mode is derived from the migration metadata persisted in flatkv. A nil @@ -1431,8 +1445,9 @@ func (cs *CompositeCommitStore) Exporter(version int64) (types.Exporter, error) return nil, fmt.Errorf("version %d out of range", version) } + flatKVPredates := cs.flatKVPredates(version) includeMemiavl := cs.memIAVL != nil - includeFlatKV := cs.flatKV != nil + includeFlatKV := cs.flatKV != nil && !flatKVPredates if includeFlatKV && exportNeedsMetadataGating(cs.config.WriteMode) { // Evaluate the hash predicates against metadata as-of the exported diff --git a/sei-db/state_db/sc/composite/store_auto_test.go b/sei-db/state_db/sc/composite/store_auto_test.go index 1fb66fe818..fa19c97d91 100644 --- a/sei-db/state_db/sc/composite/store_auto_test.go +++ b/sei-db/state_db/sc/composite/store_auto_test.go @@ -1,6 +1,8 @@ package composite import ( + "os" + "path/filepath" "testing" "github.com/sei-protocol/sei-chain/sei-db/common/keys" @@ -581,17 +583,10 @@ func TestComposite_Auto_ReadOnlyHandle(t *testing.T) { requireOracleMatches(t, ro, workload.snapshotOracle()) } -// TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails records a guarantee that was deliberately -// given up: heights predating flatkv's history used to be served memiavl-only, because flatkv kept a -// persisted record of the height its history began at and the read path consulted it. -// -// That record is gone, so nothing distinguishes "this height predates flatkv" from "flatkv failed to -// load at this height", and the read path can only take the safe branch — attempt the load and -// surface the failure. Answering the other way would serve the height from a memiavl whose migrated -// keys were deleted, fabricating a nonexistence answer, so failing here is the correct direction. -// -// In-era heights are unaffected. Delete this test only alongside a decision to restore pre-era reads. -func TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails(t *testing.T) { +// TestComposite_Auto_ReadOnlyPreFlatKVEraHeight pins the era-aware read-only path: heights that predate +// flatkv's history must remain queryable after the migration has begun, served memiavl-only with as-of-height +// values. In-era heights keep loading flatkv. +func TestComposite_Auto_ReadOnlyPreFlatKVEraHeight(t *testing.T) { dir := t.TempDir() cs := openAutoStoreWithConfig(t, dir, autoExportConfig(), 100) defer func() { _ = cs.Close() }() @@ -617,14 +612,21 @@ func TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails(t *testing.T) { require.NoError(t, err) } - // Pre-era height: the load is attempted against a flatkv that has no such height, and fails. - _, err := cs.LoadVersionReadOnly(3) - require.Error(t, err, "a pre-flatkv-era height is no longer distinguishable from a load failure") + roCommitter, err := cs.LoadVersionReadOnly(3) + require.NoError(t, err, "pre-flatkv-era heights must remain queryable") + ro, ok := roCommitter.(*CompositeCommitStore) + require.True(t, ok) + require.Equal(t, types.MemiavlOnly, ro.currentWriteMode) + require.Nil(t, ro.flatKV) + val, found, err := ro.Get(keys.BankStoreKey, []byte("k")) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, valAt(3), val, "value as-of height 3") + require.NoError(t, ro.Close()) - // In-era height: unchanged. - roCommitter, err := cs.LoadVersionReadOnly(7) + roCommitter, err = cs.LoadVersionReadOnly(7) require.NoError(t, err) - ro, ok := roCommitter.(*CompositeCommitStore) + ro, ok = roCommitter.(*CompositeCommitStore) require.True(t, ok) defer func() { _ = ro.Close() }() require.NotNil(t, ro.flatKV, "in-era heights must keep loading flatkv") @@ -657,3 +659,195 @@ func TestComposite_Auto_CopyAvailability(t *testing.T) { require.Nil(t, cs.Copy(), "Copy is unavailable once flatkv is open") } + +// TestComposite_Auto_ReadOnlyBelowSeededVersion pins historical reads across the MemiavlOnly -> MigrateEVM +// seam. flatkv is seeded at the height memiavl had reached and holds no history below it, so a read-only +// view or export there must be served from memiavl alone and match what the node served before the +// transition, instead of asking flatkv to replay blocks it never had. +func TestComposite_Auto_ReadOnlyBelowSeededVersion(t *testing.T) { + dir := t.TempDir() + workload := newMigrationWorkload(0xA078) + cfg := autoExportConfig() + cfg.MemIAVLConfig.SnapshotKeepRecent = 100 + + cs := openAutoStoreWithConfig(t, dir, cfg, 25) + defer func() { _ = cs.Close() }() + + runBlocks(t, cs, workload, 10) + pre := cs.Version() + oracle := workload.snapshotOracle() + runBlocks(t, cs, workload, 3) + + before, err := cs.LoadVersionReadOnly(pre) + require.NoError(t, err) + beforeInfo := before.(*CompositeCommitStore).LastCommitInfo() + require.NoError(t, before.(*CompositeCommitStore).Close()) + exp, err := cs.Exporter(pre) + require.NoError(t, err) + beforeItems := drainCompositeExporter(t, exp) + require.NoError(t, exp.Close()) + + require.NoError(t, cs.SetWriteMode(types.MigrateEVM)) + runBlocks(t, cs, workload, 5) + require.True(t, hasLatticeHash(cs)) + + view, err := cs.LoadVersionReadOnly(pre) + require.NoError(t, err) + ro := view.(*CompositeCommitStore) + defer func() { _ = ro.Close() }() + require.Nil(t, ro.flatKV, "flatkv holds no history below its seeded version") + require.Equal(t, beforeInfo, ro.LastCommitInfo()) + requireOracleMatches(t, ro, oracle) + + exp, err = cs.Exporter(pre) + require.NoError(t, err) + afterItems := drainCompositeExporter(t, exp) + require.NoError(t, exp.Close()) + require.Equal(t, beforeItems, afterItems) + + latest, err := cs.LoadVersionReadOnly(cs.Version()) + require.NoError(t, err) + defer func() { _ = latest.(*CompositeCommitStore).Close() }() + require.True(t, hasLatticeHash(latest.(*CompositeCommitStore)), + "a view above the seeded version must still include flatkv") + + seeded, ok := cs.flatKV.SeededVersion() + require.True(t, ok) + require.Equal(t, pre+3, seeded) + requireSeedBoundary(t, cs, seeded) +} + +// requireSeedBoundary asserts that a view at seeded is served from memiavl alone and one at seeded+1 includes +// flatkv. +func requireSeedBoundary(t *testing.T, cs *CompositeCommitStore, seeded int64) { + t.Helper() + atSeed, err := cs.LoadVersionReadOnly(seeded) + require.NoError(t, err) + defer func() { _ = atSeed.(*CompositeCommitStore).Close() }() + require.Nil(t, atSeed.(*CompositeCommitStore).flatKV, "the seeded version itself predates flatkv") + require.False(t, hasLatticeHash(atSeed.(*CompositeCommitStore))) + + afterSeed, err := cs.LoadVersionReadOnly(seeded + 1) + require.NoError(t, err) + defer func() { _ = afterSeed.(*CompositeCommitStore).Close() }() + require.NotNil(t, afterSeed.(*CompositeCommitStore).flatKV, "the first block after the seed is in flatkv's era") + require.True(t, hasLatticeHash(afterSeed.(*CompositeCommitStore))) +} + +// openFixedStore opens (or reopens) a composite store at dir in a fixed write mode. +func openFixedStore(t *testing.T, dir string, mode types.WriteMode) (*CompositeCommitStore, error) { + t.Helper() + cfg := autoConfig() + cfg.WriteMode = mode + cs, err := NewCompositeCommitStore(t.Context(), dir, cfg) + require.NoError(t, err) + require.NoError(t, cs.SetMigrationBatchSize(25)) + require.NoError(t, cs.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey})) + if err := cs.LoadLatest(); err != nil { + _ = cs.Close() + return nil, err + } + return cs, nil +} + +// seedFixedMigrateEVM runs a memiavl_only node for 13 blocks, switches it to a fixed migrate_evm configuration, +// which seeds flatkv at 13 on the first LoadLatest, and runs 5 more blocks. It returns the closed store's +// directory and the oracle of the state at height 10. +func seedFixedMigrateEVM(t *testing.T, seed int64) (string, map[migKeyPair][]byte) { + t.Helper() + dir := t.TempDir() + workload := newMigrationWorkload(seed) + cs, err := openFixedStore(t, dir, types.MemiavlOnly) + require.NoError(t, err) + runBlocks(t, cs, workload, 10) + oracle := workload.snapshotOracle() + runBlocks(t, cs, workload, 3) + require.NoError(t, cs.Close()) + + cs, err = openFixedStore(t, dir, types.MigrateEVM) + require.NoError(t, err) + runBlocks(t, cs, workload, 5) + require.NoError(t, cs.Close()) + return dir, oracle +} + +// TestComposite_FixedMigrateEVM_ReadOnlyBelowSeededVersion covers the same seam for a node switched from +// memiavl_only to a fixed migrate_evm configuration, where flatkv is seeded on the first LoadLatest. +func TestComposite_FixedMigrateEVM_ReadOnlyBelowSeededVersion(t *testing.T) { + dir, oracle := seedFixedMigrateEVM(t, 0xA079) + cs, err := openFixedStore(t, dir, types.MigrateEVM) + require.NoError(t, err) + defer func() { _ = cs.Close() }() + + view, err := cs.LoadVersionReadOnly(10) + require.NoError(t, err) + ro := view.(*CompositeCommitStore) + defer func() { _ = ro.Close() }() + require.Nil(t, ro.flatKV) + require.False(t, hasLatticeHash(ro)) + requireOracleMatches(t, ro, oracle) + + seeded, ok := cs.flatKV.SeededVersion() + require.True(t, ok) + require.Equal(t, int64(13), seeded) + requireSeedBoundary(t, cs, seeded) +} + +// TestComposite_RecordedSeededVersionRestoresPreSeedReads pins the remediation for a node seeded before the +// seeded version was recorded: without the record, heights below the seed fail to load; once it is recorded, +// they are served from memiavl alone. +func TestComposite_RecordedSeededVersionRestoresPreSeedReads(t *testing.T) { + dir, oracle := seedFixedMigrateEVM(t, 0xA07A) + flatKVDir := utils.GetFlatKVPath(dir) + require.NoError(t, os.Remove(filepath.Join(flatKVDir, "SEEDED_VERSION"))) + + cs, err := openFixedStore(t, dir, types.MigrateEVM) + require.NoError(t, err) + _, err = cs.LoadVersionReadOnly(10) + require.ErrorContains(t, err, "data loss or corruption") + _, err = cs.Exporter(10) + require.ErrorContains(t, err, "data loss or corruption") + require.NoError(t, cs.Close()) + + fkvCfg := autoConfig().FlatKVConfig + fkvCfg.DataDir = flatKVDir + stateWAL, err := flatkv.OpenStateWAL(&fkvCfg) + require.NoError(t, err) + fkv, err := flatkv.NewCommitStore(t.Context(), &fkvCfg, stateWAL) + require.NoError(t, err) + require.NoError(t, fkv.LoadLatest()) + require.ErrorContains(t, fkv.RecordSeededVersion(14), "predates it") + require.NoError(t, fkv.RecordSeededVersion(13)) + require.NoError(t, fkv.Close()) + + cs, err = openFixedStore(t, dir, types.MigrateEVM) + require.NoError(t, err) + defer func() { _ = cs.Close() }() + view, err := cs.LoadVersionReadOnly(10) + require.NoError(t, err) + defer func() { _ = view.(*CompositeCommitStore).Close() }() + require.Nil(t, view.(*CompositeCommitStore).flatKV) + requireOracleMatches(t, view.(*CompositeCommitStore), oracle) + exp, err := cs.Exporter(10) + require.NoError(t, err) + require.NotContains(t, moduleNamesOf(drainCompositeExporter(t, exp)), keys.FlatKVStoreKey) + require.NoError(t, exp.Close()) + requireSeedBoundary(t, cs, 13) +} + +// TestComposite_InvalidSeededVersionFailsLoadLatest pins that a seeded-version record flatkv's history +// contradicts stops the node at open, rather than letting it start and fail every historical read. +func TestComposite_InvalidSeededVersionFailsLoadLatest(t *testing.T) { + dir, _ := seedFixedMigrateEVM(t, 0xA07B) + recordPath := filepath.Join(utils.GetFlatKVPath(dir), "SEEDED_VERSION") + + for record, wantErr := range map[string]string{ + "abc\n": "not a positive height", + "999999\n": "above the store's version", + "15\n": "predates it", + } { + require.NoError(t, os.WriteFile(recordPath, []byte(record), 0600)) + _, err := openFixedStore(t, dir, types.MigrateEVM) + require.ErrorContains(t, err, wantErr, "record %q", record) + } +} diff --git a/sei-db/state_db/sc/composite/store_test.go b/sei-db/state_db/sc/composite/store_test.go index f120e10e0f..d47d4601af 100644 --- a/sei-db/state_db/sc/composite/store_test.go +++ b/sei-db/state_db/sc/composite/store_test.go @@ -44,6 +44,7 @@ func (f *failingEVMStore) CommitStateChanges(int64, []*proto.NamedChangeSet) err } func (f *failingEVMStore) OpenView() gigatypes.StateView { return nil } func (f *failingEVMStore) SetInitialVersion(int64) error { return nil } +func (f *failingEVMStore) SeededVersion() (int64, bool) { return 0, false } func (f *failingEVMStore) Get(string, []byte) ([]byte, bool) { return nil, false } func (f *failingEVMStore) GetBlockHeightModified(string, []byte) (int64, bool, error) { return -1, false, nil diff --git a/sei-db/state_db/sc/flatkv/snapshot.go b/sei-db/state_db/sc/flatkv/snapshot.go index 6e0dfaa20d..f25db9723b 100644 --- a/sei-db/state_db/sc/flatkv/snapshot.go +++ b/sei-db/state_db/sc/flatkv/snapshot.go @@ -33,6 +33,7 @@ import ( // account/, code/, storage/, misc/ // SNAPSHOT_BASE (records source snapshot name) // changelog/ (WAL, shared across snapshots) +// SEEDED_VERSION (height SetInitialVersion seeded at, if above genesis) const ( // snapshotPrefix is the directory name prefix for versioned snapshots. snapshotPrefix = "snapshot-" @@ -178,8 +179,8 @@ func updateCurrentSymlink(root, snapshotDir string) error { return nil } -// removeTmpDirs removes any directories ending in "-tmp" or "-removing" -// left over from interrupted snapshot writes or deletes. +// removeTmpDirs removes the directories ending in "-tmp" or "-removing" and the temporary seeded-version +// record left over from interrupted writes or deletes. func removeTmpDirs(dir string) error { entries, err := os.ReadDir(dir) if err != nil { @@ -192,6 +193,10 @@ func removeTmpDirs(dir string) error { if err := os.RemoveAll(filepath.Join(dir, name)); err != nil { errs = append(errs, fmt.Errorf("remove tmp dir %s: %w", name, err)) } + } else if name == seededVersionFile+tmpSuffix { + if err := os.Remove(filepath.Join(dir, name)); err != nil { + errs = append(errs, fmt.Errorf("remove tmp file %s: %w", name, err)) + } } } if len(errs) > 0 { diff --git a/sei-db/state_db/sc/flatkv/store.go b/sei-db/state_db/sc/flatkv/store.go index 9b99aaf74a..dee48e553b 100644 --- a/sei-db/state_db/sc/flatkv/store.go +++ b/sei-db/state_db/sc/flatkv/store.go @@ -173,6 +173,9 @@ type CommitStore struct { // readOnly marks stores opened via LoadVersionReadOnly. readOnly bool + // The version SetInitialVersion seeded this store at, 0 when none is recorded. See SeededVersion. + seededVersion int64 + readOnlyWorkDir string // Temp working dir for readonly store; removed by Close. // A work pool for reading from the DBs. @@ -423,6 +426,10 @@ func (s *CommitStore) LoadLatest() (retErr error) { if err := s.openTo(0); err != nil { return fmt.Errorf("failed to open FlatKV store: %w", err) } + if err := s.loadSeededVersion(); err != nil { + _ = s.closeDBsOnly() + return fmt.Errorf("failed to open FlatKV store: %w", err) + } return nil } @@ -1512,6 +1519,12 @@ func (s *CommitStore) resetForImport() error { return fmt.Errorf("resetForImport: remove %s: %w", currentLink, err) } + // An imported store's history starts at the imported version, not at any earlier seed. + if err := os.Remove(filepath.Join(dir, seededVersionFile)); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("resetForImport: remove %s: %w", seededVersionFile, err) + } + s.seededVersion = 0 + // The WAL is deliberately left alone. Import bypasses it, so a pre-existing WAL is stale relative to the // imported version — but a state-sync restore is a manual procedure in which the operator stops the node // and removes its data directories, changelog included, so the restored node opens an empty WAL and its diff --git a/sei-db/state_db/sc/flatkv/store_constants.go b/sei-db/state_db/sc/flatkv/store_constants.go index 285dc11b60..f952971626 100644 --- a/sei-db/state_db/sc/flatkv/store_constants.go +++ b/sei-db/state_db/sc/flatkv/store_constants.go @@ -12,6 +12,10 @@ const ( changelogDir = "changelog" lockFileName = "LOCK" + // seededVersionFile records the version SetInitialVersion seeded the store at (see SeededVersion), as a + // decimal height on a single line. Write it with `seidb flatkv-record-seeded-version`, not by hand. + seededVersionFile = "SEEDED_VERSION" + // DB subdirectories (inside each snapshot) accountDBDir = "account" codeDBDir = "code" diff --git a/sei-db/state_db/sc/flatkv/store_meta.go b/sei-db/state_db/sc/flatkv/store_meta.go index a9d79843c2..a77b9e0921 100644 --- a/sei-db/state_db/sc/flatkv/store_meta.go +++ b/sei-db/state_db/sc/flatkv/store_meta.go @@ -7,6 +7,8 @@ import ( "math" "os" "path/filepath" + "strconv" + "strings" "github.com/cockroachdb/pebble/v2" @@ -425,11 +427,138 @@ func (s *CommitStore) SetInitialVersion(initialVersion int64) error { if err := s.outOfBandSnapshot(); err != nil { return fmt.Errorf("flatkv: SetInitialVersion: write seeded snapshot: %w", err) } + // Recorded last, so a crash partway through the seed leaves no record. A missing record only makes + // heights at or below the seed fail to load, where one that outlived a torn seed would be trusted. + if err := writeSeededVersion(s.flatkvDir(), seededVersion); err != nil { + return fmt.Errorf("flatkv: SetInitialVersion: %w", err) + } + s.seededVersion = seededVersion } logger.Info("FlatKV SetInitialVersion", "initialVersion", initialVersion, "seededVersion", seededVersion) return nil } +// SeededVersion returns the version SetInitialVersion seeded this store at. The store holds no history at or +// below it. ok is false when the store was seeded at genesis, imported, or seeded before the version was +// recorded. The record is read and validated by LoadLatest. +func (s *CommitStore) SeededVersion() (version int64, ok bool) { + return s.seededVersion, s.seededVersion > 0 +} + +// RecordSeededVersion records version as the height this store was seeded at, for a store seeded before +// SetInitialVersion recorded it. It fails when the store's snapshots or WAL show history at or below version. +func (s *CommitStore) RecordSeededVersion(version int64) error { + if s.readOnly { + return errReadOnly + } + if s.isClosed() { + return fmt.Errorf("flatkv: RecordSeededVersion called before LoadLatest") + } + if err := s.validateSeededVersion(version); err != nil { + return err + } + if err := writeSeededVersion(s.flatkvDir(), version); err != nil { + return fmt.Errorf("flatkv: RecordSeededVersion: %w", err) + } + s.seededVersion = version + return nil +} + +// loadSeededVersion reads the seeded-version record of the open store, validates it and caches it. +func (s *CommitStore) loadSeededVersion() error { + version, ok, err := readSeededVersion(s.flatkvDir()) + if err != nil { + return err + } + if !ok { + s.seededVersion = 0 + return nil + } + if err := s.validateSeededVersion(version); err != nil { + return err + } + s.seededVersion = version + return nil +} + +// validateSeededVersion checks a seeded version against the history the open store holds. +func (s *CommitStore) validateSeededVersion(version int64) error { + // Only a record that is too high is dangerous: it would serve in-era heights without flatkv. One that is + // too low leaves the heights between it and the real seed failing to load, as with no record at all. The + // seeded snapshot is the floor of the store's history and the first WAL block follows it, so any history + // below the record disproves it. + if version <= 0 { + return fmt.Errorf("flatkv: %s holds %d, not a positive height", seededVersionFile, version) + } + if version > s.committedVersion { + return fmt.Errorf("flatkv: %s holds %d, above the store's version %d", + seededVersionFile, version, s.committedVersion) + } + var earliest int64 + if err := traverseSnapshots(s.flatkvDir(), true, func(v int64) (bool, error) { + earliest = v + return v > 0, nil + }); err != nil { + return fmt.Errorf("flatkv: list snapshots: %w", err) + } + // snapshot-0 is the empty genesis baseline every store starts from, so it says nothing about the seed. + if earliest > 0 && earliest < version { + return fmt.Errorf("flatkv: %s holds %d, but snapshot %d predates it", + seededVersionFile, version, earliest) + } + if s.wal != nil { + stored, first, _, err := s.wal.GetStoredRange() + if err != nil { + return fmt.Errorf("flatkv: WAL range: %w", err) + } + if stored && first <= uint64(version) { //nolint:gosec // version > 0 + return fmt.Errorf("flatkv: %s holds %d, but the WAL holds block %d", + seededVersionFile, version, first) + } + } + return nil +} + +// readSeededVersion reads the seeded-version record of the store under dir. ok is false when there is none. +func readSeededVersion(dir string) (version int64, ok bool, err error) { + data, err := os.ReadFile(filepath.Join(dir, seededVersionFile)) //nolint:gosec // path under the store root + if err != nil { + if os.IsNotExist(err) { + return 0, false, nil + } + return 0, false, fmt.Errorf("flatkv: read %s: %w", seededVersionFile, err) + } + version, err = strconv.ParseInt(strings.TrimSpace(string(data)), 10, 64) + if err != nil || version <= 0 { + return 0, false, fmt.Errorf("flatkv: %s holds %q, not a positive height", seededVersionFile, data) + } + return version, true, nil +} + +// writeSeededVersion durably records version as the height the store under dir was seeded at. +func writeSeededVersion(dir string, version int64) error { + path := filepath.Join(dir, seededVersionFile) + tmpPath := path + tmpSuffix + f, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) //nolint:gosec // path under the store root + if err != nil { + return fmt.Errorf("create %s: %w", seededVersionFile, err) + } + _, err = f.WriteString(strconv.FormatInt(version, 10) + "\n") + if err == nil { + err = f.Sync() + } + if closeErr := f.Close(); err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("write %s: %w", seededVersionFile, err) + } + if err := os.Rename(tmpPath, path); err != nil { + return fmt.Errorf("install %s: %w", seededVersionFile, err) + } + return nil +} + // GetLatestVersion returns the version a store opened on dir will report once LoadLatest has run. // A directory that has never been opened reads as 0. // diff --git a/sei-db/state_db/sc/flatkv/store_meta_test.go b/sei-db/state_db/sc/flatkv/store_meta_test.go index a32cf2b437..d7f8b5927f 100644 --- a/sei-db/state_db/sc/flatkv/store_meta_test.go +++ b/sei-db/state_db/sc/flatkv/store_meta_test.go @@ -214,6 +214,13 @@ func TestSetInitialVersion_HappyPath(t *testing.T) { target, err := os.Readlink(currentPath(s.flatkvDir())) require.NoError(t, err) require.Equal(t, snapshotName(99), target) + seeded, ok := s.SeededVersion() + require.True(t, ok) + require.Equal(t, int64(99), seeded) + onDisk, ok, err := readSeededVersion(s.flatkvDir()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, int64(99), onDisk) addr := ktype.Address{0xAA} slot := ktype.Slot{0xBB} @@ -235,6 +242,11 @@ func TestSetInitialVersion_GenesisSkipsSeededSnapshot(t *testing.T) { target, err := os.Readlink(currentPath(s.flatkvDir())) require.NoError(t, err) require.Equal(t, snapshotName(0), target) + _, ok := s.SeededVersion() + require.False(t, ok, "a genesis seed predates nothing") + _, ok, err = readSeededVersion(s.flatkvDir()) + require.NoError(t, err) + require.False(t, ok) addr := ktype.Address{0xAA} slot := ktype.Slot{0xBB} @@ -246,6 +258,156 @@ func TestSetInitialVersion_GenesisSkipsSeededSnapshot(t *testing.T) { require.Equal(t, int64(1), v, "first Commit after SetInitialVersion(1) must produce version 1") } +func TestSetInitialVersion_ImportClearsSeededVersion(t *testing.T) { + s := setupTestStore(t) + defer s.Close() + + require.NoError(t, s.SetInitialVersion(100)) + imp, err := s.Importer(200) + require.NoError(t, err) + require.NoError(t, imp.Close()) + + _, ok := s.SeededVersion() + require.False(t, ok, "an imported store's history starts at the import, not the earlier seed") + _, ok, err = readSeededVersion(s.flatkvDir()) + require.NoError(t, err) + require.False(t, ok) +} + +func TestSetInitialVersion_ReseedAfterAbortedImportRecordsNewVersion(t *testing.T) { + s := setupTestStore(t) + defer s.Close() + + require.NoError(t, s.SetInitialVersion(100)) + imp, err := s.Importer(200) + require.NoError(t, err) + require.Error(t, imp.Abort(nil)) + _, ok := s.SeededVersion() + require.False(t, ok) + + require.NoError(t, s.LoadLatest()) + require.Equal(t, int64(0), s.Version()) + require.NoError(t, s.SetInitialVersion(301)) + seeded, ok := s.SeededVersion() + require.True(t, ok) + require.Equal(t, int64(300), seeded) + onDisk, ok, err := readSeededVersion(s.flatkvDir()) + require.NoError(t, err) + require.True(t, ok) + require.Equal(t, int64(300), onDisk) +} + +// seededStoreWithBlocks returns the config of a closed store seeded at 10 that committed blocks 11-13. +func seededStoreWithBlocks(t *testing.T) *config.Config { + t.Helper() + cfg := config.DefaultTestConfig(t) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + require.NoError(t, s.SetInitialVersion(11)) + commitBlocks(t, s, 3) + require.Equal(t, int64(13), s.Version()) + require.NoError(t, s.Close()) + return cfg +} + +// openWithSeededRecord writes record as the store's seeded-version record and opens the store. +func openWithSeededRecord(t *testing.T, cfg *config.Config, record string) (*CommitStore, error) { + t.Helper() + require.NoError(t, os.WriteFile(filepath.Join(cfg.DataDir, seededVersionFile), []byte(record), 0600)) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + if err := s.LoadLatest(); err != nil { + _ = s.Close() + return nil, err + } + return s, nil +} + +func TestLoadLatest_ValidatesSeededVersion(t *testing.T) { + cfg := seededStoreWithBlocks(t) + + for _, tc := range []struct { + record string + wantErr string + }{ + {record: "abc\n", wantErr: "not a positive height"}, + {record: "0\n", wantErr: "not a positive height"}, + {record: "999999\n", wantErr: "above the store's version 13"}, + {record: "12\n", wantErr: "snapshot 10 predates it"}, + } { + _, err := openWithSeededRecord(t, cfg, tc.record) + require.ErrorContains(t, err, tc.wantErr, "record %q", tc.record) + } + + for _, tc := range []struct { + record string + want int64 + }{ + {record: "10\n", want: 10}, + {record: "9", want: 9}, // too low only leaves heights 10 and below failing to load + } { + s, err := openWithSeededRecord(t, cfg, tc.record) + require.NoError(t, err, "record %q", tc.record) + seeded, ok := s.SeededVersion() + require.True(t, ok) + require.Equal(t, tc.want, seeded) + require.NoError(t, s.Close()) + } +} + +func TestLoadLatest_RejectsSeededVersionTheWALContradicts(t *testing.T) { + cfg := config.DefaultTestConfig(t) + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + commitBlocks(t, s, 5) + require.NoError(t, s.Close()) + + _, err = openWithSeededRecord(t, cfg, "3") + require.ErrorContains(t, err, "the WAL holds block 1") +} + +func TestRecordSeededVersion(t *testing.T) { + cfg := seededStoreWithBlocks(t) + require.NoError(t, os.Remove(filepath.Join(cfg.DataDir, seededVersionFile))) + + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + require.NoError(t, s.LoadLatest()) + _, ok := s.SeededVersion() + require.False(t, ok) + + require.ErrorContains(t, s.RecordSeededVersion(13), "snapshot 10 predates it") + require.ErrorContains(t, s.RecordSeededVersion(14), "above the store's version 13") + _, ok = s.SeededVersion() + require.False(t, ok, "a rejected record must not be adopted") + + require.NoError(t, s.RecordSeededVersion(10)) + seeded, ok := s.SeededVersion() + require.True(t, ok) + require.Equal(t, int64(10), seeded) + + s = reopenStore(t, s, cfg) + defer s.Close() + require.NoError(t, s.LoadLatest()) + seeded, ok = s.SeededVersion() + require.True(t, ok) + require.Equal(t, int64(10), seeded) +} + +func TestLoadLatest_RemovesTemporarySeededVersionRecord(t *testing.T) { + cfg := seededStoreWithBlocks(t) + tmpPath := filepath.Join(cfg.DataDir, seededVersionFile+tmpSuffix) + require.NoError(t, os.WriteFile(tmpPath, []byte("1"), 0600)) + + s, err := newCommitStoreWithWAL(t.Context(), cfg) + require.NoError(t, err) + defer s.Close() + require.NoError(t, s.LoadLatest()) + require.NoFileExists(t, tmpPath) +} + func TestSetInitialVersion_RejectsAfterCommit(t *testing.T) { s := setupTestStore(t) defer s.Close() diff --git a/sei-db/tools/cmd/seidb/main.go b/sei-db/tools/cmd/seidb/main.go index ee421275a4..9e7a5b5da4 100644 --- a/sei-db/tools/cmd/seidb/main.go +++ b/sei-db/tools/cmd/seidb/main.go @@ -31,6 +31,7 @@ func main() { operations.ReplayChangelogCmd(), operations.TraceProfileReportCmd(), operations.MigrateEvmStatusCmd(), + operations.FlatKVRecordSeededVersionCmd(), operations.EvmLogicalDigestCmd(), operations.KVRepairExportCmd(), operations.HashLogCmd()) diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_record_seeded_version.go b/sei-db/tools/cmd/seidb/operations/flatkv_record_seeded_version.go new file mode 100644 index 0000000000..47ff24b112 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/flatkv_record_seeded_version.go @@ -0,0 +1,66 @@ +package operations + +import ( + "context" + "fmt" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv" + flatkvconfig "github.com/sei-protocol/sei-chain/sei-db/state_db/sc/flatkv/config" + "github.com/spf13/cobra" +) + +// FlatKVRecordSeededVersionCmd is the seidb subcommand that records the version a FlatKV store was seeded at, +// for a store seeded before the seed recorded it. The node must be stopped. +func FlatKVRecordSeededVersionCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "flatkv-record-seeded-version", + Short: "Record the version a FlatKV store was seeded at", + Long: "Validates --version against the history the stopped node's FlatKV store holds and records it as the " + + "height the store was seeded at: one below the first block FlatKV committed. Heights at or below it " + + "are then served from memIAVL alone.", + RunE: func(cmd *cobra.Command, _ []string) error { + homeDir, _ := cmd.Flags().GetString("home") + dataDir, _ := cmd.Flags().GetString("data-dir") + version, _ := cmd.Flags().GetInt64("version") + resolvedHome, err := resolveSeiHome(homeDir, dataDir) + if err != nil { + return err + } + if version <= 0 { + return fmt.Errorf("--version must be a positive height, got %d", version) + } + return recordFlatKVSeededVersion(cmd.Context(), resolvedHome, version) + }, + } + cmd.Flags().String("home", "", "Sei home directory. Defaults to $HOME/.sei") + cmd.Flags().String("data-dir", "", "Sei data directory or home directory. If the basename is data, its parent is used as home") + cmd.Flags().Int64("version", 0, "Height the FlatKV store was seeded at") + return cmd +} + +func recordFlatKVSeededVersion(ctx context.Context, homeDir string, version int64) error { + if ctx == nil { + ctx = context.Background() + } + cfg := flatkvconfig.DefaultConfig() + cfg.DataDir = utils.GetFlatKVPath(homeDir) + stateWAL, err := flatkv.OpenStateWAL(cfg) + if err != nil { + return fmt.Errorf("failed to open FlatKV state WAL: %w", err) + } + store, err := flatkv.NewCommitStore(ctx, cfg, stateWAL) + if err != nil { + _ = stateWAL.Close() + return fmt.Errorf("failed to create FlatKV store: %w", err) + } + defer func() { _ = store.Close() }() + if err := store.LoadLatest(); err != nil { + return fmt.Errorf("failed to open FlatKV store: %w", err) + } + if err := store.RecordSeededVersion(version); err != nil { + return err + } + fmt.Printf("Recorded FlatKV seeded version %d in %s\n", version, cfg.DataDir) + return nil +} diff --git a/sei-db/tools/cmd/seidb/operations/flatkv_record_seeded_version_test.go b/sei-db/tools/cmd/seidb/operations/flatkv_record_seeded_version_test.go new file mode 100644 index 0000000000..9904104816 --- /dev/null +++ b/sei-db/tools/cmd/seidb/operations/flatkv_record_seeded_version_test.go @@ -0,0 +1,28 @@ +package operations + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/sei-protocol/sei-chain/sei-db/common/utils" + "github.com/stretchr/testify/require" +) + +func TestRecordFlatKVSeededVersion(t *testing.T) { + homeDir := t.TempDir() + store := newTestFlatKVStoreAtHome(t, homeDir) + require.NoError(t, store.SetInitialVersion(11)) + require.NoError(t, store.Close()) + require.NoError(t, os.Remove(filepath.Join(utils.GetFlatKVPath(homeDir), "SEEDED_VERSION"))) + + require.ErrorContains(t, recordFlatKVSeededVersion(context.Background(), homeDir, 11), "above the store's version 10") + require.NoError(t, recordFlatKVSeededVersion(context.Background(), homeDir, 10)) + + store = newTestFlatKVStoreAtHome(t, homeDir) + defer func() { require.NoError(t, store.Close()) }() + seeded, ok := store.SeededVersion() + require.True(t, ok) + require.Equal(t, int64(10), seeded) +}