Serve historical reads below the FlatKV seed height from memIAVL

sei-db/state_db/giga/types/live_state_store.go

@@ -73,6 +73,10 @@ type LiveStateStore interface {
 	// initialVersion <= 0.
 	SetInitialVersion(initialVersion int64) error
 
+	// SeededVersion returns the version SetInitialVersion seeded this store at above genesis. The store holds
+	// no history at or below it. ok is false when no such version is recorded.
+	SeededVersion() (version int64, ok bool)
+
 	// Get returns the value for a key within the given module.
 	// For EVM keys (moduleName == "evm"), the key is a memiavl EVM key
 	// routed to account/storage/code/misc DBs internally.

sei-db/state_db/sc/composite/store.go

@@ -451,7 +451,8 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type
 		}
 	}
 
-	if cs.flatKV != nil {
+	flatKVPredates := cs.flatKVPredates(targetVersion)
+	if cs.flatKV != nil && !flatKVPredates {
 		fkv, err := cs.flatKV.LoadVersionReadOnly(targetVersion)
 		if err != nil {
 			return nil, fmt.Errorf("failed to load FlatKV version: %w", err)
@@ -475,7 +476,9 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type
 			return nil, err
 		}
 	}
-	if err := ro.resolveCurrentWriteMode(false); err != nil {
+	if flatKVPredates {
+		ro.currentWriteMode = types.MemiavlOnly
+	} else if err := ro.resolveCurrentWriteMode(false); err != nil {
 		return nil, fmt.Errorf("failed to resolve effective write mode for read-only handle: %w", err)
 	}
 	if err := ro.buildRouter(); err != nil {
@@ -487,6 +490,17 @@ func (cs *CompositeCommitStore) LoadVersionReadOnly(targetVersion int64) (_ type
 	return ro, nil
 }
 
+// flatKVPredates reports whether version is at or below the height flatKV was seeded at, so that a view or
+// export there is served from memIAVL alone. Version 0 means latest and never predates.
+func (cs *CompositeCommitStore) flatKVPredates(version int64) bool {
+	if cs.flatKV == nil || cs.memIAVL == nil || version <= 0 {
+		return false
+	}
+	// Up to the seeded height memIAVL alone made up the AppHash, and flatKV holds no history there.
+	seeded, ok := cs.flatKV.SeededVersion()
+	return ok && version <= seeded
+}
+
 // resolveCurrentWriteMode sets cs.currentWriteMode after the backends have been
 // opened. For a fixed configured mode this is a copy; for types.Auto the
 // mode is derived from the migration metadata persisted in flatkv. A nil
@@ -1431,8 +1445,9 @@ func (cs *CompositeCommitStore) Exporter(version int64) (types.Exporter, error)
 		return nil, fmt.Errorf("version %d out of range", version)
 	}
 
+	flatKVPredates := cs.flatKVPredates(version)
 	includeMemiavl := cs.memIAVL != nil
-	includeFlatKV := cs.flatKV != nil
+	includeFlatKV := cs.flatKV != nil && !flatKVPredates
 
 	if includeFlatKV && exportNeedsMetadataGating(cs.config.WriteMode) {
 		// Evaluate the hash predicates against metadata as-of the exported

sei-db/state_db/sc/composite/store_auto_test.go

@@ -1,6 +1,8 @@
 package composite
 
 import (
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/sei-protocol/sei-chain/sei-db/common/keys"
@@ -581,17 +583,10 @@ func TestComposite_Auto_ReadOnlyHandle(t *testing.T) {
 	requireOracleMatches(t, ro, workload.snapshotOracle())
 }
 
-// TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails records a guarantee that was deliberately
-// given up: heights predating flatkv's history used to be served memiavl-only, because flatkv kept a
-// persisted record of the height its history began at and the read path consulted it.
-//
-// That record is gone, so nothing distinguishes "this height predates flatkv" from "flatkv failed to
-// load at this height", and the read path can only take the safe branch — attempt the load and
-// surface the failure. Answering the other way would serve the height from a memiavl whose migrated
-// keys were deleted, fabricating a nonexistence answer, so failing here is the correct direction.
-//
-// In-era heights are unaffected. Delete this test only alongside a decision to restore pre-era reads.
-func TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails(t *testing.T) {
+// TestComposite_Auto_ReadOnlyPreFlatKVEraHeight pins the era-aware read-only path: heights that predate
+// flatkv's history must remain queryable after the migration has begun, served memiavl-only with as-of-height
+// values. In-era heights keep loading flatkv.
+func TestComposite_Auto_ReadOnlyPreFlatKVEraHeight(t *testing.T) {
 	dir := t.TempDir()
 	cs := openAutoStoreWithConfig(t, dir, autoExportConfig(), 100)
 	defer func() { _ = cs.Close() }()
@@ -617,14 +612,21 @@ func TestComposite_Auto_ReadOnlyPreFlatKVEraHeightNowFails(t *testing.T) {
 		require.NoError(t, err)
 	}
 
-	// Pre-era height: the load is attempted against a flatkv that has no such height, and fails.
-	_, err := cs.LoadVersionReadOnly(3)
-	require.Error(t, err, "a pre-flatkv-era height is no longer distinguishable from a load failure")
+	roCommitter, err := cs.LoadVersionReadOnly(3)
+	require.NoError(t, err, "pre-flatkv-era heights must remain queryable")
+	ro, ok := roCommitter.(*CompositeCommitStore)
+	require.True(t, ok)
+	require.Equal(t, types.MemiavlOnly, ro.currentWriteMode)
+	require.Nil(t, ro.flatKV)
+	val, found, err := ro.Get(keys.BankStoreKey, []byte("k"))
+	require.NoError(t, err)
+	require.True(t, found)
+	require.Equal(t, valAt(3), val, "value as-of height 3")
+	require.NoError(t, ro.Close())
 
-	// In-era height: unchanged.
-	roCommitter, err := cs.LoadVersionReadOnly(7)
+	roCommitter, err = cs.LoadVersionReadOnly(7)
 	require.NoError(t, err)
-	ro, ok := roCommitter.(*CompositeCommitStore)
+	ro, ok = roCommitter.(*CompositeCommitStore)
 	require.True(t, ok)
 	defer func() { _ = ro.Close() }()
 	require.NotNil(t, ro.flatKV, "in-era heights must keep loading flatkv")
@@ -657,3 +659,195 @@ func TestComposite_Auto_CopyAvailability(t *testing.T) {
 	require.Nil(t, cs.Copy(),
 		"Copy is unavailable once flatkv is open")
 }
+
+// TestComposite_Auto_ReadOnlyBelowSeededVersion pins historical reads across the MemiavlOnly -> MigrateEVM
+// seam. flatkv is seeded at the height memiavl had reached and holds no history below it, so a read-only
+// view or export there must be served from memiavl alone and match what the node served before the
+// transition, instead of asking flatkv to replay blocks it never had.
+func TestComposite_Auto_ReadOnlyBelowSeededVersion(t *testing.T) {
+	dir := t.TempDir()
+	workload := newMigrationWorkload(0xA078)
+	cfg := autoExportConfig()
+	cfg.MemIAVLConfig.SnapshotKeepRecent = 100
+
+	cs := openAutoStoreWithConfig(t, dir, cfg, 25)
+	defer func() { _ = cs.Close() }()
+
+	runBlocks(t, cs, workload, 10)
+	pre := cs.Version()
+	oracle := workload.snapshotOracle()
+	runBlocks(t, cs, workload, 3)
+
+	before, err := cs.LoadVersionReadOnly(pre)
+	require.NoError(t, err)
+	beforeInfo := before.(*CompositeCommitStore).LastCommitInfo()
+	require.NoError(t, before.(*CompositeCommitStore).Close())
+	exp, err := cs.Exporter(pre)
+	require.NoError(t, err)
+	beforeItems := drainCompositeExporter(t, exp)
+	require.NoError(t, exp.Close())
+
+	require.NoError(t, cs.SetWriteMode(types.MigrateEVM))
+	runBlocks(t, cs, workload, 5)
+	require.True(t, hasLatticeHash(cs))
+
+	view, err := cs.LoadVersionReadOnly(pre)
+	require.NoError(t, err)
+	ro := view.(*CompositeCommitStore)
+	defer func() { _ = ro.Close() }()
+	require.Nil(t, ro.flatKV, "flatkv holds no history below its seeded version")
+	require.Equal(t, beforeInfo, ro.LastCommitInfo())
+	requireOracleMatches(t, ro, oracle)
+
+	exp, err = cs.Exporter(pre)
+	require.NoError(t, err)
+	afterItems := drainCompositeExporter(t, exp)
+	require.NoError(t, exp.Close())
+	require.Equal(t, beforeItems, afterItems)
+
+	latest, err := cs.LoadVersionReadOnly(cs.Version())
+	require.NoError(t, err)
+	defer func() { _ = latest.(*CompositeCommitStore).Close() }()
+	require.True(t, hasLatticeHash(latest.(*CompositeCommitStore)),
+		"a view above the seeded version must still include flatkv")
+
+	seeded, ok := cs.flatKV.SeededVersion()
+	require.True(t, ok)
+	require.Equal(t, pre+3, seeded)
+	requireSeedBoundary(t, cs, seeded)
+}
+
+// requireSeedBoundary asserts that a view at seeded is served from memiavl alone and one at seeded+1 includes
+// flatkv.
+func requireSeedBoundary(t *testing.T, cs *CompositeCommitStore, seeded int64) {
+	t.Helper()
+	atSeed, err := cs.LoadVersionReadOnly(seeded)
+	require.NoError(t, err)
+	defer func() { _ = atSeed.(*CompositeCommitStore).Close() }()
+	require.Nil(t, atSeed.(*CompositeCommitStore).flatKV, "the seeded version itself predates flatkv")
+	require.False(t, hasLatticeHash(atSeed.(*CompositeCommitStore)))
+
+	afterSeed, err := cs.LoadVersionReadOnly(seeded + 1)
+	require.NoError(t, err)
+	defer func() { _ = afterSeed.(*CompositeCommitStore).Close() }()
+	require.NotNil(t, afterSeed.(*CompositeCommitStore).flatKV, "the first block after the seed is in flatkv's era")
+	require.True(t, hasLatticeHash(afterSeed.(*CompositeCommitStore)))
+}
+
+// openFixedStore opens (or reopens) a composite store at dir in a fixed write mode.
+func openFixedStore(t *testing.T, dir string, mode types.WriteMode) (*CompositeCommitStore, error) {
+	t.Helper()
+	cfg := autoConfig()
+	cfg.WriteMode = mode
+	cs, err := NewCompositeCommitStore(t.Context(), dir, cfg)
+	require.NoError(t, err)
+	require.NoError(t, cs.SetMigrationBatchSize(25))
+	require.NoError(t, cs.Initialize([]string{keys.BankStoreKey, keys.EVMStoreKey}))
+	if err := cs.LoadLatest(); err != nil {
+		_ = cs.Close()
+		return nil, err
+	}
+	return cs, nil
+}
+
+// seedFixedMigrateEVM runs a memiavl_only node for 13 blocks, switches it to a fixed migrate_evm configuration,
+// which seeds flatkv at 13 on the first LoadLatest, and runs 5 more blocks. It returns the closed store's
+// directory and the oracle of the state at height 10.
+func seedFixedMigrateEVM(t *testing.T, seed int64) (string, map[migKeyPair][]byte) {
+	t.Helper()
+	dir := t.TempDir()
+	workload := newMigrationWorkload(seed)
+	cs, err := openFixedStore(t, dir, types.MemiavlOnly)
+	require.NoError(t, err)
+	runBlocks(t, cs, workload, 10)
+	oracle := workload.snapshotOracle()
+	runBlocks(t, cs, workload, 3)
+	require.NoError(t, cs.Close())
+
+	cs, err = openFixedStore(t, dir, types.MigrateEVM)
+	require.NoError(t, err)
+	runBlocks(t, cs, workload, 5)
+	require.NoError(t, cs.Close())
+	return dir, oracle
+}
+
+// TestComposite_FixedMigrateEVM_ReadOnlyBelowSeededVersion covers the same seam for a node switched from
+// memiavl_only to a fixed migrate_evm configuration, where flatkv is seeded on the first LoadLatest.
+func TestComposite_FixedMigrateEVM_ReadOnlyBelowSeededVersion(t *testing.T) {
+	dir, oracle := seedFixedMigrateEVM(t, 0xA079)
+	cs, err := openFixedStore(t, dir, types.MigrateEVM)
+	require.NoError(t, err)
+	defer func() { _ = cs.Close() }()
+
+	view, err := cs.LoadVersionReadOnly(10)
+	require.NoError(t, err)
+	ro := view.(*CompositeCommitStore)
+	defer func() { _ = ro.Close() }()
+	require.Nil(t, ro.flatKV)
+	require.False(t, hasLatticeHash(ro))
+	requireOracleMatches(t, ro, oracle)
+
+	seeded, ok := cs.flatKV.SeededVersion()
+	require.True(t, ok)
+	require.Equal(t, int64(13), seeded)
+	requireSeedBoundary(t, cs, seeded)
+}
+
+// TestComposite_RecordedSeededVersionRestoresPreSeedReads pins the remediation for a node seeded before the
+// seeded version was recorded: without the record, heights below the seed fail to load; once it is recorded,
+// they are served from memiavl alone.
+func TestComposite_RecordedSeededVersionRestoresPreSeedReads(t *testing.T) {
+	dir, oracle := seedFixedMigrateEVM(t, 0xA07A)
+	flatKVDir := utils.GetFlatKVPath(dir)
+	require.NoError(t, os.Remove(filepath.Join(flatKVDir, "SEEDED_VERSION")))
+
+	cs, err := openFixedStore(t, dir, types.MigrateEVM)
+	require.NoError(t, err)
+	_, err = cs.LoadVersionReadOnly(10)
+	require.ErrorContains(t, err, "data loss or corruption")
+	_, err = cs.Exporter(10)
+	require.ErrorContains(t, err, "data loss or corruption")
+	require.NoError(t, cs.Close())
+
+	fkvCfg := autoConfig().FlatKVConfig
+	fkvCfg.DataDir = flatKVDir
+	stateWAL, err := flatkv.OpenStateWAL(&fkvCfg)
+	require.NoError(t, err)
+	fkv, err := flatkv.NewCommitStore(t.Context(), &fkvCfg, stateWAL)
+	require.NoError(t, err)
+	require.NoError(t, fkv.LoadLatest())
+	require.ErrorContains(t, fkv.RecordSeededVersion(14), "predates it")
+	require.NoError(t, fkv.RecordSeededVersion(13))
+	require.NoError(t, fkv.Close())
+
+	cs, err = openFixedStore(t, dir, types.MigrateEVM)
+	require.NoError(t, err)
+	defer func() { _ = cs.Close() }()
+	view, err := cs.LoadVersionReadOnly(10)
+	require.NoError(t, err)
+	defer func() { _ = view.(*CompositeCommitStore).Close() }()
+	require.Nil(t, view.(*CompositeCommitStore).flatKV)
+	requireOracleMatches(t, view.(*CompositeCommitStore), oracle)
+	exp, err := cs.Exporter(10)
+	require.NoError(t, err)
+	require.NotContains(t, moduleNamesOf(drainCompositeExporter(t, exp)), keys.FlatKVStoreKey)
+	require.NoError(t, exp.Close())
+	requireSeedBoundary(t, cs, 13)
+}
+
+// TestComposite_InvalidSeededVersionFailsLoadLatest pins that a seeded-version record flatkv's history
+// contradicts stops the node at open, rather than letting it start and fail every historical read.
+func TestComposite_InvalidSeededVersionFailsLoadLatest(t *testing.T) {
+	dir, _ := seedFixedMigrateEVM(t, 0xA07B)
+	recordPath := filepath.Join(utils.GetFlatKVPath(dir), "SEEDED_VERSION")
+
+	for record, wantErr := range map[string]string{
+		"abc\n":    "not a positive height",
+		"999999\n": "above the store's version",
+		"15\n":     "predates it",
+	} {
+		require.NoError(t, os.WriteFile(recordPath, []byte(record), 0600))
+		_, err := openFixedStore(t, dir, types.MigrateEVM)
+		require.ErrorContains(t, err, wantErr, "record %q", record)
+	}
+}

sei-db/state_db/sc/composite/store_test.go

@@ -44,6 +44,7 @@ func (f *failingEVMStore) CommitStateChanges(int64, []*proto.NamedChangeSet) err
 }
 func (f *failingEVMStore) OpenView() gigatypes.StateView     { return nil }
 func (f *failingEVMStore) SetInitialVersion(int64) error     { return nil }
+func (f *failingEVMStore) SeededVersion() (int64, bool)      { return 0, false }
 func (f *failingEVMStore) Get(string, []byte) ([]byte, bool) { return nil, false }
 func (f *failingEVMStore) GetBlockHeightModified(string, []byte) (int64, bool, error) {
 	return -1, false, nil

sei-db/state_db/sc/flatkv/snapshot.go

@@ -33,6 +33,7 @@ import (
 //	    account/, code/, storage/, misc/
 //	    SNAPSHOT_BASE                        (records source snapshot name)
 //	  changelog/                             (WAL, shared across snapshots)
+//	  SEEDED_VERSION                         (height SetInitialVersion seeded at, if above genesis)
 const (
 	// snapshotPrefix is the directory name prefix for versioned snapshots.
 	snapshotPrefix = "snapshot-"
@@ -178,8 +179,8 @@ func updateCurrentSymlink(root, snapshotDir string) error {
 	return nil
 }
 
-// removeTmpDirs removes any directories ending in "-tmp" or "-removing"
-// left over from interrupted snapshot writes or deletes.
+// removeTmpDirs removes the directories ending in "-tmp" or "-removing" and the temporary seeded-version
+// record left over from interrupted writes or deletes.
 func removeTmpDirs(dir string) error {
 	entries, err := os.ReadDir(dir)
 	if err != nil {
@@ -192,6 +193,10 @@ func removeTmpDirs(dir string) error {
 			if err := os.RemoveAll(filepath.Join(dir, name)); err != nil {
 				errs = append(errs, fmt.Errorf("remove tmp dir %s: %w", name, err))
 			}
+		} else if name == seededVersionFile+tmpSuffix {
+			if err := os.Remove(filepath.Join(dir, name)); err != nil {
+				errs = append(errs, fmt.Errorf("remove tmp file %s: %w", name, err))
+			}
 		}
 	}
 	if len(errs) > 0 {

sei-db/state_db/sc/flatkv/store.go

@@ -173,6 +173,9 @@ type CommitStore struct {
 	// readOnly marks stores opened via LoadVersionReadOnly.
 	readOnly bool
 
+	// The version SetInitialVersion seeded this store at, 0 when none is recorded. See SeededVersion.
+	seededVersion int64
+
 	readOnlyWorkDir string // Temp working dir for readonly store; removed by Close.
 
 	// A work pool for reading from the DBs.
@@ -423,6 +426,10 @@ func (s *CommitStore) LoadLatest() (retErr error) {
 	if err := s.openTo(0); err != nil {
 		return fmt.Errorf("failed to open FlatKV store: %w", err)
 	}
+	if err := s.loadSeededVersion(); err != nil {
+		_ = s.closeDBsOnly()
+		return fmt.Errorf("failed to open FlatKV store: %w", err)
+	}
 	return nil
 }
 
@@ -1512,6 +1519,12 @@ func (s *CommitStore) resetForImport() error {
 		return fmt.Errorf("resetForImport: remove %s: %w", currentLink, err)
 	}
 
+	// An imported store's history starts at the imported version, not at any earlier seed.
+	if err := os.Remove(filepath.Join(dir, seededVersionFile)); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("resetForImport: remove %s: %w", seededVersionFile, err)
+	}
+	s.seededVersion = 0
+
 	// The WAL is deliberately left alone. Import bypasses it, so a pre-existing WAL is stale relative to the
 	// imported version — but a state-sync restore is a manual procedure in which the operator stops the node
 	// and removes its data directories, changelog included, so the restored node opens an empty WAL and its

sei-db/state_db/sc/flatkv/store_constants.go

@@ -12,6 +12,10 @@ const (
 	changelogDir  = "changelog"
 	lockFileName  = "LOCK"
 
+	// seededVersionFile records the version SetInitialVersion seeded the store at (see SeededVersion), as a
+	// decimal height on a single line. Write it with `seidb flatkv-record-seeded-version`, not by hand.
+	seededVersionFile = "SEEDED_VERSION"
+
 	// DB subdirectories (inside each snapshot)
 	accountDBDir = "account"
 	codeDBDir    = "code"