From 13edc78bf0d5e932f744e7e27878e19bb6d45f65 Mon Sep 17 00:00:00 2001 From: Max Moisiuk Date: Mon, 21 Sep 2026 17:01:44 +0200 Subject: [PATCH] [Filestore] Persist resize state in the main FS tablet --- .../shard_creation_state_companion.cpp | 38 +++--- .../shard_creation_state_companion_ut.cpp | 110 +++++++++++++++ .../libs/storage/tablet/protos/tablet.proto | 5 + .../tablet/tablet_actor_unsafe_node_ops.cpp | 127 +++++++++++++++++- .../tablet/tablet_actor_updateconfig.cpp | 9 ++ .../libs/storage/tablet/tablet_state.cpp | 8 ++ .../libs/storage/tablet/tablet_state.h | 4 + .../filestore/private/api/protos/tablet.proto | 6 + 8 files changed, 281 insertions(+), 26 deletions(-) diff --git a/cloud/filestore/libs/storage/service/shard_creation_state_companion.cpp b/cloud/filestore/libs/storage/service/shard_creation_state_companion.cpp index 2e7406c85ee..25990348e93 100644 --- a/cloud/filestore/libs/storage/service/shard_creation_state_companion.cpp +++ b/cloud/filestore/libs/storage/service/shard_creation_state_companion.cpp @@ -259,27 +259,23 @@ void TShardCreationStateCompanion::UpdateShardCreationState( return; } - // Implement this block after adding ShardCreationState - // into TEvUnsafeChangeTabletStateRequest - Y_ABORT("add ShardCreationState into TEvUnsafeChangeTabletStateRequest"); - - // auto request = - // std::make_unique(); - // request->Record.SetFileSystemId(FileSystemId); - // auto* shardCreationState = request->Record.MutableShardCreationState(); - // shardCreationState->SetVersion(ShardCreationState.GetVersion()); - // shardCreationState->SetBaseShardCount( - // ShardCreationState.GetBaseShardCount()); - // shardCreationState->SetTargetShardCount( - // ShardCreationState.GetTargetShardCount()); - // shardCreationState->SetTargetShardConfigHash( - // ShardCreationState.GetTargetShardConfigHash()); - // SaveCompressedBitmap( - // *CreatedShardBitmap, - // ShardBitmapBitCount, - // *shardCreationState->MutableCreatedShardBitmap()); - - // NCloud::Send(ctx, MakeIndexTabletProxyServiceId(), std::move(request)); + auto request = + std::make_unique(); + request->Record.SetFileSystemId(FileSystemId); + auto* shardCreationState = request->Record.MutableShardCreationState(); + shardCreationState->SetVersion(ShardCreationState.GetVersion()); + shardCreationState->SetBaseShardCount( + ShardCreationState.GetBaseShardCount()); + shardCreationState->SetTargetShardCount( + ShardCreationState.GetTargetShardCount()); + shardCreationState->SetTargetShardConfigHash( + ShardCreationState.GetTargetShardConfigHash()); + SaveCompressedBitmap( + *CreatedShardBitmap, + ShardBitmapBitCount, + *shardCreationState->MutableCreatedShardBitmap()); + + NCloud::Send(ctx, MakeIndexTabletProxyServiceId(), std::move(request)); } void TShardCreationStateCompanion::OnShardCreated( diff --git a/cloud/filestore/libs/storage/service/shard_creation_state_companion_ut.cpp b/cloud/filestore/libs/storage/service/shard_creation_state_companion_ut.cpp index 05f804ee15b..12702b266bc 100644 --- a/cloud/filestore/libs/storage/service/shard_creation_state_companion_ut.cpp +++ b/cloud/filestore/libs/storage/service/shard_creation_state_companion_ut.cpp @@ -95,6 +95,17 @@ NProtoPrivate::TFileSystemShardCreationState MakeState( return MakeState(version, createdShards, 0, MakeShardConfigs()); } +std::unique_ptr LoadCreatedShardBitmap( + const NProtoPrivate::TFileSystemShardCreationState& state) +{ + auto result = LoadCompressedBitmap( + state.GetCreatedShardBitmap(), + state.GetCreatedShardBitmap().GetBitCount(), + MaxShardCount); + UNIT_ASSERT_C(!HasError(result), FormatError(result.GetError())); + return result.ExtractResult(); +} + //////////////////////////////////////////////////////////////////////////////// enum ETestEvents @@ -327,6 +338,105 @@ Y_UNIT_TEST_SUITE(TShardCreationStateCompanionTest) UNIT_ASSERT_VALUES_EQUAL(11, companion.GetShardCreationStateVersion()); } + Y_UNIT_TEST(ShouldSendUpdatedShardCreationState) + { + TActorSystem runtime; + runtime.Start(); + + const auto sender = runtime.AllocateEdgeActor(); + const auto tabletProxy = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeIndexTabletProxyServiceId(), tabletProxy); + + const auto actorId = + runtime.Register(new TActorWithCompanion(MakeState(42, {1}))); + + runtime.Send(new IEventHandle( + actorId, + sender, + new TEvOnShardCreated(3))); + + auto request = runtime.GrabEdgeEvent< + TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>( + tabletProxy); + + const auto& record = request->Get()->Record; + UNIT_ASSERT_VALUES_EQUAL(FileSystemId, record.GetFileSystemId()); + UNIT_ASSERT(record.HasShardCreationState()); + + const auto& state = record.GetShardCreationState(); + UNIT_ASSERT_VALUES_EQUAL(42, state.GetVersion()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetBaseShardCount()); + UNIT_ASSERT_VALUES_EQUAL(ShardCount, state.GetTargetShardCount()); + UNIT_ASSERT_VALUES_EQUAL( + CalculateShardCreationTargetHashForTest(0, MakeShardConfigs()), + state.GetTargetShardConfigHash()); + UNIT_ASSERT_VALUES_EQUAL( + ShardCount, + state.GetCreatedShardBitmap().GetBitCount()); + + const auto bitmap = LoadCreatedShardBitmap(state); + UNIT_ASSERT_VALUES_EQUAL(2, bitmap->Count()); + UNIT_ASSERT(!bitmap->Test(0)); + UNIT_ASSERT(bitmap->Test(1)); + UNIT_ASSERT(!bitmap->Test(2)); + UNIT_ASSERT(bitmap->Test(3)); + } + + Y_UNIT_TEST(ShouldResendMergedBitmapAfterCasConflict) + { + TActorSystem runtime; + runtime.Start(); + + const auto sender = runtime.AllocateEdgeActor(); + const auto tabletProxy = runtime.AllocateEdgeActor(); + runtime.RegisterService(MakeIndexTabletProxyServiceId(), tabletProxy); + + const auto actorId = + runtime.Register(new TActorWithCompanion(MakeState(42, {}))); + + runtime.Send(new IEventHandle( + actorId, + sender, + new TEvOnShardCreated(3))); + + auto request = runtime.GrabEdgeEvent< + TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>( + tabletProxy); + + { + const auto& state = request->Get()->Record.GetShardCreationState(); + UNIT_ASSERT_VALUES_EQUAL(42, state.GetVersion()); + + const auto bitmap = LoadCreatedShardBitmap(state); + UNIT_ASSERT_VALUES_EQUAL(1, bitmap->Count()); + UNIT_ASSERT(!bitmap->Test(1)); + UNIT_ASSERT(bitmap->Test(3)); + } + + runtime.Send(new IEventHandle( + actorId, + sender, + new TEvMergeShardCreationState(MakeState(43, {1})))); + + request = runtime.GrabEdgeEvent< + TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>( + tabletProxy); + + const auto& state = request->Get()->Record.GetShardCreationState(); + UNIT_ASSERT_VALUES_EQUAL(43, state.GetVersion()); + UNIT_ASSERT_VALUES_EQUAL(0, state.GetBaseShardCount()); + UNIT_ASSERT_VALUES_EQUAL(ShardCount, state.GetTargetShardCount()); + UNIT_ASSERT_VALUES_EQUAL( + CalculateShardCreationTargetHashForTest(0, MakeShardConfigs()), + state.GetTargetShardConfigHash()); + + const auto bitmap = LoadCreatedShardBitmap(state); + UNIT_ASSERT_VALUES_EQUAL(2, bitmap->Count()); + UNIT_ASSERT(bitmap->Test(1)); + UNIT_ASSERT(!bitmap->Test(2)); + UNIT_ASSERT(bitmap->Test(3)); + } + Y_UNIT_TEST(ShouldRejectMergingDifferentTarget) { TShardCreationStateCompanion companion( diff --git a/cloud/filestore/libs/storage/tablet/protos/tablet.proto b/cloud/filestore/libs/storage/tablet/protos/tablet.proto index 0b2da399511..a2a2aef085a 100644 --- a/cloud/filestore/libs/storage/tablet/protos/tablet.proto +++ b/cloud/filestore/libs/storage/tablet/protos/tablet.proto @@ -62,6 +62,11 @@ message TFileSystem // Compress ShardId and ShardNodeName when we write them to NodeRefs table. optional bool CompressNodeRef = 25; + + // Persisted physical shard creation progress. Intentionally retained after + // successful resize: clearing it could erase recovery bits from another + // in-flight resize. Bits below current topology size are committed history. + NProtoPrivate.TFileSystemShardCreationState ShardCreationState = 26; } //////////////////////////////////////////////////////////////////////////////// diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp index 50476289979..a5f901a1067 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp @@ -2,6 +2,10 @@ #include "helpers.h" +#include + +#include + namespace NCloud::NFileStore::NStorage { using namespace NActors; @@ -11,6 +15,84 @@ using namespace NKikimr::NTabletFlatExecutor; //////////////////////////////////////////////////////////////////////////////// +namespace { + +ui64 GetChunkDataSize(const NCloud::NProto::TCompressedBitmapData& bitmap) +{ + ui64 dataSize = 0; + for (const auto& chunk: bitmap.GetChunks()) { + dataSize += chunk.GetData().size(); + } + + return dataSize; +} + +TString DescribeShardCreationState( + const NProtoPrivate::TFileSystemShardCreationState& state) +{ + const auto& bitmap = state.GetCreatedShardBitmap(); + + TStringBuilder out; + out << "{Version: "; + if (state.HasVersion()) { + out << state.GetVersion(); + } else { + out << ""; + } + out << ", BaseShardCount: " << state.GetBaseShardCount() + << ", TargetShardCount: " << state.GetTargetShardCount() + << ", TargetShardConfigHash: " << state.GetTargetShardConfigHash() + << ", CreatedShardBitmap: {BitCount: " << bitmap.GetBitCount() + << ", Chunks: " << bitmap.ChunksSize() + << ", ChunkDataSize: " << GetChunkDataSize(bitmap) << "}}"; + + return out; +} + +TString DescribeChangeTabletStateRequest( + const NProtoPrivate::TUnsafeChangeTabletStateRequest& request) +{ + if (!request.HasShardCreationState()) { + return request.DebugString(); + } + + TStringBuilder out; + out << "{FileSystemId: " << request.GetFileSystemId(); + if (request.HasFrozen()) { + out << ", Frozen: " << (request.GetFrozen() ? "true" : "false"); + } + if (request.HasCompressNodeRef()) { + out << ", CompressNodeRef: " + << (request.GetCompressNodeRef() ? "true" : "false"); + } + out << ", ShardCreationState: " + << DescribeShardCreationState(request.GetShardCreationState()) << "}"; + + return out; +} + +TString DescribeChangeTabletStateResponse( + const NProtoPrivate::TUnsafeChangeTabletStateResponse& response) +{ + if (!response.HasShardCreationState()) { + return response.ShortUtf8DebugString(); + } + + TStringBuilder out; + out << "{"; + if (HasError(response.GetError())) { + out << "Error: " << FormatError(response.GetError()) << ", "; + } + out << "ShardCreationState: " + << DescribeShardCreationState(response.GetShardCreationState()) << "}"; + + return out; +} + +} // namespace + +//////////////////////////////////////////////////////////////////////////////// + void TIndexTabletActor::HandleUnsafeCreateNode( const TEvIndexTablet::TEvUnsafeCreateNodeRequest::TPtr& ev, const TActorContext& ctx) @@ -886,7 +968,7 @@ void TIndexTabletActor::HandleUnsafeChangeTabletState( TFileStoreComponents::TABLET, "%s UnsafeChangeTabletState: %s", LogTag.c_str(), - msg->Record.DebugString().Quote().c_str()); + DescribeChangeTabletStateRequest(msg->Record).Quote().c_str()); ExecuteTx( ctx, @@ -913,8 +995,6 @@ void TIndexTabletActor::ExecuteTx_UnsafeChangeTabletState( TTransactionContext& tx, TTxIndexTablet::TUnsafeChangeTabletState& args) { - Y_UNUSED(ctx); - auto db = CreateIndexTabletDatabase(tx.DB); if (args.Request.HasCompressNodeRef()) { @@ -924,6 +1004,35 @@ void TIndexTabletActor::ExecuteTx_UnsafeChangeTabletState( if (args.Request.HasFrozen()) { SetFrozen(*db, args.Request.GetFrozen()); } + + if (args.Request.HasShardCreationState()) { + const auto& requested = args.Request.GetShardCreationState(); + + if (requested.HasVersion() && + requested.GetVersion() == + GetFileSystem().GetShardCreationState().GetVersion()) + { + if (auto error = ValidateCompressedBitmapData( + requested.GetCreatedShardBitmap(), + Config->GetMaxShardCount()); + HasError(error)) + { + LOG_WARN( + ctx, + TFileStoreComponents::TABLET, + "%s Invalid shard creation state: %s", + LogTag.c_str(), + FormatError(error).c_str()); + + *args.Response.MutableError() = std::move(error); + return; + } + + auto newState = requested; + newState.SetVersion(requested.GetVersion() + 1); + SetShardCreationState(*db, newState); + } + } } void TIndexTabletActor::CompleteTx_UnsafeChangeTabletState( @@ -934,14 +1043,22 @@ void TIndexTabletActor::CompleteTx_UnsafeChangeTabletState( auto response = std::make_unique(); + if (HasError(args.Response)) { + *response->Record.MutableError() = args.Response.GetError(); + } + + if (args.Request.HasShardCreationState()) { + *response->Record.MutableShardCreationState() = + GetFileSystem().GetShardCreationState(); + } LOG_INFO( ctx, TFileStoreComponents::TABLET, "%s UnsafeChangeTabletState: %s, result: %s", LogTag.c_str(), - args.Request.DebugString().Quote().c_str(), - response->Record.ShortUtf8DebugString().Quote().c_str()); + DescribeChangeTabletStateRequest(args.Request).Quote().c_str(), + DescribeChangeTabletStateResponse(response->Record).Quote().c_str()); NCloud::Reply(ctx, *args.RequestInfo, std::move(response)); } diff --git a/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp b/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp index 92965f5efd8..92924a8103b 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp @@ -302,6 +302,15 @@ void TIndexTabletActor::HandleUpdateConfig( newConfig.SetStrictFileSystemSizeEnforcementEnabled( oldConfig.GetStrictFileSystemSizeEnforcementEnabled()); + // Preserve the shard creation state across AlterFileStore/resize + // UpdateConfig calls. Otherwise a retry could wipe the persisted bitmap and + // reset the CAS version while the resize actor still needs them for + // subsequent shard creation state updates. + if (oldConfig.HasShardCreationState()) { + *newConfig.MutableShardCreationState() = + oldConfig.GetShardCreationState(); + } + // Config update occured due to alter/resize. if (auto error = ValidateUpdateConfigRequest(oldConfig, newConfig)) { LOG_ERROR(ctx, TFileStoreComponents::TABLET, diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.cpp b/cloud/filestore/libs/storage/tablet/tablet_state.cpp index fd4a72afeb2..7a5b109fab5 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.cpp +++ b/cloud/filestore/libs/storage/tablet/tablet_state.cpp @@ -352,6 +352,14 @@ void TIndexTabletState::SetCompressNodeRef( db.WriteFileSystem(FileSystem); } +void TIndexTabletState::SetShardCreationState( + IIndexTabletDatabase& db, + const NProtoPrivate::TFileSystemShardCreationState& state) +{ + *FileSystem.MutableShardCreationState() = state; + db.WriteFileSystem(FileSystem); +} + const NProto::TFileStorePerformanceProfile& TIndexTabletState::GetPerformanceProfile() const { if (FileSystem.HasPerformanceProfile() && diff --git a/cloud/filestore/libs/storage/tablet/tablet_state.h b/cloud/filestore/libs/storage/tablet/tablet_state.h index 4f2d4072d02..d09cc06cb6f 100644 --- a/cloud/filestore/libs/storage/tablet/tablet_state.h +++ b/cloud/filestore/libs/storage/tablet/tablet_state.h @@ -350,6 +350,10 @@ class TIndexTabletState void SetCompressNodeRef(IIndexTabletDatabase& db, bool compressNodeRef); + void SetShardCreationState( + IIndexTabletDatabase& db, + const NProtoPrivate::TFileSystemShardCreationState& state); + protected: // // FileSystemEvents - appended to the transaction's sink, only if diff --git a/cloud/filestore/private/api/protos/tablet.proto b/cloud/filestore/private/api/protos/tablet.proto index d28b3154fc4..08d32ad2ea9 100644 --- a/cloud/filestore/private/api/protos/tablet.proto +++ b/cloud/filestore/private/api/protos/tablet.proto @@ -1146,12 +1146,18 @@ message TUnsafeChangeTabletStateRequest // Compress ShardId and ShardNodeName when we write them to NodeRefs table. optional bool CompressNodeRef = 3; + + // Update shard creation state during large operations. + TFileSystemShardCreationState ShardCreationState = 4; } message TUnsafeChangeTabletStateResponse { // Optional error, set only if error happened. NCloud.NProto.TError Error = 1; + + // Returns current shard creation state. + TFileSystemShardCreationState ShardCreationState = 2; } ////////////////////////////////////////////////////////////////////////////////