Skip to content

[Filestore] [4/N] Persist shard creation progress across throttled create and resize retries #7253

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Maximus5 wants to merge 1 commit into from
Open

[Filestore] [4/N] Persist shard creation progress across throttled create and resize retries #7253

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 17 additions & 21 deletions cloud/filestore/libs/storage/service/shard_creation_state_companion.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -259,27 +259,23 @@ void TShardCreationStateCompanion::UpdateShardCreationState(
return;
}

// Implement this block after adding ShardCreationState
// into TEvUnsafeChangeTabletStateRequest
Y_ABORT("add ShardCreationState into TEvUnsafeChangeTabletStateRequest");

// auto request =
// std::make_unique<TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>();
// request->Record.SetFileSystemId(FileSystemId);
// auto* shardCreationState = request->Record.MutableShardCreationState();
// shardCreationState->SetVersion(ShardCreationState.GetVersion());
// shardCreationState->SetBaseShardCount(
// ShardCreationState.GetBaseShardCount());
// shardCreationState->SetTargetShardCount(
// ShardCreationState.GetTargetShardCount());
// shardCreationState->SetTargetShardConfigHash(
// ShardCreationState.GetTargetShardConfigHash());
// SaveCompressedBitmap(
// *CreatedShardBitmap,
// ShardBitmapBitCount,
// *shardCreationState->MutableCreatedShardBitmap());

// NCloud::Send(ctx, MakeIndexTabletProxyServiceId(), std::move(request));
auto request =
std::make_unique<TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>();
request->Record.SetFileSystemId(FileSystemId);
auto* shardCreationState = request->Record.MutableShardCreationState();
shardCreationState->SetVersion(ShardCreationState.GetVersion());
shardCreationState->SetBaseShardCount(
ShardCreationState.GetBaseShardCount());
shardCreationState->SetTargetShardCount(
ShardCreationState.GetTargetShardCount());
shardCreationState->SetTargetShardConfigHash(
ShardCreationState.GetTargetShardConfigHash());
SaveCompressedBitmap(
*CreatedShardBitmap,
ShardBitmapBitCount,
*shardCreationState->MutableCreatedShardBitmap());

NCloud::Send(ctx, MakeIndexTabletProxyServiceId(), std::move(request));
}

void TShardCreationStateCompanion::OnShardCreated(
Expand Down
110 changes: 110 additions & 0 deletions cloud/filestore/libs/storage/service/shard_creation_state_companion_ut.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,17 @@ NProtoPrivate::TFileSystemShardCreationState MakeState(
return MakeState(version, createdShards, 0, MakeShardConfigs());
}

std::unique_ptr<NCloud::TCompressedBitmap> LoadCreatedShardBitmap(
const NProtoPrivate::TFileSystemShardCreationState& state)
{
auto result = LoadCompressedBitmap(
state.GetCreatedShardBitmap(),
state.GetCreatedShardBitmap().GetBitCount(),
MaxShardCount);
UNIT_ASSERT_C(!HasError(result), FormatError(result.GetError()));
return result.ExtractResult();
}

////////////////////////////////////////////////////////////////////////////////

enum ETestEvents
Expand Down Expand Up @@ -327,6 +338,105 @@ Y_UNIT_TEST_SUITE(TShardCreationStateCompanionTest)
UNIT_ASSERT_VALUES_EQUAL(11, companion.GetShardCreationStateVersion());
}

Y_UNIT_TEST(ShouldSendUpdatedShardCreationState)
{
TActorSystem runtime;
runtime.Start();

const auto sender = runtime.AllocateEdgeActor();
const auto tabletProxy = runtime.AllocateEdgeActor();
runtime.RegisterService(MakeIndexTabletProxyServiceId(), tabletProxy);

const auto actorId =
runtime.Register(new TActorWithCompanion(MakeState(42, {1})));

runtime.Send(new IEventHandle(
actorId,
sender,
new TEvOnShardCreated(3)));

auto request = runtime.GrabEdgeEvent<
TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>(
tabletProxy);

const auto& record = request->Get()->Record;
UNIT_ASSERT_VALUES_EQUAL(FileSystemId, record.GetFileSystemId());
UNIT_ASSERT(record.HasShardCreationState());

const auto& state = record.GetShardCreationState();
UNIT_ASSERT_VALUES_EQUAL(42, state.GetVersion());
UNIT_ASSERT_VALUES_EQUAL(0, state.GetBaseShardCount());
UNIT_ASSERT_VALUES_EQUAL(ShardCount, state.GetTargetShardCount());
UNIT_ASSERT_VALUES_EQUAL(
CalculateShardCreationTargetHashForTest(0, MakeShardConfigs()),
state.GetTargetShardConfigHash());
UNIT_ASSERT_VALUES_EQUAL(
ShardCount,
state.GetCreatedShardBitmap().GetBitCount());

const auto bitmap = LoadCreatedShardBitmap(state);
UNIT_ASSERT_VALUES_EQUAL(2, bitmap->Count());
UNIT_ASSERT(!bitmap->Test(0));
UNIT_ASSERT(bitmap->Test(1));
UNIT_ASSERT(!bitmap->Test(2));
UNIT_ASSERT(bitmap->Test(3));
}

Y_UNIT_TEST(ShouldResendMergedBitmapAfterCasConflict)
{
TActorSystem runtime;
runtime.Start();

const auto sender = runtime.AllocateEdgeActor();
const auto tabletProxy = runtime.AllocateEdgeActor();
runtime.RegisterService(MakeIndexTabletProxyServiceId(), tabletProxy);

const auto actorId =
runtime.Register(new TActorWithCompanion(MakeState(42, {})));

runtime.Send(new IEventHandle(
actorId,
sender,
new TEvOnShardCreated(3)));

auto request = runtime.GrabEdgeEvent<
TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>(
tabletProxy);

{
const auto& state = request->Get()->Record.GetShardCreationState();
UNIT_ASSERT_VALUES_EQUAL(42, state.GetVersion());

const auto bitmap = LoadCreatedShardBitmap(state);
UNIT_ASSERT_VALUES_EQUAL(1, bitmap->Count());
UNIT_ASSERT(!bitmap->Test(1));
UNIT_ASSERT(bitmap->Test(3));
}

runtime.Send(new IEventHandle(
actorId,
sender,
new TEvMergeShardCreationState(MakeState(43, {1}))));

request = runtime.GrabEdgeEvent<
TEvIndexTablet::TEvUnsafeChangeTabletStateRequest>(
tabletProxy);

const auto& state = request->Get()->Record.GetShardCreationState();
UNIT_ASSERT_VALUES_EQUAL(43, state.GetVersion());
UNIT_ASSERT_VALUES_EQUAL(0, state.GetBaseShardCount());
UNIT_ASSERT_VALUES_EQUAL(ShardCount, state.GetTargetShardCount());
UNIT_ASSERT_VALUES_EQUAL(
CalculateShardCreationTargetHashForTest(0, MakeShardConfigs()),
state.GetTargetShardConfigHash());

const auto bitmap = LoadCreatedShardBitmap(state);
UNIT_ASSERT_VALUES_EQUAL(2, bitmap->Count());
UNIT_ASSERT(bitmap->Test(1));
UNIT_ASSERT(!bitmap->Test(2));
UNIT_ASSERT(bitmap->Test(3));
}

Y_UNIT_TEST(ShouldRejectMergingDifferentTarget)
{
TShardCreationStateCompanion companion(
Expand Down
5 changes: 5 additions & 0 deletions cloud/filestore/libs/storage/tablet/protos/tablet.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ message TFileSystem

// Compress ShardId and ShardNodeName when we write them to NodeRefs table.
optional bool CompressNodeRef = 25;

// Persisted physical shard creation progress. Intentionally retained after
// successful resize: clearing it could erase recovery bits from another
// in-flight resize. Bits below current topology size are committed history.
NProtoPrivate.TFileSystemShardCreationState ShardCreationState = 26;
}

////////////////////////////////////////////////////////////////////////////////
Expand Down
127 changes: 122 additions & 5 deletions cloud/filestore/libs/storage/tablet/tablet_actor_unsafe_node_ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

#include "helpers.h"

#include <cloud/storage/core/libs/common/compressed_bitmap_serialization.h>

#include <util/string/builder.h>

namespace NCloud::NFileStore::NStorage {

using namespace NActors;
Expand All @@ -11,6 +15,84 @@ using namespace NKikimr::NTabletFlatExecutor;

////////////////////////////////////////////////////////////////////////////////

namespace {

ui64 GetChunkDataSize(const NCloud::NProto::TCompressedBitmapData& bitmap)

@SvartMetal SvartMetal Oct 23, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we put it to some reusable place?

{
ui64 dataSize = 0;
for (const auto& chunk: bitmap.GetChunks()) {
dataSize += chunk.GetData().size();
}

return dataSize;
}

TString DescribeShardCreationState(
const NProtoPrivate::TFileSystemShardCreationState& state)
{
const auto& bitmap = state.GetCreatedShardBitmap();

TStringBuilder out;
out << "{Version: ";
if (state.HasVersion()) {
out << state.GetVersion();
} else {
out << "<unset>";
}
out << ", BaseShardCount: " << state.GetBaseShardCount()
<< ", TargetShardCount: " << state.GetTargetShardCount()
<< ", TargetShardConfigHash: " << state.GetTargetShardConfigHash()
<< ", CreatedShardBitmap: {BitCount: " << bitmap.GetBitCount()
<< ", Chunks: " << bitmap.ChunksSize()
<< ", ChunkDataSize: " << GetChunkDataSize(bitmap) << "}}";

return out;
}

TString DescribeChangeTabletStateRequest(
const NProtoPrivate::TUnsafeChangeTabletStateRequest& request)
{
if (!request.HasShardCreationState()) {
return request.DebugString();
}

TStringBuilder out;
out << "{FileSystemId: " << request.GetFileSystemId();
if (request.HasFrozen()) {
out << ", Frozen: " << (request.GetFrozen() ? "true" : "false");
}
if (request.HasCompressNodeRef()) {
out << ", CompressNodeRef: "
<< (request.GetCompressNodeRef() ? "true" : "false");
}
out << ", ShardCreationState: "
<< DescribeShardCreationState(request.GetShardCreationState()) << "}";

return out;
}

TString DescribeChangeTabletStateResponse(
const NProtoPrivate::TUnsafeChangeTabletStateResponse& response)
{
if (!response.HasShardCreationState()) {
return response.ShortUtf8DebugString();
}

TStringBuilder out;
out << "{";
if (HasError(response.GetError())) {
out << "Error: " << FormatError(response.GetError()) << ", ";
}
out << "ShardCreationState: "
<< DescribeShardCreationState(response.GetShardCreationState()) << "}";

return out;
}

} // namespace

////////////////////////////////////////////////////////////////////////////////

void TIndexTabletActor::HandleUnsafeCreateNode(
const TEvIndexTablet::TEvUnsafeCreateNodeRequest::TPtr& ev,
const TActorContext& ctx)
Expand Down Expand Up @@ -886,7 +968,7 @@ void TIndexTabletActor::HandleUnsafeChangeTabletState(
TFileStoreComponents::TABLET,
"%s UnsafeChangeTabletState: %s",
LogTag.c_str(),
msg->Record.DebugString().Quote().c_str());
DescribeChangeTabletStateRequest(msg->Record).Quote().c_str());

ExecuteTx<TUnsafeChangeTabletState>(
ctx,
Expand All @@ -913,8 +995,6 @@ void TIndexTabletActor::ExecuteTx_UnsafeChangeTabletState(
TTransactionContext& tx,
TTxIndexTablet::TUnsafeChangeTabletState& args)
{
Y_UNUSED(ctx);

auto db = CreateIndexTabletDatabase(tx.DB);

if (args.Request.HasCompressNodeRef()) {
Expand All @@ -924,6 +1004,35 @@ void TIndexTabletActor::ExecuteTx_UnsafeChangeTabletState(
if (args.Request.HasFrozen()) {
SetFrozen(*db, args.Request.GetFrozen());
}

if (args.Request.HasShardCreationState()) {
const auto& requested = args.Request.GetShardCreationState();

if (requested.HasVersion() &&
requested.GetVersion() ==
GetFileSystem().GetShardCreationState().GetVersion())
{
if (auto error = ValidateCompressedBitmapData(
requested.GetCreatedShardBitmap(),
Config->GetMaxShardCount());
HasError(error))
{
LOG_WARN(
ctx,
TFileStoreComponents::TABLET,
"%s Invalid shard creation state: %s",
LogTag.c_str(),
FormatError(error).c_str());

*args.Response.MutableError() = std::move(error);
return;
}

auto newState = requested;
newState.SetVersion(requested.GetVersion() + 1);
SetShardCreationState(*db, newState);
}
}
}

void TIndexTabletActor::CompleteTx_UnsafeChangeTabletState(
Expand All @@ -934,14 +1043,22 @@ void TIndexTabletActor::CompleteTx_UnsafeChangeTabletState(

auto response =
std::make_unique<TEvIndexTablet::TEvUnsafeChangeTabletStateResponse>();
if (HasError(args.Response)) {
*response->Record.MutableError() = args.Response.GetError();
}

if (args.Request.HasShardCreationState()) {
*response->Record.MutableShardCreationState() =
GetFileSystem().GetShardCreationState();
}

LOG_INFO(
ctx,
TFileStoreComponents::TABLET,
"%s UnsafeChangeTabletState: %s, result: %s",
LogTag.c_str(),
args.Request.DebugString().Quote().c_str(),
response->Record.ShortUtf8DebugString().Quote().c_str());
DescribeChangeTabletStateRequest(args.Request).Quote().c_str(),
DescribeChangeTabletStateResponse(response->Record).Quote().c_str());

NCloud::Reply(ctx, *args.RequestInfo, std::move(response));
}
Expand Down
9 changes: 9 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_actor_updateconfig.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -302,6 +302,15 @@ void TIndexTabletActor::HandleUpdateConfig(
newConfig.SetStrictFileSystemSizeEnforcementEnabled(
oldConfig.GetStrictFileSystemSizeEnforcementEnabled());

// Preserve the shard creation state across AlterFileStore/resize
// UpdateConfig calls. Otherwise a retry could wipe the persisted bitmap and
// reset the CAS version while the resize actor still needs them for
// subsequent shard creation state updates.
if (oldConfig.HasShardCreationState()) {
*newConfig.MutableShardCreationState() =
oldConfig.GetShardCreationState();
}

// Config update occured due to alter/resize.
if (auto error = ValidateUpdateConfigRequest(oldConfig, newConfig)) {
LOG_ERROR(ctx, TFileStoreComponents::TABLET,
Expand Down
8 changes: 8 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_state.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,14 @@ void TIndexTabletState::SetCompressNodeRef(
db.WriteFileSystem(FileSystem);
}

void TIndexTabletState::SetShardCreationState(
IIndexTabletDatabase& db,
const NProtoPrivate::TFileSystemShardCreationState& state)
{
*FileSystem.MutableShardCreationState() = state;
db.WriteFileSystem(FileSystem);
}

const NProto::TFileStorePerformanceProfile& TIndexTabletState::GetPerformanceProfile() const
{
if (FileSystem.HasPerformanceProfile() &&
Expand Down
4 changes: 4 additions & 0 deletions cloud/filestore/libs/storage/tablet/tablet_state.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,10 @@ class TIndexTabletState

void SetCompressNodeRef(IIndexTabletDatabase& db, bool compressNodeRef);

void SetShardCreationState(
IIndexTabletDatabase& db,
const NProtoPrivate::TFileSystemShardCreationState& state);

protected:
//
// FileSystemEvents - appended to the transaction's sink, only if
Expand Down
Loading
Loading